Fixing Tests for Xcode 14.3 (#888)

* settting macOS to 13, and let's runa test of... the tests

* I want the artifacts of the test, might give me some more insight in Xcode

* archive artifact

* trying a thing with the xcode version

* push notifications alert makes this test fail

* test improvement

* let's try with a very long time

* let's disable autocorrection

* sleep

* 1 seconds fixed 90% of the issue, 2 should fix 100%

* waiting some more time in flaky tests and updated the content of a test that was failing

* updated a screenshot test that had a notification

* this integration test is a bit so flaky increased the timing

* controlled delay waited a bit moe for the timeline to settle

* try await

* MainActor everywhere

* milliseconds fix

* trying with a bit more waiting time after the tap and a slow velocity for the swipe

* let's try waiting more time

.github/workflows/integration-tests.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   integration_tests:
     name: Integration Tests
-    runs-on: macos-12
+    runs-on: macos-13
 
     concurrency:
       # Only allow a single run of this workflow on each branch, automatically cancelling older runs.
@@ -36,6 +36,15 @@ jobs:
           INTEGRATION_TESTS_HOST: ${{ secrets.INTEGRATION_TESTS_HOST }}
           INTEGRATION_TESTS_USERNAME: ${{ secrets.INTEGRATION_TESTS_USERNAME }}
           INTEGRATION_TESTS_PASSWORD: ${{ secrets.INTEGRATION_TESTS_PASSWORD }}
+      
+      - name: Archive artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: test-output
+          path: fastlane/test_output
+          retention-days: 7
+          if-no-files-found: ignore
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

.github/workflows/ui_tests.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   tests:
     name: Tests
-    runs-on: macos-12
+    runs-on: macos-13
 
     concurrency:
       # When running on develop, use the sha to allow all runs of this workflow to run concurrently.

ElementX/Sources/UITests/UITestsAppCoordinator.swift

@@ -254,7 +254,7 @@ class MockScreen: Identifiable {
             navigationStackCoordinator.setRootCoordinator(coordinator)
             return navigationStackCoordinator
         case .sessionVerification:
-            var sessionVerificationControllerProxy = SessionVerificationControllerProxyMock.configureMock(requestDelay: .seconds(2))
+            var sessionVerificationControllerProxy = SessionVerificationControllerProxyMock.configureMock(requestDelay: .seconds(5))
             let parameters = SessionVerificationScreenCoordinatorParameters(sessionVerificationControllerProxy: sessionVerificationControllerProxy)
             return SessionVerificationScreenCoordinator(parameters: parameters)
         case .userSessionScreen:

IntegrationTests/Sources/LoginTests.swift

@@ -40,34 +40,34 @@ class LoginTests: XCTestCase {
                 
         let getStartedButton = app.buttons[A11yIdentifiers.onboardingScreen.signIn]
         
-        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
         getStartedButton.tap()
-        
+
         let editHomeserverButton = app.buttons[A11yIdentifiers.loginScreen.changeServer]
-        XCTAssertTrue(editHomeserverButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(editHomeserverButton.waitForExistence(timeout: 10.0))
         editHomeserverButton.tap()
         
         let homeserverTextField = app.textFields[A11yIdentifiers.changeServerScreen.server]
-        XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 10.0))
         
         homeserverTextField.clearAndTypeText(app.homeserver)
                 
         let confirmButton = app.buttons[A11yIdentifiers.changeServerScreen.continue]
-        XCTAssertTrue(confirmButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(confirmButton.waitForExistence(timeout: 10.0))
         confirmButton.tap()
         
         let usernameTextField = app.textFields[A11yIdentifiers.loginScreen.emailUsername]
-        XCTAssertTrue(usernameTextField.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(usernameTextField.waitForExistence(timeout: 10.0))
         
         usernameTextField.clearAndTypeText(app.username)
         
         let passwordTextField = app.secureTextFields[A11yIdentifiers.loginScreen.password]
-        XCTAssertTrue(passwordTextField.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(passwordTextField.waitForExistence(timeout: 10.0))
         
         passwordTextField.clearAndTypeText(app.password)
         
         let nextButton = app.buttons[A11yIdentifiers.loginScreen.continue]
-        XCTAssertTrue(nextButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(nextButton.waitForExistence(timeout: 10.0))
         XCTAssertTrue(nextButton.isEnabled)
         
         nextButton.tap()
@@ -79,38 +79,38 @@ class LoginTests: XCTestCase {
         
         // Handle save password sheet
         let savePasswordButton = app.buttons["Save Password"]
-        if savePasswordButton.waitForExistence(timeout: 1.0) {
+        if savePasswordButton.waitForExistence(timeout: 10.0) {
             savePasswordButton.tap()
         }
         
         // Handle analytics prompt screen
         if app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0) {
             let enableButton = app.buttons[A11yIdentifiers.analyticsPromptScreen.enable]
-            XCTAssertTrue(enableButton.waitForExistence(timeout: 1.0))
+            XCTAssertTrue(enableButton.waitForExistence(timeout: 10.0))
             enableButton.tap()
         }
         
         // Handle the notifications permission alert https://stackoverflow.com/a/58171074/730924
         let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
         let alertAllowButton = springboard.buttons.element(boundBy: 1)
-        if alertAllowButton.waitForExistence(timeout: 1.0) {
+        if alertAllowButton.waitForExistence(timeout: 10.0) {
             alertAllowButton.tap()
         }
         
         let profileButton = app.buttons[A11yIdentifiers.homeScreen.userAvatar]
-        XCTAssertTrue(profileButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(profileButton.waitForExistence(timeout: 10.0))
 
         // `Failed to scroll to visible (by AX action) Button` https://stackoverflow.com/a/33534187/730924
         profileButton.forceTap()
         
         let menuLogoutButton = app.buttons["Sign out"]
-        XCTAssertTrue(menuLogoutButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(menuLogoutButton.waitForExistence(timeout: 10.0))
         menuLogoutButton.tap()
         
         let alertLogoutButton = app.buttons["Sign out"]
-        XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(alertLogoutButton.waitForExistence(timeout: 10.0))
         alertLogoutButton.tap()
         
-        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 1.0))
+        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 10.0))
     }
 }

Tools/Scripts/Templates/SimpleScreenExample/Tests/UI/TemplateScreenUITests.swift

@@ -17,8 +17,9 @@
 import ElementX
 import XCTest
 
+@MainActor
 class TemplateScreenUITests: XCTestCase {
-    func testRegularScreen() {
+    func testRegularScreen() async throws {
         let app = Application.launch(.simpleRegular)
         
         let title = app.staticTexts["title"]
@@ -26,10 +27,10 @@ class TemplateScreenUITests: XCTestCase {
         
         XCTAssertEqual(title.label, "Make this chat public?")
 
-        app.assertScreenshot(.simpleRegular)
+        try await app.assertScreenshot(.simpleRegular)
     }
     
-    func testUpgradeScreen() {
+    func testUpgradeScreen() async throws {
         let app = Application.launch(.simpleUpgrade)
         
         let title = app.staticTexts["title"]
@@ -37,6 +38,6 @@ class TemplateScreenUITests: XCTestCase {
         
         XCTAssertEqual(title.label, "Privacy warning")
 
-        app.assertScreenshot(.simpleUpgrade)
+        try await app.assertScreenshot(.simpleUpgrade)
     }
 }

UITests/Sources/AnalyticsPromptUITests.swift

@@ -17,10 +17,11 @@
 import ElementX
 import XCTest
 
+@MainActor
 class AnalyticsPromptUITests: XCTestCase {
     /// Verify that the prompt is displayed correctly.
-    func testAnalyticsPrompt() {
+    func testAnalyticsPrompt() async throws {
         let app = Application.launch(.analyticsPrompt)
-        app.assertScreenshot(.analyticsPrompt)
+        try await app.assertScreenshot(.analyticsPrompt)
     }
 }

UITests/Sources/AnalyticsSettingsScreenUITests.swift

@@ -17,10 +17,11 @@
 import ElementX
 import XCTest
 
+@MainActor
 class AnalyticsSettingsScreenUITests: XCTestCase {
     /// Verify that the analytics option screen is displayed correctly.
-    func testAnalyticsSettingsScreen() {
+    func testAnalyticsSettingsScreen() async throws {
         let app = Application.launch(.analyticsSettingsScreen)
-        app.assertScreenshot(.analyticsSettingsScreen)
+        try await app.assertScreenshot(.analyticsSettingsScreen)
     }
 }

UITests/Sources/Application.swift

@@ -30,16 +30,21 @@ struct Application {
 }
 
 extension XCUIApplication {
+    @MainActor
     /// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created.
     /// - Parameter identifier: Identifier of the UI test screen
     /// - Parameter step: An optional integer that can be used to take multiple snapshots per test identifier.
     /// - Parameter insets: Optional insets with which to crop the image by.
-    func assertScreenshot(_ identifier: UITestsScreenIdentifier, step: Int? = nil, insets: UIEdgeInsets? = nil) {
+    func assertScreenshot(_ identifier: UITestsScreenIdentifier, step: Int? = nil, insets: UIEdgeInsets? = nil, delayInMilliseconds: UInt = 2000, precision: Float = 0.99) async throws {
         var snapshotName = identifier.rawValue
         if let step {
             snapshotName += "-\(step)"
         }
-        
+
+        // Sometimes the CI might be too slow to load the content so let's wait the delay time
+        if delayInMilliseconds > 0 {
+            try await Task.sleep(for: .milliseconds(delayInMilliseconds))
+        }
         var snapshot = XCUIScreen.main.screenshot().image
         
         if let insets {
@@ -47,7 +52,9 @@ extension XCUIApplication {
         }
 
         let failure = verifySnapshot(matching: snapshot,
-                                     as: .image(precision: 0.99, perceptualPrecision: 0.98, scale: nil),
+                                     as: .image(precision: precision,
+                                                perceptualPrecision: 0.98,
+                                                scale: nil),
                                      named: snapshotName,
                                      testName: testName)
 

UITests/Sources/AuthenticationCoordinatorUITests.swift

@@ -20,7 +20,7 @@ import XCTest
 
 @MainActor
 class AuthenticationCoordinatorUITests: XCTestCase {
-    func testLoginWithPassword() {
+    func testLoginWithPassword() async throws {
         // Given the authentication flow.
         let app = Application.launch(.authenticationFlow)
         
@@ -32,13 +32,12 @@ class AuthenticationCoordinatorUITests: XCTestCase {
         app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("alice\n")
         app.secureTextFields[A11yIdentifiers.loginScreen.password].clearAndTypeText("12345678")
 
-        app.assertScreenshot(.authenticationFlow)
+        try await app.assertScreenshot(.authenticationFlow)
         
         // Login Screen: Tap next
         app.buttons[A11yIdentifiers.loginScreen.continue].tap()
-        
-        // Then login should succeed.
-        XCTAssertFalse(app.alerts.element.exists, "No alert should be shown when logging in with valid credentials.")
+
+        XCTAssertTrue(app.staticTexts[A11yIdentifiers.analyticsPromptScreen.title].waitForExistence(timeout: 1.0), "The analytics prompt screen should be seen after login")
     }
     
     func testLoginWithIncorrectPassword() async throws {

UITests/Sources/BugReportUITests.swift

@@ -17,35 +17,36 @@
 import ElementX
 import XCTest
 
+@MainActor
 class BugReportUITests: XCTestCase {
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.bugReport)
         
         // Initial state without a screenshot attached.
-        app.assertScreenshot(.bugReport, step: 0)
+        try await app.assertScreenshot(.bugReport, step: 0)
     }
     
-    func testReportText() {
+    func testReportText() async throws {
         let app = Application.launch(.bugReport)
         
         // Type 4 characters and the send button should be disabled.
         app.textViews[A11yIdentifiers.bugReportScreen.report].clearAndTypeText("Text")
         XCTAssert(app.switches[A11yIdentifiers.bugReportScreen.sendLogs].isOn)
-        app.assertScreenshot(.bugReport, step: 2)
+        try await app.assertScreenshot(.bugReport, step: 2)
         
         // Type more than 4 characters and send the button should become enabled.
         app.textViews[A11yIdentifiers.bugReportScreen.report].clearAndTypeText("Longer text")
         XCTAssert(app.switches[A11yIdentifiers.bugReportScreen.sendLogs].isOn)
-        app.assertScreenshot(.bugReport, step: 3)
+        try await app.assertScreenshot(.bugReport, step: 3)
     }
     
-    func testInitialStateComponentsWithScreenshot() {
+    func testInitialStateComponentsWithScreenshot() async throws {
         let app = Application.launch(.bugReportWithScreenshot)
         
         // Initial state with a screenshot attached.
         XCTAssert(app.images[A11yIdentifiers.bugReportScreen.screenshot].exists)
         XCTAssert(app.buttons[A11yIdentifiers.bugReportScreen.removeScreenshot].exists)
-        app.assertScreenshot(.bugReportWithScreenshot)
+        try await app.assertScreenshot(.bugReportWithScreenshot)
     }
 }
 

UITests/Sources/DeveloperOptionsScreenUITests.swift

@@ -17,4 +17,5 @@
 import ElementX
 import XCTest
 
+@MainActor
 class DeveloperOptionsScreenUITests: XCTestCase { }

UITests/Sources/HomeScreenUITests.swift

@@ -16,9 +16,11 @@
 
 import XCTest
 
+@MainActor
 class HomeScreenUITests: XCTestCase {
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.home)
-        app.assertScreenshot(.home)
+        // The gradient of the skeleton canges dynamically over time so the time may influence the match, better to have a lower precision for this one
+        try await app.assertScreenshot(.home, precision: 0.95)
     }
 }

UITests/Sources/InviteUsersScreenUITests.swift

@@ -17,15 +17,16 @@
 import ElementX
 import XCTest
 
+@MainActor
 class InviteUsersScreenUITests: XCTestCase {
-    func testLanding() {
+    func testLanding() async throws {
         let app = Application.launch(.inviteUsers)
-        app.assertScreenshot(.inviteUsers)
+        try await app.assertScreenshot(.inviteUsers)
     }
 
-    func testSelectedUsers() {
+    func testSelectedUsers() async throws {
         let app = Application.launch(.inviteUsers)
         app.collectionViews.firstMatch.cells.element(boundBy: 1).tap()
-        app.assertScreenshot(.inviteUsers, step: 1)
+        try await app.assertScreenshot(.inviteUsers, step: 1)
     }
 }

UITests/Sources/InvitesScreenUITests.swift

@@ -17,29 +17,30 @@
 import ElementX
 import XCTest
 
+@MainActor
 class InvitesScreenUITests: XCTestCase {
-    func testInvitesWithNoBadges() {
+    func testInvitesWithNoBadges() async throws {
         let app = Application.launch(.invites)
-        app.assertScreenshot(.invites)
+        try await app.assertScreenshot(.invites)
     }
     
-    func testInvitesWithBadges() {
+    func testInvitesWithBadges() async throws {
         let app = Application.launch(.invitesWithBadges)
-        app.assertScreenshot(.invitesWithBadges)
+        try await app.assertScreenshot(.invitesWithBadges)
     }
     
-    func testNoInvites() {
+    func testNoInvites() async throws {
         let app = Application.launch(.invitesNoInvites)
         XCTAssertTrue(app.staticTexts[A11yIdentifiers.invitesScreen.noInvites].exists)
-        app.assertScreenshot(.invitesNoInvites)
+        try await app.assertScreenshot(.invitesNoInvites)
     }
     
-    func testDeclineInvite() {
+    func testDeclineInvite() async throws {
         let app = Application.launch(.invites)
         let declineButton = app.buttons[A11yIdentifiers.invitesScreen.decline].firstMatch
         XCTAssert(declineButton.exists)
         declineButton.tap()
         XCTAssertEqual(app.alerts.count, 1)
-        app.assertScreenshot(.invites, step: 1)
+        try await app.assertScreenshot(.invites, step: 1)
     }
 }

UITests/Sources/LoginScreenUITests.swift

@@ -19,20 +19,20 @@ import XCTest
 
 @MainActor
 class LoginScreenUITests: XCTestCase {
-    func testMatrixDotOrg() {
+    func testMatrixDotOrg() async throws {
         // Given the initial login screen which defaults to matrix.org.
         let app = Application.launch(.login)
-        app.assertScreenshot(.login)
+        try await app.assertScreenshot(.login)
         
         // When typing in a username and password.
         app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("@test:matrix.org")
         app.secureTextFields[A11yIdentifiers.loginScreen.password].clearAndTypeText("12345678")
         
         // Then the form should be ready to submit.
-        app.assertScreenshot(.login, step: 0)
+        try await app.assertScreenshot(.login, step: 0)
     }
     
-    func testOIDC() {
+    func testOIDC() async throws {
         // Given the initial login screen.
         let app = Application.launch(.login)
         
@@ -40,10 +40,10 @@ class LoginScreenUITests: XCTestCase {
         app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("@test:company.com\n")
         
         // Then the screen should be configured for OIDC.
-        app.assertScreenshot(.login, step: 1)
+        try await app.assertScreenshot(.login, step: 1)
     }
     
-    func testUnsupported() {
+    func testUnsupported() async throws {
         // Given the initial login screen.
         let app = Application.launch(.login)
         
@@ -51,6 +51,6 @@ class LoginScreenUITests: XCTestCase {
         app.textFields[A11yIdentifiers.loginScreen.emailUsername].clearAndTypeText("@test:server.net\n")
         
         // Then the screen should not allow login to continue.
-        app.assertScreenshot(.login, step: 2)
+        try await app.assertScreenshot(.login, step: 2)
     }
 }

UITests/Sources/MediaUploadPreviewScreenUITests.swift

@@ -17,4 +17,5 @@
 import ElementX
 import XCTest
 
+@MainActor
 class MediaUploadPreviewScreenUITests: XCTestCase { }

UITests/Sources/OnboardingUITests.swift

@@ -18,8 +18,8 @@ import XCTest
 
 @MainActor
 class OnboardingUITests: XCTestCase {
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.onboarding)
-        app.assertScreenshot(.onboarding)
+        try await app.assertScreenshot(.onboarding)
     }
 }

UITests/Sources/ReportContentScreenUITests.swift

@@ -17,9 +17,10 @@
 import ElementX
 import XCTest
 
+@MainActor
 class ReportContentScreenUITests: XCTestCase {
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.reportContent)
-        app.assertScreenshot(.reportContent, step: 0)
+        try await app.assertScreenshot(.reportContent, step: 0)
     }
 }

UITests/Sources/RoomDetailsScreenUITests.swift

@@ -17,28 +17,29 @@
 import ElementX
 import XCTest
 
+@MainActor
 class RoomDetailsScreenUITests: XCTestCase {
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.roomDetailsScreen)
         
         XCTAssert(app.staticTexts[A11yIdentifiers.roomDetailsScreen.avatar].exists)
         XCTAssert(app.buttons[A11yIdentifiers.roomDetailsScreen.people].waitForExistence(timeout: 1))
-        app.assertScreenshot(.roomDetailsScreen)
+        try await app.assertScreenshot(.roomDetailsScreen)
     }
 
-    func testInitialStateComponentsWithRoomAvatar() {
+    func testInitialStateComponentsWithRoomAvatar() async throws {
         let app = Application.launch(.roomDetailsScreenWithRoomAvatar)
 
         XCTAssert(app.images[A11yIdentifiers.roomDetailsScreen.avatar].waitForExistence(timeout: 1))
         XCTAssert(app.buttons[A11yIdentifiers.roomDetailsScreen.people].waitForExistence(timeout: 1))
-        app.assertScreenshot(.roomDetailsScreenWithRoomAvatar)
+        try await app.assertScreenshot(.roomDetailsScreenWithRoomAvatar)
     }
 
-    func testInitialStateComponentsDmDetails() {
+    func testInitialStateComponentsDmDetails() async throws {
         let app = Application.launch(.roomDetailsScreenDmDetails)
 
         XCTAssert(app.images[A11yIdentifiers.roomDetailsScreen.dmAvatar].waitForExistence(timeout: 1))
         XCTAssertFalse(app.buttons[A11yIdentifiers.roomDetailsScreen.people].exists)
-        app.assertScreenshot(.roomDetailsScreenDmDetails)
+        try await app.assertScreenshot(.roomDetailsScreenDmDetails)
     }
 }

UITests/Sources/RoomMemberDetailsScreenUITests.swift

@@ -17,28 +17,29 @@
 import ElementX
 import XCTest
 
+@MainActor
 class RoomMemberDetailsScreenUITests: XCTestCase {
-    func testInitialStateComponentsForAccountOwner() {
+    func testInitialStateComponentsForAccountOwner() async throws {
         let app = Application.launch(.roomMemberDetailsAccountOwner)
 
         XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.ignore].exists)
         XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.unignore].exists)
-        app.assertScreenshot(.roomMemberDetailsAccountOwner)
+        try await app.assertScreenshot(.roomMemberDetailsAccountOwner)
     }
 
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.roomMemberDetails)
 
         XCTAssert(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.ignore].waitForExistence(timeout: 1))
         XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.unignore].exists)
-        app.assertScreenshot(.roomMemberDetails)
+        try await app.assertScreenshot(.roomMemberDetails)
     }
 
-    func testInitialStateComponentsForIgnoredUser() {
+    func testInitialStateComponentsForIgnoredUser() async throws {
         let app = Application.launch(.roomMemberDetailsIgnoredUser)
 
         XCTAssertFalse(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.ignore].exists)
         XCTAssert(app.buttons[A11yIdentifiers.roomMemberDetailsScreen.unignore].waitForExistence(timeout: 1))
-        app.assertScreenshot(.roomMemberDetailsIgnoredUser)
+        try await app.assertScreenshot(.roomMemberDetailsIgnoredUser)
     }
 }

UITests/Sources/RoomMembersListScreenUITests.swift

@@ -17,34 +17,35 @@
 import ElementX
 import XCTest
 
+@MainActor
 class RoomMembersListScreenUITests: XCTestCase {
-    func testJoinedMembers() {
+    func testJoinedMembers() async throws {
         let app = Application.launch(.roomMembersListScreen)
         
-        app.assertScreenshot(.roomMembersListScreen)
+        try await app.assertScreenshot(.roomMembersListScreen)
     }
     
-    func testJoinedAndInvitedMembers() {
+    func testJoinedAndInvitedMembers() async throws {
         let app = Application.launch(.roomMembersListScreenPendingInvites)
         
-        app.assertScreenshot(.roomMembersListScreenPendingInvites)
+        try await app.assertScreenshot(.roomMembersListScreenPendingInvites)
     }
     
-    func testSearchInvitedMember() {
+    func testSearchInvitedMember() async throws {
         let app = Application.launch(.roomMembersListScreenPendingInvites)
         
         let searchBar = app.searchFields.firstMatch
         searchBar.clearAndTypeText("alice\n")
         
-        app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 1)
+        try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 1)
     }
     
-    func testSearchJoinedMember() {
+    func testSearchJoinedMember() async throws {
         let app = Application.launch(.roomMembersListScreenPendingInvites)
         
         let searchBar = app.searchFields.firstMatch
         searchBar.clearAndTypeText("bob\n")
         
-        app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 2)
+        try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 2)
     }
 }

UITests/Sources/RoomScreenUITests.swift

@@ -19,29 +19,29 @@ import XCTest
 
 @MainActor
 class RoomScreenUITests: XCTestCase {
-    func testPlainNoAvatar() {
+    func testPlainNoAvatar() async throws {
         let app = Application.launch(.roomPlainNoAvatar)
 
         XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.name].exists)
         XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.avatar].exists)
 
-        app.assertScreenshot(.roomPlainNoAvatar)
+        try await app.assertScreenshot(.roomPlainNoAvatar)
     }
 
-    func testEncryptedWithAvatar() {
+    func testEncryptedWithAvatar() async throws {
         let app = Application.launch(.roomEncryptedWithAvatar)
 
         XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.name].exists)
         XCTAssert(app.images[A11yIdentifiers.roomScreen.avatar].waitForExistence(timeout: 1))
 
-        app.assertScreenshot(.roomEncryptedWithAvatar)
+        try await app.assertScreenshot(.roomEncryptedWithAvatar)
     }
     
-    func testSmallTimelineLayout() {
+    func testSmallTimelineLayout() async throws {
         let app = Application.launch(.roomSmallTimeline)
         
         // The messages should be bottom aligned.
-        app.assertScreenshot(.roomSmallTimeline)
+        try await app.assertScreenshot(.roomSmallTimeline)
     }
     
     func testSmallTimelineWithIncomingAndPagination() async throws {
@@ -57,7 +57,7 @@ class RoomScreenUITests: XCTestCase {
         try await performOperation(.paginate, using: client)
 
         // Then the 4 visible messages should stay aligned to the bottom.
-        app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination)
+        try await app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination)
     }
     
     func testSmallTimelineWithLargePagination() async throws {
@@ -72,7 +72,7 @@ class RoomScreenUITests: XCTestCase {
         try await performOperation(.paginate, using: client)
 
         // The bottom of the timeline should remain visible with more items added above.
-        app.assertScreenshot(.roomSmallTimelineLargePagination)
+        try await app.assertScreenshot(.roomSmallTimelineLargePagination)
     }
     
     func testTimelineLayoutInMiddle() async throws {
@@ -82,29 +82,29 @@ class RoomScreenUITests: XCTestCase {
         
         await client.waitForApp()
         defer { try? client.stop() }
-        
+
         // Given a timeline that is neither at the top nor the bottom.
         app.tables.element.swipeDown()
-        try await Task.sleep(for: .milliseconds(500)) // Allow the table to settle
-        app.assertScreenshot(.roomLayoutMiddle, step: 0) // Assert initial state for comparison.
+        try await Task.sleep(for: .seconds(5)) // Allow the table to settle
+        try await app.assertScreenshot(.roomLayoutMiddle, step: 0) // Assert initial state for comparison.
         
         // When a back pagination occurs.
         try await performOperation(.paginate, using: client)
         
         // Then the UI should remain unchanged.
-        app.assertScreenshot(.roomLayoutMiddle, step: 0)
+        try await app.assertScreenshot(.roomLayoutMiddle, step: 0)
         
         // When an incoming message arrives
         try await performOperation(.incomingMessage, using: client)
         
         // Then the UI should still remain unchanged.
-        app.assertScreenshot(.roomLayoutMiddle, step: 0)
+        try await app.assertScreenshot(.roomLayoutMiddle, step: 0)
         
         // When the keyboard appears for the message composer.
         try await tapMessageComposer(in: app)
         
         // Then the timeline scroll offset should remain unchanged.
-        app.assertScreenshot(.roomLayoutMiddle, step: 1)
+        try await app.assertScreenshot(.roomLayoutMiddle, step: 1)
     }
     
     func testTimelineLayoutAtTop() async throws {
@@ -120,13 +120,13 @@ class RoomScreenUITests: XCTestCase {
             app.tables.element.swipeDown()
         }
         let cropped = UIEdgeInsets(top: 150, left: 0, bottom: 0, right: 0) // Ignore the navigation bar and pagination indicator as these change.
-        app.assertScreenshot(.roomLayoutTop, insets: cropped) // Assert initial state for comparison.
+        try await app.assertScreenshot(.roomLayoutTop, insets: cropped) // Assert initial state for comparison.
         
         // When a back pagination occurs.
         try await performOperation(.paginate, using: client)
 
         // Then the bottom of the timeline should remain unchanged (with new items having been added above).
-        app.assertScreenshot(.roomLayoutTop, insets: cropped)
+        try await app.assertScreenshot(.roomLayoutTop, insets: cropped)
     }
     
     func testTimelineLayoutAtBottom() async throws {
@@ -136,18 +136,23 @@ class RoomScreenUITests: XCTestCase {
         
         await client.waitForApp()
         defer { try? client.stop() }
-        
+
+        // Some time for the timeline to settle
+        try await Task.sleep(for: .seconds(2))
         // When an incoming message arrives.
         try await performOperation(.incomingMessage, using: client)
+        // Some time for the timeline to settle
+        try await Task.sleep(for: .seconds(2))
         
         // Then the timeline should scroll down to reveal the message.
-        app.assertScreenshot(.roomLayoutBottom, step: 0)
+        try await app.assertScreenshot(.roomLayoutBottom, step: 0)
         
         // When the keyboard appears for the message composer.
         try await tapMessageComposer(in: app)
-        
+
+        try await Task.sleep(for: .seconds(2))
         // Then the timeline should still show the last message.
-        app.assertScreenshot(.roomLayoutBottom, step: 1)
+        try await app.assertScreenshot(.roomLayoutBottom, step: 1)
     }
     
     // MARK: - Helper Methods
@@ -155,11 +160,11 @@ class RoomScreenUITests: XCTestCase {
     private func performOperation(_ operation: UITestsSignal, using client: UITestsSignalling.Client) async throws {
         try client.send(operation)
         await _ = client.signals.values.first { $0 == .success }
-        try await Task.sleep(for: .milliseconds(500)) // Allow the timeline to update
+        try await Task.sleep(for: .seconds(2)) // Allow the timeline to update
     }
     
     private func tapMessageComposer(in app: XCUIApplication) async throws {
         app.textViews.element.tap()
-        try await Task.sleep(for: .milliseconds(500)) // Allow the animations to complete
+        try await Task.sleep(for: .seconds(5)) // Allow the animations to complete
     }
 }

UITests/Sources/ServerSelectionUITests.swift

@@ -19,15 +19,15 @@ import XCTest
 
 @MainActor
 class ServerSelectionUITests: XCTestCase {
-    func testNormalState() async {
+    func testNormalState() async throws {
         // Given the initial server selection screen as a modal.
         let app = Application.launch(.serverSelection)
         
         // Then it should be configured for matrix.org
-        app.assertScreenshot(.serverSelection, step: 0)
+        try await app.assertScreenshot(.serverSelection, step: 0)
     }
 
-    func testEmptyAddress() async {
+    func testEmptyAddress() async throws {
         // Given the initial server selection screen as a modal.
         let app = Application.launch(.serverSelection)
         
@@ -36,10 +36,10 @@ class ServerSelectionUITests: XCTestCase {
         app.textFields[A11yIdentifiers.changeServerScreen.server].buttons.element.tap()
         
         // Then the screen should not allow the user to continue.
-        app.assertScreenshot(.serverSelection, step: 1)
+        try await app.assertScreenshot(.serverSelection, step: 1)
     }
 
-    func testInvalidAddress() {
+    func testInvalidAddress() async throws {
         // Given the initial server selection screen as a modal.
         let app = Application.launch(.serverSelection)
         
@@ -47,16 +47,16 @@ class ServerSelectionUITests: XCTestCase {
         app.textFields[A11yIdentifiers.changeServerScreen.server].clearAndTypeText("thisisbad\n") // The tests only accept an address from LoginHomeserver.mockXYZ
         
         // Then an error should be shown and the confirmation button disabled.
-        app.assertScreenshot(.serverSelection, step: 2)
+        try await app.assertScreenshot(.serverSelection, step: 2)
         XCTAssertFalse(app.buttons[A11yIdentifiers.changeServerScreen.continue].isEnabled, "The confirm button should be disabled when there is an error.")
     }
 
-    func testNonModalPresentation() {
+    func testNonModalPresentation() async throws {
         // Given the initial server selection screen pushed onto the stack.
         let app = Application.launch(.serverSelectionNonModal)
         
         // Then the screen should be tweaked slightly to reflect the change of navigation.
-        app.assertScreenshot(.serverSelectionNonModal)
+        try await app.assertScreenshot(.serverSelectionNonModal)
         XCTAssertFalse(app.buttons[A11yIdentifiers.changeServerScreen.dismiss].exists, "The dismiss button should be hidden when not in modal presentation.")
     }
 }

UITests/Sources/SessionVerificationUITests.swift

@@ -17,6 +17,7 @@
 import ElementX
 import XCTest
 
+@MainActor
 class SessionVerificationUITests: XCTestCase {
     enum Step {
         static let initialState = 0
@@ -30,71 +31,71 @@ class SessionVerificationUITests: XCTestCase {
         static let verificationCancelled = 7
     }
     
-    func testChallengeMatches() {
+    func testChallengeMatches() async throws {
         let app = Application.launch(.sessionVerification)
-        app.assertScreenshot(.sessionVerification, step: Step.initialState)
+        try await app.assertScreenshot(.sessionVerification, step: Step.initialState)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.requestVerification].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
+        try await app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
         
-        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
+        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
+        try await app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
         
-        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
+        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.acceptingEmojis)
+        try await app.assertScreenshot(.sessionVerification, step: Step.acceptingEmojis)
         
-        XCTAssert(app.staticTexts[A11yIdentifiers.sessionVerificationScreen.verificationComplete].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.verificationComplete)
+        XCTAssert(app.staticTexts[A11yIdentifiers.sessionVerificationScreen.verificationComplete].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.verificationComplete)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
     }
     
-    func testChallengeDoesNotMatch() {
+    func testChallengeDoesNotMatch() async throws {
         let app = Application.launch(.sessionVerification)
-        app.assertScreenshot(.sessionVerification, step: Step.initialState)
+        try await app.assertScreenshot(.sessionVerification, step: Step.initialState)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.requestVerification].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
+        try await app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
         
-        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
+        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
+        try await app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
         
-        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
+        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.declineChallenge].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
+        try await app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
     }
     
-    func testSessionVerificationCancelation() {
+    func testSessionVerificationCancelation() async throws {
         let app = Application.launch(.sessionVerification)
-        app.assertScreenshot(.sessionVerification, step: Step.initialState)
+        try await app.assertScreenshot(.sessionVerification, step: Step.initialState)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.requestVerification].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
+        try await app.assertScreenshot(.sessionVerification, step: Step.waitingForOtherDevice)
         
-        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
+        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.useEmojiComparisonPrompt)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.startSasVerification].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
+        try await app.assertScreenshot(.sessionVerification, step: Step.waitingForEmojis)
         
-        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 5.0))
-        app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
+        XCTAssert(app.buttons[A11yIdentifiers.sessionVerificationScreen.acceptChallenge].waitForExistence(timeout: 10.0))
+        try await app.assertScreenshot(.sessionVerification, step: Step.compareEmojis)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
-        app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
+        try await app.assertScreenshot(.sessionVerification, step: Step.verificationCancelled)
         
         app.buttons[A11yIdentifiers.sessionVerificationScreen.close].tap()
     }

UITests/Sources/SettingsScreenUITests.swift

@@ -17,9 +17,10 @@
 import ElementX
 import XCTest
 
+@MainActor
 class SettingsScreenUITests: XCTestCase {
-    func testInitialStateComponents() {
+    func testInitialStateComponents() async throws {
         let app = Application.launch(.settings)
-        app.assertScreenshot(.settings)
+        try await app.assertScreenshot(.settings)
     }
 }

UITests/Sources/SoftLogoutUITests.swift

@@ -26,7 +26,7 @@ class SoftLogoutUITests: XCTestCase {
         app = nil
     }
 
-    func testInitialState() {
+    func testInitialState() async throws {
         app = Application.launch(.softLogout)
         
         XCTAssertTrue(app.staticTexts[A11yIdentifiers.softLogoutScreen.title].exists, "The title should be shown.")
@@ -38,6 +38,6 @@ class SoftLogoutUITests: XCTestCase {
         XCTAssertTrue(app.buttons[A11yIdentifiers.softLogoutScreen.forgotPassword].exists, "The forgot password button should be shown.")
         XCTAssertTrue(app.buttons[A11yIdentifiers.softLogoutScreen.clearData].exists, "The clear data button should be shown.")
 
-        app.assertScreenshot(.softLogout)
+        try await app.assertScreenshot(.softLogout)
     }
 }

UITests/Sources/StartChatScreenUITests.swift

@@ -17,26 +17,27 @@
 import ElementX
 import XCTest
 
+@MainActor
 class StartChatScreenUITests: XCTestCase {
-    func testLanding() {
+    func testLanding() async throws {
         let app = Application.launch(.startChat)
-        app.assertScreenshot(.startChat)
+        try await app.assertScreenshot(.startChat)
     }
     
-    func testSearchWithNoResults() {
+    func testSearchWithNoResults() async throws {
         let app = Application.launch(.startChat)
         let searchField = app.searchFields.firstMatch
         searchField.clearAndTypeText("None")
         XCTAssert(app.staticTexts[A11yIdentifiers.startChatScreen.searchNoResults].waitForExistence(timeout: 1.0))
-        app.assertScreenshot(.startChat, step: 1)
+        try await app.assertScreenshot(.startChat, step: 1)
     }
     
-    func testSearchWithResults() {
+    func testSearchWithResults() async throws {
         let app = Application.launch(.startChatWithSearchResults)
         let searchField = app.searchFields.firstMatch
         searchField.clearAndTypeText("Bob")
         XCTAssertFalse(app.staticTexts[A11yIdentifiers.startChatScreen.searchNoResults].waitForExistence(timeout: 1.0))
         XCTAssertEqual(app.collectionViews.firstMatch.cells.count, 2)
-        app.assertScreenshot(.startChat, step: 2)
+        try await app.assertScreenshot(.startChat, step: 2)
     }
 }

UITests/Sources/UserSessionScreenTests.swift

@@ -24,7 +24,7 @@ class UserSessionScreenTests: XCTestCase {
         
         let app = Application.launch(.userSessionScreen)
 
-        app.assertScreenshot(.userSessionScreen, step: 1)
+        try await app.assertScreenshot(.userSessionScreen, step: 1)
         
         app.buttons[A11yIdentifiers.homeScreen.roomName(roomName)].tap()
         
@@ -32,10 +32,10 @@ class UserSessionScreenTests: XCTestCase {
         
         try await Task.sleep(for: .seconds(1))
     
-        app.assertScreenshot(.userSessionScreen, step: 2)
+        try await app.assertScreenshot(.userSessionScreen, step: 2)
         
         app.buttons[A11yIdentifiers.roomScreen.attachmentPicker].tap()
         
-        app.assertScreenshot(.userSessionScreen, step: 3)
+        try await app.assertScreenshot(.userSessionScreen, step: 3)
     }
 }

fastlane/Fastfile

@@ -2,7 +2,7 @@ require 'yaml'
 require_relative 'changelog'
 
 before_all do
-  xcversion(version: "~> 14.1")
+  xcversion(version: "~> 14.3")
 
   ENV["FASTLANE_XCODEBUILD_SETTINGS_TIMEOUT"] = "180"
   ENV["FASTLANE_XCODE_LIST_TIMEOUT"] = "180"
@@ -142,6 +142,7 @@ lane :integration_tests do
     scheme: "IntegrationTests",
     devices: ["iPhone 13 Pro"],
     ensure_devices_found: true,
+    result_bundle: true
   )
 
   slather(