Start fixing flakey tests ❄️ (#3329)

* Wait longer on authentication flow tests.

* Move default perceptualPrecision value into the snapshot preferences.

* Delay snapshots *before* setting up the test.

* Reset the simulators on GitHub before running?

* Remove a test that is now handled by Rust.

* Fix a test that was yielding.

.github/workflows/integration-tests.yml

@@ -30,6 +30,9 @@ jobs:
         run:
           source ci_scripts/ci_common.sh && setup_github_actions_environment
 
+      - name: Reset simulators
+        run: SNAPSHOT_FORCE_DELETE=1 bundle exec fastlane snapshot reset_simulators
+
       - name: Run tests
         run: bundle exec fastlane integration_tests
         env:

.github/workflows/unit_tests.yml

@@ -36,6 +36,9 @@ jobs:
         run:
           swiftformat --lint .
 
+      - name: Reset simulators
+        run: SNAPSHOT_FORCE_DELETE=1 bundle exec fastlane snapshot reset_simulators
+      
       - name: Run tests
         run: bundle exec fastlane unit_tests
         

.github/workflows/unit_tests_enterprise.yml

@@ -45,6 +45,9 @@ jobs:
       - name: SwiftFormat
         run: swiftformat --lint .
 
+      - name: Reset simulators
+        run: SNAPSHOT_FORCE_DELETE=1 bundle exec fastlane snapshot reset_simulators
+
       - name: Run tests
         run: bundle exec fastlane unit_tests skip_previews:true
         

ElementX/Sources/Other/Extensions/Snapshotting.swift

@@ -33,7 +33,7 @@ public struct SnapshotPrecisionPreferenceKey: PreferenceKey {
 }
 
 public struct SnapshotPerceptualPrecisionPreferenceKey: PreferenceKey {
-    public static var defaultValue: Float = 1.0
+    public static var defaultValue: Float = 0.98
 
     public static func reduce(value: inout Float, nextValue: () -> Float) {
         value = nextValue()
@@ -50,7 +50,7 @@ public extension SwiftUI.View {
     ///   - precision: The percentage of pixels that must match.
     ///   - perceptualPrecision: The percentage a pixel must match the source pixel to be considered a match. 98-99% mimics the precision of the human eye.
     @inlinable
-    func snapshotPreferences(delay: TimeInterval = .zero, precision: Float = 1.0, perceptualPrecision: Float = 1.0) -> some SwiftUI.View {
+    func snapshotPreferences(delay: TimeInterval = .zero, precision: Float = 1.0, perceptualPrecision: Float = 0.98) -> some SwiftUI.View {
         preference(key: SnapshotDelayPreferenceKey.self, value: delay)
             .preference(key: SnapshotPrecisionPreferenceKey.self, value: precision)
             .preference(key: SnapshotPerceptualPrecisionPreferenceKey.self, value: perceptualPrecision)

PreviewTests/Sources/PreviewTests.swift

@@ -11,6 +11,7 @@ import XCTest
 @testable import ElementX
 @testable import SnapshotTesting
 
+@MainActor
 class PreviewTests: XCTestCase {
     private let deviceConfig: ViewImageConfig = .iPhoneX
     private let simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation
@@ -50,6 +51,22 @@ class PreviewTests: XCTestCase {
     // MARK: - Snapshots
 
     func assertSnapshots(matching preview: _Preview, testName: String = #function) {
+        let preferences = SnapshotPreferences()
+
+        let preferenceReadingView = preview.content
+            .onPreferenceChange(SnapshotDelayPreferenceKey.self) { preferences.delay = $0 }
+            .onPreferenceChange(SnapshotPrecisionPreferenceKey.self) { preferences.precision = $0 }
+            .onPreferenceChange(SnapshotPerceptualPrecisionPreferenceKey.self) { preferences.perceptualPrecision = $0 }
+        
+        // Render an image of the view in order to trigger the preference updates to occur.
+        let imageRenderer = ImageRenderer(content: preferenceReadingView)
+        _ = imageRenderer.uiImage
+        
+        // Delay the test now - a delay after creating the `snapshotView` results in the underlying view not getting updated for snapshotting.
+        if preferences.delay != .zero {
+            wait(for: preferences.delay)
+        }
+        
         for deviceName in snapshotDevices {
             guard var device = PreviewDevice(rawValue: deviceName).snapshotDevice() else {
                 fatalError("Unknown device name: \(deviceName)")
@@ -58,12 +75,13 @@ class PreviewTests: XCTestCase {
             device.safeArea = .one
             // Ignore specific device display scale
             let traits = UITraitCollection(displayScale: 2.0)
-            if let failure = assertSnapshots(matching: AnyView(preview.content),
+            if let failure = assertSnapshots(matching: preview.content,
                                              name: preview.displayName,
                                              isScreen: preview.layout == .device,
                                              device: device,
                                              testName: testName + deviceName + "-" + localeCode,
-                                             traits: traits) {
+                                             traits: traits,
+                                             preferences: preferences) {
                 XCTFail(failure)
             }
         }
@@ -85,19 +103,12 @@ class PreviewTests: XCTestCase {
     }
 
     private func assertSnapshots(matching view: AnyView,
-                                 name: String?, isScreen: Bool,
+                                 name: String?,
+                                 isScreen: Bool,
                                  device: ViewImageConfig,
                                  testName: String = #function,
-                                 traits: UITraitCollection = .init()) -> String? {
-        var delay: TimeInterval = 0
-        var precision: Float = 1
-        var perceptualPrecision: Float = 1
-
-        let view = view
-            .onPreferenceChange(SnapshotDelayPreferenceKey.self) { delay = $0 }
-            .onPreferenceChange(SnapshotPrecisionPreferenceKey.self) { precision = $0 }
-            .onPreferenceChange(SnapshotPerceptualPrecisionPreferenceKey.self) { perceptualPrecision = $0 }
-
+                                 traits: UITraitCollection = .init(),
+                                 preferences: SnapshotPreferences) -> String? {
         let matchingView = isScreen ? AnyView(view) : AnyView(view
             .frame(width: device.size?.width)
             .fixedSize(horizontal: false, vertical: true)
@@ -105,15 +116,27 @@ class PreviewTests: XCTestCase {
         
         return withSnapshotTesting(record: recordMode) {
             verifySnapshot(of: matchingView,
-                           as: .prefireImage(precision: { precision },
-                                             perceptualPrecision: { perceptualPrecision },
-                                             duration: { delay },
+                           as: .prefireImage(preferences: preferences,
                                              layout: isScreen ? .device(config: device) : .sizeThatFits,
                                              traits: traits),
                            named: name,
                            testName: testName)
         }
     }
+    
+    private func wait(for duration: TimeInterval) {
+        let expectation = XCTestExpectation(description: "Wait")
+        DispatchQueue.main.asyncAfter(deadline: .now() + duration) {
+            expectation.fulfill()
+        }
+        _ = XCTWaiter.wait(for: [expectation], timeout: duration + 1)
+    }
+}
+
+private class SnapshotPreferences {
+    var delay: TimeInterval = 0
+    var precision: Float = 1
+    var perceptualPrecision: Float = 1
 }
 
 // MARK: - SnapshotTesting + Extensions
@@ -144,9 +167,7 @@ private extension PreviewDevice {
 
 private extension Snapshotting where Value: SwiftUI.View, Format == UIImage {
     static func prefireImage(drawHierarchyInKeyWindow: Bool = false,
-                             precision: @escaping () -> Float,
-                             perceptualPrecision: @escaping () -> Float,
-                             duration: @escaping () -> TimeInterval,
+                             preferences: SnapshotPreferences,
                              layout: SwiftUISnapshotLayout = .sizeThatFits,
                              traits: UITraitCollection = .init()) -> Snapshotting {
         let config: ViewImageConfig
@@ -165,7 +186,7 @@ private extension Snapshotting where Value: SwiftUI.View, Format == UIImage {
             config = .init(safeArea: .one, size: size, traits: traits)
         }
 
-        return SimplySnapshotting<UIImage>(pathExtension: "png", diffing: .prefireImage(precision: precision, perceptualPrecision: perceptualPrecision, scale: traits.displayScale))
+        return SimplySnapshotting<UIImage>(pathExtension: "png", diffing: .prefireImage(preferences: preferences, scale: traits.displayScale))
             .asyncPullback { view in
                 var config = config
 
@@ -182,30 +203,18 @@ private extension Snapshotting where Value: SwiftUI.View, Format == UIImage {
                     controller = hostingController
                 }
                 
-                return Async<UIImage> { callback in
-                    let strategy = snapshotView(config: config,
-                                                drawHierarchyInKeyWindow: drawHierarchyInKeyWindow,
-                                                traits: traits,
-                                                view: controller.view,
-                                                viewController: controller)
-
-                    let duration = duration()
-                    if duration != .zero {
-                        let expectation = XCTestExpectation(description: "Wait")
-                        DispatchQueue.main.asyncAfter(deadline: .now() + duration) {
-                            expectation.fulfill()
-                        }
-                        _ = XCTWaiter.wait(for: [expectation], timeout: duration + 1)
-                    }
-                    strategy.run(callback)
-                }
+                return snapshotView(config: config,
+                                    drawHierarchyInKeyWindow: drawHierarchyInKeyWindow,
+                                    traits: traits,
+                                    view: controller.view,
+                                    viewController: controller)
             }
     }
 }
 
 private extension Diffing where Value == UIImage {
-    static func prefireImage(precision: @escaping () -> Float, perceptualPrecision: @escaping () -> Float, scale: CGFloat?) -> Diffing {
-        lazy var originalDiffing = Diffing.image(precision: precision(), perceptualPrecision: 0.98, scale: scale)
+    static func prefireImage(preferences: SnapshotPreferences, scale: CGFloat?) -> Diffing {
+        lazy var originalDiffing = Diffing.image(precision: preferences.precision, perceptualPrecision: preferences.perceptualPrecision, scale: scale)
         return Diffing(toData: { originalDiffing.toData($0) },
                        fromData: { originalDiffing.fromData($0) },
                        diff: { originalDiffing.diff($0, $1) })

UnitTests/Sources/ComposerToolbarViewModelTests.swift

@@ -113,11 +113,11 @@ class ComposerToolbarViewModelTests: XCTestCase {
         XCTAssertEqual(viewModel.state.suggestions, suggestions)
     }
     
-    func testSuggestionTrigger() async {
+    func testSuggestionTrigger() async throws {
+        let deferred = deferFulfillment(wysiwygViewModel.$attributedContent) { $0.plainText == "#not_implemented_yay" }
         wysiwygViewModel.setMarkdownContent("@test")
         wysiwygViewModel.setMarkdownContent("#not_implemented_yay")
-        
-        await Task.yield()
+        try await deferred.fulfill()
         
         // The first one is nil because when initialised the view model is empty
         XCTAssertEqual(completionSuggestionServiceMock.setSuggestionTriggerReceivedInvocations, [nil, .init(type: .user, text: "test", range: .init(location: 0, length: 5)), nil])

UnitTests/Sources/TimelineViewModelTests.swift

@@ -256,39 +256,6 @@ class TimelineViewModelTests: XCTestCase {
         XCTAssertEqual(arguments?.type, .read)
     }
     
-    func testSendMoreReadReceipts() async throws {
-        // Given a room with only text items in the timeline that are all read.
-        let items = [TextRoomTimelineItem(eventID: "t1"),
-                     TextRoomTimelineItem(eventID: "t2"),
-                     TextRoomTimelineItem(eventID: "t3")]
-        let (viewModel, _, timelineProxy, timelineController) = readReceiptsConfiguration(with: items)
-        viewModel.context.send(viewAction: .sendReadReceiptIfNeeded(items.last!.id))
-        try await Task.sleep(for: .milliseconds(100))
-        XCTAssertEqual(timelineProxy.sendReadReceiptForTypeCallsCount, 1)
-        var arguments = timelineProxy.sendReadReceiptForTypeReceivedArguments
-        XCTAssertEqual(arguments?.eventID, "t3")
-        XCTAssertEqual(arguments?.type, .read)
-        
-        // When sending a receipt for the first item in the timeline.
-        viewModel.context.send(viewAction: .sendReadReceiptIfNeeded(items.first!.id))
-        try await Task.sleep(for: .milliseconds(100))
-        
-        // When a new message is received and marked as read.
-        let newMessage = TextRoomTimelineItem(eventID: "t4")
-        timelineController.timelineItems.append(newMessage)
-        timelineController.callbacks.send(.updatedTimelineItems(timelineItems: timelineController.timelineItems, isSwitchingTimelines: false))
-        try await Task.sleep(for: .milliseconds(100))
-        
-        viewModel.context.send(viewAction: .sendReadReceiptIfNeeded(newMessage.id))
-        try await Task.sleep(for: .milliseconds(100))
-        
-        // Then the request should be made.
-        XCTAssertEqual(timelineProxy.sendReadReceiptForTypeCallsCount, 3)
-        arguments = timelineProxy.sendReadReceiptForTypeReceivedArguments
-        XCTAssertEqual(arguments?.eventID, "t4")
-        XCTAssertEqual(arguments?.type, .read)
-    }
-    
     func testSendReadReceiptWithoutEvents() async throws {
         // Given a room with only virtual items.
         let items = [SeparatorRoomTimelineItem(timelineID: "v1"),

UnitTests/Sources/UserSessionFlowCoordinatorTests.swift

@@ -246,7 +246,7 @@ class UserSessionFlowCoordinatorTests: XCTestCase {
     
     private func process(route: AppRoute, expectedState: UserSessionFlowCoordinatorStateMachine.State) async throws {
         // Sometimes the state machine's state changes before the coordinators have updated the stack.
-        let delayedPublisher = userSessionFlowCoordinator.statePublisher.delay(for: .milliseconds(10), scheduler: DispatchQueue.main)
+        let delayedPublisher = userSessionFlowCoordinator.statePublisher.delay(for: .milliseconds(100), scheduler: DispatchQueue.main)
         
         let deferred = deferFulfillment(delayedPublisher) { $0 == expectedState }
         userSessionFlowCoordinator.handleAppRoute(route, animated: true)