Prefire multiple devices (#2543)

* Update Prefire, setup multiple snapshot devices and languages
Squashed commits:
[28cb4ae6d] Switch to macos-14, iOS 17 and the iPhone 15 simulator
[080fc82cb] Update snapshots
[c874ca9c7] Bump Compound and Prefire, reduce perceptual precission to 0.98
[a59b8a54c] Update snapshots after adding iPad and pseudolanguage snapshotting
[f30cc3277] Setup multi-language snapshotting
[aeb11c7e7] Update prefire template, setup multiple snapshot devices

* Update ruby dependencies and fastlane plugins

* Bump Compound, switch back to the original prefire - version 2.0.4

* Remove preview test simulator version specifiers

.github/workflows/unit_tests.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   tests:
     name: Tests
-    runs-on: macos-13
+    runs-on: macos-14
 
     concurrency:
       # When running on develop, use the sha to allow all runs of this workflow to run concurrently.
@@ -39,13 +39,6 @@ jobs:
         run:
           swiftformat --lint .
       
-      - name: Link to 16.4 Simulators
-        run: |
-          echo "Creating Runtimes folder if needed..."
-          sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
-          echo "Creating symlink of the iOS 16.4 runtime..."
-          sudo ln -s /Applications/Xcode_14.3.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 16.4.simruntime
-
       - name: Run tests
         run: bundle exec fastlane unit_tests
         

.prefire.yml

@@ -1,4 +0,0 @@
-test_configuration:
-  - template_file_path: Tools/Prefire/PreviewTests.stencil
-  - simulator_device: "iPhone14"
-  - required_os: 16

ElementX.xcodeproj/project.pbxproj

@@ -7072,7 +7072,7 @@
 			repositoryURL = "https://github.com/BarredEwe/Prefire";
 			requirement = {
 				kind = upToNextMinorVersion;
-				minimumVersion = 1.5.0;
+				minimumVersion = 2.0.4;
 			};
 		};
 		395DE6AE429B7ACC7C7FE31D /* XCRemoteSwiftPackageReference "KZFileWatchers" */ = {
@@ -7240,7 +7240,7 @@
 			repositoryURL = "https://github.com/element-hq/compound-ios";
 			requirement = {
 				kind = revision;
-				revision = 19c778b6ce0b82601ecdca23ff8d4a7aa41f990f;
+				revision = 19bc4c5cf9c69b7b7aeb9ac8c937c2c81a83e8b1;
 			};
 		};
 		F76A08D0EA29A07A54F4EB4D /* XCRemoteSwiftPackageReference "swift-collections" */ = {

ElementX.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -14,7 +14,7 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/element-hq/compound-ios",
       "state" : {
-        "revision" : "19c778b6ce0b82601ecdca23ff8d4a7aa41f990f"
+        "revision" : "19bc4c5cf9c69b7b7aeb9ac8c937c2c81a83e8b1"
       }
     },
     {
@@ -175,8 +175,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/BarredEwe/Prefire",
       "state" : {
-        "revision" : "898a4a9f5d5eb0a0b07adb1a7c89daf0f068b129",
-        "version" : "1.5.0"
+        "revision" : "608e7992dedc5ee409e59b3580010371ff0cef57",
+        "version" : "2.0.4"
       }
     },
     {
@@ -263,7 +263,7 @@
     {
       "identity" : "swiftui-introspect",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/siteline/SwiftUI-Introspect",
+      "location" : "https://github.com/siteline/SwiftUI-Introspect.git",
       "state" : {
         "revision" : "b94da693e57eaf79d16464b8b7c90d09cba4e290",
         "version" : "0.9.2"

ElementX/Sources/Screens/RoomDetailsEditScreen/View/RoomDetailsEditScreen.swift

@@ -176,12 +176,12 @@ struct RoomDetailsEditScreen_Previews: PreviewProvider, TestablePreview {
             RoomDetailsEditScreen(context: viewModel.context)
         }
         .previewDisplayName("Normal")
-        .snapshot(delay: 0.1)
+        .snapshot(delay: 0.25)
         
         NavigationStack {
             RoomDetailsEditScreen(context: readOnlyViewModel.context)
         }
         .previewDisplayName("Read only")
-        .snapshot(delay: 0.1)
+        .snapshot(delay: 0.25)
     }
 }

Gemfile.lock

@@ -7,15 +7,17 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    CFPropertyList (3.0.6)
+    CFPropertyList (3.0.7)
+      base64
+      nkf
       rexml
     addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
-    artifactory (3.0.15)
+    artifactory (3.0.17)
     atomos (0.1.3)
     aws-eventstream (1.3.0)
-    aws-partitions (1.887.0)
-    aws-sdk-core (3.191.0)
+    aws-partitions (1.896.0)
+    aws-sdk-core (3.191.3)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
@@ -30,6 +32,7 @@ GEM
     aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
     babosa (1.0.4)
+    base64 (0.2.0)
     claide (1.1.0)
     colored (1.2)
     colored2 (3.1.2)
@@ -115,7 +118,7 @@ GEM
     fastlane-plugin-brew (0.1.1)
     fastlane-plugin-browserstack (0.3.2)
       rest-client (~> 2.0, >= 2.0.2)
-    fastlane-plugin-sentry (1.18.0)
+    fastlane-plugin-sentry (1.20.0)
       os (~> 1.1, >= 1.1.4)
     fastlane-plugin-xcconfig (2.0.0)
     fastlane-plugin-xcodegen (1.1.0)
@@ -137,12 +140,12 @@ GEM
       google-apis-core (>= 0.11.0, < 2.a)
     google-apis-storage_v1 (0.31.0)
       google-apis-core (>= 0.11.0, < 2.a)
-    google-cloud-core (1.6.1)
+    google-cloud-core (1.7.0)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
     google-cloud-env (1.6.0)
       faraday (>= 0.17.3, < 3.0)
-    google-cloud-errors (1.3.1)
+    google-cloud-errors (1.4.0)
     google-cloud-storage (1.47.0)
       addressable (~> 2.8)
       digest-crc (~> 0.4)
@@ -164,17 +167,19 @@ GEM
     httpclient (2.8.3)
     jmespath (1.6.2)
     json (2.7.1)
-    jwt (2.7.1)
+    jwt (2.8.1)
+      base64
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.1205)
+    mime-types-data (3.2024.0305)
     mini_magick (4.12.0)
     mini_mime (1.1.5)
     multi_json (1.15.0)
-    multipart-post (2.3.0)
+    multipart-post (2.4.0)
     nanaimo (0.3.0)
     naturally (2.2.1)
     netrc (0.11.0)
+    nkf (0.2.0)
     optparse (0.4.0)
     os (1.1.4)
     plist (3.7.1)
@@ -196,7 +201,7 @@ GEM
     rubyzip (2.3.2)
     security (0.1.3)
     semantic (1.6.1)
-    signet (0.18.0)
+    signet (0.19.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)

PreviewTests/.prefire.yml

@@ -0,0 +1,7 @@
+test_configuration:
+  - template_file_path: PreviewTests.stencil
+  - simulator_device: "iPhone15,2"
+  - required_os: 17
+  - snapshot_devices:
+      - iPhone 15
+      - iPad

PreviewTests/PreviewTests.stencil

@@ -4,7 +4,6 @@
 import XCTest
 import SwiftUI
 import Prefire
-
 @testable import SnapshotTesting
 #if canImport(AccessibilitySnapshot)
     import AccessibilitySnapshot
@@ -12,11 +11,24 @@ import Prefire
 {% if argument.mainTarget %}
 @testable import {{ argument.mainTarget }}
 {% endif %}
+{% for import in argument.imports %}
+{% if import != "last" %}
+import {{ import }}
+{% endif %}
+{% endfor %}
+{% for import in argument.testableImports %}
+{% if import != "last" %}
+@testable import {{ import }}
+{% endif %}
+{% endfor %}
 
 class PreviewTests: XCTestCase {
     private let deviceConfig: ViewImageConfig = .iPhoneX
-    private let simulatorDevice = "{{ argument.simulatorDevice|default:"iPhone15,2" }}"
-    private let requiredOSVersion = {{ argument.simulatorOSVersion|default:"16" }}
+    private var simulatorDevice: String?{% if argument.simulatorDevice %} = "{{ argument.simulatorDevice|default:nil }}"{% endif %}
+    private var requiredOSVersion: Int?{% if argument.simulatorOSVersion %} = {{ argument.simulatorOSVersion }}{% endif %}
+    private let snapshotDevices: [String]{% if argument.snapshotDevices %} = {{ argument.snapshotDevices|split:"|" }}{% else %} = []{% endif %}
+    
+    
     {% if argument.file %}
 
     private var file: StaticString { .init(stringLiteral: "{{ argument.file }}") }
@@ -29,40 +41,104 @@ class PreviewTests: XCTestCase {
         UIView.setAnimationsEnabled(false)
     }
 
+    // MARK: - PreviewProvider
+
     {% for type in types.types where (type.implements.PrefireProvider or type.based.PrefireProvider or type|annotated:"PrefireProvider") and type.name != "TestablePreview" %}
     func test_{{ type.name|lowerFirstLetter|replace:"_Previews", "" }}() {
         for preview in {{ type.name }}._allPreviews {
             assertSnapshots(matching: preview)
         }
     }
+    {%- if not forloop.last %}
 
+    {% endif %}
     {% endfor %}
+    {% if argument.previewsMacros %}
+    // MARK: - Macros
+
+    {{ argument.previewsMacros }}
+    {% endif %}
     // MARK: Private
 
     private func assertSnapshots(matching preview: _Preview, testName: String = #function) {
-        let isScreen = preview.layout == .device
-        let device = preview.device?.snapshotDevice() ?? deviceConfig
+        guard !snapshotDevices.isEmpty else {
+            if let failure = assertSnapshots(matching: AnyView(preview.content),
+                                             name: preview.displayName,
+                                             isScreen: preview.layout == .device,
+                                             device: preview.device?.snapshotDevice() ?? deviceConfig,
+                                             testName: testName) {
+                XCTFail(failure)
+            }
+            return
+        }
+        
+        for deviceName in snapshotDevices {
+            guard var device = PreviewDevice(rawValue: deviceName).snapshotDevice() else {
+                fatalError("Unknown device name: \(deviceName)")
+            }
+            
+            // Ignore specific device safe area
+            device.safeArea = .zero
+            
+            // Ignore specific device display scale
+            let traits = UITraitCollection(displayScale: 2.0)
+            
+            if let failure = assertSnapshots(matching: AnyView(preview.content),
+                                            name: preview.displayName,
+                                            isScreen: preview.layout == .device,
+                                            device: device,
+                                            testName: testName + deviceName + "-" + localeCode,
+                                            traits: traits) {
+                XCTFail(failure)
+            }
+        }
+    }
+    
+    private var localeCode: String {
+        if UserDefaults.standard.bool(forKey: "NSDoubleLocalizedStrings") {
+            return "pseudo"
+        }
+        return languageCode + "-" + regionCode
+    }
+
+    private var languageCode: String {
+        Locale.current.language.languageCode?.identifier ?? ""
+    }
+
+    private var regionCode: String {
+        Locale.current.language.region?.identifier ?? ""
+    }
+
+    private func assertSnapshots(matching view: AnyView,
+                                 name: String?, isScreen: Bool,
+                                 device: ViewImageConfig,
+                                 testName: String = #function,
+                                 traits: UITraitCollection = .init()) -> String? {
         var delay: TimeInterval = 0
         var precision: Float = 1
+        var perceptualPrecision: Float = 1
 
-        let view = preview.content
+        let view = view
             .onPreferenceChange(DelayPreferenceKey.self) { delay = $0 }
             .onPreferenceChange(PrecisionPreferenceKey.self) { precision = $0 }
+            .onPreferenceChange(PerceptualPrecisionPreferenceKey.self) { perceptualPrecision = $0 }
 
         let matchingView = isScreen ? AnyView(view) : AnyView(view
             .frame(width: device.size?.width)
             .fixedSize(horizontal: false, vertical: true)
         )
 
-        let failure = verifySnapshot(of: matchingView,
-                                    as: .prefireImage(precision: { precision }, duration: { delay }, layout: isScreen ? .device(config: device) : .sizeThatFits),
-                                    named: preview.displayName{% if argument.file %},
-                                    file: file{% endif %},
-                                    testName: testName)
-
-        if let failure {
-            XCTFail(failure)
-        }
+        let failure = verifySnapshot(
+            of: matchingView,
+            as: .prefireImage(precision: { precision },
+                              perceptualPrecision: { perceptualPrecision },
+                              duration: { delay },
+                              layout: isScreen ? .device(config: device) : .sizeThatFits,
+                              traits: traits),
+            named: name{% if argument.file %},
+            file: file{% endif %},
+            testName: testName
+        )
 
         #if canImport(AccessibilitySnapshot)
             let vc = UIHostingController(rootView: matchingView)
@@ -70,27 +146,28 @@ class PreviewTests: XCTestCase {
             assertSnapshot(
                 matching: vc,
                 as: .wait(for: delay, on: .accessibilityImage(showActivationPoints: .always)),
-                named: preview.displayName.map { $0 + ".accessibility" }{% if argument.file %},
+                named: name.flatMap { $0 + ".accessibility" }{% if argument.file %},
                 file: file{% endif %},
                 testName: testName
             )
         #endif
+        return failure
     }
 
     /// Check environments to avoid problems with snapshots on different devices or OS.
     private func checkEnvironments() {
-        let deviceModel = ProcessInfo().environment["SIMULATOR_MODEL_IDENTIFIER"]
-        let osVersion = ProcessInfo().operatingSystemVersion
-        guard deviceModel?.contains(simulatorDevice) ?? false else {
-            fatalError("Switch to using \(simulatorDevice) for these tests.")
-        }
-	
-        guard osVersion.majorVersion == requiredOSVersion else {
-            fatalError("Switch to iOS \(requiredOSVersion) for these tests.")
+        if let simulatorDevice {
+            let deviceModel = ProcessInfo().environment["SIMULATOR_MODEL_IDENTIFIER"]
+            guard deviceModel?.contains(simulatorDevice) ?? false else {
+                fatalError("Switch to using \(simulatorDevice) for these tests.")
+            }
         }
 
-        guard UITraitCollection.current.userInterfaceStyle == .light else {
-            fatalError("Switch to light mode for these tests.")
+        if let requiredOSVersion {
+            let osVersion = ProcessInfo().operatingSystemVersion
+            guard osVersion.majorVersion == requiredOSVersion else {
+                fatalError("Switch to iOS \(requiredOSVersion) for these tests.")
+            }
         }
     }
 }
@@ -100,7 +177,7 @@ class PreviewTests: XCTestCase {
 private extension PreviewDevice {
     func snapshotDevice() -> ViewImageConfig? {
         switch rawValue {
-        case "iPhone 14", "iPhone 13", "iPhone 12", "iPhone 11", "iPhone 10":
+        case "iPhone 15", "iPhone 14", "iPhone 13", "iPhone 12", "iPhone 11", "iPhone 10":
             return .iPhoneX
         case "iPhone 6", "iPhone 6s", "iPhone 7", "iPhone 8":
             return .iPhone8
@@ -108,6 +185,14 @@ private extension PreviewDevice {
             return .iPhone8Plus
         case "iPhone SE (1st generation)", "iPhone SE (2nd generation)":
             return .iPhoneSe
+        case "iPad":
+            return .iPad10_2
+        case "iPad Mini":
+            return .iPadMini
+        case "iPad Pro 11":
+            return .iPadPro11
+        case "iPad Pro 12.9":
+            return .iPadPro12_9
         default: return nil
         }
     }
@@ -117,6 +202,7 @@ private extension Snapshotting where Value: SwiftUI.View, Format == UIImage {
     static func prefireImage(
         drawHierarchyInKeyWindow: Bool = false,
         precision: @escaping () -> Float,
+        perceptualPrecision: @escaping () -> Float,
         duration: @escaping () -> TimeInterval,
         layout: SwiftUISnapshotLayout = .sizeThatFits,
         traits: UITraitCollection = .init()
@@ -135,7 +221,7 @@ private extension Snapshotting where Value: SwiftUI.View, Format == UIImage {
             config = .init(safeArea: .zero, size: size, traits: traits)
         }
 
-        return SimplySnapshotting<UIImage>(pathExtension: "png", diffing: .prefireImage(precision: precision, scale: traits.displayScale))
+        return SimplySnapshotting<UIImage>(pathExtension: "png", diffing: .prefireImage(precision: precision, perceptualPrecision: perceptualPrecision, scale: traits.displayScale))
             .asyncPullback { view in
                 var config = config
 
@@ -176,7 +262,7 @@ private extension Snapshotting where Value: SwiftUI.View, Format == UIImage {
 }
 
 private extension Diffing where Value == UIImage {
-    static func prefireImage(precision: @escaping () -> Float, scale: CGFloat?) -> Diffing {
+    static func prefireImage(precision: @escaping () -> Float, perceptualPrecision: @escaping () -> Float, scale: CGFloat?) -> Diffing {
         lazy var originalDiffing = Diffing.image(precision: precision(), perceptualPrecision: 0.98, scale: scale)
         return Diffing(
             toData: { originalDiffing.toData($0) },

PreviewTests/SupportingFiles/PreviewTests.xctestplan

@@ -2,9 +2,26 @@
   "configurations" : [
     {
       "id" : "D789E5F1-30B8-4A23-B20E-281ACE05CFD6",
-      "name" : "Default",
+      "name" : "English",
       "options" : {
-
+        "language" : "en",
+        "locationScenario" : {
+          "identifier" : "London, England",
+          "referenceType" : "built-in"
+        },
+        "region" : "GB"
+      }
+    },
+    {
+      "id" : "059C4F1C-EAB6-4D96-BA8F-30DF92A365E9",
+      "name" : "Pseudolanguage",
+      "options" : {
+        "language" : "IDELaunchSchemeLanguageDoubleLocalizedStrings",
+        "locationScenario" : {
+          "identifier" : "London, England",
+          "referenceType" : "built-in"
+        },
+        "region" : "DE"
       }
     }
   ],