Use an environment variable to record failures.

This commit is contained in:
Doug 2024-09-19 10:30:56 +01:00 committed by Doug
parent d41dd47b51
commit d2d3b28dc7
4 changed files with 34 additions and 27 deletions

View File

@ -11,18 +11,19 @@ import XCTest
@testable import ElementX @testable import ElementX
@testable import SnapshotTesting @testable import SnapshotTesting
#if canImport(AccessibilitySnapshot)
import AccessibilitySnapshot
#endif
class PreviewTests: XCTestCase { class PreviewTests: XCTestCase {
private let deviceConfig: ViewImageConfig = .iPhoneX private let deviceConfig: ViewImageConfig = .iPhoneX
private var simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation private let simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation
private var requiredOSVersion = (major: 18, minor: 0) private let requiredOSVersion = (major: 18, minor: 0)
private let snapshotDevices = ["iPhone 16", "iPad"] private let snapshotDevices = ["iPhone 16", "iPad"]
private var recordMode: SnapshotTestingConfiguration.Record = .missing
override func setUp() { override func setUp() {
super.setUp() super.setUp()
if ProcessInfo().environment["RECORD_FAILURES"].map(Bool.init) == true {
recordMode = .failed
}
checkEnvironments() checkEnvironments()
UIView.setAnimationsEnabled(false) UIView.setAnimationsEnabled(false)
@ -102,7 +103,7 @@ class PreviewTests: XCTestCase {
.fixedSize(horizontal: false, vertical: true) .fixedSize(horizontal: false, vertical: true)
) )
let failure = withSnapshotTesting(record: .missing) { return withSnapshotTesting(record: recordMode) {
verifySnapshot(of: matchingView, verifySnapshot(of: matchingView,
as: .prefireImage(precision: { precision }, as: .prefireImage(precision: { precision },
perceptualPrecision: { perceptualPrecision }, perceptualPrecision: { perceptualPrecision },
@ -112,16 +113,6 @@ class PreviewTests: XCTestCase {
named: name, named: name,
testName: testName) testName: testName)
} }
#if canImport(AccessibilitySnapshot)
let vc = UIHostingController(rootView: matchingView)
vc.view.frame = UIScreen.main.bounds
assertSnapshot(matching: vc,
as: .wait(for: delay, on: .accessibilityImage(showActivationPoints: .always)),
named: name.flatMap { $0 + ".accessibility" },
testName: testName)
#endif
return failure
} }
} }

View File

@ -31,6 +31,11 @@
{ {
"key" : "IS_RUNNING_UNIT_TESTS", "key" : "IS_RUNNING_UNIT_TESTS",
"value" : "1" "value" : "1"
},
{
"enabled" : false,
"key" : "RECORD_FAILURES",
"value" : "true"
} }
], ],
"testExecutionOrdering" : "random", "testExecutionOrdering" : "random",

View File

@ -10,6 +10,10 @@ import XCTest
enum Application { enum Application {
static func launch(_ identifier: UITestsScreenIdentifier, disableTimelineAccessibility: Bool = true) -> XCUIApplication { static func launch(_ identifier: UITestsScreenIdentifier, disableTimelineAccessibility: Bool = true) -> XCUIApplication {
if ProcessInfo().environment["RECORD_FAILURES"].map(Bool.init) == true {
XCUIApplication.recordMode = .failed
}
checkEnvironments() checkEnvironments()
let app = XCUIApplication() let app = XCUIApplication()
@ -47,6 +51,8 @@ enum Application {
} }
extension XCUIApplication { extension XCUIApplication {
static var recordMode: SnapshotTestingConfiguration.Record = .missing
@MainActor @MainActor
/// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created. /// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created.
/// - Parameter identifier: Identifier of the UI test screen /// - Parameter identifier: Identifier of the UI test screen
@ -69,17 +75,17 @@ extension XCUIApplication {
snapshot = snapshot.inset(by: insets) snapshot = snapshot.inset(by: insets)
} }
let failure = verifySnapshot(of: snapshot, let failure = withSnapshotTesting(record: Self.recordMode) {
as: .image(precision: precision, verifySnapshot(of: snapshot,
perceptualPrecision: 0.98, as: .image(precision: precision,
scale: nil), perceptualPrecision: 0.98,
// use any kind of suffix here to snapshot the same file multiple times and avoid countering on the library side scale: nil),
named: "UI", // use any kind of suffix here to snapshot the same file multiple times and avoid countering on the library side
testName: snapshotName) named: "UI",
testName: snapshotName)
}
if let failure, if let failure {
!failure.contains("No reference was found on disk."),
!failure.contains("to test against the newly-recorded snapshot") {
XCTFail(failure) XCTFail(failure)
} }
} }

View File

@ -27,6 +27,11 @@
{ {
"key" : "SNAPSHOT_ARTIFACTS", "key" : "SNAPSHOT_ARTIFACTS",
"value" : "\/tmp\/__FailedScreenshots__" "value" : "\/tmp\/__FailedScreenshots__"
},
{
"enabled" : false,
"key" : "RECORD_FAILURES",
"value" : "true"
} }
], ],
"locationScenario" : { "locationScenario" : {