Use an environment variable to record failures.

This commit is contained in:
Doug 2024-09-19 10:30:56 +01:00 committed by Doug
parent d41dd47b51
commit d2d3b28dc7
4 changed files with 34 additions and 27 deletions

View File

@ -11,19 +11,20 @@ import XCTest
@testable import ElementX
@testable import SnapshotTesting
#if canImport(AccessibilitySnapshot)
import AccessibilitySnapshot
#endif
class PreviewTests: XCTestCase {
private let deviceConfig: ViewImageConfig = .iPhoneX
private var simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation
private var requiredOSVersion = (major: 18, minor: 0)
private let simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation
private let requiredOSVersion = (major: 18, minor: 0)
private let snapshotDevices = ["iPhone 16", "iPad"]
private var recordMode: SnapshotTestingConfiguration.Record = .missing
override func setUp() {
super.setUp()
if ProcessInfo().environment["RECORD_FAILURES"].map(Bool.init) == true {
recordMode = .failed
}
checkEnvironments()
UIView.setAnimationsEnabled(false)
}
@ -102,7 +103,7 @@ class PreviewTests: XCTestCase {
.fixedSize(horizontal: false, vertical: true)
)
let failure = withSnapshotTesting(record: .missing) {
return withSnapshotTesting(record: recordMode) {
verifySnapshot(of: matchingView,
as: .prefireImage(precision: { precision },
perceptualPrecision: { perceptualPrecision },
@ -112,16 +113,6 @@ class PreviewTests: XCTestCase {
named: name,
testName: testName)
}
#if canImport(AccessibilitySnapshot)
let vc = UIHostingController(rootView: matchingView)
vc.view.frame = UIScreen.main.bounds
assertSnapshot(matching: vc,
as: .wait(for: delay, on: .accessibilityImage(showActivationPoints: .always)),
named: name.flatMap { $0 + ".accessibility" },
testName: testName)
#endif
return failure
}
}

View File

@ -31,6 +31,11 @@
{
"key" : "IS_RUNNING_UNIT_TESTS",
"value" : "1"
},
{
"enabled" : false,
"key" : "RECORD_FAILURES",
"value" : "true"
}
],
"testExecutionOrdering" : "random",

View File

@ -10,6 +10,10 @@ import XCTest
enum Application {
static func launch(_ identifier: UITestsScreenIdentifier, disableTimelineAccessibility: Bool = true) -> XCUIApplication {
if ProcessInfo().environment["RECORD_FAILURES"].map(Bool.init) == true {
XCUIApplication.recordMode = .failed
}
checkEnvironments()
let app = XCUIApplication()
@ -47,6 +51,8 @@ enum Application {
}
extension XCUIApplication {
static var recordMode: SnapshotTestingConfiguration.Record = .missing
@MainActor
/// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created.
/// - Parameter identifier: Identifier of the UI test screen
@ -69,17 +75,17 @@ extension XCUIApplication {
snapshot = snapshot.inset(by: insets)
}
let failure = verifySnapshot(of: snapshot,
let failure = withSnapshotTesting(record: Self.recordMode) {
verifySnapshot(of: snapshot,
as: .image(precision: precision,
perceptualPrecision: 0.98,
scale: nil),
// use any kind of suffix here to snapshot the same file multiple times and avoid countering on the library side
named: "UI",
testName: snapshotName)
}
if let failure,
!failure.contains("No reference was found on disk."),
!failure.contains("to test against the newly-recorded snapshot") {
if let failure {
XCTFail(failure)
}
}

View File

@ -27,6 +27,11 @@
{
"key" : "SNAPSHOT_ARTIFACTS",
"value" : "\/tmp\/__FailedScreenshots__"
},
{
"enabled" : false,
"key" : "RECORD_FAILURES",
"value" : "true"
}
],
"locationScenario" : {