Improve test architecture so failures don't crash the simulator

nicklockwood · nicklockwood · commit 6573d256ba6c · 2015-06-17T07:09:23.000-08:00
diff --git a/Examples/UIExplorer/UIExplorerIntegrationTests/IntegrationTestsTests.m b/Examples/UIExplorer/UIExplorerIntegrationTests/IntegrationTestsTests.m
@@ -32,10 +32,6 @@ - (void)setUp
   NSOperatingSystemVersion version = [[NSProcessInfo processInfo] operatingSystemVersion];
   RCTAssert(version.majorVersion == 8 || version.minorVersion == 3, @"Tests should be run on iOS 8.3, found %zd.%zd.%zd", version.majorVersion, version.minorVersion, version.patchVersion);
   _runner = RCTInitRunnerForApp(@"Examples/UIExplorer/UIExplorerIntegrationTests/js/IntegrationTestsApp");
-
-  // If tests have changes, set recordMode = YES below and run the affected
-  // tests on an iPhone5, iOS 8.3 simulator.
-  _runner.recordMode = NO;
 }
 
 #pragma mark Logic Tests
@@ -53,8 +49,7 @@ - (void)testTheTester_waitOneFrame
   expectErrorBlock:nil];
 }
 
-// TODO: this seems to stall forever - figure out why
-- (void)DISABLED_testTheTester_ExpectError
+- (void)testTheTester_ExpectError
 {
   [_runner runTest:_cmd
             module:@"IntegrationTestHarnessTest"
@@ -91,12 +86,9 @@ - (void)testPromises
 
 - (void)testSimpleSnapshot
 {
+  // If tests have changes, set recordMode = YES below and re-run
+  _runner.recordMode = NO;
   [_runner runTest:_cmd module:@"SimpleSnapshotTest"];
 }
 
-- (void)testZZZ_NotInRecordMode
-{
-  RCTAssert(_runner.recordMode == NO, @"Don't forget to turn record mode back to NO before commit.");
-}
-
 @end
diff --git a/Examples/UIExplorer/UIExplorerIntegrationTests/UIExplorerIntegrationTests.m b/Examples/UIExplorer/UIExplorerIntegrationTests/UIExplorerIntegrationTests.m
@@ -21,8 +21,6 @@
 #import "RCTRedBox.h"
 #import "RCTRootView.h"
 
-#define TIMEOUT_SECONDS 240
-
 @interface UIExplorerTests : XCTestCase
 {
   RCTTestRunner *_runner;
@@ -40,52 +38,6 @@ - (void)setUp
   NSString *version = [[UIDevice currentDevice] systemVersion];
   RCTAssert([version isEqualToString:@"8.3"], @"Snapshot tests should be run on iOS 8.3, found %@", version);
   _runner = RCTInitRunnerForApp(@"Examples/UIExplorer/UIExplorerApp.ios");
-
-  // If tests have changes, set recordMode = YES below and run the affected
-  // tests on an iPhone5, iOS 8.3 simulator.
-  _runner.recordMode = NO;
-}
-
-- (BOOL)findSubviewInView:(UIView *)view matching:(BOOL(^)(UIView *view))test
-{
-  if (test(view)) {
-    return YES;
-  }
-  for (UIView *subview in [view subviews]) {
-    if ([self findSubviewInView:subview matching:test]) {
-      return YES;
-    }
-  }
-  return NO;
-}
-
-// Make sure this test runs first because the other tests will tear out the rootView
-- (void)testAAA_RootViewLoadsAndRenders
-{
-  // TODO (t7296305) Fix and Re-Enable this UIExplorer Test
-  return;
-
-  UIViewController *vc = [UIApplication sharedApplication].delegate.window.rootViewController;
-  RCTAssert([vc.view isKindOfClass:[RCTRootView class]], @"This test must run first.");
-  NSDate *date = [NSDate dateWithTimeIntervalSinceNow:TIMEOUT_SECONDS];
-  BOOL foundElement = NO;
-  NSString *redboxError = nil;
-
-  while ([date timeIntervalSinceNow] > 0 && !foundElement && !redboxError) {
-    [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
-    [[NSRunLoop mainRunLoop] runMode:NSRunLoopCommonModes beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
-
-    redboxError = [[RCTRedBox sharedInstance] currentErrorMessage];
-    foundElement = [self findSubviewInView:vc.view matching:^(UIView *view) {
-      if ([view.accessibilityLabel isEqualToString:@"<View>"]) {
-        return YES;
-      }
-      return NO;
-    }];
-  }
-
-  XCTAssertNil(redboxError, @"RedBox error: %@", redboxError);
-  XCTAssertTrue(foundElement, @"Couldn't find element with '<View>' text in %d seconds", TIMEOUT_SECONDS);
 }
 
 #define RCT_SNAPSHOT_TEST(name, reRecord) \
@@ -102,10 +54,4 @@ - (void)test##name##Snapshot              \
 RCT_SNAPSHOT_TEST(SliderExample, NO)
 RCT_SNAPSHOT_TEST(TabBarExample, NO)
 
-// Make sure this test runs last
-- (void)testZZZ_NotInRecordMode
-{
-  RCTAssert(_runner.recordMode == NO, @"Don't forget to turn record mode back to NO before commit.");
-}
-
 @end
diff --git a/Examples/UIExplorer/UIExplorerIntegrationTests/js/PromiseTest.js b/Examples/UIExplorer/UIExplorerIntegrationTests/js/PromiseTest.js
@@ -22,7 +22,7 @@ var PromiseTest = React.createClass({
     Promise.all([
       this.testShouldResolve(),
       this.testShouldReject(),
-    ]).then(() => RCTTestModule.finish(
+    ]).then(() => RCTTestModule.markTestPassed(
       this.shouldResolve && this.shouldReject
     ));
   },
@@ -42,7 +42,7 @@ var PromiseTest = React.createClass({
   },
 
   render() {
-      return <React.View />;
+    return <React.View />;
   }
 
 });
diff --git a/Examples/UIExplorer/UIExplorerIntegrationTests/js/SimpleSnapshotTest.js b/Examples/UIExplorer/UIExplorerIntegrationTests/js/SimpleSnapshotTest.js
@@ -24,8 +24,8 @@ var SimpleSnapshotTest = React.createClass({
     requestAnimationFrame(() => TestModule.verifySnapshot(this.done));
   },
 
-  done() {
-    TestModule.markTestCompleted();
+  done(success) {
+    TestModule.markTestPassed(success);
   },
 
   render() {
diff --git a/Examples/UIExplorer/UIExplorerList.js b/Examples/UIExplorer/UIExplorerList.js
@@ -115,7 +115,7 @@ COMPONENTS.concat(APIS).forEach((Example) => {
         // View is still blank after first RAF :\
         global.requestAnimationFrame(() =>
           global.requestAnimationFrame(() => TestModule.verifySnapshot(
-            TestModule.markTestCompleted
+            TestModule.markTestPassed
           )
         ));
       },
diff --git a/Libraries/RCTTest/RCTTestModule.h b/Libraries/RCTTest/RCTTestModule.h
@@ -12,6 +12,12 @@
 #import "RCTBridgeModule.h"
 #import "RCTDefines.h"
 
+typedef NS_ENUM(NSInteger, RCTTestStatus) {
+  RCTTestStatusPending = 0,
+  RCTTestStatusPassed,
+  RCTTestStatusFailed
+};
+
 @class FBSnapshotTestController;
 
 @interface RCTTestModule : NSObject <RCTBridgeModule>
@@ -32,8 +38,8 @@
 @property (nonatomic, assign) SEL testSelector;
 
 /**
- * This is typically polled while running the runloop until true.
+ * This is polled while running the runloop until true.
  */
-@property (nonatomic, readonly, getter=isDone) BOOL done;
+@property (nonatomic, readonly) RCTTestStatus status;
 
 @end
diff --git a/Libraries/RCTTest/RCTTestModule.m b/Libraries/RCTTest/RCTTestModule.m
@@ -51,16 +51,7 @@ - (instancetype)init
                                              selector:_testSelector
                                            identifier:_snapshotCounter[testName]
                                                 error:&error];
-
-    RCTAssert(success, @"Snapshot comparison failed: %@", error);
-    callback(@[]);
-  }];
-}
-
-RCT_EXPORT_METHOD(markTestCompleted)
-{
-  [_bridge.uiManager addUIBlock:^(RCTUIManager *uiManager, RCTSparseArray *viewRegistry) {
-    _done = YES;
+    callback(@[@(success)]);
   }];
 }
 
@@ -79,11 +70,16 @@ - (instancetype)init
   reject(nil);
 }
 
-RCT_EXPORT_METHOD(finish:(BOOL)success)
+RCT_EXPORT_METHOD(markTestCompleted)
 {
-  RCTAssert(success, @"RCTTestModule finished without success");
-  [self markTestCompleted];
+  [self markTestPassed:YES];
 }
 
+RCT_EXPORT_METHOD(markTestPassed:(BOOL)success)
+{
+  [_bridge.uiManager addUIBlock:^(RCTUIManager *uiManager, RCTSparseArray *viewRegistry) {
+    _status = success ? RCTTestStatusPassed : RCTTestStatusFailed;
+  }];
+}
 
 @end
diff --git a/Libraries/RCTTest/RCTTestRunner.m b/Libraries/RCTTest/RCTTestRunner.m
@@ -16,7 +16,7 @@
 #import "RCTTestModule.h"
 #import "RCTUtils.h"
 
-#define TIMEOUT_SECONDS 240
+#define TIMEOUT_SECONDS 60
 
 @interface RCTBridge (RCTTestRunner)
 
@@ -93,7 +93,7 @@ - (void)runTest:(SEL)test module:(NSString *)moduleName initialProps:(NSDictiona
 
   NSDate *date = [NSDate dateWithTimeIntervalSinceNow:TIMEOUT_SECONDS];
   NSString *error = [[RCTRedBox sharedInstance] currentErrorMessage];
-  while ([date timeIntervalSinceNow] > 0 && ![testModule isDone] && error == nil) {
+  while ([date timeIntervalSinceNow] > 0 && testModule.status == RCTTestStatusPending && error == nil) {
     [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
     [[NSRunLoop mainRunLoop] runMode:NSRunLoopCommonModes beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
     error = [[RCTRedBox sharedInstance] currentErrorMessage];
@@ -104,11 +104,12 @@ - (void)runTest:(SEL)test module:(NSString *)moduleName initialProps:(NSDictiona
   [[RCTRedBox sharedInstance] dismiss];
   if (expectErrorBlock) {
     RCTAssert(expectErrorBlock(error), @"Expected an error but nothing matched.");
-  } else if (error) {
-    RCTAssert(error == nil, @"RedBox error: %@", error);
   } else {
-    RCTAssert([testModule isDone], @"Test didn't finish within %d seconds", TIMEOUT_SECONDS);
+    RCTAssert(error == nil, @"RedBox error: %@", error);
+    RCTAssert(testModule.status != RCTTestStatusPending, @"Test didn't finish within %d seconds", TIMEOUT_SECONDS);
+    RCTAssert(testModule.status == RCTTestStatusPassed, @"Test failed");
   }
+  RCTAssert(self.recordMode == NO, @"Don't forget to turn record mode back to NO before commit.");
 }
 
 @end

Original file line number	Diff line number	Diff line change
`@@ -115,7 +115,7 @@ COMPONENTS.concat(APIS).forEach((Example) => {`
`115`	`115`	`// View is still blank after first RAF :\`
`116`	`116`	`global.requestAnimationFrame(() =>`
`117`	`117`	`global.requestAnimationFrame(() => TestModule.verifySnapshot(`
`118`		`- TestModule.markTestCompleted`
	`118`	`+ TestModule.markTestPassed`
`119`	`119`	`)`
`120`	`120`	`));`
`121`	`121`	`},`