Don't ignore pending tests when executing specs under XCTest

This means that the pending tests, although not executed, will still be given to the reporter, so the pending test summary info can be printed. Note that the pending tests aren't exposed to the XCTest runner at all.
cedarbdd · Jan 24, 2016 · 8c2b850 · 8c2b850
1 parent 0a51ec8
commit 8c2b850
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 20 deletions.
diff --git a/Source/Headers/Project/XCTest/NSInvocation+CDRXExample.h b/Source/Headers/Project/XCTest/NSInvocation+CDRXExample.h
@@ -6,7 +6,9 @@
 @interface NSInvocation (CDRXExample)
 
 @property (nonatomic, retain, setter=cdr_setDispatcher:) CDRReportDispatcher *cdr_dispatcher;
-@property (nonatomic, retain, setter=cdr_setExample:) CDRExample *cdr_example;
+@property (nonatomic, retain, setter=cdr_setExamples:) NSArray *cdr_examples;
 @property (nonatomic, retain, setter=cdr_setSpecClassName:) NSString *cdr_specClassName;
 
+- (void)cdr_addSupplementaryExample:(CDRExample *)example;
+
 @end
diff --git a/Source/XCTest/CDRSpec+XCTestSupport.m b/Source/XCTest/CDRSpec+XCTestSupport.m
@@ -46,8 +46,15 @@ - (id)testSuiteWithRandomSeed:(unsigned int)seed dispatcher:(CDRReportDispatcher
     NSArray *examples = [self allExamplesToRun];
 
     NSMutableArray *testInvocations = [NSMutableArray array];
+    NSMutableArray *unusedPendingExamples = [NSMutableArray array];
     for (CDRExample *example in examples) {
-        if (!example.isPending) {
+        if (example.isPending) {
+            if (testInvocations.count > 0) {
+                [[testInvocations lastObject] cdr_addSupplementaryExample:example];
+            } else {
+                [unusedPendingExamples addObject:example];
+            }
+        } else {
             NSString *methodName = [namer methodNameForExample:example withClassName:NSStringFromClass([self class])];
             SEL selector = NSSelectorFromString(methodName);
             NSMethodSignature *methodSignature = [newXCTestSubclass instanceMethodSignatureForSelector:selector];
@@ -60,9 +67,11 @@ - (id)testSuiteWithRandomSeed:(unsigned int)seed dispatcher:(CDRReportDispatcher
             NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:methodSignature];
             invocation.selector = selector;
             invocation.cdr_dispatcher = dispatcher;
-            invocation.cdr_example = example;
+            invocation.cdr_examples = [unusedPendingExamples arrayByAddingObject:example];
             invocation.cdr_specClassName = className;
             [testInvocations addObject:invocation];
+
+            [unusedPendingExamples removeAllObjects];
         }
     }
 
@@ -109,13 +118,17 @@ - (NSArray *)allExamplesToRun {
     return examples;
 }
 
-- (void)createTestMethodForSelector:(SEL)selector onClass:(Class)aClass {
-    IMP imp = imp_implementationWithBlock(^(id instance){
-        CDRExample *example = [[instance invocation] cdr_example];
+static void testMethodImplementation(id instance, SEL _cmd) {
+    NSMutableSet *alreadyReportedExampleGroups = [NSMutableSet set];
+    CDRReportDispatcher *theDispatcher = [[instance invocation] cdr_dispatcher];
+
+    for (CDRExample *example in [[instance invocation] cdr_examples]) {
         CDRExampleGroup *parentGroup = (CDRExampleGroup *)example.parent;
-        CDRReportDispatcher *theDispatcher = [[instance invocation] cdr_dispatcher];
         while (![parentGroup isEqual:example.spec.rootGroup]) {
-            [theDispatcher runWillStartExampleGroup:parentGroup];
+            if (![alreadyReportedExampleGroups containsObject:parentGroup]) {
+                [theDispatcher runWillStartExampleGroup:parentGroup];
+            }
+
             parentGroup = (CDRExampleGroup *)[parentGroup parent];
         }
 
@@ -133,14 +146,20 @@ - (void)createTestMethodForSelector:(SEL)selector onClass:(Class)aClass {
 
         parentGroup = (CDRExampleGroup *)example.parent;
         while (![parentGroup isEqual:example.spec.rootGroup]) {
-            [theDispatcher runDidFinishExampleGroup:parentGroup];
+            if (![alreadyReportedExampleGroups containsObject:parentGroup]) {
+                [theDispatcher runDidFinishExampleGroup:parentGroup];
+                [alreadyReportedExampleGroups addObject:parentGroup];
+            }
+
             parentGroup = (CDRExampleGroup *)[parentGroup parent];
         }
-    });
-    Method m = class_getInstanceMethod([self class], @selector(defineBehaviors));
+    }
+}
 
+- (void)createTestMethodForSelector:(SEL)selector onClass:(Class)aClass {
+    Method m = class_getInstanceMethod([self class], @selector(defineBehaviors));
     const char *encoding = method_getTypeEncoding(m);
-    class_addMethod(aClass, selector, imp, encoding);
+    class_addMethod(aClass, selector, (IMP)testMethodImplementation, encoding);
 }
 
 @end
diff --git a/Source/XCTest/CDRXCTestCase.m b/Source/XCTest/CDRXCTestCase.m
@@ -52,7 +52,8 @@ + (void)setTestInvocations:(NSArray *)array {
 } while(0);
 
 - (void)recordFailureWithDescription:(NSString *)description inFile:(NSString *)filename atLine:(NSUInteger)lineNumber expected:(BOOL)expected {
-    if (self.invocation.cdr_example.state == CDRExampleStateIncomplete) {
+    CDRExample *example = self.invocation.cdr_examples.firstObject;
+    if (example.state == CDRExampleStateIncomplete) {
         [[CDRSpecFailure specFailureWithReason:description fileName:filename lineNumber:(int)lineNumber] raise];
     } else {
         super_recordFailure(description, filename, lineNumber, expected);

diff --git a/Source/XCTest/NSInvocation+CDRXExample.m b/Source/XCTest/NSInvocation+CDRXExample.m
@@ -2,7 +2,7 @@
 #import <objc/runtime.h>
 
 const char *CDRXDispatcherKey;
-const char *CDRXExampleKey;
+const char *CDRXExamplesKey;
 const char *CDRXSpecClassNameKey;
 
 
@@ -16,12 +16,12 @@ - (void)cdr_setDispatcher:(CDRReportDispatcher *)dispatcher {
     objc_setAssociatedObject(self, &CDRXDispatcherKey, dispatcher, OBJC_ASSOCIATION_RETAIN_NONATOMIC);
 }
 
-- (CDRExample *)cdr_example {
-    return objc_getAssociatedObject(self, &CDRXExampleKey);
+- (NSArray *)cdr_examples {
+    return objc_getAssociatedObject(self, &CDRXExamplesKey);
 }
 
-- (void)cdr_setExample:(CDRExample *)example {
-    objc_setAssociatedObject(self, &CDRXExampleKey, example, OBJC_ASSOCIATION_RETAIN_NONATOMIC);
+- (void)cdr_setExamples:(NSArray *)examples {
+    objc_setAssociatedObject(self, &CDRXExamplesKey, examples, OBJC_ASSOCIATION_RETAIN_NONATOMIC);
 }
 
 - (NSString *)cdr_specClassName {
@@ -32,4 +32,9 @@ - (void)cdr_setSpecClassName:(NSString *)specClassName {
     objc_setAssociatedObject(self, &CDRXSpecClassNameKey, specClassName, OBJC_ASSOCIATION_RETAIN_NONATOMIC);
 }
 
+- (void)cdr_addSupplementaryExample:(CDRExample *)example {
+    NSArray *existingExamples = self.cdr_examples ?: @[];
+    self.cdr_examples = [existingExamples arrayByAddingObject:example];
+}
+
 @end
diff --git a/Spec/XCTest/CDRXCTestSuiteSpec.mm b/Spec/XCTest/CDRXCTestSuiteSpec.mm
@@ -21,17 +21,25 @@
     beforeEach(^{
         reporter = [TestReporter new];
         dispatcher = [[CDRReportDispatcher alloc] initWithReporters:@[reporter]];
-    });
 
-    it(@"should report that each parent example group has started and ended", ^{
         CDRSpec *simulatedSpec = [[NSClassFromString(@"CDRXCSimulatedTestSuiteSpec") alloc] init];
         [simulatedSpec defineBehaviors];
         subject = [simulatedSpec testSuiteWithRandomSeed:0 dispatcher:dispatcher];
         [subject performTest:nil];
+    });
 
+    it(@"should report that each parent example group has started and ended", ^{
         reporter.startedExampleGroups.count should equal(4);
         reporter.finishedExampleGroups.count  should equal(4);
     });
+
+    it(@"should report that pending examples have started and ended", ^{
+        NSPredicate *pendingPredicate = [NSPredicate predicateWithBlock:^BOOL(CDRExample *example, NSDictionary *_) {
+            return example.state == CDRExampleStatePending;
+        }];
+        [reporter.startedExamples filteredArrayUsingPredicate:pendingPredicate].count should equal(2);
+        [reporter.finishedExamples filteredArrayUsingPredicate:pendingPredicate].count should equal(2);
+    });
 });
 
 SPEC_END
@@ -43,9 +51,17 @@
     describe(@"with nested groups", ^{
         describe(@"lots of nested groups", ^{
             describe(@"no really, lots of nested groups", ^{
+                xit(@"should report pending examples before the first test to run", ^{
+                    1 should equal(2);
+                });
+
                 it(@"should start and finish each example group", ^{
                     // nothing to see here
                 });
+
+                xit(@"should report pending examples after the last test to run", ^{
+                    1 should equal(2);
+                });
             });
         });
     });