improve IndexStore::listTests to include inherited tests (#335)

motivation: test discivery on linux depends on index store test listing

changes:
* refactor code to also scan for class inheritance
* rollup test across inheritance in different files
* deprecate single file API as it cannot effectively take inheritance into account

rdar://59655518

Author: tomerd

Sources/TSCUtility/IndexStore.swift

@@ -41,6 +41,11 @@ public final class IndexStore {
         return IndexStore(impl)
     }
 
+    public func listTests(in objectFiles: [AbsolutePath]) throws -> [TestCaseClass] {
+        return try impl.listTests(in: objectFiles)
+    }
+
+    @available(*, deprecated, message: "use listTests(in:) instead")
     public func listTests(inObjectFile object: AbsolutePath) throws -> [TestCaseClass] {
         return try impl.listTests(inObjectFile: object)
     }
@@ -58,13 +63,10 @@ public final class IndexStoreAPI {
 }
 
 private final class IndexStoreImpl {
-
     typealias TestCaseClass = IndexStore.TestCaseClass
 
     let api: IndexStoreAPIImpl
 
-    var fn: indexstore_functions_t { api.fn }
-
     let store: indexstore_t
 
     private init(store: indexstore_t, api: IndexStoreAPIImpl) {
@@ -79,47 +81,156 @@ private final class IndexStoreImpl {
         throw StringError("Unable to open store at \(path)")
     }
 
+    public func listTests(in objectFiles: [AbsolutePath]) throws -> [TestCaseClass] {
+        var inheritance = [String: [String: String]]()
+        var testMethods = [String: [String: [(name: String, async: Bool)]]]()
+
+        for objectFile in objectFiles {
+            // Get the records of this object file.
+            let unitReader = try self.api.call{ self.api.fn.unit_reader_create(store, unitName(object: objectFile), &$0) }
+            let records = try getRecords(unitReader: unitReader)
+            let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str
+            for record in records {
+                // get tests info
+                let testsInfo = try self.getTestsInfo(record: record)
+                // merge results across module
+                for (className, parentClassName) in testsInfo.inheritance {
+                    inheritance[moduleName, default: [:]][className] = parentClassName
+                }
+                for (className, classTestMethods) in testsInfo.testMethods {
+                    testMethods[moduleName, default: [:]][className, default: []].append(contentsOf: classTestMethods)
+                }
+            }
+        }
+
+        // merge across inheritance in module boundries
+        func flatten(moduleName: String, className: String) -> [String: (name: String, async: Bool)] {
+            var allMethods = [String: (name: String, async: Bool)]()
+
+            if let parentClassName = inheritance[moduleName]?[className] {
+                let parentMethods = flatten(moduleName: moduleName, className: parentClassName)
+                allMethods.merge(parentMethods, uniquingKeysWith:  { (lhs, _) in lhs })
+            }
+
+            for method in testMethods[moduleName]?[className] ?? [] {
+                allMethods[method.name] = (name: method.name, async: method.async)
+            }
+
+            return allMethods
+        }
+
+        var testCaseClasses = [TestCaseClass]()
+        for (moduleName, classMethods) in testMethods {
+            for className in classMethods.keys {
+                let methods = flatten(moduleName: moduleName, className: className)
+                    .map { (name, info) in TestCaseClass.TestMethod(name: name, isAsync: info.async) }
+                    .sorted()
+                testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name)))
+            }
+        }
+
+        return testCaseClasses
+    }
+
+
+    @available(*, deprecated, message: "use listTests(in:) instead")
     public func listTests(inObjectFile object: AbsolutePath) throws -> [TestCaseClass] {
         // Get the records of this object file.
-        let unitReader = try api.call{ fn.unit_reader_create(store, unitName(object: object), &$0) }
+        let unitReader = try api.call{ self.api.fn.unit_reader_create(store, unitName(object: object), &$0) }
         let records = try getRecords(unitReader: unitReader)
 
         // Get the test classes.
-        let testCaseClasses = try records.flatMap{ try self.getTestCaseClasses(forRecord: $0) }
-
-        // Fill the module name and return.
-        let module = fn.unit_reader_get_module_name(unitReader).str
-        return testCaseClasses.map {
-            var c = $0
-            c.module = module
-            return c
+        var inheritance = [String: String]()
+        var testMethods = [String: [(name: String, async: Bool)]]()
+
+        for record in records {
+            let testsInfo = try self.getTestsInfo(record: record)
+            inheritance.merge(testsInfo.inheritance, uniquingKeysWith: { (lhs, _) in lhs })
+            testMethods.merge(testsInfo.testMethods, uniquingKeysWith: { (lhs, _) in lhs })
+        }
+
+        func flatten(className: String) -> [(method: String, async: Bool)] {
+            var results = [(String, Bool)]()
+            if let parentClassName = inheritance[className] {
+                let parentMethods = flatten(className: parentClassName)
+                results.append(contentsOf: parentMethods)
+            }
+            if let methods = testMethods[className] {
+                results.append(contentsOf: methods)
+            }
+            return results
         }
+
+        let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str
+
+        var testCaseClasses = [TestCaseClass]()
+        for className in testMethods.keys {
+            let methods = flatten(className: className)
+                .map { TestCaseClass.TestMethod(name: $0.method, isAsync: $0.async) }
+                .sorted()
+            testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name)))
+        }
+
+        return testCaseClasses
     }
 
-    private func getTestCaseClasses(forRecord record: String) throws -> [TestCaseClass] {
-        let recordReader = try api.call{ fn.record_reader_create(store, record, &$0) }
+    private func getTestsInfo(record: String) throws -> (inheritance: [String: String], testMethods: [String: [(name: String, async: Bool)]] ) {
+        let recordReader = try api.call{ self.api.fn.record_reader_create(store, record, &$0) }
 
-        class TestCaseBuilder {
-            var classToMethods: [String: Set<TestCaseClass.TestMethod>] = [:]
+        // scan for inheritance
 
-            func add(className: String, method: TestCaseClass.TestMethod) {
-                classToMethods[className, default: []].insert(method)
+        let inheritanceRef = Ref([String: String](), api: self.api)
+        let inheritancePointer = unsafeBitCast(Unmanaged.passUnretained(inheritanceRef), to: UnsafeMutableRawPointer.self)
+
+        _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, inheritancePointer) { inheritancePointer , occ -> Bool in
+            let inheritanceRef = Unmanaged<Ref<[String: String?]>>.fromOpaque(inheritancePointer!).takeUnretainedValue()
+            let fn = inheritanceRef.api.fn
+
+            // Get the symbol.
+            let sym = fn.occurrence_get_symbol(occ)
+            let symbolProperties = fn.symbol_get_properties(sym)
+            // We only care about symbols that are marked unit tests and are instance methods.
+            if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 {
+                return true
+            }
+            if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_CLASS{
+                return true
             }
 
-            func build() -> [TestCaseClass] {
-                return classToMethods.map {
-                    let testMethods = Array($0.value).sorted()
-                    return TestCaseClass(name: $0.key, module: "", testMethods: testMethods, methods: testMethods.map(\.name))
+            let parentClassName = fn.symbol_get_name(sym).str
+            
+            let childClassNameRef = Ref("", api: inheritanceRef.api)
+            let childClassNamePointer = unsafeBitCast(Unmanaged.passUnretained(childClassNameRef), to: UnsafeMutableRawPointer.self)
+            _ = fn.occurrence_relations_apply_f(occ!, childClassNamePointer) { childClassNamePointer, relation in
+                guard let relation = relation else { return true }
+                let childClassNameRef = Unmanaged<Ref<String>>.fromOpaque(childClassNamePointer!).takeUnretainedValue()
+                let fn = childClassNameRef.api.fn
+
+                // Look for the base class.
+                if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_BASEOF.rawValue) {
+                    return true
                 }
+
+                let childClassNameSym = fn.symbol_relation_get_symbol(relation)
+                childClassNameRef.instance = fn.symbol_get_name(childClassNameSym).str
+                return true
             }
+
+            if !childClassNameRef.instance.isEmpty {
+                inheritanceRef.instance[childClassNameRef.instance] = parentClassName
+            }
+
+            return true
         }
 
-        let builder = Ref(TestCaseBuilder(), api: api)
+        // scan for methods
 
-        let ctx = unsafeBitCast(Unmanaged.passUnretained(builder), to: UnsafeMutableRawPointer.self)
-        _ = fn.record_reader_occurrences_apply_f(recordReader, ctx) { ctx , occ -> Bool in
-            let builder = Unmanaged<Ref<TestCaseBuilder>>.fromOpaque(ctx!).takeUnretainedValue()
-            let fn = builder.api.fn
+        let testMethodsRef = Ref([String: [(name: String, async: Bool)]](), api: api)
+        let testMethodsPointer = unsafeBitCast(Unmanaged.passUnretained(testMethodsRef), to: UnsafeMutableRawPointer.self)
+
+        _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, testMethodsPointer) { testMethodsPointer , occ -> Bool in
+            let testMethodsRef = Unmanaged<Ref<[String: [(name: String, async: Bool)]]>>.fromOpaque(testMethodsPointer!).takeUnretainedValue()
+            let fn = testMethodsRef.api.fn
 
             // Get the symbol.
             let sym = fn.occurrence_get_symbol(occ)
@@ -132,41 +243,45 @@ private final class IndexStoreImpl {
                 return true
             }
 
-            let className = Ref("", api: builder.api)
-            let ctx = unsafeBitCast(Unmanaged.passUnretained(className), to: UnsafeMutableRawPointer.self)
+            let classNameRef = Ref("", api: testMethodsRef.api)
+            let classNamePointer = unsafeBitCast(Unmanaged.passUnretained(classNameRef), to: UnsafeMutableRawPointer.self)
 
-            _ = fn.occurrence_relations_apply_f(occ!, ctx) { ctx, relation in
+            _ = fn.occurrence_relations_apply_f(occ!, classNamePointer) { classNamePointer, relation in
                 guard let relation = relation else { return true }
-                let className = Unmanaged<Ref<String>>.fromOpaque(ctx!).takeUnretainedValue()
-                let fn = className.api.fn
+                let classNameRef = Unmanaged<Ref<String>>.fromOpaque(classNamePointer!).takeUnretainedValue()
+                let fn = classNameRef.api.fn
 
                 // Look for the class.
                 if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF.rawValue) {
                     return true
                 }
 
-                let sym = fn.symbol_relation_get_symbol(relation)
-                className.instance = fn.symbol_get_name(sym).str
+                let classNameSym = fn.symbol_relation_get_symbol(relation)
+                classNameRef.instance = fn.symbol_get_name(classNameSym).str
                 return true
             }
 
-            if !className.instance.isEmpty {
+            if !classNameRef.instance.isEmpty {
                 let methodName = fn.symbol_get_name(sym).str
                 let isAsync = symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC.rawValue) != 0
-                builder.instance.add(className: className.instance, method: TestCaseClass.TestMethod(name: methodName, isAsync: isAsync))
+                testMethodsRef.instance[classNameRef.instance, default: []].append((name: methodName, async: isAsync))
             }
 
             return true
         }
 
-        return builder.instance.build()
+        return (
+            inheritance: inheritanceRef.instance,
+            testMethods: testMethodsRef.instance
+        )
+
     }
 
     private func getRecords(unitReader: indexstore_unit_reader_t?) throws -> [String] {
         let builder = Ref([String](), api: api)
 
         let ctx = unsafeBitCast(Unmanaged.passUnretained(builder), to: UnsafeMutableRawPointer.self)
-        _ = fn.unit_reader_dependencies_apply_f(unitReader, ctx) { ctx , unit -> Bool in
+        _ = self.api.fn.unit_reader_dependencies_apply_f(unitReader, ctx) { ctx , unit -> Bool in
             let store = Unmanaged<Ref<[String]>>.fromOpaque(ctx!).takeUnretainedValue()
             let fn = store.api.fn
             if fn.unit_dependency_get_kind(unit) == INDEXSTORE_UNIT_DEPENDENCY_RECORD {
@@ -181,12 +296,12 @@ private final class IndexStoreImpl {
     private func unitName(object: AbsolutePath) -> String {
         let initialSize = 64
         var buf = UnsafeMutablePointer<CChar>.allocate(capacity: initialSize)
-        let len = fn.store_get_unit_name_from_output_path(store, object.pathString, buf, initialSize)
+        let len = self.api.fn.store_get_unit_name_from_output_path(store, object.pathString, buf, initialSize)
 
         if len + 1 > initialSize {
             buf.deallocate()
             buf = UnsafeMutablePointer<CChar>.allocate(capacity: len + 1)
-            _ = fn.store_get_unit_name_from_output_path(store, object.pathString, buf, len + 1)
+            _ = self.api.fn.store_get_unit_name_from_output_path(store, object.pathString, buf, len + 1)
         }
 
         defer {