Drop SemanticDB from javac compiler plug-in; emit SCIP shards directly

Rebased onto origin/main (post #899's semanticdb-shared extraction).

- semanticdb-javac: replace SemanticDB protobuf emission with SCIP shards
  (ScipShardWriter/Aggregator/Occurrences/Symbols/Signatures + ScipVisitor).
- semanticdb-kotlinc: add the SCIP shard infrastructure alongside the
  existing SemanticDB code path (dropped fully in stacked PR B).
- scip-semanticdb: ScipShardAggregator walks per-source shards, applies
  SymbolRewriter, and merges into a single Index.
- Use the shared semanticdb-shared module's symbol/builder utilities.

Author: jupblb

build.sbt

@@ -137,6 +137,8 @@ lazy val javacPlugin = project
     fatjarPackageSettings,
     javaOnlySettings,
     moduleName := "semanticdb-javac",
+    libraryDependencies +=
+      "org.scip-code" % "scip-java-bindings" % V.scipBindings,
     // Scoped to compile so doc tasks (which reject -g) are unaffected.
     Compile / compile / javacOptions += "-g",
     // JDK 14+ ServiceLoader-scans the classpath for Plugin providers; our
@@ -358,8 +360,14 @@ lazy val semanticdbKotlinc = project
     // classpath via Provided so the assembled fat-jar does not bundle it.
     libraryDependencies +=
       "org.jetbrains.kotlin" % "kotlin-stdlib" % V.kotlinVersion % Provided,
-    // The SemanticDB proto schema and the generated Java classes live in
-    // semanticdbShared; we get them transitively via .dependsOn below.
+    // protobuf java codegen — proto file lives at src/main/proto/...
+    Compile / PB.protoSources :=
+      Seq((Compile / sourceDirectory).value / "proto"),
+    Compile / PB.targets :=
+      Seq(PB.gens.java(V.protobuf) -> (Compile / sourceManaged).value),
+    libraryDependencies += "com.google.protobuf" % "protobuf-java" % V.protobuf,
+    libraryDependencies +=
+      "org.scip-code" % "scip-java-bindings" % V.scipBindings,
     // kotlin-compiler-embeddable is supplied by kotlinc at runtime
     libraryDependencies +=
       "org.jetbrains.kotlin" % "kotlin-compiler-embeddable" % V.kotlinVersion %
@@ -432,7 +440,6 @@ lazy val semanticdbKotlinc = project
       Attributed.blank(dir)
     }
   )
-  .dependsOn(semanticdbShared)
 
 // `semanticdbKotlincMinimized` mirrors the (still-present) Gradle build at
 // semanticdb-kotlinc/minimized/build.gradle.kts. It compiles a small set of
@@ -517,15 +524,19 @@ lazy val semanticdbKotlincMinimized = project
           val snapDir =
             (baseDirectory.value / "src" / "generatedSnapshots" / "resources")
               .getAbsolutePath
-          val scipOut = s"$tgtRoot/index.scip"
+          // Write `index.scip` outside the shard-scanned targetroot to avoid re-ingestion.
+          val scipOut = (target.value / "scip-index" / "index.scip")
+            .getAbsolutePath
           val mainCls = "com.sourcegraph.scip_java.ScipJava"
           Def.sequential(
             Compile / compile,
             (cli / Compile / runMain).toTask(
-              s" $mainCls index-semanticdb --no-emit-inverse-relationships --cwd $srcRoot --output $scipOut $tgtRoot"
+              s" $mainCls index-semanticdb --no-emit-inverse-relationships --use-scip-shards --cwd $srcRoot --output $scipOut $tgtRoot"
             ),
             (cli / Compile / runMain).toTask(
-              s" $mainCls snapshot --cwd $srcRoot --output $snapDir $tgtRoot"
+              s" $mainCls snapshot --cwd $srcRoot --output $snapDir ${file(
+                  scipOut
+                ).getParentFile.getAbsolutePath}"
             )
           )
         }

scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexSemanticdbCommand.scala

@@ -12,6 +12,7 @@ import com.sourcegraph.scip_java.buildtools.ClasspathEntry
 import com.sourcegraph.scip_semanticdb.ConsoleScipSemanticdbReporter
 import com.sourcegraph.scip_semanticdb.ScipSemanticdb
 import com.sourcegraph.scip_semanticdb.ScipSemanticdbOptions
+import com.sourcegraph.scip_semanticdb.ScipShardAggregator
 import moped.annotations._
 import moped.cli.Application
 import moped.cli.Command
@@ -59,6 +60,11 @@ final case class IndexSemanticdbCommand(
         "Maven->Maven or Gradle->Gradle projects because those build tools compile sources to classfiles inside directories."
     )
     allowExportingGlobalSymbolsFromDirectoryEntries: Boolean = true,
+    @Description(
+      "If true, aggregate *.scip shards under META-INF/scip/ instead of *.semanticdb files. " +
+        "Pass --use-scip-shards=false to fall back to the SemanticDB-based aggregator."
+    )
+    useScipShards: Boolean = true,
     @Inline()
     app: Application = Application.default
 ) extends Command {
@@ -96,7 +102,10 @@ final case class IndexSemanticdbCommand(
         allowEmptyIndex,
         allowExportingGlobalSymbolsFromDirectoryEntries
       )
-    ScipSemanticdb.run(options)
+    if (useScipShards)
+      ScipShardAggregator.run(options)
+    else
+      ScipSemanticdb.run(options)
     postPackages(packages)
     if (!app.reporter.hasErrors()) {
       app.info(options.output.toString)

scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotCommand.scala

@@ -51,31 +51,37 @@ case class SnapshotCommand(
               attrs: BasicFileAttributes
           ): FileVisitResult = {
             if (scipPattern.matches(file)) {
-              foundScipFile = true
               val index = Index.parseFrom(Files.readAllBytes(file))
-              val root = URI.create(index.getMetadata.getProjectRoot)
-              index
-                .getDocumentsList
-                .asScala
-                .foreach { doc =>
-                  val sourcepath = Paths.get(root.resolve(doc.getRelativePath))
-                  val source =
-                    new String(
-                      Files.readAllBytes(sourcepath),
-                      StandardCharsets.UTF_8
+              // Skip per-source shards (no project_root); only the aggregator output carries it.
+              val rawProjectRoot = index.getMetadata.getProjectRoot
+              if (rawProjectRoot.nonEmpty) {
+                foundScipFile = true
+                val projectRoot = URI.create(rawProjectRoot)
+                index
+                  .getDocumentsList
+                  .asScala
+                  .foreach { doc =>
+                    val sourcepath = Paths.get(
+                      projectRoot.resolve(doc.getRelativePath)
                     )
-                  val document = ScipPrinters.printTextDocument(
-                    doc,
-                    source,
-                    CommentSyntax.default
-                  )
-                  val snapshotOutput = output.resolve(doc.getRelativePath)
-                  Files.createDirectories(snapshotOutput.getParent)
-                  Files.write(
-                    snapshotOutput,
-                    document.getBytes(StandardCharsets.UTF_8)
-                  )
-                }
+                    val source =
+                      new String(
+                        Files.readAllBytes(sourcepath),
+                        StandardCharsets.UTF_8
+                      )
+                    val document = ScipPrinters.printTextDocument(
+                      doc,
+                      source,
+                      CommentSyntax.default
+                    )
+                    val snapshotOutput = output.resolve(doc.getRelativePath)
+                    Files.createDirectories(snapshotOutput.getParent)
+                    Files.write(
+                      snapshotOutput,
+                      document.getBytes(StandardCharsets.UTF_8)
+                    )
+                  }
+              }
             }
             super.visitFile(file, attrs)
           }