Add a new binary for tracing

Author: Sauyon Lee

Makefile

@@ -14,11 +14,11 @@ CODEQL_PLATFORM = osx64
 endif
 endif
 
-CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh)
+CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh linux64 osx64 win64)
 
 EXTRACTOR_PACK_OUT = build/codeql-extractor-go
 
-BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap go-gen-dbscheme
+BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme
 
 .PHONY: tools tools-codeql tools-codeql-full clean autoformat \
 	tools-linux64 tools-osx64 tools-win64 check-formatting

codeql-tools/autobuild.cmd

@@ -4,7 +4,12 @@ SETLOCAL EnableDelayedExpansion
 rem Some legacy environment variables for the autobuilder.
 set LGTM_SRC=%CD%
 
-type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
+if "%CODEQL_EXTRACTOR_GO_BUILD_TRACING%"=="on" (
+  echo "Tracing enabled"
+  type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-build-runner.exe"
+) else (
+  type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
+)
 exit /b %ERRORLEVEL%
 
 ENDLOCAL

codeql-tools/autobuild.sh

@@ -11,4 +11,9 @@ fi
 LGTM_SRC="$(pwd)"
 export LGTM_SRC
 
-"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
+if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" == "on" ]; then
+  echo "Tracing enabled"
+  "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner"
+else
+  "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
+fi

codeql-tools/linux64/compiler-tracing.spec

@@ -0,0 +1,7 @@
+**/go-autobuilder:
+  order compiler
+  trace no
+**/go:
+  invoke ${config_dir}/go-extractor
+  prepend --mimic
+  prepend "${compiler}"

codeql-tools/osx64/compiler-tracing.spec

@@ -0,0 +1,7 @@
+**/go-autobuilder:
+  order compiler
+  trace no
+**/go:
+  invoke ${config_dir}/go-extractor
+  prepend --mimic
+  prepend "${compiler}"

codeql-tools/win64/compiler-tracing.spec

@@ -0,0 +1,7 @@
+**/go-autobuilder.exe:
+  order compiler
+  trace no
+**/go.exe:
+  invoke ${config_dir}/go-extractor.exe
+  prepend --mimic
+  prepend "${compiler}"

extractor/cli/go-build-runner/go-build-runner.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/github/codeql-go/extractor/util"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+
+	"github.com/github/codeql-go/extractor/autobuilder"
+)
+
+func main() {
+	// check if a build command has successfully extracted something
+	autobuilder.CheckExtracted = true
+	if autobuilder.Autobuild() {
+		return
+	}
+
+	// if the autobuilder fails, invoke the extractor manually
+	// we cannot simply call `go build` here, because the tracer is not able to trace calls made by
+	// this binary
+	log.Printf("No build commands succeeded, falling back to go build ./...")
+
+	mypath, err := os.Executable()
+	if err != nil {
+		log.Fatalf("Could not determine path of extractor: %v.\n", err)
+	}
+	extractor := filepath.Join(filepath.Dir(mypath), "go-extractor")
+	if runtime.GOOS == "windows" {
+		extractor = extractor + ".exe"
+	}
+
+	util.RunCmd(exec.Command(extractor, "./..."))
+}