shim layering

we need our shims to massage our build variables on linux properly (otherwise, we have to recreate logic every time we use gcc). this adds proper layered fallback to included tooling. obviously envvars can still override.

Author: jhheider

build/build.ts

@@ -161,46 +161,84 @@ if (ghout) {
 
 ///////////////////////////////////////////////////////////////////
 function make_toolchain() {
-  const deps = new Set(config.deps.dry.build.concat(config.deps.dry.runtime).map(x => x.project))
+  if (host().platform == "darwin") return
 
-  if (deps.has('llvm.org') || deps.has('gnu.org/gcc')) {
+  if (yml?.build?.skip === 'shims' || yml?.build?.skip?.includes?.('shims')) {
     return
   }
 
-  if (host().platform != "darwin") {
-    // rm ∵ // https://github.com/pkgxdev/brewkit/issues/303
-    const d = config.path.home.join('toolchain').rm({ recursive: true }).mkdir('p')
-
-    const symlink = (names: string[], {to}: {to: string}) => {
-      for (const name of names) {
-        const path = d.join(name)
-        if (path.exists()) continue
-        const target = useConfig().prefix.join('llvm.org/v*/bin', to)
-        path.ln('s', { target })
-      }
+  const deps = new Set(config.deps.dry.build.concat(config.deps.dry.runtime).map(x => x.project))
+  const has_gcc = deps.has('gnu.org/gcc')
+  const has_llvm = deps.has('llvm.org')
+  const has_binutils = deps.has('gnu.org/binutils')
+
+  // rm ∵ // https://github.com/pkgxdev/brewkit/issues/303
+  const d = config.path.home.join('toolchain').rm({ recursive: true }).mkdir('p')
+  const prefix = useConfig().prefix
+
+  const llvm = (bin: string) => prefix.join('llvm.org/v*/bin', bin)
+  const gcc = (bin: string) => prefix.join('gnu.org/gcc/v*/bin', bin)
+  const binutils = (bin: string) => prefix.join('gnu.org/binutils/v*/bin', bin)
+
+  const symlink = (names: string[], target: Path) => {
+    for (const name of names) {
+      const path = d.join(name)
+      if (path.exists()) continue
+      path.ln('s', { target })
     }
+  }
+
+  // compilers
+  if (has_gcc && has_llvm) {
+    symlink(["cc", "gcc"], gcc("gcc"))
+    symlink(["c++", "g++"], gcc("g++"))
+    symlink(["clang"], llvm("clang"))
+    symlink(["clang++"], llvm("clang++"))
+    symlink(["cpp"], gcc("cpp"))
+  } else if (has_gcc) {
+    symlink(["cc", "gcc"], gcc("gcc"))
+    symlink(["c++", "g++"], gcc("g++"))
+    symlink(["cpp"], gcc("cpp"))
+  } else {
+    // llvm is default; build-script.ts adds +llvm.org to env if not already a dep
+    symlink(["cc", "gcc", "clang"], llvm("clang"))
+    symlink(["c++", "g++", "clang++"], llvm("clang++"))
+    symlink(["cpp"], llvm("clang-cpp"))
+  }
 
-    symlink(["cc", "gcc", "clang"], {to: "clang"})
-    symlink(["c++", "g++", "clang++"], {to: "clang++"})
-    symlink(["cpp"], {to: "clang-cpp"})
+  // linker
+  if (has_binutils) {
+    symlink(["ld"], binutils("ld"))
+  } else if (host().platform == "linux") {
+    symlink(["ld"], llvm("ld.lld"))
+  } else if (host().platform == "windows") {
+    symlink(["ld"], llvm("lld-link"))
+  }
 
-    if (host().platform == "linux") {
-      symlink(["ld"], {to: "ld.lld"})
-    } else if (host().platform == "windows") {
-      symlink(["ld"], {to: "lld-link"})
-    }
+  if (has_llvm || !has_gcc) {
+    symlink(["ld.lld"], llvm("ld.lld"))
+    symlink(["lld-link"], llvm("lld-link"))
+  }
 
-    symlink(["ld.lld"], {to: "ld.lld"})
-    symlink(["lld-link"], {to: "lld-link"})
-
-    symlink(["ar"], {to: "llvm-ar"})
-    symlink(["as"], {to: "llvm-as"})
-    symlink(["nm"], {to: "llvm-nm"})
-    symlink(["objcopy"], {to: "llvm-objcopy"})
-    symlink(["ranlib"], {to: "llvm-ranlib"})
-    symlink(["readelf"], {to: "llvm-readelf"})
-    symlink(["strings"], {to: "llvm-strings"})
-    symlink(["strip"], {to: "llvm-strip"})
+  // utilities
+  if (has_binutils) {
+    symlink(["ar"], binutils("ar"))
+    symlink(["as"], binutils("as"))
+    symlink(["nm"], binutils("nm"))
+    symlink(["objcopy"], binutils("objcopy"))
+    symlink(["ranlib"], binutils("ranlib"))
+    symlink(["readelf"], binutils("readelf"))
+    symlink(["strings"], binutils("strings"))
+    symlink(["strip"], binutils("strip"))
+  } else {
+    symlink(["ar"], llvm("llvm-ar"))
+    symlink(["as"], llvm("llvm-as"))
+    symlink(["nm"], llvm("llvm-nm"))
+    symlink(["objcopy"], llvm("llvm-objcopy"))
+    symlink(["ranlib"], llvm("llvm-ranlib"))
+    symlink(["readelf"], llvm("llvm-readelf"))
+    symlink(["strings"], llvm("llvm-strings"))
+    symlink(["strip"], llvm("llvm-strip"))
   }
 
   return new Path(new URL(import.meta.url).pathname).join("../../share/toolchain/bin")

lib/porcelain/build-script.ts

@@ -9,7 +9,12 @@ const { host } = utils
 export default async function(config: Config, PATH?: Path): Promise<string> {
   const depset = new Set(config.deps.gas.map(x => x.pkg.project))
   const depstr = (deps: PackageRequirement[]) => deps.map(x => `"+${utils.pkg.str(x)}"`).join(' ')
-  const env_plus = `${depstr(config.deps.dry.runtime)} ${depstr(config.deps.dry.build)}`.trim()
+  let env_plus = `${depstr(config.deps.dry.runtime)} ${depstr(config.deps.dry.build)}`.trim()
+
+  // if no compiler is an explicit dep, add llvm as default (was previously done per-invocation in the shim)
+  if (host().platform != 'darwin' && !depset.has('llvm.org') && !depset.has('gnu.org/gcc')) {
+    env_plus = `${env_plus} "+llvm.org"`.trim()
+  }
   const user_script = await usePantry().getScript(config.pkg, 'build', config.deps.gas, config)
 
   const pkgx = find_in_PATH('pkgx')
@@ -51,10 +56,10 @@ export default async function(config: Config, PATH?: Path): Promise<string> {
 
     ${gum} format "## env"
       export PKGX_HOME="$HOME"
+      export PATH="${brewkit_PATHs}:$PATH"
       set -a
       ${env_plus ? `eval "$(CLICOLOR_FORCE=1 ${pkgx} ${env_plus})"` : ''}
       set +a
-      export PATH="${brewkit_PATHs}:$PATH"
 
       export PKGX="${pkgx}"
       export HOME=${config.path.home.string}

share/toolchain/shim

@@ -3,32 +3,18 @@
 tool=$(basename "$0")
 
 if [ "$(uname)" != Darwin ]; then
-  if [ -x /usr/local/bin/pkgx ]; then
-    # removed from PATH deliberately
-    #TODO like, probs we should set PKGX or something before removing it from PATH
-    pkgx=/usr/local/bin/pkgx
-  else
-    # if not the above probs this is running in pkgx CI/CD
-    pkgx="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/v*/bin/pkgx"
-  fi
-  # prevent fork bombs (shouldn't be possible but who knows)
-  export PATH="/usr/bin:/bin:/usr/sbin:/sbin"
-
-  # NOTE this slows down configure scripts a shit tonne
-  # 1. a fix is speeding up pkgx resolution by caching the pantry
-  # 2. or do this once and store the env to a file that we can then source
-  set -a
-  eval "$("$pkgx" +llvm.org)"
+  # ensure $HOME/toolchain (real binaries) is found before shim dir to prevent recursion
+  export PATH="$HOME/toolchain:$PATH"
 
   if printf "%s\n" "$@" | grep -qxF -- '-shared'; then
-      has_shared=1
+    has_shared=1
   fi
 
   filtered_args=()
   for arg in "$@"; do
     # -shared and -pie conflict. libs aren't executables, so accept the -shared
     if [ "$arg" = "-pie" ] && [ "$has_shared" = "1" ]; then
-        continue
+      continue
     fi
 
     if [ "$arg" != -Werror ]; then