Merge pull request #881 from lightninglabs/reproducible-build

build: make builds fully reproducible across systems

Author: guggero

.github/workflows/main.yml

@@ -12,7 +12,7 @@ env:
   # If you change this value, please change it in the following files as well:
   # /Dockerfile
   # /dev.Dockerfile
-  GO_VERSION: 1.22.3
+  GO_VERSION: 1.22.6
 
 jobs:
   ########################

.gitignore

@@ -13,4 +13,16 @@ itest/litd-itest
 itest/lnd-itest
 itest/itest.test
 itest/.logs
-itest/*.log
+itest/*.log
+
+vendor
+*.idea
+*.run
+*.iml
+profile.cov
+profile.tmp
+
+.DS_Store
+
+.vscode
+*.code-workspace

.golangci.yml

@@ -31,7 +31,7 @@ linters-settings:
       - G402 # Look for bad TLS connection settings.
       - G306 # Poor file permissions used when writing to a new file.
   staticcheck:
-    go: "1.18"
+    go: "1.22.6"
     checks: ["-SA1019"]
 
 linters:

Dockerfile

@@ -1,5 +1,5 @@
 # Start with a NodeJS base image that also contains yarn.
-FROM node:22.8.0-alpine as nodejsbuilder
+FROM node:22.8.0-alpine@sha256:bec0ea49c2333c429b62e74e91f8ba1201b060110745c3a12ff957cd51b363c6 as nodejsbuilder
 
 # Pass a tag, branch or a commit using build-arg. This allows a docker image to
 # be built from a specified Git state. The default image will use the Git tip of
@@ -32,10 +32,7 @@ RUN apk add --no-cache --update alpine-sdk \
 
 # The first stage is already done and all static assets should now be generated
 # in the app/build sub directory.
-# If you change this value, please also update:
-# /dev.Dockerfile
-# /.github/workflows/main.yml
-FROM golang:1.22.3-alpine as golangbuilder
+FROM golang:1.22.6-alpine@sha256:1a478681b671001b7f029f94b5016aed984a23ad99c707f6a0ab6563860ae2f3 as golangbuilder
 
 # Instead of checking out from git again, we just copy the whole working
 # directory of the previous stage that includes the generated static assets.
@@ -53,7 +50,7 @@ RUN apk add --no-cache --update alpine-sdk \
   && make go-install-cli
 
 # Start a new, final image to reduce size.
-FROM alpine as final
+FROM alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d as final
 
 # Define a root volume for data persistence.
 VOLUME /root/.lnd

Makefile

@@ -18,6 +18,11 @@ COMMIT := $(shell git describe --abbrev=40 --dirty --tags)
 COMMIT_HASH := $(shell git rev-parse HEAD)
 PUBLIC_URL := 
 
+# GO_VERSION is the Go version used for the release build, docker files, and
+# GitHub Actions. This is the reference version for the project. All other Go
+# versions are checked against this version.
+GO_VERSION = 1.22.6
+
 LOOP_COMMIT := $(shell cat go.mod | \
 		grep $(LOOP_PKG) | \
 		head -n1 | \
@@ -159,9 +164,25 @@ app-build: yarn-install
 	@$(call print, "Building production app.")
 	cd app; yarn build
 
-release: app-build
+docker-app-build:
+	@$(call print, "Building production app in docker.")
+	cd app; ./gen_app_docker.sh
+
+release: docker-app-build go-release
+
+go-release:
 	@$(call print, "Creating release of lightning-terminal.")
-	./release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(LND_RELEASE_TAGS)" "$(RELEASE_LDFLAGS)"
+	./scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(LND_RELEASE_TAGS)" "$(RELEASE_LDFLAGS)" "$(GO_VERSION)"
+
+docker-release: docker-app-build
+	@$(call print, "Building release helper docker image.")
+	if [ "$(tag)" = "" ]; then echo "Must specify tag=<commit_or_tag>!"; exit 1; fi
+
+	docker build -t litd-release-helper -f make/builder.Dockerfile make/
+
+	# Run the actual compilation inside the docker image. We pass in all flags
+	# that we might want to overwrite in manual tests.
+	$(DOCKER_RELEASE_HELPER) make go-release tag="$(tag)" sys="$(sys)" COMMIT="$(COMMIT)" 
 
 docker-tools:
 	@$(call print, "Building tools docker image.")
@@ -226,7 +247,17 @@ fmt: $(GOIMPORTS_BIN)
 	@$(call print, "Formatting source.")
 	gofmt -l -w -s $(GOFILES_NOVENDOR)
 
-lint: docker-tools
+check-go-version-yaml:
+	@$(call print, "Checking for target Go version (v$(GO_VERSION)) in YAML files (*.yaml, *.yml)")
+	./scripts/check-go-version-yaml.sh $(GO_VERSION)
+
+check-go-version-dockerfile:
+	@$(call print, "Checking for target Go version (v$(GO_VERSION)) in Dockerfile files (*Dockerfile)")
+	./scripts/check-go-version-dockerfile.sh $(GO_VERSION)
+
+check-go-version: check-go-version-dockerfile check-go-version-yaml
+
+lint: check-go-version docker-tools
 	@$(call print, "Linting source.")
 	$(DOCKER_TOOLS) golangci-lint run -v $(LINT_WORKERS)
 
@@ -267,3 +298,11 @@ clean: clean-itest
 	$(RM) ./litcli-debug
 	$(RM) ./litd-debug
 	$(RM) coverage.txt
+
+# Prevent make from interpreting any of the defined goals as folders or files to
+# include in the build process.
+.PHONY: default all yarn-install build install go-build go-build-noui \
+	go-install go-install-noui go-install-cli app-build release go-release \
+	docker-release docker-tools scratch check unit unit-cover unit-race \
+	clean-itest build-itest itest-only itest flake-unit fmt lint mod mod-check \
+	list rpc protos protos-check rpc-js-compile clean

app/Dockerfile

@@ -0,0 +1,12 @@
+# Start with a NodeJS base image that also contains yarn.
+FROM node:22.8.0-bookworm@sha256:bd00c03095f7586432805dbf7989be10361d27987f93de904b1fc003949a4794 as nodejsbuilder
+
+RUN apt-get update && apt-get install -y git
+
+ENV HOME=/tmp
+
+RUN mkdir /build
+
+WORKDIR /build
+
+CMD ["/bin/bash", "-c", "chown $(id -u):$(id -g) /build && cd app && rm -rf node_modules && yarn cache clean && yarn install && yarn build"]

app/gen_app_docker.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+# Directory of the script file, independent of where it's called from.
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+echo "Building app compiler docker image..."
+docker build -q -t lit-app-builder .
+
+echo "Compiling app files..."
+docker run \
+  --rm \
+  --user $(id -u):$(id -g) \
+  -v "$DIR/../:/build" \
+  lit-app-builder

app/package.json

@@ -8,7 +8,7 @@
     "start": "BROWSER=none react-scripts start",
     "develop": "REACT_APP_USE_SAMPLE_DATA=true yarn start",
     "build": "react-scripts build",
-    "postbuild": "echo '# Keep directory in git.' > build/.gitkeep",
+    "postbuild": "git restore build/.gitkeep",
     "test": "react-scripts test --env=jest-environment-jsdom --transformIgnorePatterns \"node_modules/(?!d3)/\"",
     "test:ci": "cross-env CI=true yarn test --coverage",
     "eject": "react-scripts eject",

autopilotserverrpc/autopilotserver.pb.go

@@ -1,5 +1,5 @@
 # Start with a NodeJS base image that also contains yarn.
-FROM node:22.8.0-alpine as nodejsbuilder
+FROM node:22.8.0-alpine@sha256:bec0ea49c2333c429b62e74e91f8ba1201b060110745c3a12ff957cd51b363c6 as nodejsbuilder
 
 # Copy in the local repository to build from.
 COPY . /go/src/github.com/lightninglabs/lightning-terminal
@@ -10,10 +10,7 @@ RUN cd /go/src/github.com/lightninglabs/lightning-terminal/app \
 
 # The first stage is already done and all static assets should now be generated
 # in the app/build sub directory.
-# If you change this value, please also update:
-# /Dockerfile
-# /.github/workflows/main.yml
-FROM golang:1.22.3-alpine as golangbuilder
+FROM golang:1.22.6-alpine@sha256:1a478681b671001b7f029f94b5016aed984a23ad99c707f6a0ab6563860ae2f3 as golangbuilder
 
 # Instead of checking out from git again, we just copy the whole working
 # directory of the previous stage that includes the generated static assets.
@@ -31,7 +28,7 @@ RUN apk add --no-cache --update alpine-sdk \
   && make go-install-cli
 
 # Start a new, final image to reduce size.
-FROM alpine as final
+FROM alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d as final
 
 # Define a root volume for data persistence.
 VOLUME /root/.lnd