Merge pull request #292 from MatrixAI/qa-testing

Adding QA jobs and expanding test jobs for CI/CD.

Author: CMCDragonkai

.gitignore

@@ -3,6 +3,7 @@
 .env*
 !.env.example
 /result*
+/builds
 
 # Logs
 logs

.gitlab-ci.yml

@@ -1,70 +1,192 @@
 variables:
-  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  # Cache .npm
+  NPM_CONFIG_CACHE: "./tmp/npm"
+  # Prefer offline node module installation
+  NPM_CONFIG_PREFER_OFFLINE: "true"
+  # `ts-node` has its own cache
+  # It must use an absolute path, otherwise ts-node calls will CWD
+  TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
+  TS_CACHED_TRANSPILE_PORTABLE: "true"
+
+# Cached directories shared between jobs & pipelines per-branch
+cache:
+  key: $CI_COMMIT_REF_SLUG
+  paths:
+    - ./tmp/npm/
+    - ./tmp/ts-node-cache/
+    # `jest` cache is configured in jest.config.js
+    - ./tmp/jest/
 
 stages:
   - check
+  - test
   - build
+  - quality
   - release
 
-image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
-
 lint:
+  image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
   stage: check
   interruptible: true
   script:
     - >
         nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run '
-        npm install;
+        npm ci;
         npm run lint;
         '
 
-test:
-  stage: check
-  interruptible: true
-  script:
-    - >
-        nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run '
-        npm install;
-        npm run test;
-        '
-
 nix-dry:
   stage: check
-  interruptible: true
+  image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
   script:
     - nix-build -v -v --dry-run ./release.nix --attr application
     - nix-build -v -v --dry-run ./release.nix --attr docker
     - nix-build -v -v --dry-run ./release.nix --attr package.linux.x64.elf
     - nix-build -v -v --dry-run ./release.nix --attr package.windows.x64.exe
     - nix-build -v -v --dry-run ./release.nix --attr package.macos.x64.macho
 
+test-generate:
+  image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
+  stage: check
+  interruptible: true
+  script:
+    - mkdir -p ./tmp
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages bash --run '
+        ./scripts/test-pipelines.sh > ./tmp/test-pipelines.yml
+        '
+  artifacts:
+    paths:
+      - ./tmp/test-pipelines.yml
+
+test:
+  stage: test
+  # Don't implicitly inherit top-level variables in child pipeline
+  # All inherited variables should be explicitly defined here
+  # Note that variables defined here override any variables defined in the child pipeline
+  # This causes a bug with $CI_PROJECT_DIR, which is expanded into an empty string
+  inherit:
+    variables: false
+  trigger:
+    include:
+      - artifact: tmp/test-pipelines.yml
+        job: test-generate
+    strategy: depend
+
 nix:
   stage: build
-  interruptible: true
+  image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
   script:
+    - mkdir -p ./builds
+    # nix-specific application target
+    - >
+        build_application="$(nix-build \
+          --max-jobs "$(nproc)" --cores "$(nproc)" \
+          ./release.nix  \
+          --attr application \
+        )"
+    - >
+        nix-store --export $( \
+          nix-store --query --requisites "$build_application" \
+        ) | gzip > ./builds/js-polykey.closure.gz
+    # non-nix targets
     - >
-        nix-build ./release.nix
-        --max-jobs $(nproc)
-        --attr application
-        --attr docker
+        builds="$(nix-build \
+          --max-jobs "$(nproc)" --cores "$(nproc)" \
+          ./release.nix \
+          --attr docker \
+          --attr package.linux.x64.elf \
+          --attr package.windows.x64.exe \
+          --attr package.macos.x64.macho)"
+    - cp -r $builds ./builds/
+  only:
+    - master
+  artifacts:
+    paths:
+      - ./builds/
+
+application run:
+  stage: quality
+  image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
+  dependencies:
+    - nix
+  script:
+    - >
+        build_application="$( \
+          gunzip -c ./builds/js-polykey.closure.gz | \
+          nix-store --import | \
+          tail -1 \
+        )"
+    - $build_application/bin/polykey
+  only:
+    - master
+
+docker run:
+  stage: quality
+  image: docker:20.10.11
+  dependencies:
+    - nix
+  services:
+    - docker:20.10.11-dind
+  variables:
+    DOCKER_TLS_CERTDIR: "/certs"
+  before_script:
+    - docker info
+  script:
+    - image="$(docker load --input ./builds/*docker* | cut -d' ' -f3)"
+    - docker run "$image"
+  only:
+    - master
+
+linux run:
+  stage: quality
+  image: ubuntu:latest
+  dependencies:
+    - nix
+  script:
+    - for f in ./builds/*-linux-*; do "$f"; done
+  only:
+    - master
+
+windows run:
+  stage: quality
+  dependencies:
+    - nix
+  script:
+    - Get-ChildItem -File ./builds/*-win32-* | ForEach {& $_.FullName}
+  tags:
+    - windows
   only:
     - master
 
+macos run:
+   stage: quality
+   image: macos-11-xcode-12
+   dependencies:
+     - nix
+   script:
+      - for f in ./builds/*-macos-*; do "$f"; done
+   only:
+     - master
+   tags:
+     - shared-macos-amd64
+
 packages:
   stage: release
-  interruptible: true
+  image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
+  dependencies:
+    - nix
   script:
     - >
         nix-shell -I nixpkgs=./pkgs.nix --packages git gitAndTools.gh --run '
-        builds="$(nix-build \
-          --max-jobs $(nproc) --cores $(nproc) \
-          ./release.nix \
-          --attr package.linux.x64.elf \
-          --attr package.windows.x64.exe \
-          --attr package.macos.x64.macho)";
         commit="$(git rev-parse --short HEAD)";
         gh release \
-          create "$commit" $builds \
+          create "$commit" \
+          builds/*.closure.gz \
+          builds/*-linux-* \
+          builds/*-win32-* \
+          builds/*-macos-* \
           --title "Build-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
           --prerelease \
           --notes "" \

.npmignore

@@ -13,3 +13,4 @@
 /tmp
 /docs
 /benches
+/builds

README.md

@@ -67,7 +67,7 @@ Once you update the `src/proto/schemas` files, run this to update the `src/proto
 npm run proto-generate
 ```
 
-### Executing Commands
+### Calling Commands
 
 When calling commands in development, use this style:
 
@@ -85,7 +85,7 @@ npm run docs
 
 See the docs at: https://matrixai.github.io/Polykey/
 
-### Publishing
+### Publishing to NPM
 
 ```sh
 # npm login
@@ -96,6 +96,18 @@ git push
 git push --tags
 ```
 
+### Packaging Cross-Platform Executables
+
+We use `pkg` to package the source code into executables.
+
+This requires a specific version of `pkg` and also `node-gyp-build`.
+
+Configuration for `pkg` is done in:
+
+* `package.json` - Pins `pkg` and `node-gyp-build`, and configures assets and scripts.
+* `utils.nix` - Pins `pkg` for Nix usage
+* `release.nix` - Build expressions for executables
+
 ## Deployment
 
 ### Deploying to AWS ECS:

default.nix

@@ -12,7 +12,9 @@ let
     }
     ''
     mkdir -p $out/lib/node_modules/${utils.node2nixDev.packageName}
-    # copy only the dist
+    # copy the package.json
+    cp ${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}/package.json $out/lib/node_modules/${utils.node2nixDev.packageName}/
+    # copy the dist
     cp -r ${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}/dist $out/lib/node_modules/${utils.node2nixDev.packageName}/
     # copy over the production dependencies
     if [ -d "${utils.node2nixProd}/lib/node_modules" ]; then

jest.config.js

@@ -1,3 +1,7 @@
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+const process = require('process');
 const { pathsToModuleNameMapper } = require('ts-jest/utils');
 const { compilerOptions } = require('./tsconfig');
 
@@ -10,8 +14,33 @@ const moduleNameMapper = pathsToModuleNameMapper(
 // https://github.com/panva/jose/discussions/105
 moduleNameMapper['^jose/(.*)$'] = "<rootDir>/node_modules/jose/dist/node/cjs/$1";
 
+// Global variables that are shared across the jest worker pool
+// These variables must be static and serialisable
+const globals = {
+  // Absolute directory to the project root
+  projectDir: __dirname,
+  // Absolute directory to the test root
+  testDir: path.join(__dirname, 'tests'),
+  // Default global data directory
+  dataDir: fs.mkdtempSync(
+    path.join(os.tmpdir(), 'polykey-test-global-'),
+  ),
+  // Default asynchronous test timeout
+  defaultTimeout: 20000,
+  polykeyStartupTimeout: 30000,
+  failedConnectionTimeout: 50000,
+  // Timeouts rely on setTimeout which takes 32 bit numbers
+  maxTimeout: Math.pow(2, 31) - 1,
+};
+
+// The `globalSetup` and `globalTeardown` cannot access the `globals`
+// They run in their own process context
+// They can receive process environment
+process.env['GLOBAL_DATA_DIR'] = globals.dataDir;
+
 module.exports = {
   testEnvironment: "node",
+  cacheDirectory: '<rootDir>/tmp/jest',
   verbose: true,
   roots: [
     "<rootDir>/tests"
@@ -23,13 +52,21 @@ module.exports = {
     "^.+\\.tsx?$": "ts-jest",
     "^.+\\.jsx?$": "babel-jest"
   },
+  globals,
+  // Global setup script executed once before all test files
+  globalSetup: "<rootDir>/tests/globalSetup.ts",
+  // Global teardown script executed once after all test files
+  globalTeardown: "<rootDir>/tests/globalTeardown.ts",
+  // Setup files are executed before each test file
+  // Can access globals
   setupFiles: [
     "<rootDir>/tests/setup.ts"
   ],
+  // Setup files after env are executed before each test file
+  // after the jest test environment is installed
+  // Can access globals
   setupFilesAfterEnv: [
     "<rootDir>/tests/setupAfterEnv.ts"
   ],
-  globalSetup: "<rootDir>/tests/globalSetup.ts",
-  globalTeardown: "<rootDir>/tests/globalTeardown.ts",
   moduleNameMapper: moduleNameMapper
 };