From 99ef736fd4b44db9605415a4743cc95f26f2dd60 Mon Sep 17 00:00:00 2001 From: Ignas Samoilovas Date: Wed, 18 Mar 2026 16:09:32 +0200 Subject: [PATCH 1/4] cleanup for github actions pipeline changes --- .circleci/config.yml | 81 ------------------- .github/workflows/main.yml | 20 ++--- README.md | 4 +- package.json | 4 - packages/apollo-datasource-axios/package.json | 5 ++ packages/avro-logical-types/package.json | 5 ++ packages/avro-stream/package.json | 5 ++ packages/axios-logger/package.json | 5 ++ packages/bigquery-pg-sink/package.json | 5 ++ packages/config-file/package.json | 5 ++ packages/datadog-metrics-tracker/package.json | 5 ++ packages/influx-metrics-tracker/package.json | 5 ++ packages/kafka-avro-cli/package.json | 5 ++ packages/kafka-consumer/package.json | 5 ++ packages/kafka-pg-sink/package.json | 5 ++ packages/re-pipeline/package.json | 5 ++ packages/schema-registry-api/package.json | 5 ++ packages/winston-logger/package.json | 5 ++ 18 files changed, 82 insertions(+), 97 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index d95ce8a..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,81 +0,0 @@ -# version: 2 -# -# npmrc-git: &npmrc-git -# run: echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" > ~/.npmrc -# -# restore-cache: &restore-cache -# restore_cache: -# keys: -# - v1-{{ checksum "yarn.lock" }} -# -# save-cache: &save-cache -# save_cache: -# key: v1-{{ checksum "yarn.lock" }} -# paths: -# - ~/.cache -# -# jobs: -# test: -# docker: -# - image: circleci/node:12 -# - image: confluentinc/cp-zookeeper:5.0.1 -# environment: -# ZOOKEEPER_CLIENT_PORT: 2181 -# ZOOKEEPER_TICK_TIME: 2000 -# - image: confluentinc/cp-kafka:5.0.1 -# environment: -# KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 -# KAFKA_ZOOKEEPER_CONNECT: 'localhost:2181' -# KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT -# KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_HOST://localhost:29092 -# - image: confluentinc/cp-schema-registry:5.0.1 -# environment: -# SCHEMA_REGISTRY_HOST_NAME: localhost -# SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'localhost:2181' -# - image: circleci/postgres:10-alpine -# environment: -# POSTGRES_PASSWORD: dev-pass -# steps: -# - checkout -# - *restore-cache -# - run: yarn config set yarn-offline-mirror ~/.cache/yarn -# - run: yarn -# - *save-cache -# - run: yarn build -# - run: .circleci/wait-for-server.sh -# - run: yarn test -# -# publish: -# docker: -# - image: circleci/node:12 -# steps: -# - checkout -# - *restore-cache -# - *npmrc-git -# - run: yarn config set yarn-offline-mirror ~/.cache/yarn -# - run: yarn --frozen-lockfile -# - run: yarn build -# - run: git config user.email "ovotech-ci@ovoenergy.com" -# - run: git config user.name "Ovotech CI" -# - run: yarn lerna publish from-package --yes --registry https://npm.pkg.github.com -# -# workflows: -# version: 2 -# test: -# jobs: -# - test: -# context: boost-btt -# filters: -# branches: -# ignore: master -# deploy: -# jobs: -# - test: -# context: boost-btt -# filters: -# branches: -# only: master -# - publish: -# context: boost-btt -# requires: -# - test diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3ea243d..186075a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,15 +42,15 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: '12' - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/yarn key: v1-${{ hashFiles('**/yarn.lock') }} @@ -78,15 +78,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: '12' - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/yarn key: v1-${{ hashFiles('**/yarn.lock') }} @@ -104,7 +104,7 @@ jobs: - name: List changed packages run: | - echo "### Following packages will be published" >> $GITHUB_STEP_SUMMARY + echo "### Following pending deployments found" >> $GITHUB_STEP_SUMMARY yarn lerna exec --loglevel silent --concurrency 1 -- \ 'VERSION=$(node -p "require(\"./package.json\").version"); \ if [ "$(npm view $LERNA_PACKAGE_NAME version --registry=https://npm.pkg.github.com/ 2>/dev/null || echo "0.0.0")" != "$VERSION" ]; then \ @@ -119,10 +119,10 @@ jobs: name: github-npm-registry steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: '12' @@ -130,7 +130,7 @@ jobs: run: echo "//npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}" > ~/.npmrc - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/yarn key: v1-${{ hashFiles('**/yarn.lock') }} diff --git a/README.md b/README.md index 31726a3..9b5bdb0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # BIT Kafka Tools -[![CircleCI](https://circleci.com/gh/ovotech/bit-node-tools.svg?style=svg&circle-token=ae40b0f9ff7943343688a0319478e70091e37fbe)](https://circleci.com/gh/ovotech/bit-node-tools) +[![Build Status](https://github.com/ovotech/bit-node-tools/actions/workflows/main.yml/badge.svg)](https://github.com/ovotech/bit-node-tools/actions/workflows/main.yml) BIT Team tools for working with Kafka, Avro and other misc stuff. They are split into several independent packages that can be imported separately. @@ -84,7 +84,7 @@ yarn lint ## Deployment 1. MANUALLY bump the package version along with your changes -2. On merge to master lerna will pick up on the changes and output what will be published in the GitHub actions job `prepare-publish` +2. On merge to master lerna will pick up on the changes and output what will be published in the GitHub actions pipeline under `prepare-publish summary` 3. MANUALLY approve deployment for the GitHub actions job `publish` ## Built With diff --git a/package.json b/package.json index 19aa9af..5aa4e73 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,4 @@ "devDependencies": { "lerna": "^3.14.1" }, - "repository": { - "url": "https://github.com/ovotech/bit-node-tools", - "type": "git" - } } diff --git a/packages/apollo-datasource-axios/package.json b/packages/apollo-datasource-axios/package.json index 4f93741..eb7ae3c 100644 --- a/packages/apollo-datasource-axios/package.json +++ b/packages/apollo-datasource-axios/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "peerDependencies": { "axios": "*" }, diff --git a/packages/avro-logical-types/package.json b/packages/avro-logical-types/package.json index 42c0738..aed803d 100644 --- a/packages/avro-logical-types/package.json +++ b/packages/avro-logical-types/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "peerDependencies": { "avsc": "*" }, diff --git a/packages/avro-stream/package.json b/packages/avro-stream/package.json index 2427816..862d850 100644 --- a/packages/avro-stream/package.json +++ b/packages/avro-stream/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "dependencies": { "@ovotech/schema-registry-api": "^1.0.3", "avsc": "^5.4.10" diff --git a/packages/axios-logger/package.json b/packages/axios-logger/package.json index c5b9596..6c21b38 100644 --- a/packages/axios-logger/package.json +++ b/packages/axios-logger/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "peerDependencies": { "axios": "*" }, diff --git a/packages/bigquery-pg-sink/package.json b/packages/bigquery-pg-sink/package.json index 5ad30e6..46ae648 100644 --- a/packages/bigquery-pg-sink/package.json +++ b/packages/bigquery-pg-sink/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Sam Clift ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "lint-prettier": "prettier --list-different {src,test}/**/*.ts", "lint-tslint": "tslint --config tslint.json '{src,test}/**/*.ts'", diff --git a/packages/config-file/package.json b/packages/config-file/package.json index 235630f..bd78e18 100644 --- a/packages/config-file/package.json +++ b/packages/config-file/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "test": "jest --runInBand", "lint-prettier": "prettier --list-different {src,test}/**/*.ts", diff --git a/packages/datadog-metrics-tracker/package.json b/packages/datadog-metrics-tracker/package.json index dfff6a0..a31eb43 100644 --- a/packages/datadog-metrics-tracker/package.json +++ b/packages/datadog-metrics-tracker/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Manikant Sharma ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "test": "jest --runInBand", "start": "node ./src/index.ts", diff --git a/packages/influx-metrics-tracker/package.json b/packages/influx-metrics-tracker/package.json index d0af170..4dac6d6 100644 --- a/packages/influx-metrics-tracker/package.json +++ b/packages/influx-metrics-tracker/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Hayden Field ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "test": "jest --runInBand", "lint-prettier": "prettier --list-different {src,test}/**/*.ts", diff --git a/packages/kafka-avro-cli/package.json b/packages/kafka-avro-cli/package.json index 7e9aaeb..940ac09 100644 --- a/packages/kafka-avro-cli/package.json +++ b/packages/kafka-avro-cli/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "bin": { "kac": "./scripts/kac" }, diff --git a/packages/kafka-consumer/package.json b/packages/kafka-consumer/package.json index be07895..9e71fd6 100644 --- a/packages/kafka-consumer/package.json +++ b/packages/kafka-consumer/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "lint-prettier": "prettier --list-different {src,test}/**/*.ts", "lint-tslint": "tslint --config tslint.json '{src,test}/**/*.ts'", diff --git a/packages/kafka-pg-sink/package.json b/packages/kafka-pg-sink/package.json index 680cd5e..8d6802b 100644 --- a/packages/kafka-pg-sink/package.json +++ b/packages/kafka-pg-sink/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "test": "jest --runInBand", "lint-prettier": "prettier --list-different {src,test}/**/*.ts", diff --git a/packages/re-pipeline/package.json b/packages/re-pipeline/package.json index f722544..13b70cd 100644 --- a/packages/re-pipeline/package.json +++ b/packages/re-pipeline/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "scripts": { "test": "jest --runInBand", "lint-prettier": "prettier --list-different {src,test}/**/*.ts", diff --git a/packages/schema-registry-api/package.json b/packages/schema-registry-api/package.json index 0e085cf..c564373 100644 --- a/packages/schema-registry-api/package.json +++ b/packages/schema-registry-api/package.json @@ -7,6 +7,11 @@ "types": "dist/index.d.ts", "author": "Ivan Kerin ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "dependencies": { "node-fetch": "^2.6.0" }, diff --git a/packages/winston-logger/package.json b/packages/winston-logger/package.json index c6f6216..1a3a4bd 100644 --- a/packages/winston-logger/package.json +++ b/packages/winston-logger/package.json @@ -7,6 +7,11 @@ "description": "Winston logger wrapper", "author": "Boost Internal Tools ", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/ovotech/bit-node-tools.git", + "directory": "packages/keycloak-auth" + }, "devDependencies": { "@types/jest": "^24.0.13", "@types/node": "^11.11.4", From af6dc28d8398f2b3462cb522e4b7d6efea138f1c Mon Sep 17 00:00:00 2001 From: Ignas Samoilovas Date: Wed, 18 Mar 2026 16:11:57 +0200 Subject: [PATCH 2/4] lint package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5aa4e73..b945775 100644 --- a/package.json +++ b/package.json @@ -14,5 +14,5 @@ }, "devDependencies": { "lerna": "^3.14.1" - }, + } } From bc021971713c72cbe87dca97dd794d03eaef7c14 Mon Sep 17 00:00:00 2001 From: Ignas Samoilovas Date: Wed, 18 Mar 2026 18:00:30 +0200 Subject: [PATCH 3/4] skip publish if no changes detected --- .github/workflows/main.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 186075a..4bd246e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,6 +76,8 @@ jobs: needs: test if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.check_changes.outputs.has_changes }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -103,17 +105,26 @@ jobs: run: echo "//npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}" > ~/.npmrc - name: List changed packages + id: check_changes run: | echo "### Following pending deployments found" >> $GITHUB_STEP_SUMMARY yarn lerna exec --loglevel silent --concurrency 1 -- \ 'VERSION=$(node -p "require(\"./package.json\").version"); \ if [ "$(npm view $LERNA_PACKAGE_NAME version --registry=https://npm.pkg.github.com/ 2>/dev/null || echo "0.0.0")" != "$VERSION" ]; then \ echo "- **$LERNA_PACKAGE_NAME** - will publish version $VERSION" >> $GITHUB_STEP_SUMMARY; \ + echo "true" > $GITHUB_WORKSPACE/.has_changes; \ fi' + if [ -f $GITHUB_WORKSPACE/.has_changes ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + rm $GITHUB_WORKSPACE/.has_changes + else + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "- None" >> $GITHUB_STEP_SUMMARY + fi publish: needs: prepare-publish - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' && needs.prepare-publish.outputs.has_changes == 'true' runs-on: ubuntu-latest environment: name: github-npm-registry From 9bca55950bed8c85f90ff33fbbebeeede6fa0d91 Mon Sep 17 00:00:00 2001 From: Ignas Samoilovas Date: Thu, 19 Mar 2026 11:52:38 +0200 Subject: [PATCH 4/4] simplify changes detection --- .github/workflows/main.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4bd246e..57d3a54 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -108,15 +108,14 @@ jobs: id: check_changes run: | echo "### Following pending deployments found" >> $GITHUB_STEP_SUMMARY - yarn lerna exec --loglevel silent --concurrency 1 -- \ + OUTPUT=$(yarn lerna exec --loglevel silent --concurrency 1 -- \ 'VERSION=$(node -p "require(\"./package.json\").version"); \ if [ "$(npm view $LERNA_PACKAGE_NAME version --registry=https://npm.pkg.github.com/ 2>/dev/null || echo "0.0.0")" != "$VERSION" ]; then \ - echo "- **$LERNA_PACKAGE_NAME** - will publish version $VERSION" >> $GITHUB_STEP_SUMMARY; \ - echo "true" > $GITHUB_WORKSPACE/.has_changes; \ - fi' - if [ -f $GITHUB_WORKSPACE/.has_changes ]; then + echo "- **$LERNA_PACKAGE_NAME** - will publish version $VERSION"; \ + fi' | grep '\- \*\*' || true) + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" >> $GITHUB_STEP_SUMMARY echo "has_changes=true" >> $GITHUB_OUTPUT - rm $GITHUB_WORKSPACE/.has_changes else echo "has_changes=false" >> $GITHUB_OUTPUT echo "- None" >> $GITHUB_STEP_SUMMARY