From 8ec55cfa3a75d26369001658bf8bc5f32e90d8fd Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Wed, 27 May 2026 20:02:42 +0300 Subject: [PATCH 1/3] Move BQ update to kubernetes --- Dockerfile.bq-updater | 21 +++++ deployment/bq-updater/cronjob.yaml | 97 ++++++++++++++++++++++++ deployment/bq-updater/kustomization.yaml | 11 +++ scripts/run_bq_update.sh | 15 ++-- scripts/run_update_epoch.sh | 8 +- 5 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 Dockerfile.bq-updater create mode 100644 deployment/bq-updater/cronjob.yaml create mode 100644 deployment/bq-updater/kustomization.yaml diff --git a/Dockerfile.bq-updater b/Dockerfile.bq-updater new file mode 100644 index 0000000..09e8eae --- /dev/null +++ b/Dockerfile.bq-updater @@ -0,0 +1,21 @@ +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + coreutils \ + jq \ + postgresql-client \ + sed \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app/scripts + +COPY scripts/ /app/scripts/ +COPY schema/tables/ /app/schema/tables/ + +RUN find /app/scripts -maxdepth 1 -type f -name "*.sh" -exec chmod +x {} \; + +ENTRYPOINT ["bash", "-lc"] +CMD ["./run_bq_update.sh"] diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml new file mode 100644 index 0000000..fbe4d85 --- /dev/null +++ b/deployment/bq-updater/cronjob.yaml @@ -0,0 +1,97 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cardano-bq-slot-sync + labels: + app: cardano-bq-slot-sync +spec: + suspend: true + schedule: "3,33 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 600 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: cardano-bq-slot-sync + spec: + imagePullSecrets: + - name: pullsecret + restartPolicy: OnFailure + containers: + - name: cardano-bq-slot-sync + image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 + imagePullPolicy: Always + command: ["bash", "-lc"] + args: + - gcloud config set project blockchain-analytics-392322 && ./run_bq_update.sh + env: + - name: BQ_DRYRUN + value: "false" + - name: PUBSUB_TOPIC_NAME + value: "bca-alerts" + envFrom: + - secretRef: + name: bq-config + - secretRef: + name: pg-config + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cardano-bq-epoch-sync + labels: + app: cardano-bq-epoch-sync +spec: + suspend: true + schedule: "16 5 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 1800 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: cardano-bq-epoch-sync + spec: + imagePullSecrets: + - name: pullsecret + restartPolicy: OnFailure + containers: + - name: cardano-bq-epoch-sync + image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 + imagePullPolicy: Always + command: ["bash", "-lc"] + args: + - gcloud config set project blockchain-analytics-392322 && ./run_update_epoch.sh + env: + - name: BQ_DRYRUN + value: "false" + - name: PUBSUB_TOPIC_NAME + value: "bca-alerts" + envFrom: + - secretRef: + name: bq-config + - secretRef: + name: pg-config + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi diff --git a/deployment/bq-updater/kustomization.yaml b/deployment/bq-updater/kustomization.yaml new file mode 100644 index 0000000..964ab79 --- /dev/null +++ b/deployment/bq-updater/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: cardano-bq + +resources: + - cronjob.yaml + +images: + - name: code.blockchain-applied.com/bca/cardano-bq-updater + newTag: "1.0.3" diff --git a/scripts/run_bq_update.sh b/scripts/run_bq_update.sh index 9b2caf0..b917b16 100755 --- a/scripts/run_bq_update.sh +++ b/scripts/run_bq_update.sh @@ -46,16 +46,19 @@ Q+="COMMIT TRANSACTION;" # Print the query for debugging (optional) echo -e "$Q" > query.txt -# DRYRUN="--dry_run" -DRYRUN= +# Set BQ_DRYRUN=true to skip all writes (useful for testing) +[ "${BQ_DRYRUN}" = "true" ] && DRYRUN="--dry_run" || DRYRUN="" # Execute the transaction ${BQ} query --bigqueryrc=$(pwd)/dot.bigqueryrc ${DRYRUN} --dataset_id=${DATASETID} --nouse_legacy_sql < query.txt 2> logs/transaction-query.err > logs/transaction-query.out - -echo "Updating db-sync slot_no to ${ENDING_SLOT} and epoch_no to ${PG_EPOCH} in BigQuery" -Q="UPDATE ${BQ_PROJECT}.db_sync.last_index set last_slot_no=${ENDING_SLOT}, last_epoch_no=${PG_EPOCH} WHERE tablename='db-sync';" -${BQ} query --nouse_legacy_sql "${Q}" +if [ "${BQ_DRYRUN}" = "true" ]; then + echo "DRY RUN: skipping last_index update and Pub/Sub publish" +else + echo "Updating db-sync slot_no to ${ENDING_SLOT} and epoch_no to ${PG_EPOCH} in BigQuery" + Q="UPDATE ${BQ_PROJECT}.db_sync.last_index set last_slot_no=${ENDING_SLOT}, last_epoch_no=${PG_EPOCH} WHERE tablename='db-sync';" + ${BQ} query --nouse_legacy_sql "${Q}" +fi rm ${TEMPDIR}/key.json rmdir ${TEMPDIR} diff --git a/scripts/run_update_epoch.sh b/scripts/run_update_epoch.sh index fdefcbd..aaf4831 100755 --- a/scripts/run_update_epoch.sh +++ b/scripts/run_update_epoch.sh @@ -43,14 +43,18 @@ do if [ "${PG_EPOCH_NO}" -gt "${BQ_EPOCH_NO}" ]; then SCRIPT="./update_epoch_${TABLE}.sh" echo "Updating ${TABLENAME}. BigQuery epoch: ${BQ_EPOCH_NO} - Postgres epoch: ${PG_EPOCH_NO}" - ${SCRIPT} ${BQ_EPOCH_NO} ${PG_EPOCH_NO} + [ "${BQ_DRYRUN}" = "true" ] && echo "DRY RUN: skipping ${SCRIPT}" || ${SCRIPT} ${BQ_EPOCH_NO} ${PG_EPOCH_NO} fi done rm ${TEMPDIR}/key.json rmdir ${TEMPDIR} -gcloud pubsub topics publish ${PUBSUB_TOPIC_NAME} --message "Updated BQ epoch tables to epoch_no ${PG_EPOCH_NO}" --project $BQ_PROJECT +if [ "${BQ_DRYRUN}" = "true" ]; then + echo "DRY RUN: skipping Pub/Sub publish" +else + gcloud pubsub topics publish ${PUBSUB_TOPIC_NAME} --message "Updated BQ epoch tables to epoch_no ${PG_EPOCH_NO}" --project $BQ_PROJECT +fi echo "All done." From 576b0f870a9b86e9eb60113a62df90f0da4d8081 Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Wed, 27 May 2026 20:19:26 +0300 Subject: [PATCH 2/3] Enable cronjobs --- deployment/bq-updater/cronjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index fbe4d85..57e593d 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -5,7 +5,7 @@ metadata: labels: app: cardano-bq-slot-sync spec: - suspend: true + suspend: false schedule: "3,33 * * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 @@ -54,7 +54,7 @@ metadata: labels: app: cardano-bq-epoch-sync spec: - suspend: true + suspend: false schedule: "16 5 * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 From d44819f4f5b1534eec5857d5e19a6dd6ae91acf2 Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Wed, 27 May 2026 20:21:42 +0300 Subject: [PATCH 3/3] change restartPolicy --- deployment/bq-updater/cronjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index 57e593d..afdf662 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -21,7 +21,7 @@ spec: spec: imagePullSecrets: - name: pullsecret - restartPolicy: OnFailure + restartPolicy: Never containers: - name: cardano-bq-slot-sync image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 @@ -70,7 +70,7 @@ spec: spec: imagePullSecrets: - name: pullsecret - restartPolicy: OnFailure + restartPolicy: Never containers: - name: cardano-bq-epoch-sync image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0