Hotfix k8s bq updater

Dockerfile.bq-updater

@@ -0,0 +1,21 @@
+FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        bash \
+        ca-certificates \
+        coreutils \
+        jq \
+        postgresql-client \
+        sed \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app/scripts
+
+COPY scripts/ /app/scripts/
+COPY schema/tables/ /app/schema/tables/
+
+RUN find /app/scripts -maxdepth 1 -type f -name "*.sh" -exec chmod +x {} \;
+
+ENTRYPOINT ["bash", "-lc"]
+CMD ["./run_bq_update.sh"]

deployment/bq-updater/cronjob.yaml

@@ -0,0 +1,97 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cardano-bq-slot-sync
+  labels:
+    app: cardano-bq-slot-sync
+spec:
+  suspend: false
+  schedule: "3,33 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 600
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app: cardano-bq-slot-sync
+        spec:
+          imagePullSecrets:
+            - name: pullsecret
+          restartPolicy: Never
+          containers:
+            - name: cardano-bq-slot-sync
+              image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0
+              imagePullPolicy: Always
+              command: ["bash", "-lc"]
+              args:
+                - gcloud config set project blockchain-analytics-392322 && ./run_bq_update.sh
+              env:
+                - name: BQ_DRYRUN
+                  value: "false"
+                - name: PUBSUB_TOPIC_NAME
+                  value: "bca-alerts"
+              envFrom:
+                - secretRef:
+                    name: bq-config
+                - secretRef:
+                    name: pg-config
+              resources:
+                requests:
+                  cpu: 200m
+                  memory: 512Mi
+                limits:
+                  cpu: 1000m
+                  memory: 2Gi
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cardano-bq-epoch-sync
+  labels:
+    app: cardano-bq-epoch-sync
+spec:
+  suspend: false
+  schedule: "16 5 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 1800
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app: cardano-bq-epoch-sync
+        spec:
+          imagePullSecrets:
+            - name: pullsecret
+          restartPolicy: Never
+          containers:
+            - name: cardano-bq-epoch-sync
+              image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0
+              imagePullPolicy: Always
+              command: ["bash", "-lc"]
+              args:
+                - gcloud config set project blockchain-analytics-392322 && ./run_update_epoch.sh
+              env:
+                - name: BQ_DRYRUN
+                  value: "false"
+                - name: PUBSUB_TOPIC_NAME
+                  value: "bca-alerts"
+              envFrom:
+                - secretRef:
+                    name: bq-config
+                - secretRef:
+                    name: pg-config
+              resources:
+                requests:
+                  cpu: 200m
+                  memory: 512Mi
+                limits:
+                  cpu: 1000m
+                  memory: 2Gi

deployment/bq-updater/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: cardano-bq
+
+resources:
+  - cronjob.yaml
+
+images:
+  - name: code.blockchain-applied.com/bca/cardano-bq-updater
+    newTag: "1.0.3"

scripts/run_bq_update.sh

@@ -46,16 +46,19 @@ Q+="COMMIT TRANSACTION;"
 # Print the query for debugging (optional)
 echo -e "$Q" > query.txt
 
-# DRYRUN="--dry_run"
-DRYRUN=
+# Set BQ_DRYRUN=true to skip all writes (useful for testing)
+[ "${BQ_DRYRUN}" = "true" ] && DRYRUN="--dry_run" || DRYRUN=""
 
 # Execute the transaction
 ${BQ} query --bigqueryrc=$(pwd)/dot.bigqueryrc ${DRYRUN} --dataset_id=${DATASETID} --nouse_legacy_sql < query.txt 2> logs/transaction-query.err > logs/transaction-query.out
 
-
-echo "Updating db-sync slot_no to ${ENDING_SLOT} and epoch_no to ${PG_EPOCH} in BigQuery"
-Q="UPDATE ${BQ_PROJECT}.db_sync.last_index set last_slot_no=${ENDING_SLOT}, last_epoch_no=${PG_EPOCH} WHERE tablename='db-sync';"
-${BQ} query --nouse_legacy_sql "${Q}"
+if [ "${BQ_DRYRUN}" = "true" ]; then
+  echo "DRY RUN: skipping last_index update and Pub/Sub publish"
+else
+  echo "Updating db-sync slot_no to ${ENDING_SLOT} and epoch_no to ${PG_EPOCH} in BigQuery"
+  Q="UPDATE ${BQ_PROJECT}.db_sync.last_index set last_slot_no=${ENDING_SLOT}, last_epoch_no=${PG_EPOCH} WHERE tablename='db-sync';"
+  ${BQ} query --nouse_legacy_sql "${Q}"
+fi
 
 rm ${TEMPDIR}/key.json
 rmdir ${TEMPDIR}

scripts/run_update_epoch.sh

@@ -43,14 +43,18 @@ do
   if [ "${PG_EPOCH_NO}" -gt "${BQ_EPOCH_NO}" ]; then
     SCRIPT="./update_epoch_${TABLE}.sh"
     echo "Updating ${TABLENAME}. BigQuery epoch: ${BQ_EPOCH_NO} - Postgres epoch: ${PG_EPOCH_NO}"
-    ${SCRIPT} ${BQ_EPOCH_NO} ${PG_EPOCH_NO}
+    [ "${BQ_DRYRUN}" = "true" ] && echo "DRY RUN: skipping ${SCRIPT}" || ${SCRIPT} ${BQ_EPOCH_NO} ${PG_EPOCH_NO}
   fi
 done
 
 rm ${TEMPDIR}/key.json
 rmdir ${TEMPDIR}
 
-gcloud pubsub topics publish ${PUBSUB_TOPIC_NAME} --message "Updated BQ epoch tables to epoch_no ${PG_EPOCH_NO}" --project $BQ_PROJECT
+if [ "${BQ_DRYRUN}" = "true" ]; then
+  echo "DRY RUN: skipping Pub/Sub publish"
+else
+  gcloud pubsub topics publish ${PUBSUB_TOPIC_NAME} --message "Updated BQ epoch tables to epoch_no ${PG_EPOCH_NO}" --project $BQ_PROJECT
+fi
 
 echo "All done."