diff --git a/.circleci/config.yml b/.circleci/config.yml index 146ba4e8c..a38599b3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,7 +77,7 @@ jobs: - run: name: Check that the CHANGELOG has been modified in the current branch command: | - git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG + git log --name-only --pretty=format: origin..HEAD | grep CHANGELOG # Check that the CHANGELOG max line length does not exceed 80 characters lint-changelog: @@ -378,16 +378,26 @@ jobs: command: | helm repo add elastic https://helm.elastic.co helm repo update - helm install elastic-operator elastic/eck-operator + # Pin the ECK operator: an unpinned chart drifts and breaks CI + helm install elastic-operator elastic/eck-operator --version 2.16.1 + + # The operator must be running before it can reconcile the + # Elasticsearch resource, otherwise no pods are ever created. + kubectl rollout status statefulset/elastic-operator --timeout=300s # Deploy a two-nodes elasticsearch cluster kubectl apply -f src/helm/manifests/data-lake.yml - # Wait for pods to be ready - # Still need to wait for pods to be created before waiting for them - # to be ready (see https://github.com/kubernetes/kubectl/issues/1516) - sleep 5 - kubectl wait --for=condition=ready pod --selector=common.k8s.elastic.co/type=elasticsearch --timeout=120s + # The operator writes `.status` asynchronously, so `kubectl wait + # --for=jsonpath` errors with "status is not found" when the field + # does not exist yet. Poll until the cluster reports a Ready phase. + for i in $(seq 1 60); do + phase=$(kubectl get elasticsearch data-lake -o jsonpath='{.status.phase}' 2>/dev/null || true) + echo "Elasticsearch phase: ${phase:-} ($i/60)" + if [ "$phase" = "Ready" ]; then break; fi + sleep 10 + done + [ "$phase" = "Ready" ] # Store elastic user password echo 'ELASTIC_PASSWORD="$(kubectl get secret data-lake-es-elastic-user -o jsonpath="{.data.elastic}" | base64 -d)"' >> $BASH_ENV diff --git a/.k3d-cluster.env.sh b/.k3d-cluster.env.sh index 1b968b8a2..0d02cf292 100644 --- a/.k3d-cluster.env.sh +++ b/.k3d-cluster.env.sh @@ -4,7 +4,7 @@ echo -n "Loading k3d cluster environment... " export ARNOLD_DEFAULT_VAULT_PASSWORD=arnold export ANSIBLE_VAULT_PASSWORD="${ARNOLD_DEFAULT_VAULT_PASSWORD}" -export ARNOLD_IMAGE_TAG=master +export ARNOLD_IMAGE_TAG=6.23.0 export K3D_BIND_HOST_PORT_HTTP=80 export K3D_BIND_HOST_PORT_HTTPS=443 export K3D_CLUSTER_NAME=ralph diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f3b6ce9f..2d09a386a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ and this project adheres to ### Fixed - Fix type of `statement.result.score.scaled` from `int` to `Decimal` +- Fix `check-changelog` CI job by replacing the removed `git whatchanged` + command with `git log --name-only` +- Make MongoDB delete-failure tests tolerant to pymongo error message changes +- Pin Arnold to `6.23.0` (was `master`) to fix the CircleCI `tray` job broken + by the ansible-core `2.14.18` vault handling change +- Pin the ECK operator and poll the Elasticsearch resource status in the + `test-helm` CI job to fix flaky "no matching resources"/"status not found" ## [5.0.1] - 2024-07-11 diff --git a/Makefile b/Makefile index 185f78db2..9bd09638c 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ ES_INDEX = statements ES_URL = $(ES_PROTOCOL)://$(ES_HOST):$(ES_PORT) # -- Arnold -ARNOLD = ARNOLD_IMAGE_TAG=master bin/arnold +ARNOLD = ARNOLD_IMAGE_TAG=6.23.0 bin/arnold ARNOLD_APP = ralph ARNOLD_APP_VARS = group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/main.yml ARNOLD_CUSTOMER ?= ralph @@ -56,11 +56,11 @@ K8S_NAMESPACE = $(ARNOLD_ENVIRONMENT)-$(ARNOLD_CUSTOMER) default: help bin/arnold: - curl -Lo "bin/arnold" "https://raw.githubusercontent.com/openfun/arnold/master/bin/arnold" + curl -Lo "bin/arnold" "https://raw.githubusercontent.com/openfun/arnold/v6.23.0/bin/arnold" chmod +x bin/arnold bin/init-cluster: - curl -Lo "bin/init-cluster" "https://raw.githubusercontent.com/openfun/arnold/master/bin/init-cluster" + curl -Lo "bin/init-cluster" "https://raw.githubusercontent.com/openfun/arnold/v6.23.0/bin/init-cluster" chmod +x bin/init-cluster .env: diff --git a/tests/backends/data/test_async_mongo.py b/tests/backends/data/test_async_mongo.py index bad3734ae..af0608264 100644 --- a/tests/backends/data/test_async_mongo.py +++ b/tests/backends/data/test_async_mongo.py @@ -785,12 +785,11 @@ async def test_backends_data_async_mongo_write_with_delete_operation_failure( """ backend = async_mongo_backend() - msg = ( - "Failed to delete document chunk: Invalid document " - "{'q': {'_source.id': {'$in': []}}, 'limit': 0} | " - "cannot encode object: , of type: " - ) - with pytest.raises(BackendException, match=msg): + # The tail of the error message is produced by pymongo/bson and its exact + # wording changes across pymongo versions; only assert on the stable prefix + # controlled by Ralph. + msg_prefix = "Failed to delete document chunk: " + with pytest.raises(BackendException, match=re.escape(msg_prefix)): await backend.write([{"id": object}], operation_type=BaseOperationType.DELETE) # Given `ignore_errors` argument set to `True`, the `write` method should not raise @@ -805,11 +804,12 @@ async def test_backends_data_async_mongo_write_with_delete_operation_failure( == 0 ) - assert ( - "ralph.backends.data.async_mongo", - logging.WARNING, - msg, - ) in caplog.record_tuples + assert any( + name == "ralph.backends.data.async_mongo" + and level == logging.WARNING + and message.startswith(msg_prefix) + for name, level, message in caplog.record_tuples + ) @pytest.mark.anyio diff --git a/tests/backends/data/test_mongo.py b/tests/backends/data/test_mongo.py index 7caf52de3..60d47d41d 100644 --- a/tests/backends/data/test_mongo.py +++ b/tests/backends/data/test_mongo.py @@ -620,16 +620,20 @@ def test_backends_data_mongo_write_with_delete_operation_failure( """ backend = mongo_backend() - msg = ( - "Failed to delete document chunk: Invalid document {'q': {'_source.id': {'$in':" - " []}}, 'limit': 0} | cannot encode object: , " - "of type: " - ) + # The tail of the error message is produced by pymongo/bson and its exact + # wording changes across pymongo versions; only assert on the stable prefix + # controlled by Ralph. + msg_prefix = "Failed to delete document chunk: " with caplog.at_level(logging.ERROR): - with pytest.raises(BackendException, match=msg): + with pytest.raises(BackendException, match=re.escape(msg_prefix)): backend.write([{"id": object}], operation_type=BaseOperationType.DELETE) - assert ("ralph.backends.data.mongo", logging.ERROR, msg) in caplog.record_tuples + assert any( + name == "ralph.backends.data.mongo" + and level == logging.ERROR + and message.startswith(msg_prefix) + for name, level, message in caplog.record_tuples + ) # Given `ignore_errors` argument set to `True`, the `write` method should not raise # an exception. @@ -643,7 +647,12 @@ def test_backends_data_mongo_write_with_delete_operation_failure( == 0 ) - assert ("ralph.backends.data.mongo", logging.WARNING, msg) in caplog.record_tuples + assert any( + name == "ralph.backends.data.mongo" + and level == logging.WARNING + and message.startswith(msg_prefix) + for name, level, message in caplog.record_tuples + ) backend.close()