ETL Tests

.gitattributes

@@ -1,7 +1,7 @@
 ###############################
 # Git Line Endings            #
 # Set default behaviour to automatically normalize line endings.
-* text eolf=lf
+* text eol=lf
 # Force batch scripts to always use CRLF line endings so that if a repo is accessed
 # in Windows via a file share from Linux, the scripts will work.
 *.{cmd,[cC][mM][dD]} text eol=crlf

.github/workflows/_docker-template.yml

@@ -114,7 +114,7 @@ jobs:
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Docker metadata (again)
-        id: meta
+        id: meta_manifest
         uses: docker/metadata-action@v5
         with:
           images: ${{ inputs.image }}
@@ -141,4 +141,4 @@ jobs:
               --tag "${img}:${tag}" \
               "${img}:${tag}-amd64" \
               "${img}:${tag}-arm64"
-          done < <(printf "%s" "${{ steps.meta.outputs.tags }}")
+          done < <(printf "%s" "${{ steps.meta_manifest.outputs.tags }}")

.gitignore

@@ -5,9 +5,18 @@
 .vscode
 .venv
 .ruff_cache
+.mypy_cache
 venv
 **__pycache__
 **/venv
+*.pyc 
+.pyc
+build
+
+# Ignore setuptools metadata
+*.egg-info/
+*.egg-info
+**/*.egg-info/
 
 # keys and certificates
 *.pem
@@ -37,6 +46,7 @@ security/templates/**
 docs/build/*
 
 # Ignore all .env files at any level
+.env
 *.env
 **/*.env
 !*.env.template

deploy/database.env

@@ -1,9 +1,13 @@
-# production db name
-POSTGRES_DATABANK_DB=cogstack
-
 POSTGRES_DB_MAX_CONNECTIONS=100
 
 # Prefix of file names to load the DB schema for in /services/cogstack-db/(pgsql/mssql)/schemas/ folder
 POSTGRES_DB_SCHEMA_PREFIX="cogstack_db"
 
-POSTGRES_SHM_SIZE="1g"
+# production db name
+DATABASE_DB_NAME=cogstack
+
+DATABASE_DOCKER_SHM_SIZE=1g
+
+DATABASE_DOCKER_CPU_MIN=1
+DATABASE_DOCKER_CPU_MAX=1
+DATABASE_DOCKER_RAM=1g

deploy/elasticsearch.env

@@ -9,10 +9,10 @@ ELASTICSEARCH_VERSION=opensearch
 # possible values :
 #   - elasticsearch                         : docker.elastic.co/elasticsearch/elasticsearch:8.18.2
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest 
-#   - opensearch                            : opensearchproject/opensearch:3.2.0
+#   - opensearch                            : opensearchproject/opensearch:3.3.0
 # the custom cogstack image is always based on the last image of ES native 
 
-ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.2.0
+ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.3.0
 
 ELASTICSEARCH_LOG_LEVEL=INFO
 
@@ -88,9 +88,14 @@ ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup
 ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
 
 # MEMORY CONFIG
-ELASTICSEARCH_JAVA_OPTS="-Xms2048m -Xmx2048m -Des.failure_store_feature_flag_enabled=true"
+ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true"
+
+ELASTICSEARCH_DOCKER_CPU_MIN=1
+ELASTICSEARCH_DOCKER_CPU_MAX=1
+ELASTICSEARCH_DOCKER_RAM=1g
+
+ELASTICSEARCH_DOCKER_SHM_SIZE=512m
 
-ELASTICSEARCH_SHM_SIZE="1g"
 ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
 ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
 
@@ -140,9 +145,6 @@ ELASTICSEARCH_HOSTS='["https://elasticsearch-1:9200","https://elasticsearch-2:92
 
 KIBANA_HOST="https://kibana:5601"
 
-KIBANA_SERVER_NAME="cogstack-kibana"
-
-
 ########################################################################## KIBANA Env vars ###########################################################################
 # NOTE: some variables from the Elasticsearch section are used
 #       - ${ELASTICSEARCH_VERSION} is used for certificate paths, as well as kibana.yml config path.
@@ -158,23 +160,27 @@ KIBANA_VERSION=opensearch-dashboards
 #   - kibana
 #   - opensearch_dashboards  # make note of the underscore...
 
-KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards 
+KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards
 
 # possible values:
 #   - elasticsearch                         : docker.elastic.co/kibana/kibana:8.18.2
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest
-#   - opensearch                            : opensearchproject/opensearch-dashboards:3.2.0
+#   - opensearch                            : opensearchproject/opensearch-dashboards:3.3.0
 # the custom cogstack image is always based on the last image of ES native 
 
-ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.2.0
+ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.3.0
 
 KIBANA_SERVER_NAME="cogstack-kibana"
 KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601"
 
 KIBANA_SERVER_HOST="0.0.0.0"
 KIBANA_SERVER_OUTPUT_PORT=5601
 
-KIBANA_SHM_SIZE="1g"
+KIBANA_DOCKER_SHM_SIZE=512m
+KIBANA_DOCKER_CPU_MIN=1
+KIBANA_DOCKER_CPU_MAX=1
+KIBANA_DOCKER_RAM=1g
+
 
 # this is used in Kibana
 # it needs to be generated via the API 
@@ -201,6 +207,10 @@ ELASTICSEARCH_XPACK_SECURITY_REPORTING_ENCRYPTION_KEY="e0Y1gTxHWOopIWMTtpjQsDS6K
 
 METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:8.18.2"
 
+METRICBEAT_DOCKER_SHM=512m
+METRICBEAT_DOCKER_CPU_MIN=1
+METRICBEAT_DOCKER_CPU_MAX=1
+METRICBEAT_DOCKER_RAM=1g
 
 ########################################################################## FILEBEAT Env vars ###########################################################################
 
@@ -213,3 +223,9 @@ FILEBEAT_STARTUP_COMMAND="-e --strict.perms=false"
 FILEBEAT_HOST="https://elasticsearch-1:9200"
 
 FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:8.18.2"
+
+
+FILEBEAT_DOCKER_SHM=512m
+FILEBEAT_DOCKER_CPU_MIN=1
+FILEBEAT_DOCKER_CPU_MAX=1
+FILEBEAT_DOCKER_RAM=1g

deploy/export_env_vars.sh

@@ -3,12 +3,15 @@
 # Enable strict mode (without -e to avoid exit-on-error)
 set -uo pipefail
 
+# Support being sourced in shells where BASH_SOURCE is unset (e.g. zsh)
+SCRIPT_SOURCE="${BASH_SOURCE[0]-$0}"
+SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)"
+SCRIPT_NAME="$(basename "$SCRIPT_SOURCE")"
 
-echo "🔧 Running $(basename "${BASH_SOURCE[0]}")..."
+echo "🔧 Running $SCRIPT_NAME..."
 
 set -a
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 DEPLOY_DIR="$SCRIPT_DIR"
 SECURITY_DIR="$SCRIPT_DIR/../security/env"
 SERVICES_DIR="$SCRIPT_DIR/../services"
@@ -38,6 +41,18 @@ env_files=(
   "$SERVICES_DIR/cogstack-nlp/medcat-service/env/medcat.env"
 )
 
+LINT_SCRIPT="$SCRIPT_DIR/../nifi/user_scripts/utils/lint_env.py"
+
+if [ -x "$LINT_SCRIPT" ]; then
+  echo "🔍 Validating env files..."
+  if ! python3 "$LINT_SCRIPT" "${env_files[@]}"; then
+    echo "❌ Env validation failed. Fix the errors above before continuing."
+    exit 1
+  fi
+else
+  echo "⚠️  Skipping env validation; $LINT_SCRIPT not found or not executable."
+fi
+
 for env_file in "${env_files[@]}"; do
   if [ -f "$env_file" ]; then
     echo "✅ Sourcing $env_file"
@@ -56,4 +71,4 @@ set +a
 
 # Restore safe defaults for interactive/dev shell
 set +u
-set +o pipefail
+set +o pipefail

deploy/gitea.env

@@ -23,3 +23,8 @@ GITEA_LOCAL_PUB_KEY_PATH="$GITEA_LOCAL_KEY_PATH.pub"
 GITEA_LOCAL_KEY_TITLE="gitea-cogstack-$(hostname)-$(date +%s)"
 
 GITEA_DEFAULT_MAIN_REMOTE_NAME="cogstack-gitea"
+
+GITEA_DOCKER_SHM_SIZE=512m
+GITEA_DOCKER_CPU_MIN=1
+GITEA_DOCKER_CPU_MAX=1
+GITEA_DOCKER_RAM=1g

deploy/nginx.env

@@ -1,3 +1,9 @@
 NGINX_KIBANA_HOST=kibana
 NGINX_KIBANA_PROXY_PORT=5601
 NGINX_ES_NODE_SOURCE_INSTANCE_NAME="elasticsearch-1"
+
+
+NGINX_SHM_SIZE=1g
+NGINX_DOCKER_CPU_MIN=1
+NGINX_DOCKER_CPU_MAX=1
+NGINX_DOCKER_RAM=1g

deploy/nifi.env

@@ -1,3 +1,29 @@
+
+
+##############################################################################################################################
+# IMPORTANT SETTINGS FOR DEPLOYMENTS RESOURCE SCOPED
+##############################################################################################################################
+NIFI_JVM_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom"
+NIFI_JVM_HEAP_INIT=768m
+NIFI_JVM_HEAP_MAX=1g
+
+
+NIFI_DOCKER_SHM_SIZE=1g
+NIFI_DOCKER_REGISTRY_SHM_SIZE=1g
+
+NIFI_DOCKER_CPU_MIN=1
+NIFI_DOCKER_CPU_MAX=1
+NIFI_DOCKER_RAM=1g
+
+NIFI_REGISTRY_DOCKER_CPU_MIN=1
+NIFI_REGISTRY_DOCKER_CPU_MAX=1
+NIFI_REGISTRY_DOCKER_RAM=1g
+
+NIFI_DOCKER_LOG_SIZE_PER_FILE="250m"
+NIFI_DOCKER_LOG_NUM_FILES=10
+
+##############################################################################################################################
+
 # NiFi
 NIFI_ENV_FILE="./nifi.env"
 NIFI_SECURITY_DIR="../security/certificates/nifi/"
@@ -6,11 +32,6 @@ NIFI_DATA_PATH="../data/"
 NIFI_VERSION="2.7.2"
 NIFI_TOOLKIT_VERSION=$NIFI_VERSION
 
-NIFI_SHM_SIZE="1g"
-NIFI_REGISTRY_SHM_SIZE="1g"
-NIFI_DOCKER_LOG_SIZE_PER_FILE="250m"
-NIFI_DOCKER_LOG_NUM_FILES=10
-
 #### Port and network settings
 NIFI_WEB_PROXY_CONTEXT_PATH="/nifi"