ETL Tests

.gitattributes

@@ -1,7 +1,7 @@
 ###############################
 # Git Line Endings            #
 # Set default behaviour to automatically normalize line endings.
-* text eolf=lf
+* text eol=lf
 # Force batch scripts to always use CRLF line endings so that if a repo is accessed
 # in Windows via a file share from Linux, the scripts will work.
 *.{cmd,[cC][mM][dD]} text eol=crlf

.github/workflows/_docker-template.yml

@@ -114,7 +114,7 @@ jobs:
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Docker metadata (again)
-        id: meta
+        id: meta_manifest
         uses: docker/metadata-action@v5
         with:
           images: ${{ inputs.image }}
@@ -141,4 +141,4 @@ jobs:
               --tag "${img}:${tag}" \
               "${img}:${tag}-amd64" \
               "${img}:${tag}-arm64"
-          done < <(printf "%s" "${{ steps.meta.outputs.tags }}")
+          done < <(printf "%s" "${{ steps.meta_manifest.outputs.tags }}")

.github/workflows/docker-nifi.yml

@@ -17,7 +17,7 @@ jobs:
     uses: ./.github/workflows/_docker-template.yml
     with:
       image: cogstacksystems/cogstack-nifi
-      context: nifi
+      context: .
       dockerfile: nifi/Dockerfile
       cache_scope: nifi
     secrets: inherit

.github/workflows/shell-script-enforce-exec-check.yml

@@ -0,0 +1,33 @@
+name: shell-script-enforce-exec-check
+
+on:
+  pull_request:
+  push:
+    branches: ['**']
+
+permissions:
+  contents: read
+
+jobs:
+  exec-bit:
+    name: Enforce executable bit on tracked .sh scripts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Verify shell scripts are executable
+        run: |
+          set -euo pipefail
+          echo "Checking tracked .sh files are executable."
+          echo "This prevents CI/runtime failures when scripts are invoked."
+          fail=0
+          while IFS= read -r -d '' file; do
+            if [[ ! -x "$file" ]]; then
+              echo "Non-executable .sh: $file"
+              fail=1
+            fi
+          done < <(git ls-files -z '*.sh')
+          if [[ "$fail" -ne 0 ]]; then
+            echo "Fix with: git update-index --chmod=+x <file>"
+            exit 1
+          fi

.github/workflows/smoke-nifi-services.yml

@@ -0,0 +1,71 @@
+name: smoke-nifi-services
+
+on:
+  pull_request:
+  push:
+    branches: ['**']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  smoke:
+    name: Build and smoke-test NiFi services
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Generate certificates
+        working-directory: security/scripts
+        run: |
+          set -euo pipefail
+          ./create_root_ca_cert.sh
+          ./create_nifi_certs.sh
+
+      - name: Start NiFi services (build)
+        run: |
+          set -euo pipefail
+          source deploy/export_env_vars.sh
+          set -euo pipefail
+          docker compose -f deploy/services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
+
+      - name: Smoke tests
+        run: |
+          set -euo pipefail
+          echo "Running smoke checks against NiFi, NiFi Registry, and nginx."
+          retries=30
+          delay=15
+          for attempt in $(seq 1 $retries); do
+            if ./scripts/smoke_nifi_services.sh; then
+              exit 0
+            fi
+            echo "Attempt ${attempt}/${retries} failed. Sleeping ${delay}s..."
+            sleep "$delay"
+          done
+          echo "Smoke tests failed after ${retries} attempts."
+          exit 1
+
+      - name: Dump container logs
+        if: failure()
+        run: |
+          set -euo pipefail
+          source deploy/export_env_vars.sh
+          set -euo pipefail
+          docker compose -f deploy/services.dev.yml ps
+          docker compose -f deploy/services.dev.yml logs --no-color nifi nifi-nginx nifi-registry-flow
+
+      - name: Shutdown stack
+        if: always()
+        run: |
+          set -euo pipefail
+          source deploy/export_env_vars.sh
+          set -euo pipefail
+          docker compose -f deploy/services.dev.yml down -v

.gitignore

@@ -5,9 +5,18 @@
 .vscode
 .venv
 .ruff_cache
+.mypy_cache
 venv
 **__pycache__
 **/venv
+*.pyc 
+.pyc
+build
+
+# Ignore setuptools metadata
+*.egg-info/
+*.egg-info
+**/*.egg-info/
 
 # keys and certificates
 *.pem
@@ -37,6 +46,7 @@ security/templates/**
 docs/build/*
 
 # Ignore all .env files at any level
+.env
 *.env
 **/*.env
 !*.env.template

deploy/Makefile

@@ -31,6 +31,12 @@ show-env:
 start-nifi:
 	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
 
+start-nifi-dev:
+	$(WITH_ENV) docker compose -f services.dev.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
+
+start-nifi-dev-build:
+	$(WITH_ENV) docker compose -f services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
+
 start-elastic:
 	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-1 elasticsearch-2 kibana
 
@@ -95,14 +101,17 @@ start-data-infra: start-nifi start-elastic start-samples
 
 start-all: start-data-infra start-jupyter start-medcat-service start-ocr-services
 
-.PHONY: start-all start-data-infra start-nifi start-elastic start-samples start-jupyter
+.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build start-elastic start-samples start-jupyter
 
 
 # stop services
 #
 stop-nifi:
 	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) nifi nifi-nginx nifi-registry-flow
 
+stop-nifi-dev:
+	$(WITH_ENV) docker compose -f services.dev.yml $(DC_STOP_CMD) nifi nifi-nginx nifi-registry-flow
+
 stop-elastic:
 	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) elasticsearch-1 elasticsearch-2 kibana
 
@@ -164,7 +173,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples
 
 stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services
 
-.PHONY: stop-data-infra stop-nifi stop-elastic stop-samples stop-jupyter
+.PHONY: stop-data-infra stop-nifi stop-nifi-dev stop-elastic stop-samples stop-jupyter
 
 
 # cleanup

deploy/database.env

@@ -1,9 +1,13 @@
-# production db name
-POSTGRES_DATABANK_DB=cogstack
-
 POSTGRES_DB_MAX_CONNECTIONS=100
 
 # Prefix of file names to load the DB schema for in /services/cogstack-db/(pgsql/mssql)/schemas/ folder
 POSTGRES_DB_SCHEMA_PREFIX="cogstack_db"
 
-POSTGRES_SHM_SIZE="1g"
+# production db name
+DATABASE_DB_NAME=cogstack
+
+DATABASE_DOCKER_SHM_SIZE=1g
+
+DATABASE_DOCKER_CPU_MIN=1
+DATABASE_DOCKER_CPU_MAX=1
+DATABASE_DOCKER_RAM=1g

deploy/elasticsearch.env

@@ -9,10 +9,10 @@ ELASTICSEARCH_VERSION=opensearch
 # possible values :
 #   - elasticsearch                         : docker.elastic.co/elasticsearch/elasticsearch:8.18.2
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest 
-#   - opensearch                            : opensearchproject/opensearch:3.2.0
+#   - opensearch                            : opensearchproject/opensearch:3.4.0
 # the custom cogstack image is always based on the last image of ES native 
 
-ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.2.0
+ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.4.0
 
 ELASTICSEARCH_LOG_LEVEL=INFO
 
@@ -88,9 +88,14 @@ ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup
 ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
 
 # MEMORY CONFIG
-ELASTICSEARCH_JAVA_OPTS="-Xms2048m -Xmx2048m -Des.failure_store_feature_flag_enabled=true"
+ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true"
+
+ELASTICSEARCH_DOCKER_CPU_MIN=1
+ELASTICSEARCH_DOCKER_CPU_MAX=1
+ELASTICSEARCH_DOCKER_RAM=1g
+
+ELASTICSEARCH_DOCKER_SHM_SIZE=512m
 
-ELASTICSEARCH_SHM_SIZE="1g"
 ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
 ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
 
@@ -140,9 +145,6 @@ ELASTICSEARCH_HOSTS='["https://elasticsearch-1:9200","https://elasticsearch-2:92
 
 KIBANA_HOST="https://kibana:5601"
 
-KIBANA_SERVER_NAME="cogstack-kibana"
-
-
 ########################################################################## KIBANA Env vars ###########################################################################
 # NOTE: some variables from the Elasticsearch section are used
 #       - ${ELASTICSEARCH_VERSION} is used for certificate paths, as well as kibana.yml config path.
@@ -158,23 +160,27 @@ KIBANA_VERSION=opensearch-dashboards
 #   - kibana
 #   - opensearch_dashboards  # make note of the underscore...
 
-KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards 
+KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards
 
 # possible values:
 #   - elasticsearch                         : docker.elastic.co/kibana/kibana:8.18.2
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest
-#   - opensearch                            : opensearchproject/opensearch-dashboards:3.2.0
+#   - opensearch                            : opensearchproject/opensearch-dashboards:3.4.0
 # the custom cogstack image is always based on the last image of ES native 
 
-ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.2.0
+ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.4.0
 
 KIBANA_SERVER_NAME="cogstack-kibana"
 KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601"
 
 KIBANA_SERVER_HOST="0.0.0.0"
 KIBANA_SERVER_OUTPUT_PORT=5601
 
-KIBANA_SHM_SIZE="1g"
+KIBANA_DOCKER_SHM_SIZE=512m
+KIBANA_DOCKER_CPU_MIN=1
+KIBANA_DOCKER_CPU_MAX=1
+KIBANA_DOCKER_RAM=1g
+
 
 # this is used in Kibana
 # it needs to be generated via the API 
@@ -201,6 +207,10 @@ ELASTICSEARCH_XPACK_SECURITY_REPORTING_ENCRYPTION_KEY="e0Y1gTxHWOopIWMTtpjQsDS6K
 
 METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:8.18.2"
 
+METRICBEAT_DOCKER_SHM=512m
+METRICBEAT_DOCKER_CPU_MIN=1
+METRICBEAT_DOCKER_CPU_MAX=1
+METRICBEAT_DOCKER_RAM=1g
 
 ########################################################################## FILEBEAT Env vars ###########################################################################
 
@@ -213,3 +223,9 @@ FILEBEAT_STARTUP_COMMAND="-e --strict.perms=false"
 FILEBEAT_HOST="https://elasticsearch-1:9200"
 
 FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:8.18.2"
+
+
+FILEBEAT_DOCKER_SHM=512m
+FILEBEAT_DOCKER_CPU_MIN=1
+FILEBEAT_DOCKER_CPU_MAX=1
+FILEBEAT_DOCKER_RAM=1g

deploy/export_env_vars.sh

@@ -3,12 +3,15 @@
 # Enable strict mode (without -e to avoid exit-on-error)
 set -uo pipefail
 
+# Support being sourced in shells where BASH_SOURCE is unset (e.g. zsh)
+SCRIPT_SOURCE="${BASH_SOURCE[0]-$0}"
+SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)"
+SCRIPT_NAME="$(basename "$SCRIPT_SOURCE")"
 
-echo "🔧 Running $(basename "${BASH_SOURCE[0]}")..."
+echo "🔧 Running $SCRIPT_NAME..."
 
 set -a
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 DEPLOY_DIR="$SCRIPT_DIR"
 SECURITY_DIR="$SCRIPT_DIR/../security/env"
 SERVICES_DIR="$SCRIPT_DIR/../services"
@@ -38,6 +41,22 @@ env_files=(
   "$SERVICES_DIR/cogstack-nlp/medcat-service/env/medcat.env"
 )
 
+LINT_SCRIPT="$SCRIPT_DIR/../nifi/user_scripts/utils/lint_env.py"
+
+if [ -e "$LINT_SCRIPT" ]; then
+    chmod +x $LINT_SCRIPT
+fi
+
+if [ -x "$LINT_SCRIPT" ]; then
+  echo "🔍 Validating env files..."
+  if ! python3 "$LINT_SCRIPT" "${env_files[@]}"; then
+    echo "❌ Env validation failed. Fix the errors above before continuing."
+    exit 1
+  fi
+else
+  echo "⚠️  Skipping env validation; $LINT_SCRIPT not found or not executable."
+fi
+
 for env_file in "${env_files[@]}"; do
   if [ -f "$env_file" ]; then
     echo "✅ Sourcing $env_file"
@@ -56,4 +75,4 @@ set +a
 
 # Restore safe defaults for interactive/dev shell
 set +u
-set +o pipefail
+set +o pipefail

deploy/gitea.env

@@ -23,3 +23,8 @@ GITEA_LOCAL_PUB_KEY_PATH="$GITEA_LOCAL_KEY_PATH.pub"
 GITEA_LOCAL_KEY_TITLE="gitea-cogstack-$(hostname)-$(date +%s)"
 
 GITEA_DEFAULT_MAIN_REMOTE_NAME="cogstack-gitea"
+
+GITEA_DOCKER_SHM_SIZE=512m
+GITEA_DOCKER_CPU_MIN=1
+GITEA_DOCKER_CPU_MAX=1
+GITEA_DOCKER_RAM=1g

deploy/nginx.env

@@ -1,3 +1,11 @@
 NGINX_KIBANA_HOST=kibana
 NGINX_KIBANA_PROXY_PORT=5601
 NGINX_ES_NODE_SOURCE_INSTANCE_NAME="elasticsearch-1"
+
+NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+
+
+NGINX_SHM_SIZE=512mb
+NGINX_DOCKER_CPU_MIN=1
+NGINX_DOCKER_CPU_MAX=1
+NGINX_DOCKER_RAM=1g