Enhance security features and logging in NGINX configuration and Docker setup

Author: Robbie1977

.github/workflows/docker.yml

@@ -32,7 +32,16 @@ jobs:
         DNS_RESOLVER: 8.8.8.8
       run: |
         envsubst '${UPSTREAM_SERVER} ${CACHE_MAX_SIZE} ${CACHE_STALE_TIME} ${DNS_RESOLVER}' < nginx.conf.template > test-nginx.conf
-        docker run --rm -v $(pwd)/test-nginx.conf:/tmp/nginx.conf nginx:1.26-alpine nginx -t -c /tmp/nginx.conf
+        mkdir -p test-logs/hacks
+        touch test-logs/blocked.txt test-logs/whitelist.txt test-logs/blocked-ips.map test-logs/whitelisted-ips.map
+        docker run --rm \
+          -v $(pwd)/test-nginx.conf:/tmp/nginx.conf \
+          -v $(pwd)/test-logs:/logs \
+          -v $(pwd)/test-logs/blocked-ips.map:/etc/nginx/blocked-ips.map \
+          -v $(pwd)/test-logs/whitelisted-ips.map:/etc/nginx/whitelisted-ips.map \
+          nginx:1.26-alpine nginx -t -c /tmp/nginx.conf
+        rm -f test-nginx.conf
+        rm -rf test-logs
 
     - name: Build test Docker image
       run: docker build --no-cache . --file Dockerfile --tag test-image
@@ -71,4 +80,4 @@ jobs:
         password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
     - name: Push Docker image
-      run: docker push virtualflybrain/owl_cache:${{ steps.meta.outputs.tag }}
+      run: docker push virtualflybrain/owl_cache:${{ steps.meta.outputs.tag }}

Dockerfile

@@ -11,11 +11,16 @@ ENV DNS_RESOLVER=8.8.8.8
 ARG NGINX_CONF=nginx.conf.template
 COPY $NGINX_CONF /etc/nginx/nginx.conf.template
 COPY health-monitor.sh /usr/local/bin/health-monitor.sh
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-RUN mkdir -p /var/cache/nginx/owlery && chown -R nginx:nginx /var/cache/nginx && \
-    chmod +x /usr/local/bin/health-monitor.sh && \
+RUN mkdir -p /var/cache/nginx/owlery /logs/hacks && \
+    touch /logs/blocked.txt /logs/whitelist.txt /etc/nginx/blocked-ips.map /etc/nginx/whitelisted-ips.map && \
+    chown -R nginx:nginx /var/cache/nginx /logs && \
+    chmod +x /usr/local/bin/health-monitor.sh /usr/local/bin/docker-entrypoint.sh && \
     apk add --no-cache gettext
 
 EXPOSE 80 8080
 
-CMD ["/bin/sh", "-c", "export UPSTREAM_SERVER=$UPSTREAM_SERVER && export CACHE_MAX_SIZE=$CACHE_MAX_SIZE && export CACHE_STALE_TIME=$CACHE_STALE_TIME && export DNS_RESOLVER=\"$DNS_RESOLVER\" && envsubst '${UPSTREAM_SERVER} ${CACHE_MAX_SIZE} ${CACHE_STALE_TIME} ${DNS_RESOLVER}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf && /usr/local/bin/health-monitor.sh & nginx -g 'daemon off;'"]
+VOLUME ["/var/cache/nginx", "/logs"]
+
+CMD ["/usr/local/bin/docker-entrypoint.sh"]

README.md

@@ -4,6 +4,8 @@
 
 A high-performance caching proxy server that sits in front of OWL reasoning services to dramatically speed up query responses. Built on NGINX Alpine with a 6-month cache TTL, stale-while-revalidate pattern, and 5-year disk retention so a cached response is always available.
 
+The proxy also includes security guardrails to refuse common scanner/probing requests before they reach Owlery, with optional IP block and whitelist files under `/logs`.
+
 ## Usage Examples
 
 ### Basic Usage
@@ -29,6 +31,9 @@ services:
     ports:
       - "80:80"
       - "8080:8080"
+    volumes:
+      - /cache:/var/cache/nginx
+      - /logs:/logs
     environment:
       - UPSTREAM_SERVER=owl:8080  # For production with owl service
       - CACHE_MAX_SIZE=1t         # 1TB cache size for high-traffic deployments
@@ -93,6 +98,35 @@ Example response:
 - `STATUS_POLL_INTERVAL`: Seconds between `/status` refreshes (default: `5`)
 - `HEALTH_LOG_INTERVAL`: Seconds between periodic upstream health log lines when state is unchanged (default: `300`)
 
+### Security Filtering and Blocking
+
+- **Probe filtering**: Requests matching common probing signatures (for example `*.php`, `wp-login.php`, `.env`, `phpmyadmin`, path traversal payloads) are immediately refused with HTTP `403` and are **not** forwarded upstream.
+- **Probe log output**: Refused probe requests are logged to `/logs/hacks/probes.log`, including both raw `X-Forwarded-For` and the extracted left-most client IP.
+- **Manual IP blocklist**: Add one IPv4/IPv6 address per line in `/logs/blocked.txt` (comments allowed with `#`).
+- **Manual IP whitelist**: Add one IPv4/IPv6 address per line in `/logs/whitelist.txt` (comments allowed with `#`).
+
+Example `/logs/blocked.txt`:
+
+```txt
+203.0.113.10
+# office VPN egress
+2001:db8::1234
+```
+
+Example `/logs/whitelist.txt`:
+
+```txt
+203.0.113.50
+# trusted monitoring source
+2001:db8::beef
+```
+
+Blocked IP requests return HTTP `403` and are logged to `/logs/hacks/blocked.log`.
+
+Whitelist entries take precedence over both the blocklist and probe filter.
+
+Blocklist/whitelist entries are loaded when the container starts. If you update `/logs/blocked.txt` or `/logs/whitelist.txt`, restart the container to apply changes.
+
 ### Cache Headers
 
 The proxy adds helpful headers to responses:
@@ -151,6 +185,11 @@ docker pull virtualflybrain/owl_cache:latest
 mkdir -p /cache
 chown -R 101:101 /cache
 
+# Create persistent logs + blocklist file
+mkdir -p /logs/hacks
+touch /logs/blocked.txt
+touch /logs/whitelist.txt
+
 # Deploy with compose
 docker-compose up -d
 

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - UPSTREAM_SERVER=owl:8080
     volumes:
       - /cache:/var/cache/nginx
+      - /logs:/logs
     networks:
       - owlery_network
     ports:
@@ -26,4 +27,4 @@ services:
 
 networks:
   owlery_network:
-    driver: bridge
+    driver: bridge

docker-entrypoint.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+set -eu
+
+BLOCKLIST_SOURCE="/logs/blocked.txt"
+BLOCKLIST_MAP="/etc/nginx/blocked-ips.map"
+WHITELIST_SOURCE="/logs/whitelist.txt"
+WHITELIST_MAP="/etc/nginx/whitelisted-ips.map"
+
+prepare_log_paths() {
+    # Ensure required runtime directories exist, including fresh bind mounts/volumes.
+    mkdir -p \
+        /logs \
+        /logs/hacks \
+        /var/cache/nginx \
+        /var/cache/nginx/owlery \
+        /var/log/nginx \
+        /etc/nginx
+
+    touch "$BLOCKLIST_SOURCE"
+    touch "$WHITELIST_SOURCE"
+    touch "$BLOCKLIST_MAP"
+    touch "$WHITELIST_MAP"
+}
+
+generate_ip_map() {
+    source_file="$1"
+    target_map="$2"
+    label="$3"
+    tmp_map="$(mktemp /tmp/${label}-ips.XXXXXX)"
+    : > "$tmp_map"
+
+    while IFS= read -r raw_line || [ -n "$raw_line" ]; do
+        line="$(printf '%s' "$raw_line" | tr -d '\r' | sed 's/#.*//;s/^[[:space:]]*//;s/[[:space:]]*$//')"
+        [ -z "$line" ] && continue
+
+        if printf '%s' "$line" | grep -Eq '^[0-9A-Fa-f:.]+$'; then
+            printf '%s 1;\n' "$line" >> "$tmp_map"
+        else
+            printf 'Ignoring invalid %s IP entry in %s: %s\n' "$label" "$source_file" "$raw_line" >&2
+        fi
+    done < "$source_file"
+
+    mv "$tmp_map" "$target_map"
+}
+
+export UPSTREAM_SERVER="${UPSTREAM_SERVER:-owl.virtualflybrain.org:80}"
+export CACHE_MAX_SIZE="${CACHE_MAX_SIZE:-20g}"
+export CACHE_STALE_TIME="${CACHE_STALE_TIME:-6M}"
+export DNS_RESOLVER="${DNS_RESOLVER:-8.8.8.8}"
+
+prepare_log_paths
+generate_ip_map "$BLOCKLIST_SOURCE" "$BLOCKLIST_MAP" "blocked"
+generate_ip_map "$WHITELIST_SOURCE" "$WHITELIST_MAP" "whitelisted"
+
+envsubst '${UPSTREAM_SERVER} ${CACHE_MAX_SIZE} ${CACHE_STALE_TIME} ${DNS_RESOLVER}' \
+    < /etc/nginx/nginx.conf.template \
+    > /etc/nginx/nginx.conf
+
+/usr/local/bin/health-monitor.sh &
+exec nginx -g 'daemon off;'

nginx.conf.template

@@ -11,13 +11,56 @@ http {
     keepalive_timeout 65s;
     keepalive_requests 10000;
 
+    # Extract the left-most X-Forwarded-For IP (original client) when present.
+    map $http_x_forwarded_for $client_ip_for_security {
+        default $remote_addr;
+        "~^\s*([^,\s]+)" $1;
+    }
+
+    # Block common automated probing paths before they reach the upstream.
+    map $request_uri $is_probe_request {
+        default 0;
+        ~*(?:^|/).*\.(?:php\d*|phtml|phar|asp|aspx|jsp|cgi|pl)(?:$|[/?]) 1;
+        ~*(?:^|/)(?:wp-admin|wp-login\.php|xmlrpc\.php|phpmyadmin|pma|adminer|cgi-bin|vendor/phpunit)(?:$|[/?]) 1;
+        ~*(?:^|/)\.(?:env|git|svn|hg|DS_Store)(?:$|[/?]) 1;
+        ~*(?:\.\./|etc/passwd|proc/self/environ|php://|auto_prepend_file|allow_url_include|%00) 1;
+    }
+
+    # Generated at container startup from /logs/whitelist.txt (one IP per line).
+    map $client_ip_for_security $is_whitelisted_ip {
+        default 0;
+        include /etc/nginx/whitelisted-ips*.map;
+    }
+
+    # Generated at container startup from /logs/blocked.txt (one IP per line).
+    map $client_ip_for_security $is_blocked_ip {
+        default 0;
+        include /etc/nginx/blocked-ips*.map;
+    }
+
+    # Whitelist wins over blocklist and probe detection.
+    map "$is_whitelisted_ip:$is_blocked_ip" $should_block_ip {
+        default 0;
+        "0:1" 1;
+    }
+    map "$is_whitelisted_ip:$is_probe_request" $should_block_probe {
+        default 0;
+        "0:1" 1;
+    }
+
     log_format cache '$remote_addr - $remote_user [$time_local] "$request" '
                         '$status $body_bytes_sent "$http_referer" '
                         '"$http_user_agent" "$http_x_forwarded_for" '
                         '$upstream_cache_status $request_time $upstream_response_time';
+    log_format hack '$time_iso8601 client_ip="$client_ip_for_security" '
+                    'remote_addr="$remote_addr" xff="$http_x_forwarded_for" '
+                    'request="$request" status=$status host="$host" '
+                    'ua="$http_user_agent"';
 
     access_log /dev/stdout cache;
     access_log /var/log/nginx/cache-access.log cache;
+    access_log /logs/hacks/probes.log hack if=$should_block_probe;
+    access_log /logs/hacks/blocked.log hack if=$should_block_ip;
     error_log /dev/stderr warn;
 
     proxy_cache_path /var/cache/nginx/owlery levels=1:2 keys_zone=owlery_cache:100m max_size=${CACHE_MAX_SIZE} inactive=5y use_temp_path=off;
@@ -40,6 +83,14 @@ http {
     server {
         listen 80;
 
+        if ($should_block_ip) {
+            return 403;
+        }
+
+        if ($should_block_probe) {
+            return 403;
+        }
+
         location = /status {
             access_log off;
             default_type application/json;
@@ -95,6 +146,14 @@ http {
     server {
         listen 8080;
 
+        if ($should_block_ip) {
+            return 403;
+        }
+
+        if ($should_block_probe) {
+            return 403;
+        }
+
         location = /status {
             access_log off;
             default_type application/json;