Tune nginx for high traffic and upstream instability

Author: Robbie1977

nginx.conf.template

@@ -1,12 +1,15 @@
 worker_processes auto;
 
 events {
-    worker_connections 1024;
+    worker_connections 4096;
+    multi_accept on;
 }
 
 http {
     resolver ${DNS_RESOLVER} valid=30s ipv6=off;
     resolver_timeout 5s;
+    keepalive_timeout 65s;
+    keepalive_requests 10000;
 
     log_format cache '$remote_addr - $remote_user [$time_local] "$request" '
                         '$status $body_bytes_sent "$http_referer" '
@@ -30,6 +33,8 @@ http {
     proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504 http_404;
     proxy_cache_background_update on;
     proxy_cache_lock on;
+    proxy_cache_lock_timeout 30s;
+    proxy_cache_lock_age 120s;
 
     server {
         listen 80;
@@ -53,9 +58,14 @@ http {
             proxy_buffering on;
             proxy_http_version 1.1;
             proxy_set_header Connection "";
-            proxy_connect_timeout 6h;
+            # Fail fast when the upstream is unreachable; long-running work is in read/send.
+            proxy_connect_timeout 5s;
             proxy_read_timeout 6h;
             proxy_send_timeout 6h;
+            proxy_socket_keepalive on;
+            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+            proxy_next_upstream_timeout 20s;
+            proxy_next_upstream_tries 3;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
@@ -88,9 +98,14 @@ http {
             proxy_buffering on;
             proxy_http_version 1.1;
             proxy_set_header Connection "";
-            proxy_connect_timeout 6h;
+            # Fail fast when the upstream is unreachable; long-running work is in read/send.
+            proxy_connect_timeout 5s;
             proxy_read_timeout 6h;
             proxy_send_timeout 6h;
+            proxy_socket_keepalive on;
+            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+            proxy_next_upstream_timeout 20s;
+            proxy_next_upstream_tries 3;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;

nginx.conf.template.backup

@@ -1,12 +1,15 @@
 worker_processes auto;
 
 events {
-    worker_connections 1024;
+    worker_connections 4096;
+    multi_accept on;
 }
 
 http {
     resolver ${DNS_RESOLVER} valid=30s;
     resolver_timeout 5s;
+    keepalive_timeout 65s;
+    keepalive_requests 10000;
 
     upstream owlery {
         server ${UPSTREAM_SERVER};
@@ -31,9 +34,11 @@ http {
     proxy_cache_valid 200 7776000s;
     proxy_cache_valid 404 600s;
 
-    proxy_cache_use_stale updating;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504 http_404;
     proxy_cache_background_update on;
     proxy_cache_lock on;
+    proxy_cache_lock_timeout 30s;
+    proxy_cache_lock_age 120s;
 
     server {
         listen 80;
@@ -57,16 +62,18 @@ http {
             proxy_buffering on;
             proxy_http_version 1.1;
             proxy_set_header Connection "";
-            proxy_connect_timeout 6h;
+            # Fail fast when the upstream is unreachable; long-running work is in read/send.
+            proxy_connect_timeout 5s;
             proxy_read_timeout 6h;
             proxy_send_timeout 6h;
+            proxy_socket_keepalive on;
             proxy_set_header Host $http_host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-            proxy_next_upstream_timeout 30s;
-            proxy_next_upstream_tries 2;
+            proxy_next_upstream_timeout 20s;
+            proxy_next_upstream_tries 3;
             add_header X-Cache-Status $upstream_cache_status;
             add_header X-Cache-Key "$request_method$request_uri";
             proxy_ignore_headers Cache-Control Expires Set-Cookie;
@@ -96,16 +103,18 @@ http {
             proxy_buffering on;
             proxy_http_version 1.1;
             proxy_set_header Connection "";
-            proxy_connect_timeout 6h;
+            # Fail fast when the upstream is unreachable; long-running work is in read/send.
+            proxy_connect_timeout 5s;
             proxy_read_timeout 6h;
             proxy_send_timeout 6h;
+            proxy_socket_keepalive on;
             proxy_set_header Host $http_host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-            proxy_next_upstream_timeout 30s;
-            proxy_next_upstream_tries 2;
+            proxy_next_upstream_timeout 20s;
+            proxy_next_upstream_tries 3;
             add_header X-Cache-Status $upstream_cache_status;
             add_header X-Cache-Key "$request_method$request_uri";
             proxy_ignore_headers Cache-Control Expires Set-Cookie;