escalate: POST body escalation fix

This adds a test for POST body failover via the escalate plugin. This
also fixes the tunnel logic so the body can be sent, especially for
small POST bodies.

Author: bneradt

doc/admin-guide/plugins/escalate.en.rst

@@ -59,6 +59,14 @@ when the origin server in the remap rule returns a 401,
   option overrides the default behavior and enables escalation for non-GET
   requests in addition to GET.
 
+  .. note::
+
+    For POST body buffering to work with escalation,
+    :ts:cv:`proxy.config.http.post_copy_size` must be set large enough to buffer
+    the expected POST body sizes (e.g., 2048 bytes or larger). This enables
+    Traffic Server to buffer POST bodies before sending to the origin, so they
+    can be replayed if escalation is needed.
+
 Installation
 ------------
 

plugins/escalate/escalate.cc

@@ -209,6 +209,7 @@ EscalateResponse(TSCont cont, TSEvent event, void *edata)
 
   // Now update the Redirect URL, if set
   if (url_str) {
+    Dbg(dbg_ctl, "Calling TSHttpTxnRedirectUrlSet for status %d with URL: %.*s", status, url_len, url_str);
     TSHttpTxnRedirectUrlSet(txn, url_str, url_len); // Transfers ownership
 
     // Add our x-escalate-redirect header marker if it doesn't already exist and the option is enabled.
@@ -320,6 +321,11 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri */)
 {
   EscalationState *es = static_cast<EscalationState *>(instance);
 
+  // Note: For POST body buffering to work with escalation,
+  // proxy.config.http.post_copy_size should be set large enough to buffer the
+  // expected POST body sizes. This enables ATS to buffer POST bodies before
+  // sending to the origin so they can be replayed if escalation is needed.
+
   TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, es->cont);
   TSHttpTxnHookAdd(txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, es->cont);
   return TSREMAP_NO_REMAP;

src/proxy/http/HttpTunnel.cc

@@ -1028,8 +1028,12 @@ HttpTunnel::producer_run(HttpTunnelProducer *p)
         return;
       }
     } else {
-      body_bytes_copied  += sm->postbuf_copy_partial_data(body_bytes_to_copy);
-      body_bytes_to_copy  = 0;
+      // For POST redirect buffering, we need to copy data that's already in the buffer.
+      // When POST data is pre-buffered (common for small POSTs), producer_n (ntodo) is 0,
+      // so body_bytes_to_copy calculated earlier would be 0. Instead, copy what's available.
+      int64_t bytes_to_copy_now  = p->buffer_start->read_avail();
+      body_bytes_copied         += sm->postbuf_copy_partial_data(bytes_to_copy_now);
+      body_bytes_to_copy         = 0;
     }
   } // end of added logic for partial POST
 

tests/gold_tests/pluginTest/escalate/escalate.test.py

@@ -205,6 +205,8 @@ def _setup_servers(self, tr: 'TestRun') -> None:
             'uuid: GET_failed', "Verify the origin server received the failed GET request.")
         self._server_origin.Streams.All += Testers.ContainsExpression(
             'uuid: POST_success', "Verify the origin server received the successful POST request.")
+        self._server_origin.Streams.All += Testers.ContainsExpression(
+            'uuid: POST_fail_escalated', "Verify the origin server received the POST request that will be escalated.")
         self._server_origin.Streams.All += Testers.ContainsExpression(
             'uuid: HEAD_fail_escalated', "Verify the origin server received the HEAD request that will be escalated.")
 
@@ -217,6 +219,9 @@ def _setup_servers(self, tr: 'TestRun') -> None:
             'uuid: GET_failed', "Verify the failover server received the failed GET request.")
         self._server_failover.Streams.All += Testers.ContainsExpression(
             'uuid: GET_down_origin', "Verify the failover server received the down origin GET request.")
+        # With --escalate-non-get-methods, the POST request should now be escalated
+        self._server_failover.Streams.All += Testers.ContainsExpression(
+            'uuid: POST_fail_escalated', "Verify the failover server received the POST that is now escalated.")
         # With --escalate-non-get-methods, the HEAD request should now be escalated
         self._server_failover.Streams.All += Testers.ContainsExpression(
             'uuid: HEAD_fail_escalated', "Verify the failover server received the HEAD that is now escalated.")
@@ -231,11 +236,12 @@ def _setup_ts(self, tr: 'TestRun') -> None:
         self._ts.Disk.records_config.update(
             {
                 'proxy.config.diags.debug.enabled': 1,
-                'proxy.config.diags.debug.tags': 'http|escalate',
+                'proxy.config.diags.debug.tags': 'http|escalate|http_redirect',
                 'proxy.config.dns.nameservers': f'127.0.0.1:{self._dns.Variables.Port}',
                 'proxy.config.dns.resolv_conf': 'NULL',
                 'proxy.config.http.redirect.actions': 'self:follow',
                 'proxy.config.http.number_of_redirections': 4,
+                'proxy.config.http.post_copy_size': 400000,
             })
 
         # Set up a dead port for the down origin scenario
@@ -266,6 +272,8 @@ def _setup_client(self, tr: 'TestRun') -> None:
         client.Streams.All += Testers.ContainsExpression('x-response: third', 'Verify third GET response received (escalated).')
         client.Streams.All += Testers.ContainsExpression('x-response: fourth', 'Verify fourth GET response received (escalated).')
         client.Streams.All += Testers.ContainsExpression('x-response: post_success', 'Verify successful POST response received.')
+        client.Streams.All += Testers.ContainsExpression(
+            'x-response: post_fail_escalated', 'Verify escalated POST response received.')
         client.Streams.All += Testers.ContainsExpression(
             'x-response: head_fail_escalated', 'Verify escalated HEAD response received.')
 

tests/gold_tests/pluginTest/escalate/escalate_failover.replay.yaml

@@ -125,22 +125,50 @@ sessions:
       content:
         size: 320000
 
+  # POST request response for escalated requests (with --escalate-non-get-methods)
+  - client-request:
+      method: "POST"
+      version: "1.1"
+      url: /api/post/data
+      headers:
+        fields:
+        - [ Host, origin.server.com ]
+        - [ Content-Type, "application/json" ]
+        - [ Content-Length, 320000 ]
+        - [ X-Request, post_fail_escalated ]
+        - [ uuid, POST_fail_escalated ]
+
+    proxy-request:
+      method: "POST"
+      headers:
+        fields:
+        - [ X-Request, { value: post_fail_escalated, as: equal } ]
+        - [ Content-Length, { value: 320000, as: equal } ]
+
+    server-response:
+      status: 200
+      reason: OK
+      headers:
+        fields:
+        - [ Content-Length, 320000 ]
+        - [ X-Response, post_fail_escalated ]
+
   # HEAD request response for escalated requests (with --escalate-non-get-methods)
   - client-request:
       method: "HEAD"
       version: "1.1"
-      url: /api/head/data
+      url: /path/head_fail
       headers:
         fields:
         - [ Host, origin.server.com ]
-        - [ X-Request, head_fail_escalated ]
+        - [ X-Request, head_fail ]
         - [ uuid, HEAD_fail_escalated ]
 
     proxy-request:
       method: "HEAD"
       headers:
         fields:
-        - [ X-Request, { value: head_fail_escalated, as: equal } ]
+        - [ X-Request, { value: head_fail, as: equal } ]
 
     server-response:
       status: 200

tests/gold_tests/pluginTest/escalate/escalate_non_get_methods.replay.yaml

@@ -19,7 +19,7 @@ meta:
 
 sessions:
 - transactions:
-  # Original GET transactions from escalate_original.replay.yaml
+  # Non-escalated GET request.
   - client-request:
       method: "GET"
       version: "1.1"
@@ -51,6 +51,7 @@ sessions:
         fields:
         - [ X-Response, { value: first, as: equal } ]
 
+  # Non-escalated GET request with a chunked response.
   - client-request:
       method: "GET"
       version: "1.1"
@@ -84,6 +85,7 @@ sessions:
         fields:
         - [ X-Response, { value: second, as: equal } ]
 
+  # GET request that will fail and be escalated
   - client-request:
       method: "GET"
       version: "1.1"
@@ -107,13 +109,16 @@ sessions:
       headers:
         fields:
         - [ Content-Length, 0 ]
+        - [ X-Response, third ]
 
+    # With escalation, the failover responds with 200
     proxy-response:
-      # The failover server should reply with a 200 OK (GET requests are escalated with --escalate-non-get-methods).
       status: 200
       headers:
         fields:
         - [ X-Response, { value: third, as: equal } ]
+      content:
+        size: 320000
 
   # This will not make it to the origin server since the Host is set to a
   # non-responsive server. But the failover server should reply with a 200 OK.
@@ -134,6 +139,7 @@ sessions:
         fields:
         - [ X-Request, { value: fourth, as: equal } ]
 
+    # This server-response is just for replay file validity - origin server won't receive this transaction
     server-response:
       status: 200
       reason: OK
@@ -147,34 +153,32 @@ sessions:
         fields:
         - [ X-Response, { value: fourth, as: equal } ]
 
-  # POST request with sizable body that gets proxied normally.
+  # POST request that succeeds without escalation
   - client-request:
       method: "POST"
       version: "1.1"
-      url: /api/upload/data
+      url: /api/post/success
       headers:
         fields:
         - [ Host, origin.server.com ]
         - [ Content-Type, "application/json" ]
-        - [ Content-Length, 32 ]
+        - [ Content-Length, 320000 ]
         - [ X-Request, post_success ]
         - [ uuid, POST_success ]
-      content:
-        encoding: plain
-        size: 32
 
     proxy-request:
       method: "POST"
       headers:
         fields:
         - [ X-Request, { value: post_success, as: equal } ]
+        - [ Content-Length, { value: 320000, as: equal } ]
 
     server-response:
       status: 200
       reason: OK
       headers:
         fields:
-        - [ Content-Length, 32 ]
+        - [ Content-Length, 0 ]
         - [ X-Response, post_success ]
 
     proxy-response:
@@ -183,30 +187,66 @@ sessions:
         fields:
         - [ X-Response, { value: post_success, as: equal } ]
 
-  # A HEAD request that will be escalated with --escalate-non-get-methods
+  # POST request with body that will be escalated with --escalate-non-get-methods
+  - client-request:
+      method: "POST"
+      version: "1.1"
+      url: /api/post/data
+      headers:
+        fields:
+        - [ Host, origin.server.com ]
+        - [ Content-Type, "application/json" ]
+        - [ Content-Length, 320000 ]
+        - [ X-Request, post_fail_escalated ]
+        - [ uuid, POST_fail_escalated ]
+
+    proxy-request:
+      method: "POST"
+      headers:
+        fields:
+        - [ X-Request, { value: post_fail_escalated, as: equal } ]
+        - [ Content-Length, { value: 320000, as: equal } ]
+
+    server-response:
+      status: 502
+      reason: Bad Gateway
+      headers:
+        fields:
+        - [ Content-Length, 0 ]
+        - [ X-Response, post_fail_escalated ]
+
+    # With --escalate-non-get-methods, POST request is escalated to failover.
+    proxy-response:
+      status: 200
+      headers:
+        fields:
+        - [ X-Response, { value: post_fail_escalated, as: equal } ]
+
+  # HEAD request that will be escalated with --escalate-non-get-methods
   - client-request:
       method: "HEAD"
       version: "1.1"
-      url: /api/head/data
+      url: /path/head_fail
       headers:
         fields:
         - [ Host, origin.server.com ]
-        - [ X-Request, head_fail_escalated ]
+        - [ Content-Length, 0 ]
+        - [ X-Request, head_fail ]
         - [ uuid, HEAD_fail_escalated ]
 
     proxy-request:
       method: "HEAD"
       headers:
         fields:
-        - [ X-Request, { value: head_fail_escalated, as: equal } ]
+        - [ X-Request, { value: head_fail, as: equal } ]
 
     server-response:
       status: 502
       reason: Bad Gateway
       headers:
         fields:
         - [ Content-Length, 0 ]
-        - [ X-Response, head_fail_escalated ]
+        - [ X-Response, head_fail ]
 
     # With --escalate-non-get-methods, HEAD request is escalated to failover.
     proxy-response: