Add origin on_connect failure simulation

README.md

@@ -231,6 +231,8 @@ take the following nodes:
    such as `200` or `404`.
 1. `reason`: This takes a string that describes the status, such as `"OK"` or
    `"Not Found"`.
+1. `on_connect`: This controls the transport behavior after the request is
+   read. Supported values are `accept` (default), `refuse`, and `reset`.
 
 Here's an example of an HTTP/1 `server-response` with a status of 200, four
 fields, and a body of size 3,432 bytes:
@@ -279,6 +281,25 @@ in this case) as opposed to the generated content specified by the
           * Points
 ```
 
+The `on_connect` directive can be used to simulate origin-side connection
+failures without sending an HTTP response. This is useful when testing proxy
+behavior for upstream errors. Supported values are:
+
+* `accept`: Normal behavior. The Verifier server sends the configured HTTP
+  response. This is the default.
+* `refuse`: The Verifier server closes the upstream connection after it reads
+  and validates the request, without sending a response.
+* `reset`: The Verifier server aborts the upstream connection after it reads
+  and validates the request, without sending a response.
+
+When `on_connect` is set to `refuse` or `reset`, the `status` field becomes
+optional because no response is sent. For example:
+
+```YAML
+  server-response:
+    on_connect: refuse
+```
+
 #### Server Response Lookup
 
 The `client-request` and `server-response` nodes are all that is required to
@@ -881,7 +902,8 @@ Note that this example specifies the following delays:
 * The client also delays 15 milliseconds before sending the client
   request.
 * The server delays 17 milliseconds (17,000 microseconds) before sending the
-  corresponding response after receiving the request.
+  corresponding response after receiving the request. This same delay is
+  applied before `server-response.on_connect: refuse` or `reset` actions.
 
 Be aware of the following characteristics of the `delay` node:
 

schema/replay_schema.json

@@ -246,8 +246,28 @@
       "title": "response",
       "description": "HTTP response.",
       "type": "object",
-      "required": ["status"],
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "on_connect": {
+                "enum": ["refuse", "reset"]
+              }
+            },
+            "required": ["on_connect"]
+          },
+          "then": {},
+          "else": {
+            "required": ["status"]
+          }
+        }
+      ],
       "properties": {
+        "on_connect": {
+          "description": "Connection action to take after reading the request.",
+          "type": "string",
+          "enum": ["accept", "refuse", "reset"]
+        },
         "version": {
           "description": "HTTP version",
           "type": "string",

src/core/YamlParser.cc

@@ -314,6 +314,36 @@ get_delay_time(YAML::Node const &node)
   return zret;
 }
 
+swoc::Rv<Txn::ConnectAction>
+get_on_connect_action(YAML::Node const &node)
+{
+  swoc::Rv<Txn::ConnectAction> zret{Txn::ConnectAction::ACCEPT};
+  auto on_connect_node{node[YAML_HTTP_ON_CONNECT_KEY]};
+  if (!on_connect_node) {
+    return zret;
+  }
+  if (!on_connect_node.IsScalar()) {
+    zret.note(S_ERROR, R"("{}" key that is not a scalar.)", YAML_HTTP_ON_CONNECT_KEY);
+    return zret;
+  }
+
+  auto const action = on_connect_node.Scalar();
+  if (action == "accept") {
+    zret = Txn::ConnectAction::ACCEPT;
+  } else if (action == "refuse") {
+    zret = Txn::ConnectAction::REFUSE;
+  } else if (action == "reset") {
+    zret = Txn::ConnectAction::RESET;
+  } else {
+    zret.note(
+        S_ERROR,
+        R"(Unrecognized "{}" value "{}". Expected one of: accept, refuse, reset.)",
+        YAML_HTTP_ON_CONNECT_KEY,
+        action);
+  }
+  return zret;
+}
+
 Errata
 validate_psuedo_headers(const HttpHeader &hdr, int number_of_pseudo_headers)
 {

src/core/YamlParser.h

@@ -17,6 +17,8 @@
 
 #include "yaml-cpp/yaml.h"
 
+#include "core/http.h"
+
 #include "swoc/BufferWriter.h"
 #include "swoc/Errata.h"
 #include "swoc/MemArena.h"
@@ -71,6 +73,7 @@ static const std::string YAML_HTTP_REASON_KEY{"reason"};
 static const std::string YAML_HTTP_METHOD_KEY{"method"};
 static const std::string YAML_HTTP_SCHEME_KEY{"scheme"};
 static const std::string YAML_HTTP_VERSION_KEY{"version"};
+static const std::string YAML_HTTP_ON_CONNECT_KEY{"on_connect"};
 static const std::string YAML_HTTP_AWAIT_KEY{"await"};
 static const std::string YAML_HTTP2_KEY{"http2"};
 static const std::string YAML_HTTP2_PSEUDO_METHOD_KEY{":method"};
@@ -132,6 +135,14 @@ swoc::Rv<std::chrono::microseconds> interpret_delay_string(swoc::TextView delay)
  */
 swoc::Rv<std::chrono::microseconds> get_delay_time(YAML::Node const &node);
 
+/** Parse the value of a server-response "on_connect" directive.
+ *
+ * @param[in] node The server-response node containing the directive.
+ * @return The parsed connect action. If the directive is absent, ACCEPT is
+ * returned. Errors are returned for non-scalar or unrecognized values.
+ */
+swoc::Rv<Txn::ConnectAction> get_on_connect_action(YAML::Node const &node);
+
 struct VerificationConfig
 {
   std::shared_ptr<HttpFields> txn_rules;

src/core/http.h

@@ -638,10 +638,17 @@ struct Txn
 {
   Txn(bool verify_strictly) : _req{verify_strictly}, _rsp{verify_strictly} { }
 
+  enum class ConnectAction {
+    ACCEPT,
+    REFUSE,
+    RESET,
+  };
+
   std::chrono::nanoseconds _start; ///< The delay since the beginning of the session.
 
   /// How long the user said to delay for this transaction.
   std::chrono::microseconds _user_specified_delay_duration{0};
+  ConnectAction _connect_action{ConnectAction::ACCEPT};
   HttpHeader _req;         ///< Request to send.
   HttpHeader _rsp;         ///< Rules for response to expect.
   HttpHeader _rsp_trailer; ///< Rules for response trailer to expect.

src/server/verifier-server.cc

@@ -128,6 +128,23 @@ get_not_found_response(int64_t stream_id, HTTP_PROTOCOL_TYPE protocol, std::stri
   return response;
 }
 
+namespace
+{
+void
+perform_connect_action(Session &session, Txn::ConnectAction action)
+{
+  if (action == Txn::ConnectAction::RESET) {
+    if (session.get_fd() >= 0) {
+      struct linger l;
+      l.l_onoff = 1;
+      l.l_linger = 0;
+      setsockopt(session.get_fd(), SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(l));
+    }
+  }
+  session.close();
+}
+} // namespace
+
 std::thread
 ServerThreadPool::make_thread(std::thread *t)
 {
@@ -504,7 +521,10 @@ ServerReplayFileHandler::server_response(YAML::Node const &node)
 {
   swoc::Errata errata;
   errata.note(YamlParser::populate_http_message(node, _txn._rsp));
-  if (_txn._rsp._status == 0) {
+  auto &&[connect_action, on_connect_errata] = get_on_connect_action(node);
+  errata.note(std::move(on_connect_errata));
+  _txn._connect_action = connect_action;
+  if (_txn._rsp._status == 0 && _txn._connect_action == Txn::ConnectAction::ACCEPT) {
     errata.note(
         S_ERROR,
         R"(server-response node without a status at "{}":{}.)",
@@ -733,9 +753,20 @@ TF_Serve_Connection(std::thread *t)
       if (specified_transaction._user_specified_delay_duration > 0us) {
         sleep_for(specified_transaction._user_specified_delay_duration);
       }
-      auto &&[bytes_written, write_errata] =
-          thread_info._session->write(specified_transaction._rsp);
-      thread_errata.note(std::move(write_errata));
+      if (specified_transaction._connect_action == Txn::ConnectAction::ACCEPT) {
+        auto &&[bytes_written, write_errata] =
+            thread_info._session->write(specified_transaction._rsp);
+        thread_errata.note(std::move(write_errata));
+      } else {
+        thread_errata.note(
+            S_DIAG,
+            R"(Applying "{}" on_connect action for key {}.)",
+            specified_transaction._connect_action == Txn::ConnectAction::REFUSE ? "refuse" :
+                                                                                  "reset",
+            key);
+        perform_connect_action(*thread_info._session, specified_transaction._connect_action);
+        break;
+      }
     }
 
     // cleanup and get ready for another session.

tests/autests/gold_tests/on_connect/on_connect.test.py

@@ -0,0 +1,54 @@
+'''
+Verify server-response on_connect behavior.
+'''
+# @file
+#
+# Copyright 2022, Verizon Media
+# SPDX-License-Identifier: Apache-2.0
+#
+
+Test.Summary = '''
+Verify server-response on_connect behavior.
+'''
+
+r = Test.AddTestRun("Verify on_connect accept, refuse, and reset behavior")
+client = r.AddClientProcess("client", "replay_files/on_connect.yaml", configure_https=False)
+server = r.AddServerProcess("server", "replay_files/on_connect.yaml", configure_https=False)
+proxy = r.AddProxyProcess("proxy", listen_port=client.Variables.http_port,
+                          server_port=server.Variables.http_port)
+
+client.Streams.stdout += Testers.ContainsExpression(
+    "3 transactions in 3 sessions", "The client should have parsed all three transactions.")
+
+client.Streams.stdout += Testers.ContainsExpression(
+    "Received an HTTP/1 200 response for key 1", "The accept case should return a normal response.")
+
+client.Streams.stdout += Testers.ContainsExpression(
+    "Received an HTTP/1 502 response for key 2", "The refuse case should surface as a proxy 502.")
+
+client.Streams.stdout += Testers.ContainsExpression(
+    "Received an HTTP/1 502 response for key 3", "The reset case should surface as a proxy 502.")
+
+client.Streams.stdout += Testers.ExcludesExpression("Violation:",
+                                                    "There should be no verification errors.")
+
+server.Streams.stdout += Testers.ContainsExpression(
+    'Applying "refuse" on_connect action for key 2.', "The server should apply the refuse action.")
+
+server.Streams.stdout += Testers.ContainsExpression('Applying "reset" on_connect action for key 3.',
+                                                    "The server should apply the reset action.")
+
+server.Streams.stdout += Testers.ContainsExpression(
+    "Sent the following HTTP/1 response headers for key 1",
+    "The accept case should still write a normal response.")
+
+server.Streams.stdout += Testers.ExcludesExpression(
+    "Sent the following HTTP/1 response headers for key 2",
+    "The refuse case should not write response headers.")
+
+server.Streams.stdout += Testers.ExcludesExpression(
+    "Sent the following HTTP/1 response headers for key 3",
+    "The reset case should not write response headers.")
+
+server.Streams.stdout += Testers.ExcludesExpression("Violation:",
+                                                    "There should be no verification errors.")

tests/autests/gold_tests/on_connect/replay_files/on_connect.yaml

@@ -0,0 +1,62 @@
+# @file
+#
+# Copyright 2022, Verizon Media
+# SPDX-License-Identifier: Apache-2.0
+#
+
+meta:
+  version: '1.0'
+
+sessions:
+- transactions:
+  - client-request:
+      method: GET
+      url: http://example.com/accept
+      version: '1.1'
+      headers:
+        fields:
+        - [ Host, example.com ]
+        - [ uuid, 1 ]
+
+    proxy-response:
+      status: 200
+
+    server-response:
+      status: 200
+      reason: OK
+      headers:
+        fields:
+        - [ Content-Length, '0' ]
+        - [ uuid, 1 ]
+
+- transactions:
+  - client-request:
+      method: GET
+      url: http://example.com/refuse
+      version: '1.1'
+      headers:
+        fields:
+        - [ Host, example.com ]
+        - [ uuid, 2 ]
+
+    proxy-response:
+      status: 502
+
+    server-response:
+      on_connect: refuse
+
+- transactions:
+  - client-request:
+      method: GET
+      url: http://example.com/reset
+      version: '1.1'
+      headers:
+        fields:
+        - [ Host, example.com ]
+        - [ uuid, 3 ]
+
+    proxy-response:
+      status: 502
+
+    server-response:
+      on_connect: reset