Handle response body fallback when no length headers present

src/http/client.mbt

@@ -25,6 +25,7 @@ struct Client {
   transport : ClientTransport
   tls : @tls.Tls?
   sender : Sender
+  mut request_method : RequestMethod?
 }
 
 ///|
@@ -45,7 +46,7 @@ pub suberror ProxyError {
 /// so `headers` should not be used by the caller later.
 /// The following headers is automatically set,
 /// and must not be specified in `headers`:
-/// 
+///
 /// - Host
 /// - Content-Length, Transfer-Encoding
 ///
@@ -106,7 +107,7 @@ pub async fn Client::connect(
       (Some(tls), Reader::new(tls), Sender::new(tls, headers~))
     }
   }
-  { transport, tls, reader, sender }
+  { transport, tls, reader, sender, request_method: None }
 }
 
 ///|
@@ -122,7 +123,7 @@ pub async fn Client::connect(
 /// so `headers` should not be used by the caller later.
 /// The following headers is automatically set,
 /// and must not be specified in `headers`:
-/// 
+///
 /// - Host
 /// - Content-Length, Transfer-Encoding
 ///
@@ -211,7 +212,9 @@ pub async fn Client::flush(self : Client) -> Unit {
 pub async fn Client::end_request(self : Client) -> Response {
   self.sender.end_body()
   self.reader.skip_body()
-  self.reader.read_response()
+  let response = self.reader.read_response(request_method=self.request_method)
+  self.request_method = None
+  response
 }
 
 ///|
@@ -228,7 +231,7 @@ pub async fn Client::end_request(self : Client) -> Response {
 /// extra HTTP headers can be passed via `extra_headers`.
 /// The following headers is automatically set by `request`,
 /// and must not be specified in `extra_headers`:
-/// 
+///
 /// - Host
 /// - Content-Length, Transfer-Encoding
 pub async fn Client::request(
@@ -238,6 +241,7 @@ pub async fn Client::request(
   extra_headers? : Map[String, String] = {},
 ) -> Unit {
   self.sender.send_request(meth, path, extra_headers~)
+  self.request_method = Some(meth)
 }
 
 ///|

src/http/parser.mbt

@@ -18,6 +18,7 @@ priv enum BodyKind {
   Fixed(Int)
   Chunked(Int)
   PassThrough
+  WaitConnectionClose
 }
 
 ///|
@@ -90,12 +91,13 @@ impl @io.Reader for Reader with _direct_read(self, buf, offset~, max_len~) {
       n
     }
     PassThrough => self.transport._direct_read(buf, offset~, max_len~)
+    WaitConnectionClose => self.transport._direct_read(buf, offset~, max_len~)
   }
 }
 
 ///|
 impl @io.Reader for Reader with _get_internal_buffer(self) {
-  if self.body is PassThrough {
+  if self.body is PassThrough || self.body is WaitConnectionClose {
     self.transport._get_internal_buffer()
   } else {
     self.read_buf
@@ -185,6 +187,38 @@ async fn Reader::read_request(self : Reader) -> Request {
   { meth, path, headers }
 }
 
+///|
+/// Determines if the response body should be read until connection close.
+/// Returns true when:
+/// - No Content-Length or Transfer-Encoding header is present
+/// - Body has not been determined by other means (still Empty)
+/// - Status code is not 1xx, 204, 205, or 304 (which MUST NOT have a message body)
+/// - Not a successful (2xx) response to a CONNECT request (which becomes a tunnel)
+/// - Not a response to a HEAD request (which never has a message body)
+fn should_read_body_until_close(
+  code : Int,
+  headers : Map[String, String],
+  current_body : BodyKind,
+  request_method : RequestMethod?,
+) -> Bool {
+  // HEAD responses never have a message body, regardless of headers
+  if request_method is Some(Head) {
+    return false
+  }
+  // Successful CONNECT responses (2xx) switch connection to tunnel mode
+  // and have no HTTP message body
+  if request_method is Some(Connect) && code >= 200 && code < 300 {
+    return false
+  }
+  current_body is Empty &&
+  headers.get("content-length") is None &&
+  headers.get("transfer-encoding") is None &&
+  code >= 200 &&
+  code != 204 &&
+  code != 205 &&
+  code != 304
+}
+
 ///|
 /// Read the header of a HTTP/1.1 response from a HTTP reader.
 /// This function must be called after the body of the last response is consumed,
@@ -196,7 +230,10 @@ async fn Reader::read_request(self : Reader) -> Request {
 ///
 /// HTTP header fields are case insensitive.
 /// In the result of `read_response`, all header fields will be in lower case.
-async fn Reader::read_response(self : Reader) -> Response {
+async fn Reader::read_response(
+  self : Reader,
+  request_method~ : RequestMethod?,
+) -> Response {
   guard self.body is Empty
   guard self.transport.read_until("\r\n") is Some(response_line) else {
     raise @io.ReaderClosed
@@ -213,6 +250,13 @@ async fn Reader::read_response(self : Reader) -> Response {
   }
   let reason = response_line[code_len + 1:].to_string()
   let headers = self.read_headers()
+
+  // HTTP/1.1 Spec (RFC 7230 Section 3.3.3): Message Body Length Determination
+  // Other cases are handled in `read_headers` when parsing the headers, which will set the body kind accordingly.
+  if should_read_body_until_close(code, headers, self.body, request_method) {
+    self.body = WaitConnectionClose // Read until connection close
+  }
+
   { code, reason, headers }
 }