diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index b74a7d52186d..8753a6f6ac18 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -46,6 +46,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Utils.wrapInQuotes; import com.google.common.collect.ImmutableMap; +import com.google.common.io.FileBackedOutputStream; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; @@ -198,12 +199,18 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr final long startNanos = context.getStartNanos(); String copyHeader = null; + FileBackedOutputStream spooledBody = null; MultiDigestInputStream multiDigestInputStream = null; try { OzoneVolume volume = context.getVolume(); OzoneBucket bucket = context.getBucket(); final String lengthHeader = getHeaders().getHeaderString(HttpHeaders.CONTENT_LENGTH); long length = lengthHeader != null ? Long.parseLong(lengthHeader) : 0; + if (lengthHeader == null && body != null) { + spooledBody = new FileBackedOutputStream(32); + length = IOUtils.copyLarge(body, spooledBody, new byte[getIOBufferSize(0)]); + body = spooledBody.asByteSource().openStream(); + } if (uploadID != null && !uploadID.equals("")) { if (getHeaders().getHeaderString(COPY_SOURCE_HEADER) == null) { @@ -343,6 +350,9 @@ customMetadata, tags, multiDigestInputStream, getHeaders(), if (multiDigestInputStream != null) { multiDigestInputStream.resetDigests(); } + if (spooledBody != null) { + spooledBody.reset(); + } } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 15398577b58b..653c8f06ebe8 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -54,6 +54,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -63,10 +64,12 @@ import java.util.Base64; import java.util.Map; import java.util.stream.Stream; +import javax.ws.rs.HttpMethod; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; +import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -246,6 +249,21 @@ void testPutObjectWithSignedChunks() throws Exception { assertEquals(15, keyDetails.getDataSize()); } + @Test + void testPutObjectWithChunkedTransferEncoding() throws Exception { + when(objectEndpoint.getContext().getMethod()).thenReturn(HttpMethod.PUT); + when(headers.getHeaderString(HttpHeaders.CONTENT_LENGTH)).thenReturn(null); + when(headers.getHeaderString("Transfer-Encoding")).thenReturn("chunked"); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn(sha256Hex(CONTENT)); + + try (InputStream body = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8))) { + assertSucceeds(() -> objectEndpoint.put(BUCKET_NAME, KEY_NAME, body)); + } + + OzoneKeyDetails keyDetails = assertKeyContent(bucket, KEY_NAME, CONTENT); + assertEquals(CONTENT.length(), keyDetails.getDataSize()); + } + @Test public void testPutObjectMessageDigestResetDuringException() { MessageDigest messageDigest = mock(MessageDigest.class); @@ -672,4 +690,10 @@ private Response putObject(String bucketName, String keyName) throws IOException private Response putObject(String content) throws IOException, OS3Exception { return put(objectEndpoint, BUCKET_NAME, KEY_NAME, content); } + + private static String sha256Hex(String content) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance(OzoneConsts.FILE_HASH); + byte[] contentDigest = digest.digest(content.getBytes(StandardCharsets.UTF_8)); + return DatatypeConverter.printHexBinary(contentDigest).toLowerCase(); + } }