diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 3bd8388f9500..0e8ea9d677fb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -318,6 +318,7 @@ public final class OzoneConsts { public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; public static final String TOKEN = "token"; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 566427424223..89e029a9a9f2 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -510,6 +510,44 @@ public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyG return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata); } + /** + * Creates a key only if it does not exist (S3 If-None-Match: * semantics). + * + * @param keyName Name of the key + * @param size Size of the data + * @param replicationConfig Replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream createKeyIfNotExists(String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return proxy.createKeyIfNotExists(volumeName, name, keyName, size, + replicationConfig, metadata, tags); + } + + /** + * Rewrites a key only if its ETag matches (S3 If-Match semantics). + * + * @param keyName Name of the key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig Replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, + String expectedETag, ReplicationConfig replicationConfig, + Map metadata, Map tags) + throws IOException { + return proxy.rewriteKeyIfMatch(volumeName, name, keyName, size, + expectedETag, replicationConfig, metadata, tags); + } + /** * Creates a new key in the bucket, with default replication type RATIS and * with replication factor THREE. @@ -1029,8 +1067,8 @@ public List listStatus(String keyName, boolean recursive, * * @param prefix Optional string to filter for the selected keys. */ - public OzoneMultipartUploadList listMultipartUploads(String prefix, - String keyMarker, String uploadIdMarker, int maxUploads) + public OzoneMultipartUploadList listMultipartUploads(String prefix, + String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { return proxy.listMultipartUploads(volumeName, getName(), prefix, keyMarker, uploadIdMarker, maxUploads); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index e3a575896347..19d8f0d75047 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -379,6 +379,44 @@ OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyNam long size, long existingKeyGeneration, ReplicationConfig replicationConfig, Map metadata) throws IOException; + /** + * Creates a key only if it does not exist (S3 If-None-Match: * semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * @throws OMException with KEY_ALREADY_EXISTS if key exists + */ + OzoneOutputStream createKeyIfNotExists(String volumeName, String bucketName, + String keyName, long size, ReplicationConfig replicationConfig, + Map metadata, Map tags) + throws IOException; + + /** + * Rewrites a key only if its ETag matches (S3 If-Match semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND + */ + @SuppressWarnings("checkstyle:parameternumber") + OzoneOutputStream rewriteKeyIfMatch(String volumeName, String bucketName, + String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + /** * Writes a key in an existing bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3947e4b6818b..39c5c8a50cb0 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -677,7 +677,7 @@ public void createBucket( builder.setDefaultReplicationConfig(defaultReplicationConfig); } - String replicationType = defaultReplicationConfig == null + String replicationType = defaultReplicationConfig == null ? "server-side default replication type" : defaultReplicationConfig.getType().toString(); @@ -1315,7 +1315,7 @@ public List listBuckets(String volumeName, String bucketPrefix, List buckets = ozoneManagerClient.listBuckets( volumeName, prevBucket, bucketPrefix, maxListResult, hasSnapshot); - return buckets.stream().map(bucket -> + return buckets.stream().map(bucket -> OzoneBucket.newBuilder(conf, this) .setVolumeName(bucket.getVolumeName()) .setName(bucket.getBucketName()) @@ -1428,6 +1428,68 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String return createOutputStream(openKey); } + @Override + public OzoneOutputStream createKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support atomic key creation."); + } + + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(size) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .addAllTags(tags) + .setLatestVersionLocation(getLatestVersionLocation) + .setExpectedDataGeneration( + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + if (isS3GRequest.get() && size == 0) { + openKey.getKeyInfo().setDataSize(0); + } + return createOutputStream(openKey); + } + + @Override + @SuppressWarnings("checkstyle:parameternumber") + public OzoneOutputStream rewriteKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support conditional key rewrite."); + } + + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(size) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .addAllTags(tags) + .setLatestVersionLocation(getLatestVersionLocation) + .setExpectedETag(expectedETag); + + OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + if (isS3GRequest.get() && size == 0) { + openKey.getKeyInfo().setDataSize(0); + } + return createOutputStream(openKey); + } + private void createKeyPreChecks(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig) throws IOException { verifyVolumeName(volumeName); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 596eb1276560..dc9d16188c57 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -208,7 +208,7 @@ public enum ResultCodes { USER_MISMATCH, // Error code when requested user name passed is different // from remote user. - INVALID_PART, // When part name is not found or not matching with partname + INVALID_PART, // When part name is not found or not matching with partname // in OM MPU partInfo. INVALID_PART_ORDER, // When list of parts mentioned to complete MPU are not @@ -267,7 +267,7 @@ public enum ResultCodes { UNAUTHORIZED, S3_SECRET_ALREADY_EXISTS, - + INVALID_PATH, TOO_MANY_BUCKETS, KEY_UNDER_LEASE_RECOVERY, @@ -275,5 +275,9 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, + + ETAG_MISMATCH, + + ETAG_NOT_AVAILABLE, } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index 395425b069e9..da982a568b15 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@ -62,6 +62,7 @@ public final class OmKeyArgs implements Auditable { // This allows a key to be created an committed atomically if the original has not // been modified. private Long expectedDataGeneration = null; + private final String expectedETag; private OmKeyArgs(Builder b) { this.volumeName = b.volumeName; @@ -83,6 +84,7 @@ private OmKeyArgs(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags; this.expectedDataGeneration = b.expectedDataGeneration; + this.expectedETag = b.expectedETag; } public boolean getIsMultipartKey() { @@ -169,6 +171,10 @@ public Long getExpectedDataGeneration() { return expectedDataGeneration; } + public String getExpectedETag() { + return expectedETag; + } + @Override public Map toAuditMap() { Map auditMap = new LinkedHashMap<>(); @@ -214,6 +220,9 @@ public KeyArgs toProtobuf() { if (expectedDataGeneration != null) { builder.setExpectedDataGeneration(expectedDataGeneration); } + if (expectedETag != null) { + builder.setExpectedETag(expectedETag); + } return builder.build(); } @@ -240,6 +249,7 @@ public static class Builder { private boolean forceUpdateContainerCacheFromSCM; private final Map tags = new HashMap<>(); private Long expectedDataGeneration = null; + private String expectedETag; public Builder() { this(AclListBuilder.empty()); @@ -267,6 +277,7 @@ public Builder(OmKeyArgs obj) { this.forceUpdateContainerCacheFromSCM = obj.forceUpdateContainerCacheFromSCM; this.expectedDataGeneration = obj.expectedDataGeneration; + this.expectedETag = obj.expectedETag; this.metadata.putAll(obj.metadata); this.tags.putAll(obj.tags); this.acls = AclListBuilder.of(obj.acls); @@ -385,6 +396,11 @@ public Builder setExpectedDataGeneration(long generation) { return this; } + public Builder setExpectedETag(String eTag) { + this.expectedETag = eTag; + return this; + } + public OmKeyArgs build() { return new OmKeyArgs(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index b0e26c49d695..09d94ccbd1fe 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -110,6 +110,7 @@ public final class OmKeyInfo extends WithParentObjectId // This allows a key to be created an committed atomically if the original has not // been modified. private Long expectedDataGeneration = null; + private String expectedETag; private OmKeyInfo(Builder b) { super(b); @@ -129,6 +130,7 @@ private OmKeyInfo(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags.build(); this.expectedDataGeneration = b.expectedDataGeneration; + this.expectedETag = b.expectedETag; } private static Codec newCodec(boolean ignorePipeline) { @@ -189,6 +191,14 @@ public Long getExpectedDataGeneration() { return expectedDataGeneration; } + public void setExpectedETag(String eTag) { + this.expectedETag = eTag; + } + + public String getExpectedETag() { + return expectedETag; + } + public String getOwnerName() { return ownerName; } @@ -492,6 +502,7 @@ public static class Builder extends WithParentObjectId.Builder { private boolean isFile; private final MapBuilder tags; private Long expectedDataGeneration = null; + private String expectedETag; public Builder() { this.acls = AclListBuilder.empty(); @@ -514,6 +525,7 @@ public Builder(OmKeyInfo obj) { this.fileChecksum = obj.fileChecksum; this.isFile = obj.isFile; this.expectedDataGeneration = obj.expectedDataGeneration; + this.expectedETag = obj.expectedETag; this.tags = MapBuilder.of(obj.tags); obj.keyLocationVersions.forEach(keyLocationVersion -> this.omKeyLocationInfoGroups.add( @@ -685,6 +697,11 @@ public Builder setExpectedDataGeneration(Long existingGeneration) { return this; } + public Builder setExpectedETag(String eTag) { + this.expectedETag = eTag; + return this; + } + @Override protected void validate() { super.validate(); @@ -804,6 +821,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, if (expectedDataGeneration != null) { kb.setExpectedDataGeneration(expectedDataGeneration); } + if (expectedETag != null) { + kb.setExpectedETag(expectedETag); + } if (ownerName != null) { kb.setOwnerName(ownerName); } @@ -857,6 +877,9 @@ public static Builder builderFromProtobuf(KeyInfo keyInfo) { if (keyInfo.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration()); } + if (keyInfo.hasExpectedETag()) { + builder.setExpectedETag(keyInfo.getExpectedETag()); + } if (keyInfo.hasOwnerName()) { builder.setOwnerName(keyInfo.getOwnerName()); diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot new file mode 100644 index 000000000000..11bb0abafc61 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 Conditional Put (If-None-Match / If-Match) tests +Library OperatingSystem +Library String +Library Process +Resource ../commonlib.robot +Resource ./commonawslib.robot +Test Timeout 5 minutes +Suite Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${BUCKET} generated + +*** Test Cases *** + +Conditional Put If-None-Match Star Creates New Key + [Documentation] If-None-Match: * should succeed when key does not exist + ${key} = Set Variable condput-ifnonematch-new + Execute echo "test-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match * + Should contain ${result} ETag + +Conditional Put If-None-Match Star Fails For Existing Key + [Documentation] If-None-Match: * should fail with 412 when key already exists + ${key} = Set Variable condput-ifnonematch-existing + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Now try again with If-None-Match: * + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match * + Should contain ${result} PreconditionFailed + +Conditional Put If-Match With Correct ETag Succeeds + [Documentation] If-Match with correct ETag should succeed + ${key} = Set Variable condput-ifmatch-success + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Extract the ETag value + ${etag} = Execute And Ignore Error echo '${result}' | python3 -c "import sys,json; print(json.loads(sys.stdin.read())['ETag'])" + ${etag} = Get From List ${etag} 1 + # Rewrite with matching ETag + Execute echo "updated-content" > /tmp/${key}-updated + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}-updated --if-match ${etag} + Should contain ${result} ETag + +Conditional Put If-Match With Wrong ETag Fails + [Documentation] If-Match with wrong ETag should fail with 412 + ${key} = Set Variable condput-ifmatch-fail + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Try to rewrite with a wrong ETag + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match "wrong-etag" + Should contain ${result} PreconditionFailed + +Conditional Put If-Match On Non-Existent Key Fails + [Documentation] If-Match on a key that does not exist should fail with 412 + ${key} = Set Variable condput-ifmatch-nonexistent + Execute echo "test-content" > /tmp/${key} + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match "some-etag" + Should contain ${result} PreconditionFailed diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 7fb2f885725e..379ec2749c13 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -377,6 +377,84 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutObjectIfNoneMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-None-Match", "*"); + + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, metadata); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); + } + + @Test + public void testPutObjectIfNoneMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-None-Match", "*"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(bucketName, keyName, is2, metadata)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + } + + @Test + public void testPutObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + String etag = putObjectResult.getETag(); + + InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-Match", etag); + + PutObjectResult putObjectResult2 = s3Client.putObject(bucketName, keyName, is2, metadata); + assertNotNull(putObjectResult2.getETag()); + } + + @Test + public void testPutObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-Match", "wrong-etag"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(bucketName, keyName, is2, metadata)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + } + @Test public void testPutObjectWithMD5Header() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 175dcea265f8..4537bcccdfa8 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -236,6 +236,79 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testPutObjectIfNoneMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifNoneMatch("*"), + RequestBody.fromString(content)); + + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfNoneMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifNoneMatch("*"), + RequestBody.fromString(content))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + } + + @Test + public void testPutObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch(initialResponse.eTag()), + RequestBody.fromString("bar2")); + + assertNotNull(putObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch("wrong-etag"), + RequestBody.fromString("bar2"))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + } + @Test public void testPutObjectEmpty() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 31c826fabb04..04952c8e95c4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -35,11 +35,13 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.DEFAULT_OM_UPDATE_ID; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; +import static org.apache.hadoop.ozone.OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS; import static org.apache.hadoop.ozone.OzoneConsts.GB; import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; @@ -1367,6 +1369,54 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE assertUnchanged(keyInfo, ozoneManager.lookupKey(keyArgs)); } + @ParameterizedTest + @EnumSource + void rewriteFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails key1Details = createTestKey(bucket, "key1", "value".getBytes(UTF_8)); + OzoneOutputStream key2Out = openTestKey(bucket, "key2", "value"); + OzoneOutputStream key3Out = openTestKey(bucket, "key3", "value"); + + // Test 1: Rewrite with -1 fails when key is already committed + OMException e = assertThrows(OMException.class, () -> { + bucket.rewriteKey( + key1Details.getName(), + key1Details.getDataSize(), + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + key1Details.getMetadata()); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + + // Test 2: Rewrite with -1 succeeds when key is open but not yet committed + assertDoesNotThrow(() -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + key2Out.close(); + + // Test 3: After rewrite completes, attempting to rewrite again with -1 fails + key3Out.write("value".getBytes(UTF_8)); + key3Out.close(); + + e = assertThrows(OMException.class, () -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + } + @ParameterizedTest @EnumSource void cannotRewriteDeletedKey(BucketLayout layout) throws IOException { @@ -1393,6 +1443,100 @@ void cannotRewriteRenamedKey(BucketLayout layout) throws IOException { assertThat(e).hasMessageContaining("not found"); } + @ParameterizedTest + @EnumSource + void testCreateKeyIfNotExistsSuccess(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + String keyName = "create-if-not-exists-" + layout.name(); + byte[] content = "test-content".getBytes(UTF_8); + + try (OzoneOutputStream out = bucket.createKeyIfNotExists( + keyName, content.length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(content); + } + + assertKeyContent(bucket, keyName, content); + } + + @ParameterizedTest + @EnumSource + void testCreateKeyIfNotExistsFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "new-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.createKeyIfNotExists( + keyDetails.getName(), newContent.length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchSuccess(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + String etag = keyDetails.getMetadata().get(OzoneConsts.ETAG); + assertNotNull(etag, "Key should have an ETag"); + + byte[] newContent = "rewritten-content".getBytes(UTF_8); + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, etag, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + + assertKeyContent(bucket, keyDetails.getName(), newContent); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWithWrongETag(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "rewritten-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, "wrong-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(OMException.ResultCodes.ETAG_MISMATCH, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWhenKeyNotFound(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + byte[] content = "content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + "nonexistent-key", content.length, "some-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(content); + } + }); + assertEquals(KEY_NOT_FOUND, e.getResult()); + } + private static void rewriteKey( OzoneBucket bucket, OzoneKeyDetails keyDetails, byte[] newContent ) throws IOException { @@ -4382,6 +4526,12 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } + private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { + return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", RandomStringUtils.secure().nextAscii(10))); + } + private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..057874944670 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,6 +566,10 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; + + ETAG_MISMATCH = 99; + + ETAG_NOT_AVAILABLE = 100; } /** @@ -1082,6 +1086,11 @@ message KeyArgs { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 23; + + // expectedETag, when set, indicates that the existing key must have + // the given ETag for the operation to succeed. This is used for + // S3 conditional writes with the If-Match header. + optional string expectedETag = 24; } message KeyLocation { @@ -1173,6 +1182,11 @@ message KeyInfo { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 22; + + // expectedETag, when set, indicates that the existing key must have + // the given ETag for the operation to succeed. This is used for + // S3 conditional writes with the If-Match header. + optional string expectedETag = 23; } // KeyInfoProtoLight is a lightweight subset of KeyInfo message containing diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index be0935d909d8..dfbacbc661b4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -303,6 +303,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() .setExpectedDataGeneration(null) + .setExpectedETag(null) .addAllMetadata(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())) .setUpdateID(trxnLogIndex) @@ -616,14 +617,42 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (toCommit.getExpectedDataGeneration() != null) { // These values are not passed in the request keyArgs, so add them into the auditMap if they are present // in the open key entry. - auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(toCommit.getExpectedDataGeneration())); + Long expectedGen = toCommit.getExpectedDataGeneration(); + auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); + + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { + if (existing != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + if (existing == null) { + throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); + } + if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { + throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", + KEY_NOT_FOUND); + } + } + } + + if (toCommit.getExpectedETag() != null) { + String expectedETag = toCommit.getExpectedETag(); + auditMap.put("expectedETag", expectedETag); + if (existing == null) { - throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); + throw new OMException("Key not found for If-Match at commit", + OMException.ResultCodes.KEY_NOT_FOUND); } - if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { - throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + - ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_NOT_FOUND); + String currentETag = existing.getMetadata().get(OzoneConsts.ETAG); + if (currentETag == null) { + throw new OMException("Key does not have an ETag at commit", + OMException.ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!currentETag.equals(expectedETag)) { + throw new OMException("ETag changed during write (concurrent modification)", + OMException.ResultCodes.ETAG_MISMATCH); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index 25b5a4b15d41..5aa4fbeb36f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -246,6 +246,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Optimistic locking validation has passed. Now set the rewrite fields to null so they are // not persisted in the key table. omKeyInfo.setExpectedDataGeneration(null); + omKeyInfo.setExpectedETag(null); long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace shouldn't check and increase. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index d34320ecb8db..af766094e40b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.UniqueId; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -189,7 +190,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs.Builder finalNewKeyArgs = newKeyArgs; KeyArgs resolvedKeyArgs = - captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), + captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), () -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager, IAccessAuthorizer.ACLType.CREATE)); newCreateKeyRequest = @@ -369,7 +370,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } else { perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency); } - + if (acquireLock) { mergeOmLockDetails(ozoneLockStrategy .releaseWriteLock(omMetadataManager, volumeName, @@ -471,12 +472,39 @@ public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { - // If a key does not exist, or if it exists but the updateID do not match, then fail this request. + long expectedGen = keyArgs.getExpectedDataGeneration(); + // If expectedGen is EXPECTED_GEN_CREATE_IF_NOT_EXISTS, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { + if (dbKeyInfo != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + // If a key does not exist, or if it exists but the updateID do not match, then fail this request. + if (dbKeyInfo == null) { + throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } + if (dbKeyInfo.getUpdateID() != expectedGen) { + throw new OMException("Generation mismatch during expected rewrite", + OMException.ResultCodes.KEY_NOT_FOUND); + } + } + } + + if (keyArgs.hasExpectedETag()) { + String expectedETag = keyArgs.getExpectedETag(); if (dbKeyInfo == null) { - throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + throw new OMException("Key not found for If-Match", + OMException.ResultCodes.KEY_NOT_FOUND); + } + String existingETag = dbKeyInfo.getMetadata().get(OzoneConsts.ETAG); + if (existingETag == null) { + throw new OMException("Key does not have an ETag", + OMException.ResultCodes.ETAG_NOT_AVAILABLE); } - if (dbKeyInfo.getUpdateID() != keyArgs.getExpectedDataGeneration()) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + if (!existingETag.equals(expectedETag)) { + throw new OMException("ETag mismatch", + OMException.ResultCodes.ETAG_MISMATCH); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 7df2619e9e46..13dc904476dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -993,6 +993,9 @@ protected OmKeyInfo prepareFileInfo( if (keyArgs.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); } + if (keyArgs.hasExpectedETag()) { + builder.setExpectedETag(keyArgs.getExpectedETag()); + } return builder.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 02d913160bc1..1f40b0f72a00 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -279,6 +280,213 @@ public void testAtomicRewrite() throws Exception { assertEquals(acls, committedKey.getAcls()); } + @Test + public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + assertNull(closedKeyTable.get(getOzonePathKey())); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedDataGeneration()); + } + + @Test + public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + OmKeyInfo existingClosedKey = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), existingClosedKey); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testCommitWithExpectedETagSuccess() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + String expectedETag = "matching-etag"; + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag(expectedETag); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key with matching ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())) + .addMetadata(OzoneConsts.ETAG, expectedETag).build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedETag()); + } + + @Test + public void testCommitWithExpectedETagMismatch() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag("expected-etag"); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key with different ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())) + .addMetadata(OzoneConsts.ETAG, "different-etag").build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_MISMATCH, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testCommitWithExpectedETagNoETagOnKey() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag("expected-etag"); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key WITHOUT ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE, + omClientResponse.getOMResponse().getStatus()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { @@ -456,7 +664,7 @@ private Map doKeyCommit(boolean isHSync, .collect(Collectors.toList()); String openKey = addKeyToOpenKeyTable(allocatedBlockList); String ozoneKey = getOzonePathKey(); - + OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); assertEquals(OK, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 1666f4cb38e6..f9364d87710d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createOmKeyInfo; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.NOT_A_FILE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; @@ -146,6 +147,209 @@ public void preExecuteRejectsInvalidReplication() { assertEquals(OMException.ResultCodes.INVALID_REQUEST, e.getResult()); } + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyMissing( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + checkResponse(modifiedOmRequest, response, id, false, getBucketLayout()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyAlreadyExists( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + long expectedGen = 1L; + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, expectedGen)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(2L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagKeyNotFound( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("some-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagNoETagOnKey( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("some-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key without ETag metadata + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE, + response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("expected-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key with a different ETag + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L) + .addMetadata(OzoneConsts.ETAG, "different-etag") + .build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_MISMATCH, + response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagSuccess( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + String expectedETag = "matching-etag"; + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag(expectedETag)); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key with matching ETag + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L) + .addMetadata(OzoneConsts.ETAG, expectedETag) + .build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, response.getOMResponse().getStatus()); + + // Verify open key was created with expectedETag + OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + .get(getOpenKey(id)); + assertNotNull(openKeyInfo); + assertEquals(expectedETag, openKeyInfo.getExpectedETag()); + // Creation time should remain the same on rewrite + assertEquals(existingKeyInfo.getCreationTime(), + openKeyInfo.getCreationTime()); + } + @ParameterizedTest @MethodSource("data") public void testValidateAndUpdateCache( @@ -825,6 +1029,30 @@ private OMRequest createKeyRequest( .setCreateKeyRequest(createKeyRequest).build(); } + private OMRequest createKeyRequestWithExpectedETag(String expectedETag) { + KeyArgs.Builder keyArgs = KeyArgs.newBuilder() + .setVolumeName(volumeName).setBucketName(bucketName) + .setKeyName(keyName).setIsMultipartKey(false) + .setFactor( + ((RatisReplicationConfig) replicationConfig) + .getReplicationFactor()) + .setType(replicationConfig.getReplicationType()) + .setLatestVersionLocation(true) + .setDataSize(100L); + + if (expectedETag != null) { + keyArgs.setExpectedETag(expectedETag); + } + + CreateKeyRequest createKeyRequest = + CreateKeyRequest.newBuilder().setKeyArgs(keyArgs).build(); + + return OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey) + .setClientId(UUID.randomUUID().toString()) + .setCreateKeyRequest(createKeyRequest).build(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testKeyCreateWithFileSystemPathsEnabled( diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index d6d9b101c523..9aa7a5ed463c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -188,8 +188,18 @@ public Response put( throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex); } else if (ex.getResult() == ResultCodes.FILE_ALREADY_EXISTS) { throw newError(S3ErrorTable.NO_OVERWRITE, keyPath, ex); + } else if (ex.getResult() == ResultCodes.KEY_ALREADY_EXISTS) { + throw newError(PRECOND_FAILED, keyPath, ex); + } else if (ex.getResult() == ResultCodes.ETAG_MISMATCH) { + throw newError(PRECOND_FAILED, keyPath, ex); + } else if (ex.getResult() == ResultCodes.ETAG_NOT_AVAILABLE) { + throw newError(PRECOND_FAILED, keyPath, ex); } else if (ex.getResult() == ResultCodes.INVALID_REQUEST) { throw newError(S3ErrorTable.INVALID_REQUEST, keyPath); + } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND + && getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER) != null) { + // If-Match failed because the key doesn't exist + throw newError(PRECOND_FAILED, keyPath, ex); } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { throw newError(S3ErrorTable.NO_SUCH_KEY, keyPath); } else if (ex.getResult() == ResultCodes.NOT_SUPPORTED_OPERATION) { @@ -231,7 +241,6 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr copyHeader = getHeaders().getHeaderString(COPY_SOURCE_HEADER); - // Normal put object ReplicationConfig replicationConfig = getReplicationConfig(bucket); boolean enableEC = false; @@ -272,6 +281,15 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr return Response.ok().status(HttpStatus.SC_OK).build(); } + String ifNoneMatch = getHeaders().getHeaderString( + S3Consts.IF_NONE_MATCH_HEADER); + String ifMatch = getHeaders().getHeaderString( + S3Consts.IF_MATCH_HEADER); + + if (ifNoneMatch != null && ifMatch != null) { + throw newError(INVALID_REQUEST, keyPath); + } + // Normal put object S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body, length, amzDecodedLength, keyPath); @@ -282,21 +300,24 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); Map tags = getTaggingFromHeaders(getHeaders()); + boolean hasConditionalHeaders = ifNoneMatch != null || ifMatch != null; long putLength; final String md5Hash; - if (isDatastreamEnabled() && !enableEC && length > getDatastreamMinLength()) { + if (isDatastreamEnabled() && !enableEC + && length > getDatastreamMinLength() && !hasConditionalHeaders) { perf.appendStreamMode(); Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, getChunkSize(), - customMetadata, tags, multiDigestInputStream, getHeaders(), signatureInfo.isSignPayload(), perf); + customMetadata, tags, multiDigestInputStream, getHeaders(), + signatureInfo.isSignPayload(), perf); md5Hash = keyWriteResult.getKey(); putLength = keyWriteResult.getValue(); } else { final String amzContentSha256Header = validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload()); - try (OzoneOutputStream output = getClientProtocol().createKey( - volume.getName(), bucketName, keyPath, length, replicationConfig, - customMetadata, tags)) { + try (OzoneOutputStream output = openKeyForPut( + volume.getName(), bucketName, bucket, keyPath, length, + replicationConfig, customMetadata, tags, ifNoneMatch, ifMatch)) { long metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); perf.appendMetaLatencyNanos(metadataLatencyNs); @@ -1302,6 +1323,47 @@ private Response deleteObjectTagging(OzoneVolume volume, String bucketName, Stri return Response.noContent().build(); } + /** + * Opens a key for put, applying conditional write logic based on + * If-None-Match and If-Match headers. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, + OzoneBucket bucket, String keyPath, long length, + ReplicationConfig replicationConfig, Map customMetadata, + Map tags, String ifNoneMatch, String ifMatch) + throws IOException { + if (ifNoneMatch != null && "*".equals(ifNoneMatch.trim())) { + return getClientProtocol().createKeyIfNotExists( + volumeName, bucketName, keyPath, length, replicationConfig, + customMetadata, tags); + } else if (ifMatch != null) { + String expectedETag = parseETag(ifMatch); + return getClientProtocol().rewriteKeyIfMatch( + volumeName, bucketName, keyPath, length, expectedETag, + replicationConfig, customMetadata, tags); + } else { + return getClientProtocol().createKey( + volumeName, bucketName, keyPath, length, replicationConfig, + customMetadata, tags); + } + } + + /** + * Parses an ETag from a conditional header value, removing surrounding + * quotes if present. + */ + static String parseETag(String headerValue) { + if (headerValue == null) { + return null; + } + String etag = headerValue.trim(); + if (etag.startsWith("\"") && etag.endsWith("\"")) { + return etag.substring(1, etag.length() - 1); + } + return etag; + } + /** Request context shared among {@code ObjectOperationHandler}s. */ final class ObjectRequestContext { private final String bucketName; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 797ca1f36712..e90612cc56ed 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -98,6 +98,10 @@ public final class S3Consts { public static final String CHECKSUM_HEADER = "Content-MD5"; + // Conditional request headers + public static final String IF_NONE_MATCH_HEADER = "If-None-Match"; + public static final String IF_MATCH_HEADER = "If-Match"; + //Never Constructed private S3Consts() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 739babce1d06..c5d2025b8012 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -249,6 +249,25 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String .rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata); } + @Override + public OzoneOutputStream createKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .createKeyIfNotExists(keyName, size, replicationConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteKeyIfMatch(keyName, size, expectedETag, replicationConfig, + metadata, tags); + } + @Override public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 8037f65fda1c..a91030f6b77a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -192,6 +192,39 @@ public void close() throws IOException { return new OzoneOutputStream(byteArrayOutputStream, null); } + @Override + public OzoneOutputStream createKeyIfNotExists(String keyName, long size, + ReplicationConfig rConfig, Map metadata, + Map tags) throws IOException { + if (keyDetails.containsKey(keyName)) { + throw new OMException("Key already exists", + ResultCodes.KEY_ALREADY_EXISTS); + } + return createKey(keyName, size, rConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, + String expectedETag, ReplicationConfig rConfig, + Map metadata, Map tags) + throws IOException { + OzoneKeyDetails existing = keyDetails.get(keyName); + if (existing == null) { + throw new OMException("Key not found for If-Match", + ResultCodes.KEY_NOT_FOUND); + } + String currentETag = existing.getMetadata().get(ETAG); + if (currentETag == null) { + throw new OMException("Key does not have an ETag", + ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!currentETag.equals(expectedETag)) { + throw new OMException("ETag mismatch", + ResultCodes.ETAG_MISMATCH); + } + return createKey(keyName, size, rConfig, metadata, tags); + } + @Override public OzoneDataStreamOutput createStreamKey(String key, long size, ReplicationConfig rConfig, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index c2456dd068fd..e9d1263a3797 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -555,6 +555,85 @@ private HttpHeaders newMockHttpHeaders() { return httpHeaders; } + @Test + void testIfNoneMatchKeyDoesNotExistSuccess() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + assertSucceeds(() -> putObject(CONTENT)); + assertKeyContent(bucket, KEY_NAME, CONTENT); + } + + @Test + void testIfNoneMatchKeyExistsPreconditionFailed() throws Exception { + // First create the key + assertSucceeds(() -> putObject(CONTENT)); + + // Now try to create again with If-None-Match: * + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testIfMatchETagMatchesSuccess() throws Exception { + // First create the key to get an ETag + Response response = putObject(CONTENT); + String etag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(etag); + + // Strip quotes from the ETag + String rawEtag = etag.replace("\"", ""); + + // Now try to rewrite with matching ETag + when(headers.getHeaderString("If-Match")).thenReturn(etag); + + assertSucceeds(() -> putObject("new-content")); + assertKeyContent(bucket, KEY_NAME, "new-content"); + } + + @Test + void testIfMatchETagMismatchPreconditionFailed() throws Exception { + // First create the key + assertSucceeds(() -> putObject(CONTENT)); + + // Try to rewrite with wrong ETag + when(headers.getHeaderString("If-Match")).thenReturn("\"wrong-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject("new-content")); + assertNotNull(ex); + } + + @Test + void testIfMatchKeyNotFoundPreconditionFailed() throws Exception { + // Try If-Match on a non-existent key + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testBothHeadersProvidedInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testParseETag() { + assertEquals("abc123", ObjectEndpoint.parseETag("\"abc123\"")); + assertEquals("abc123", ObjectEndpoint.parseETag("abc123")); + assertEquals("abc123", ObjectEndpoint.parseETag(" \"abc123\" ")); + assertEquals(null, ObjectEndpoint.parseETag(null)); + } + /** Put object at {@code bucketName}/{@code keyName} with pre-defined {@link #CONTENT}. */ private Response putObject(String bucketName, String keyName) throws IOException, OS3Exception { return put(objectEndpoint, bucketName, keyName, CONTENT);