diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java index 0da411b34d44..e6919b41e3b3 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java @@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import picocli.CommandLine; @@ -97,8 +98,8 @@ public class ContainerToKeyMapping extends AbstractSubcommand implements Callabl private Table directoryTable; private Table fileTable; private Table keyTable; - private Table openFileTable; - private Table openKeyTable; + private Table openFileTable; + private Table openKeyTable; private Table multipartInfoTable; private DBStore dirTreeDbStore; private Table dirTreeTable; @@ -273,11 +274,11 @@ private void processOBSKeys(Set containerIds, Map> cont } private void processOpenFiles(Set containerIds, Map> containerToOpenKeysMap) { - try (TableIterator> fileIterator = + try (TableIterator> fileIterator = openFileTable.iterator()) { while (fileIterator.hasNext()) { - Table.KeyValue entry = fileIterator.next(); - addOpenKeyToContainerMap(entry.getKey(), entry.getValue(), containerIds, containerToOpenKeysMap); + Table.KeyValue entry = fileIterator.next(); + addOpenKeyToContainerMap(entry.getKey(), entry.getValue().getKeyInfo(), containerIds, containerToOpenKeysMap); } } catch (Exception e) { err().println("Exception occurred reading openFileTable (FSO keys), " + e); @@ -285,11 +286,11 @@ private void processOpenFiles(Set containerIds, Map> co } private void processOpenKeys(Set containerIds, Map> containerToOpenKeysMap) { - try (TableIterator> keyIterator = + try (TableIterator> keyIterator = openKeyTable.iterator()) { while (keyIterator.hasNext()) { - Table.KeyValue entry = keyIterator.next(); - addOpenKeyToContainerMap(entry.getKey(), entry.getValue(), containerIds, containerToOpenKeysMap); + Table.KeyValue entry = keyIterator.next(); + addOpenKeyToContainerMap(entry.getKey(), entry.getValue().getKeyInfo(), containerIds, containerToOpenKeysMap); } } catch (Exception e) { err().println("Exception occurred reading openKeyTable (OBS keys), " + e); diff --git a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java index 4cad62cd719c..7d189c88ab3d 100644 --- a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java +++ b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; @@ -281,14 +282,16 @@ private void createTestData() throws Exception { "openFile", OPEN_FILE_ID, DIR_ID, CONTAINER_ID_1); String openFileKey = omMetadataManager.getOpenFileName( VOLUME_ID, FSO_BUCKET_ID, DIR_ID, "openFile", 1L); - omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED).put(openFileKey, openFileInfo); + omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED).put(openFileKey, + new OmOpenKeyInfo.Builder().setKeyInfo(openFileInfo).build()); // Create open OBS key with a block in container 2 OmKeyInfo openKeyInfo = createOBSKeyInfo( "openKey", OPEN_KEY_ID, CONTAINER_ID_2); String openKey = omMetadataManager.getOzoneKey( VOLUME_NAME, OBS_BUCKET_NAME, "openKey"); - omMetadataManager.getOpenKeyTable(BucketLayout.OBJECT_STORE).put(openKey, openKeyInfo); + omMetadataManager.getOpenKeyTable(BucketLayout.OBJECT_STORE).put(openKey, + new OmOpenKeyInfo.Builder().setKeyInfo(openKeyInfo).build()); // Create MPU (multipart upload) for OBS bucket with parts in container 5 createMultipartUpload(); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index e388df27b4b3..6eb469ec0e2b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -103,15 +103,6 @@ public final class OmKeyInfo extends WithParentObjectId */ private final ImmutableMap tags; - // expectedDataGeneration, when used in key creation indicates that a - // key with the same keyName should exist with the given generation. - // For a key commit to succeed, the original key should still be present with the - // generation unchanged. - // This allows a key to be created an committed atomically if the original has not - // been modified. - private Long expectedDataGeneration = null; - private String expectedETag; - private OmKeyInfo(Builder b) { super(b); this.volumeName = b.volumeName; @@ -129,8 +120,6 @@ private OmKeyInfo(Builder b) { this.isFile = b.isFile; this.ownerName = b.ownerName; this.tags = b.tags.build(); - this.expectedDataGeneration = b.expectedDataGeneration; - this.expectedETag = b.expectedETag; } private static Codec newCodec(boolean ignorePipeline) { @@ -183,22 +172,6 @@ public String getFileName() { return fileName; } - public void setExpectedDataGeneration(Long generation) { - this.expectedDataGeneration = generation; - } - - public Long getExpectedDataGeneration() { - return expectedDataGeneration; - } - - public void setExpectedETag(String eTag) { - this.expectedETag = eTag; - } - - public String getExpectedETag() { - return expectedETag; - } - public String getOwnerName() { return ownerName; } @@ -501,8 +474,6 @@ public static class Builder extends WithParentObjectId.Builder { private boolean isFile; private final MapBuilder tags; - private Long expectedDataGeneration = null; - private String expectedETag; public Builder() { this.acls = AclListBuilder.empty(); @@ -524,8 +495,6 @@ public Builder(OmKeyInfo obj) { this.fileName = obj.fileName; this.fileChecksum = obj.fileChecksum; this.isFile = obj.isFile; - this.expectedDataGeneration = obj.expectedDataGeneration; - this.expectedETag = obj.expectedETag; this.tags = MapBuilder.of(obj.tags); obj.keyLocationVersions.forEach(keyLocationVersion -> this.omKeyLocationInfoGroups.add( @@ -692,16 +661,6 @@ public Builder addAllTags(Map keyTags) { return this; } - public Builder setExpectedDataGeneration(Long existingGeneration) { - this.expectedDataGeneration = existingGeneration; - return this; - } - - public Builder setExpectedETag(String eTag) { - this.expectedETag = eTag; - return this; - } - @Override protected void validate() { super.validate(); @@ -818,12 +777,6 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); } kb.setIsFile(isFile); - if (expectedDataGeneration != null) { - kb.setExpectedDataGeneration(expectedDataGeneration); - } - if (expectedETag != null) { - kb.setExpectedETag(expectedETag); - } if (ownerName != null) { kb.setOwnerName(ownerName); } @@ -874,12 +827,6 @@ public static Builder builderFromProtobuf(KeyInfo keyInfo) { if (keyInfo.hasIsFile()) { builder.setFile(keyInfo.getIsFile()); } - if (keyInfo.hasExpectedDataGeneration()) { - builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration()); - } - if (keyInfo.hasExpectedETag()) { - builder.setExpectedETag(keyInfo.getExpectedETag()); - } if (keyInfo.hasOwnerName()) { builder.setOwnerName(keyInfo.getOwnerName()); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmOpenKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmOpenKeyInfo.java new file mode 100644 index 000000000000..2aedca779c21 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmOpenKeyInfo.java @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.CopyObject; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Wrapper for OmKeyInfo with additional fields that are only meaningful + * for open keys stored in openKeyTable/openFileTable. These fields are used + * during key creation and commit, but should not be persisted in keyTable. + * + *

This class uses composition to wrap {@link OmKeyInfo} and adds: + *

    + *
  • expectedDataGeneration - for conditional writes based on key generation
    • + *
  • expectedETag - for conditional writes based on S3 ETag (If-Match header)
    • + *
+ */ +public final class OmOpenKeyInfo implements CopyObject { + private static final Logger LOG = LoggerFactory.getLogger(OmOpenKeyInfo.class); + + private static final Codec CODEC_TRUE = newCodec(true); + private static final Codec CODEC_FALSE = newCodec(false); + + private final OmKeyInfo keyInfo; + private final Long expectedDataGeneration; + private final String expectedETag; + + private OmOpenKeyInfo(Builder builder) { + this.keyInfo = Objects.requireNonNull(builder.keyInfo, "keyInfo cannot be null"); + this.expectedDataGeneration = builder.expectedDataGeneration; + this.expectedETag = builder.expectedETag; + } + + /** + * Creates a backward-compatible codec that can read both old and new formats. + * Old format: raw KeyInfo proto bytes (for existing data migration). + * New format: OpenKeyInfo proto bytes. + */ + private static Codec newCodec(boolean ignorePipeline) { + return new BackwardCompatibleCodec(ignorePipeline); + } + + public static Codec getCodec(boolean ignorePipeline) { + LOG.debug("OmOpenKeyInfo.getCodec ignorePipeline = {}", ignorePipeline); + return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; + } + + /** + * Returns the wrapped OmKeyInfo. + * Use this when you need to extract the key info for commit operations + * or when interacting with keyTable. + */ + public OmKeyInfo getKeyInfo() { + return keyInfo; + } + + public Long getExpectedDataGeneration() { + return expectedDataGeneration; + } + + public String getExpectedETag() { + return expectedETag; + } + + public String getVolumeName() { + return keyInfo.getVolumeName(); + } + + public String getBucketName() { + return keyInfo.getBucketName(); + } + + public String getKeyName() { + return keyInfo.getKeyName(); + } + + public void setKeyName(String keyName) { + keyInfo.setKeyName(keyName); + } + + public long getDataSize() { + return keyInfo.getDataSize(); + } + + public void setDataSize(long size) { + keyInfo.setDataSize(size); + } + + public long getCreationTime() { + return keyInfo.getCreationTime(); + } + + public long getModificationTime() { + return keyInfo.getModificationTime(); + } + + public void setModificationTime(long modificationTime) { + keyInfo.setModificationTime(modificationTime); + } + + public ReplicationConfig getReplicationConfig() { + return keyInfo.getReplicationConfig(); + } + + public void setReplicationConfig(ReplicationConfig repConfig) { + keyInfo.setReplicationConfig(repConfig); + } + + public List getKeyLocationVersions() { + return keyInfo.getKeyLocationVersions(); + } + + public void setKeyLocationVersions(List keyLocationVersions) { + keyInfo.setKeyLocationVersions(keyLocationVersions); + } + + public OmKeyLocationInfoGroup getLatestVersionLocations() { + return keyInfo.getLatestVersionLocations(); + } + + public FileEncryptionInfo getFileEncryptionInfo() { + return keyInfo.getFileEncryptionInfo(); + } + + public void setFileEncryptionInfo(FileEncryptionInfo encInfo) { + keyInfo.setFileEncryptionInfo(encInfo); + } + + public FileChecksum getFileChecksum() { + return keyInfo.getFileChecksum(); + } + + public List getAcls() { + return keyInfo.getAcls(); + } + + public Map getMetadata() { + return keyInfo.getMetadata(); + } + + public Map getTags() { + return keyInfo.getTags(); + } + + public long getObjectID() { + return keyInfo.getObjectID(); + } + + public long getUpdateID() { + return keyInfo.getUpdateID(); + } + + public long getParentObjectID() { + return keyInfo.getParentObjectID(); + } + + public String getFileName() { + return keyInfo.getFileName(); + } + + public boolean isFile() { + return keyInfo.isFile(); + } + + public boolean isHsync() { + return keyInfo.isHsync(); + } + + public String getOwnerName() { + return keyInfo.getOwnerName(); + } + + public long getGeneration() { + return keyInfo.getGeneration(); + } + + public long getReplicatedSize() { + return keyInfo.getReplicatedSize(); + } + + public String getPath() { + return keyInfo.getPath(); + } + + public boolean hasBlocks() { + return keyInfo.hasBlocks(); + } + + /** + * Append new blocks to the latest version. + */ + public void appendNewBlocks(List newLocationList, boolean updateTime) { + keyInfo.appendNewBlocks(newLocationList, updateTime); + } + + /** + * Add a new version with the given blocks. + */ + public long addNewVersion(List newLocationList, + boolean updateTime, boolean keepOldVersions) { + return keyInfo.addNewVersion(newLocationList, updateTime, keepOldVersions); + } + + /** + * Updates the location info list with committed blocks. + */ + public List updateLocationInfoList( + List locationInfoList, boolean isMpu) { + return keyInfo.updateLocationInfoList(locationInfoList, isMpu); + } + + public List updateLocationInfoList( + List locationInfoList, boolean isMpu, boolean skipBlockIDCheck) { + return keyInfo.updateLocationInfoList(locationInfoList, isMpu, skipBlockIDCheck); + } + + // ========== Proto conversion ========== + + public KeyInfo getNetworkProtobuf(int clientVersion, boolean latestVersion) { + return keyInfo.getNetworkProtobuf(clientVersion, latestVersion); + } + + public OpenKeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) { + OpenKeyInfo.Builder builder = OpenKeyInfo.newBuilder() + .setKeyInfo(keyInfo.getProtobuf(ignorePipeline, clientVersion)); + + if (expectedDataGeneration != null) { + builder.setExpectedDataGeneration(expectedDataGeneration); + } + if (expectedETag != null) { + builder.setExpectedETag(expectedETag); + } + return builder.build(); + } + + public static OmOpenKeyInfo getFromProtobuf(OpenKeyInfo proto) { + if (proto == null) { + return null; + } + return new Builder() + .setKeyInfo(OmKeyInfo.getFromProtobuf(proto.getKeyInfo())) + .setExpectedDataGeneration( + proto.hasExpectedDataGeneration() ? proto.getExpectedDataGeneration() : null) + .setExpectedETag( + proto.hasExpectedETag() ? proto.getExpectedETag() : null) + .build(); + } + + /** + * Creates an OmOpenKeyInfo from a legacy KeyInfo proto. + * This is used for backward compatibility when reading old data. + */ + public static OmOpenKeyInfo fromLegacyKeyInfo(KeyInfo keyInfo) { + if (keyInfo == null) { + return null; + } + OmKeyInfo omKeyInfo = OmKeyInfo.getFromProtobuf(keyInfo); + return new Builder() + .setKeyInfo(omKeyInfo) + // Legacy KeyInfo may have these fields - extract them if present + .setExpectedDataGeneration( + keyInfo.hasExpectedDataGeneration() ? keyInfo.getExpectedDataGeneration() : null) + .setExpectedETag( + keyInfo.hasExpectedETag() ? keyInfo.getExpectedETag() : null) + .build(); + } + + @Override + public OmOpenKeyInfo copyObject() { + return new Builder() + .setKeyInfo(keyInfo.copyObject()) + .setExpectedDataGeneration(expectedDataGeneration) + .setExpectedETag(expectedETag) + .build(); + } + + /** + * Creates a new Builder initialized with values from this object. + */ + public Builder toBuilder() { + return new Builder() + .setKeyInfo(keyInfo.copyObject()) + .setExpectedDataGeneration(expectedDataGeneration) + .setExpectedETag(expectedETag); + } + + @Override + public String toString() { + return "OmOpenKeyInfo{" + + "keyInfo=" + keyInfo + + ", expectedDataGeneration=" + expectedDataGeneration + + ", expectedETag='" + expectedETag + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OmOpenKeyInfo that = (OmOpenKeyInfo) o; + return Objects.equals(keyInfo, that.keyInfo) && + Objects.equals(expectedDataGeneration, that.expectedDataGeneration) && + Objects.equals(expectedETag, that.expectedETag); + } + + @Override + public int hashCode() { + return Objects.hash(keyInfo, expectedDataGeneration, expectedETag); + } + + /** + * Builder for OmOpenKeyInfo. + */ + public static class Builder { + private OmKeyInfo keyInfo; + private OmKeyInfo.Builder keyInfoBuilder; + private Long expectedDataGeneration; + private String expectedETag; + + public Builder() { + } + + public Builder setKeyInfo(OmKeyInfo keyInfo) { + this.keyInfo = keyInfo; + this.keyInfoBuilder = null; + return this; + } + + public Builder setExpectedDataGeneration(Long expectedDataGeneration) { + this.expectedDataGeneration = expectedDataGeneration; + return this; + } + + public Builder setExpectedETag(String expectedETag) { + this.expectedETag = expectedETag; + return this; + } + + /** + * Adds metadata to the underlying keyInfo. + * For use in toBuilder() chains. + */ + public Builder addMetadata(String key, String value) { + ensureKeyInfoBuilder(); + keyInfoBuilder.addMetadata(key, value); + return this; + } + + /** + * Sets the update ID on the underlying keyInfo. + */ + public Builder setUpdateID(long updateID) { + ensureKeyInfoBuilder(); + keyInfoBuilder.setUpdateID(updateID); + return this; + } + + private void ensureKeyInfoBuilder() { + if (keyInfoBuilder == null && keyInfo != null) { + keyInfoBuilder = keyInfo.toBuilder(); + keyInfo = null; + } + } + + public OmOpenKeyInfo build() { + if (keyInfoBuilder != null) { + keyInfo = keyInfoBuilder.build(); + } + return new OmOpenKeyInfo(this); + } + } + + /** + * Backward-compatible codec that can read both old KeyInfo format + * and new OpenKeyInfo format. + */ + private static class BackwardCompatibleCodec implements Codec { + private final boolean ignorePipeline; + + BackwardCompatibleCodec(boolean ignorePipeline) { + this.ignorePipeline = ignorePipeline; + } + + @Override + public byte[] toPersistedFormatImpl(OmOpenKeyInfo object) { + return object.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION).toByteArray(); + } + + @Override + public OmOpenKeyInfo fromPersistedFormatImpl(byte[] rawData) throws InvalidProtocolBufferException { + // Try to parse as OpenKeyInfo first + try { + OpenKeyInfo openKeyInfo = OpenKeyInfo.parseFrom(rawData); + // Check if this is actually an OpenKeyInfo (has nested KeyInfo with valid fields) + if (openKeyInfo.hasKeyInfo() && openKeyInfo.getKeyInfo().hasVolumeName()) { + return getFromProtobuf(openKeyInfo); + } + } catch (InvalidProtocolBufferException e) { + // Fall through to try KeyInfo parsing + LOG.debug("Failed to parse as OpenKeyInfo, trying KeyInfo format", e); + } + + // Try to parse as legacy KeyInfo format + KeyInfo keyInfo = KeyInfo.parseFrom(rawData); + if (keyInfo.hasVolumeName()) { + return fromLegacyKeyInfo(keyInfo); + } + + throw new InvalidProtocolBufferException( + "Invalid data format: could not parse as OpenKeyInfo or KeyInfo"); + } + + @Override + public OmOpenKeyInfo copyObject(OmOpenKeyInfo object) { + return object.copyObject(); + } + + @Override + public Class getTypeClass() { + return OmOpenKeyInfo.class; + } + } +} diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java index 285853a3a766..6ef03ceacbc5 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java @@ -70,7 +70,6 @@ public void protobufConversion() throws IOException { key = key.withMetadataMutations( metadata -> metadata.put(OzoneConsts.HSYNC_CLIENT_ID, "clientid")); assertTrue(key.isHsync()); - assertEquals(5678L, key.getExpectedDataGeneration()); } @Test @@ -129,7 +128,6 @@ private OmKeyInfo createOmKeyInfo(ReplicationConfig replicationConfig) { .addMetadata("key2", "value2") .addTag("tagKey1", "tagValue1") .addTag("tagKey2", "tagValue2") - .setExpectedDataGeneration(5678L) .build(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index 9643598e7e96..11258d002390 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -115,6 +115,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; import org.apache.hadoop.security.UserGroupInformation; @@ -245,7 +246,7 @@ public void testKeyMetadata() throws Exception { cluster.getOzoneManager().getMetadataManager(); // Expect empty OpenKeyTable and KeyTable before key creation - Table openKeyTable = omMetadataManager.getOpenKeyTable(BUCKET_LAYOUT); + Table openKeyTable = omMetadataManager.getOpenKeyTable(BUCKET_LAYOUT); assertTrue(openKeyTable.isEmpty()); Table keyTable = omMetadataManager.getKeyTable(BUCKET_LAYOUT); assertTrue(keyTable.isEmpty()); @@ -255,7 +256,7 @@ public void testKeyMetadata() throws Exception { // Wait for double buffer flush to avoid flakiness because RDB iterator bypasses table cache cluster.getOzoneManager().awaitDoubleBufferFlush(); // OpenKeyTable key should NOT have HSYNC_CLIENT_ID - OmKeyInfo keyInfo = getFirstKeyInTable(keyName, openKeyTable); + OmKeyInfo keyInfo = getFirstOpenKeyInTable(keyName, openKeyTable).getKeyInfo(); assertFalse(keyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID)); // KeyTable should still be empty assertTrue(keyTable.isEmpty()); @@ -263,7 +264,7 @@ public void testKeyMetadata() throws Exception { os.hsync(); cluster.getOzoneManager().awaitDoubleBufferFlush(); // OpenKeyTable key should have HSYNC_CLIENT_ID now - keyInfo = getFirstKeyInTable(keyName, openKeyTable); + keyInfo = getFirstOpenKeyInTable(keyName, openKeyTable).getKeyInfo(); assertTrue(keyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID)); // KeyTable key should be there and have HSYNC_CLIENT_ID keyInfo = getFirstKeyInTable(keyName, keyTable); @@ -272,7 +273,7 @@ public void testKeyMetadata() throws Exception { // hsync again, metadata should not change os.hsync(); cluster.getOzoneManager().awaitDoubleBufferFlush(); - keyInfo = getFirstKeyInTable(keyName, openKeyTable); + keyInfo = getFirstOpenKeyInTable(keyName, openKeyTable).getKeyInfo(); assertTrue(keyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID)); keyInfo = getFirstKeyInTable(keyName, keyTable); assertTrue(keyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID)); @@ -606,12 +607,12 @@ public void testHSyncOpenKeyDeletionWhileDeleteDirectory() throws Exception { private List getOpenKeyInfo(BucketLayout bucketLayout) { List omKeyInfo = new ArrayList<>(); - Table openFileTable = + Table openFileTable = cluster.getOzoneManager().getMetadataManager().getOpenKeyTable(bucketLayout); - try (Table.KeyValueIterator + try (Table.KeyValueIterator iterator = openFileTable.iterator()) { while (iterator.hasNext()) { - omKeyInfo.add(iterator.next().getValue()); + omKeyInfo.add(iterator.next().getValue().getKeyInfo()); } } catch (Exception e) { } @@ -1127,13 +1128,13 @@ public void testDisableHsync() throws Exception { } /** - * Helper method to check and get the first key in the OpenKeyTable. + * Helper method to check and get the first key in the KeyTable. * @param keyName expect key name to contain this string - * @param openKeyTable Table + * @param keyTable Table * @return OmKeyInfo */ - private OmKeyInfo getFirstKeyInTable(String keyName, Table openKeyTable) throws IOException { - try (Table.KeyValueIterator it = openKeyTable.iterator()) { + private OmKeyInfo getFirstKeyInTable(String keyName, Table keyTable) throws IOException { + try (Table.KeyValueIterator it = keyTable.iterator()) { assertTrue(it.hasNext()); Table.KeyValue kv = it.next(); String dbOpenKey = kv.getKey(); @@ -1143,6 +1144,24 @@ private OmKeyInfo getFirstKeyInTable(String keyName, Table op } } + /** + * Helper method to check and get the first key in the OpenKeyTable. + * @param keyName expect key name to contain this string + * @param openKeyTable Table + * @return OmOpenKeyInfo + */ + private OmOpenKeyInfo getFirstOpenKeyInTable(String keyName, Table openKeyTable) + throws IOException { + try (Table.KeyValueIterator it = openKeyTable.iterator()) { + assertTrue(it.hasNext()); + Table.KeyValue kv = it.next(); + String dbOpenKey = kv.getKey(); + assertNotNull(dbOpenKey); + assertTrue(dbOpenKey.contains(keyName)); + return kv.getValue(); + } + } + private void testEncryptedStreamCapabilities(boolean isEC) throws IOException, GeneralSecurityException { KeyOutputStream kos; @@ -1303,7 +1322,7 @@ public void testNormalKeyOverwriteHSyncKey() throws Exception { cleanupDeletedTable(ozoneManager); cleanupOpenKeyTable(ozoneManager, BUCKET_LAYOUT); OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); - Table openKeyTable = metadataManager.getOpenKeyTable(BUCKET_LAYOUT); + Table openKeyTable = metadataManager.getOpenKeyTable(BUCKET_LAYOUT); Table deletedTable = metadataManager.getDeletedTable(); assertTrue(openKeyTable.isEmpty()); assertTrue(deletedTable.isEmpty()); @@ -1402,7 +1421,7 @@ public void testHSyncKeyOverwriteNormalKey() throws Exception { cleanupDeletedTable(ozoneManager); cleanupOpenKeyTable(ozoneManager, BUCKET_LAYOUT); OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); - Table openKeyTable = metadataManager.getOpenKeyTable(BUCKET_LAYOUT); + Table openKeyTable = metadataManager.getOpenKeyTable(BUCKET_LAYOUT); Table deletedTable = metadataManager.getDeletedTable(); assertTrue(openKeyTable.isEmpty()); assertTrue(deletedTable.isEmpty()); @@ -1477,7 +1496,7 @@ public void testHSyncKeyOverwriteHSyncKey() throws Exception { cleanupDeletedTable(ozoneManager); cleanupOpenKeyTable(ozoneManager, BUCKET_LAYOUT); OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); - Table openKeyTable = metadataManager.getOpenKeyTable(BUCKET_LAYOUT); + Table openKeyTable = metadataManager.getOpenKeyTable(BUCKET_LAYOUT); Table deletedTable = metadataManager.getDeletedTable(); assertTrue(openKeyTable.isEmpty()); assertTrue(deletedTable.isEmpty()); @@ -1541,13 +1560,13 @@ public void testHSyncKeyOverwriteHSyncKey() throws Exception { } } - private Map getAllOpenKeys(Table table) throws IOException { + private Map getAllOpenKeys(Table table) throws IOException { Map keys = new HashMap(); - try (Table.KeyValueIterator tableIter = table.iterator()) { + try (Table.KeyValueIterator tableIter = table.iterator()) { while (tableIter.hasNext()) { - Table.KeyValue kv = tableIter.next(); + Table.KeyValue kv = tableIter.next(); String key = kv.getKey(); - keys.put(key, kv.getValue()); + keys.put(key, kv.getValue().getKeyInfo()); } } return keys; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index 7ac80ef40584..3aa6ed5fefb0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; /** @@ -289,11 +290,11 @@ public static void cleanupDeletedTable(OzoneManager ozoneManager) throws IOExcep } public static void cleanupOpenKeyTable(OzoneManager ozoneManager, BucketLayout bucketLayout) throws IOException { - Table openKeyTable = ozoneManager.getMetadataManager().getOpenKeyTable(bucketLayout); + Table openKeyTable = ozoneManager.getMetadataManager().getOpenKeyTable(bucketLayout); List nameList = new ArrayList<>(); - try (Table.KeyValueIterator keyIter = openKeyTable.iterator()) { + try (Table.KeyValueIterator keyIter = openKeyTable.iterator()) { while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); + Table.KeyValue kv = keyIter.next(); nameList.add(kv.getKey()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index 548c0acf3689..c025a54efd53 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -83,6 +83,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -548,11 +549,11 @@ public void testAbortUploadSuccessWithParts() throws Exception { String multipartOpenKey = metadataMgr.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = metadataMgr.getOpenKeyTable(bucketLayout).get(multipartOpenKey); OmMultipartKeyInfo omMultipartKeyInfo = metadataMgr.getMultipartInfoTable().get(multipartKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); assertNull(omMultipartKeyInfo); // Since deleteTable operation is performed via @@ -624,11 +625,11 @@ public void testAbortMultipartUploadSuccessWithMissingParentDirectories() throws // Abort multipart upload with missing parent directory bucket.abortMultipartUpload(keyName, uploadID); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = metadataMgr.getOpenKeyTable(bucketLayout).get(multipartOpenKey); OmMultipartKeyInfo omMultipartKeyInfo = metadataMgr.getMultipartInfoTable().get(multipartKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); assertNull(omMultipartKeyInfo); } @@ -1035,13 +1036,14 @@ private String verifyUploadedPart(String uploadID, String partName, String multipartKey = metadataMgr.getMultipartKey(volumeName, bucketName, keyName, uploadID); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = metadataMgr.getOpenKeyTable(bucketLayout).get(multipartOpenKey); OmMultipartKeyInfo omMultipartKeyInfo = metadataMgr.getMultipartInfoTable().get(multipartKey); - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); assertNotNull(omMultipartKeyInfo); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); assertEquals(keyName, omKeyInfo.getKeyName()); assertEquals(OzoneFSUtils.getFileName(keyName), omKeyInfo.getFileName()); assertEquals(uploadID, omMultipartKeyInfo.getUploadID()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index dcdb3828ae6d..a0624f1969ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -120,6 +120,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; @@ -322,7 +323,7 @@ public void allocateBlockFailureInSafeMode() throws Exception { .setFileEncryptionInfo(null).build(); metadataManager.getOpenKeyTable(getDefaultBucketLayout()).put( metadataManager.getOpenKey(VOLUME_NAME, BUCKET_NAME, KEY_NAME, 1L), - omKeyInfo); + new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build()); OMException omException = assertThrows(OMException.class, () -> writeClient.allocateBlock(keyArgs, 1L, new ExcludeList())); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index e6d3985a34d3..cbe7dd089cb0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java @@ -71,6 +71,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.util.StringUtils; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.NonHATests; @@ -157,12 +158,12 @@ public void testCreateKey() throws Exception { OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); assertEquals(bucketName, ozoneBucket.getName()); - Table openFileTable = + Table openFileTable = cluster.getOzoneManager().getMetadataManager() .getOpenKeyTable(getBucketLayout()); // before file creation - verifyKeyInFileTable(openFileTable, file, 0, true); + assertTrue(openFileTable.isEmpty(), "openFileTable is not empty!"); String data = "random data"; OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, @@ -225,12 +226,12 @@ public void testDeleteBucketWithKeys() throws Exception { OzoneBucket ozoneBucket = ozoneVolume.getBucket(testBucketName); assertEquals(ozoneBucket.getName(), testBucketName); - Table openFileTable = + Table openFileTable = cluster.getOzoneManager().getMetadataManager() .getOpenKeyTable(getBucketLayout()); // before file creation - verifyKeyInFileTable(openFileTable, file, 0, true); + assertTrue(openFileTable.isEmpty(), "openFileTable is not empty!"); // Create a key. ozoneBucket.createKey(key, 10).close(); @@ -277,7 +278,7 @@ public void testLookupKey() throws Exception { OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); assertEquals(bucketName, ozoneBucket.getName()); - Table openFileTable = + Table openFileTable = cluster.getOzoneManager().getMetadataManager() .getOpenKeyTable(getBucketLayout()); @@ -769,7 +770,7 @@ private void verifyKeyInFileTable(Table fileTable, } } - private void verifyKeyInOpenFileTable(Table openFileTable, + private void verifyKeyInOpenFileTable(Table openFileTable, long clientID, String fileName, long parentID, boolean isEmpty) throws IOException, TimeoutException, InterruptedException { final OMMetadataManager omMetadataManager = @@ -783,8 +784,8 @@ private void verifyKeyInOpenFileTable(Table openFileTable, // wait for DB updates GenericTestUtils.waitFor(() -> { try { - OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); - return omKeyInfo == null; + OmOpenKeyInfo omOpenKeyInfo = openFileTable.get(dbOpenFileKey); + return omOpenKeyInfo == null; } catch (IOException e) { fail("DB failure!"); return false; @@ -792,8 +793,9 @@ private void verifyKeyInOpenFileTable(Table openFileTable, }, 1000, 120000); } else { - OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); - assertNotNull(omKeyInfo, "Table is empty!"); + OmOpenKeyInfo omOpenKeyInfo = openFileTable.get(dbOpenFileKey); + assertNotNull(omOpenKeyInfo, "Table is empty!"); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); assertEquals(omKeyInfo.getFileName(), fileName, "Invalid file name: " + omKeyInfo.getObjectInfo()); assertEquals(parentID, omKeyInfo.getParentObjectID(), "Invalid Key"); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index a972e87ddc1b..cbd2f7b5063f 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1199,6 +1199,15 @@ message KeyInfo { optional string expectedETag = 23; } +// OpenKeyInfo wraps KeyInfo with additional fields that are only meaningful +// for open keys stored in openKeyTable/openFileTable. These fields are used +// during key creation and commit, but should not be persisted in keyTable. +message OpenKeyInfo { + required KeyInfo keyInfo = 1; + optional uint64 expectedDataGeneration = 2; + optional string expectedETag = 3; +} + // KeyInfoProtoLight is a lightweight subset of KeyInfo message containing // selected fields only, while maintaining the same field indices as KeyInfo // for compatibility and consistency. diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index be66ffc195b5..a61fb1cef748 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -424,7 +425,7 @@ List getExpiredMultipartUploads( * * @return Table. */ - Table getOpenKeyTable(BucketLayout buckLayout); + Table getOpenKeyTable(BucketLayout buckLayout); /** * Gets the DelegationTokenTable. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 08b6d6abbf18..78c1525f448e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -159,6 +159,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPartInfo; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@ -1185,16 +1186,16 @@ public OmMultipartUploadListParts listParts(String volumeName, OMMultipartUploadUtils.getMultipartOpenKey(volumeName, bucketName, keyName, uploadID, metadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED); } - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = metadataManager.getOpenKeyTable(bucketLayout) .get(multipartKey); - if (omKeyInfo == null) { + if (omOpenKeyInfo == null) { throw new IllegalStateException( "Open key is missing for multipart upload " + multipartKey); } - replicationConfig = omKeyInfo.getReplicationConfig(); + replicationConfig = omOpenKeyInfo.getReplicationConfig(); } Objects.requireNonNull(replicationConfig, "ReplicationConfig can't be identified"); @@ -1340,8 +1341,9 @@ public boolean checkAccess(OzoneObj ozObject, RequestContext context) // For Acl Type "WRITE", the key can only be found in // OpenKeyTable since appends to existing keys are not supported. if (context.getAclRights() == IAccessAuthorizer.ACLType.WRITE) { - keyInfo = + OmOpenKeyInfo omOpenKeyInfo = metadataManager.getOpenKeyTable(bucketLayout).get(objectKey); + keyInfo = omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } else { // Recursive check is done only for ACL_TYPE DELETE // Rename and delete operations will send ACL_TYPE DELETE diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1797acefa283..aaff04b37f5b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -111,6 +111,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; @@ -160,14 +161,14 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private Table bucketTable; private Table keyTable; - private Table openKeyTable; + private Table openKeyTable; private Table multipartInfoTable; private Table multipartPartsTable; private Table deletedTable; private Table dirTable; private Table fileTable; - private Table openFileTable; + private Table openFileTable; private Table deletedDirTable; private Table s3SecretTable; @@ -367,7 +368,7 @@ public Table getDeletedDirTable() { } @Override - public Table getOpenKeyTable(BucketLayout bucketLayout) { + public Table getOpenKeyTable(BucketLayout bucketLayout) { if (bucketLayout.isFileSystemOptimized()) { return openFileTable; } @@ -989,13 +990,13 @@ public ListOpenFilesResult listOpenFiles(BucketLayout bucketLayout, // listKeys do. But that complicates the iteration logic by quite a bit. // And if we do that, we need to refactor listKeys as well to dedup. - final Table okTable; + final Table okTable; okTable = getOpenKeyTable(bucketLayout); // No lock required since table iterator creates a "snapshot" - try (TableIterator> + try (TableIterator> openKeyIter = okTable.iterator()) { - KeyValue kv; + KeyValue kv; kv = openKeyIter.seek(dbContTokenPrefix); if (hasContToken && kv.getKey().equals(dbContTokenPrefix)) { // Skip one entry when cont token is specified and the current entry @@ -1007,7 +1008,7 @@ public ListOpenFilesResult listOpenFiles(BucketLayout bucketLayout, if (kv != null && kv.getKey().startsWith(dbOpenKeyPrefix)) { String dbKey = kv.getKey(); long clientID = OMMetadataManager.getClientIDFromOpenKeyDBKey(dbKey); - OmKeyInfo omKeyInfo = kv.getValue(); + OmKeyInfo omKeyInfo = kv.getValue().getKeyInfo(); // Note with HDDS-10077, there is no need to check KeyTable for hsync metadata openKeySessionList.add( new OpenKeySession(clientID, omKeyInfo, @@ -1018,7 +1019,7 @@ public ListOpenFilesResult listOpenFiles(BucketLayout bucketLayout, // Set hasMore flag as a hint for client-side pagination if (openKeyIter.hasNext()) { - KeyValue nextKv = openKeyIter.next(); + KeyValue nextKv = openKeyIter.next(); hasMore = nextKv != null && nextKv.getKey().startsWith(dbOpenKeyPrefix); } else { hasMore = false; @@ -1420,7 +1421,7 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, // Only check for expired keys in the open key table, not its cache. // If a key expires while it is in the cache, it will be cleaned // up after the cache is flushed. - try (TableIterator> + try (TableIterator> keyValueTableIterator = getOpenKeyTable(bucketLayout).iterator()) { final long expiredCreationTimestamp = @@ -1431,12 +1432,12 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, int num = 0; while (num < count && keyValueTableIterator.hasNext()) { - KeyValue openKeyValue = keyValueTableIterator.next(); + KeyValue openKeyValue = keyValueTableIterator.next(); String dbOpenKeyName = openKeyValue.getKey(); final int lastPrefix = dbOpenKeyName.lastIndexOf(OM_KEY_PREFIX); final String dbKeyName = dbOpenKeyName.substring(0, lastPrefix); - OmKeyInfo openKeyInfo = openKeyValue.getValue(); + OmKeyInfo openKeyInfo = openKeyValue.getValue().getKeyInfo(); if (isOpenMultipartKey(openKeyInfo, dbOpenKeyName)) { continue; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 323f11926af9..acd66a47e5c3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -218,11 +219,11 @@ public final class OMDBDefinition extends DBDefinition.WithMap { RepeatedOmKeyInfo.getCodec(true)); public static final String OPEN_KEY_TABLE = "openKeyTable"; - /** openKeyTable: /volume/bucket/key/id :- KeyInfo. */ - public static final DBColumnFamilyDefinition OPEN_KEY_TABLE_DEF + /** openKeyTable: /volume/bucket/key/id :- OpenKeyInfo. */ + public static final DBColumnFamilyDefinition OPEN_KEY_TABLE_DEF = new DBColumnFamilyDefinition<>(OPEN_KEY_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmOpenKeyInfo.getCodec(true)); public static final String MULTIPART_INFO_TABLE = "multipartInfoTable"; /** multipartInfoTable: /volume/bucket/key/uploadId :- parts. */ @@ -248,11 +249,11 @@ public final class OMDBDefinition extends DBDefinition.WithMap { OmKeyInfo.getCodec(true)); public static final String OPEN_FILE_TABLE = "openFileTable"; - /** openFileTable: /volumeId/bucketId/parentId/fileName/id :- KeyInfo. */ - public static final DBColumnFamilyDefinition OPEN_FILE_TABLE_DEF + /** openFileTable: /volumeId/bucketId/parentId/fileName/id :- OpenKeyInfo. */ + public static final DBColumnFamilyDefinition OPEN_FILE_TABLE_DEF = new DBColumnFamilyDefinition<>(OPEN_FILE_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmOpenKeyInfo.getCodec(true)); public static final String DIRECTORY_TABLE = "directoryTable"; /** directoryTable: /volumeId/bucketId/parentId/dirName :- DirInfo. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 9788cfbafe17..3ee89a11da7f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; @@ -278,8 +279,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Add to cache entry can be done outside of lock for this openKey. // Even if bucket gets deleted, when commitKey we shall identify if // bucket gets deleted. + OmOpenKeyInfo.Builder openKeyInfoBuilder = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo); + if (keyArgs.hasExpectedDataGeneration()) { + openKeyInfoBuilder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); + } + if (keyArgs.hasExpectedETag()) { + openKeyInfoBuilder.setExpectedETag(keyArgs.getExpectedETag()); + } + OmOpenKeyInfo omOpenKeyInfo = openKeyInfoBuilder.build(); omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( - dbOpenKeyName, omKeyInfo, trxnLogIndex); + dbOpenKeyName, omOpenKeyInfo, trxnLogIndex); // Add cache entries for the prefix directories. // Skip adding for the file key itself, until Key Commit. @@ -295,7 +305,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setOpenVersion(openVersion).build()) .setCmdType(CreateFile); omClientResponse = new OMFileCreateResponse(omResponse.build(), - omKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject()); + omOpenKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java index 6036fe90dbb7..5c3c966a069e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponseWithFSO; @@ -197,8 +198,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Add to cache entry can be done outside of lock for this openKey. // Even if bucket gets deleted, when commitKey we shall identify if // bucket gets deleted. + OmOpenKeyInfo.Builder openKeyInfoBuilder = new OmOpenKeyInfo.Builder() + .setKeyInfo(omFileInfo); + if (keyArgs.hasExpectedDataGeneration()) { + openKeyInfoBuilder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); + } + if (keyArgs.hasExpectedETag()) { + openKeyInfoBuilder.setExpectedETag(keyArgs.getExpectedETag()); + } + OmOpenKeyInfo omOpenKeyInfo = openKeyInfoBuilder.build(); OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, - dbOpenFileName, omFileInfo, keyName, trxnLogIndex); + dbOpenFileName, omOpenKeyInfo, keyName, trxnLogIndex); // Add cache entries for the prefix directories. // Skip adding for the file key itself, until Key Commit. @@ -215,7 +225,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setOpenVersion(openVersion).build()) .setCmdType(Type.CreateFile); omClientResponse = new OMFileCreateResponseWithFSO(omResponse.build(), - omFileInfo, missingParentInfos, clientID, + omOpenKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject(), volumeId); result = Result.SUCCESS; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index aa1402052f3a..468bffada61c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.slf4j.Logger; @@ -458,22 +459,23 @@ public static void addDirectoryTableCacheEntries( * * @param omMetadataManager OM Metadata Manager * @param dbOpenFileName open file name key - * @param omFileInfo key info + * @param omOpenFileInfo open key info + * @param keyName key name * @param trxnLogIndex transaction log index */ public static void addOpenFileTableCacheEntry( OMMetadataManager omMetadataManager, String dbOpenFileName, - @Nullable OmKeyInfo omFileInfo, String keyName, long trxnLogIndex) { + @Nullable OmOpenKeyInfo omOpenFileInfo, String keyName, long trxnLogIndex) { - final Table table = omMetadataManager.getOpenKeyTable( + final Table table = omMetadataManager.getOpenKeyTable( BucketLayout.FILE_SYSTEM_OPTIMIZED); - if (omFileInfo != null) { + if (omOpenFileInfo != null) { // For example, the user given key path is '/a/b/c/d/e/file1', then in DB // keyName field stores full path, which is '/a/b/c/d/e/file1'. // This is required as in some cases like hsync, Keys inside openKeyTable is used for auto commit after expiry. // (Full key path is required in commit key request) - omFileInfo.setKeyName(keyName); - table.addCacheEntry(dbOpenFileName, omFileInfo, trxnLogIndex); + omOpenFileInfo.getKeyInfo().setKeyName(keyName); + table.addCacheEntry(dbOpenFileName, omOpenFileInfo, trxnLogIndex); } else { table.addCacheEntry(dbOpenFileName, trxnLogIndex); } @@ -511,43 +513,47 @@ public static void addFileTableCacheEntry( * * @param omMetadataMgr OM Metadata Manager * @param batchOp batch of db operations - * @param omFileInfo omKeyInfo + * @param omOpenFileInfo open key info * @param openKeySessionID clientID + * @param volumeId volume ID + * @param bucketId bucket ID * @throws IOException DB failure */ public static void addToOpenFileTable(OMMetadataManager omMetadataMgr, - BatchOperation batchOp, OmKeyInfo omFileInfo, long openKeySessionID, + BatchOperation batchOp, OmOpenKeyInfo omOpenFileInfo, long openKeySessionID, long volumeId, long bucketId) throws IOException { String dbOpenFileKey = omMetadataMgr.getOpenFileName(volumeId, bucketId, - omFileInfo.getParentObjectID(), omFileInfo.getFileName(), + omOpenFileInfo.getParentObjectID(), omOpenFileInfo.getFileName(), openKeySessionID); omMetadataMgr.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .putWithBatch(batchOp, dbOpenFileKey, omFileInfo); + .putWithBatch(batchOp, dbOpenFileKey, omOpenFileInfo); } /** * Adding multipart omKeyInfo to open file table. * - * @param omMetadataMgr OM Metadata Manager - * @param batchOp batch of db operations - * @param omFileInfo omKeyInfo - * @param uploadID uploadID + * @param omMetadataMgr OM Metadata Manager + * @param batchOp batch of db operations + * @param omOpenFileInfo open key info + * @param uploadID uploadID + * @param volumeId volume ID + * @param bucketId bucket ID * @return multipartFileKey * @throws IOException DB failure */ public static String addToOpenFileTableForMultipart( OMMetadataManager omMetadataMgr, - BatchOperation batchOp, OmKeyInfo omFileInfo, String uploadID, + BatchOperation batchOp, OmOpenKeyInfo omOpenFileInfo, String uploadID, long volumeId, long bucketId) throws IOException { String multipartFileKey = omMetadataMgr.getMultipartKey(volumeId, - bucketId, omFileInfo.getParentObjectID(), - omFileInfo.getFileName(), uploadID); + bucketId, omOpenFileInfo.getParentObjectID(), + omOpenFileInfo.getFileName(), uploadID); omMetadataMgr.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .putWithBatch(batchOp, multipartFileKey, omFileInfo); + .putWithBatch(batchOp, multipartFileKey, omOpenFileInfo); return multipartFileKey; } @@ -591,9 +597,10 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, OmKeyInfo dbOmKeyInfo; if (openFileTable) { - dbOmKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataMgr.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) .get(dbOpenFileKey); + dbOmKeyInfo = omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } else { dbOmKeyInfo = omMetadataMgr.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) @@ -610,6 +617,28 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, return dbOmKeyInfo; } + /** + * Get OmOpenKeyInfo from the open file table. + * + * @param omMetadataMgr metadata manager + * @param dbOpenFileKey open file key + * @param keyName key name + * @return OmOpenKeyInfo or null if not found + * @throws IOException DB failure + */ + public static OmOpenKeyInfo getOmOpenKeyInfoFromOpenFileTable( + OMMetadataManager omMetadataMgr, String dbOpenFileKey, String keyName) + throws IOException { + OmOpenKeyInfo omOpenKeyInfo = + omMetadataMgr.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .get(dbOpenFileKey); + if (omOpenKeyInfo != null) { + // Set the full key name (DB stores only the file name) + omOpenKeyInfo.setKeyName(keyName); + } + return omOpenKeyInfo; + } + /** * Gets OmKeyInfo if exists and validate bucket and volume. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java index ca1ea07ad6ed..6c0a44570f10 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -75,7 +76,7 @@ public class OMRecoverLeaseRequest extends OMKeyRequest { private String volumeName; private String bucketName; private String keyName; - private OmKeyInfo openKeyInfo; + private OmOpenKeyInfo openKeyInfo; private String dbOpenFileKey; private boolean force; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index b692cf9d55eb..27fe8e0ed670 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -41,8 +41,8 @@ import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; @@ -180,7 +180,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut getOmRequest()); OMClientResponse omClientResponse = null; - OmKeyInfo openKeyInfo = null; + OmOpenKeyInfo openKeyInfo = null; Exception exception = null; OmBucketInfo omBucketInfo = null; boolean acquiredLock = false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java index dba523bed48d..f76b319f9d48 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java @@ -40,8 +40,8 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmFSOFile; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; @@ -104,7 +104,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut getOmRequest()); OMClientResponse omClientResponse = null; - OmKeyInfo openKeyInfo = null; + OmOpenKeyInfo openKeyInfo = null; Exception exception = null; OmBucketInfo omBucketInfo = null; boolean acquiredLock = false; @@ -199,10 +199,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut return omClientResponse; } - private OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, + private OmOpenKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, String openKeyName, String keyName) throws IOException { String fileName = OzoneFSUtils.getFileName(keyName); - return OMFileRequest.getOmKeyInfoFromFileTable(true, + return OMFileRequest.getOmOpenKeyInfoFromOpenFileTable( omMetadataManager, openKeyName, fileName); } @@ -221,16 +221,16 @@ private String getOpenKeyName(String volumeName, String bucketName, private void addOpenTableCacheEntry(long trxnLogIndex, OMMetadataManager omMetadataManager, String openKeyName, String keyName, - OmKeyInfo openKeyInfo) { + OmOpenKeyInfo openKeyInfo) { OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, openKeyName, openKeyInfo, keyName, trxnLogIndex); } @Nonnull private OMClientResponse getOmClientResponse(long clientID, - OMResponse.Builder omResponse, OmKeyInfo openKeyInfo, + OMResponse.Builder omResponse, OmOpenKeyInfo omOpenKeyInfo, OmBucketInfo omBucketInfo, long volumeId) { - return new OMAllocateBlockResponseWithFSO(omResponse.build(), openKeyInfo, + return new OMAllocateBlockResponseWithFSO(omResponse.build(), omOpenKeyInfo, clientID, getBucketLayout(), volumeId, omBucketInfo.getObjectID()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 0da27c7c2d69..1feadec46ef8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -91,7 +92,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut purgeDirsRequest.getDeletedPathList(); Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); - Map openKeyInfoMap = new HashMap<>(); + Map openKeyInfoMap = new HashMap<>(); OMMetrics omMetrics = ozoneManager.getMetrics(); DeletingServiceMetrics deletingServiceMetrics = ozoneManager.getDeletionMetrics(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( @@ -172,11 +173,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut long parentId = processed.keyInfo.getParentObjectID(); dbOpenKey = omMetadataManager.getOpenFileName(path.getVolumeId(), path.getBucketId(), parentId, processed.keyInfo.getFileName(), hsyncClientId); - OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey); - if (openKeyInfo != null) { - openKeyInfo = openKeyInfo.withMetadataMutations( + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey); + if (omOpenKeyInfo != null) { + OmKeyInfo openKeyInfo = omOpenKeyInfo.getKeyInfo().withMetadataMutations( metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); - openKeyInfoMap.put(dbOpenKey, openKeyInfo); + OmOpenKeyInfo updatedOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(openKeyInfo).build(); + openKeyInfoMap.put(dbOpenKey, updatedOpenKeyInfo); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 28435bedf88a..4f9635ca3ae6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -49,6 +49,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -253,13 +254,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, writerClientId); - omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey); - if (omKeyInfo == null) { + if (omOpenKeyInfo == null) { String action = isRecovery ? "recovery" : isHSync ? "hsync" : "commit"; throw new OMException("Failed to " + action + " key, as " + dbOpenKey + " entry is not found in the OpenKey table", KEY_NOT_FOUND); - } else if (omKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY) || + } + + omKeyInfo = omOpenKeyInfo.getKeyInfo(); + if (omKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY) || omKeyInfo.getMetadata().containsKey(OzoneConsts.OVERWRITTEN_HSYNC_KEY)) { throw new OMException("Open Key " + keyName + " is already deleted/overwritten", KEY_NOT_FOUND); @@ -273,7 +277,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } } - OmKeyInfo openKeyToDelete = null; + OmOpenKeyInfo openKeyToDelete = null; String dbOpenKeyToDeleteKey = null; if (isOverwrittenHsyncKey) { // find the overwritten openKey and add OVERWRITTEN_HSYNC_KEY to it. @@ -291,25 +295,25 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime()); // non-null indicates it is necessary to update the open key - OmKeyInfo newOpenKeyInfo = null; + OmOpenKeyInfo newOpenKeyInfo = null; if (isHSync) { if (!OmKeyHSyncUtil.isHSyncedPreviously(omKeyInfo, clientIdString, dbOpenKey)) { // Update open key as well if it is the first hsync of this key omKeyInfo = omKeyInfo.withMetadataMutations( metadata -> metadata.put(OzoneConsts.HSYNC_CLIENT_ID, clientIdString)); - newOpenKeyInfo = omKeyInfo.copyObject(); + newOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo.copyObject()) + .setExpectedDataGeneration(omOpenKeyInfo.getExpectedDataGeneration()) + .setExpectedETag(omOpenKeyInfo.getExpectedETag()) + .build(); } } - validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); - // Optimistic locking validation has passed. Now set the rewrite fields to null so they are - // not persisted in the key table. - // Combination + validateAtomicRewrite(keyToDelete, omOpenKeyInfo, auditMap); + // Optimistic locking validation has passed. // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() - .setExpectedDataGeneration(null) - .setExpectedETag(null) .addAllMetadata(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())) .setUpdateID(trxnLogIndex) @@ -618,12 +622,12 @@ public static OMRequest disallowRecovery( return req; } - protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map auditMap) + protected void validateAtomicRewrite(OmKeyInfo existing, OmOpenKeyInfo openKeyInfo, Map auditMap) throws OMException { - if (toCommit.getExpectedDataGeneration() != null) { + if (openKeyInfo.getExpectedDataGeneration() != null) { // These values are not passed in the request keyArgs, so add them into the auditMap if they are present // in the open key entry. - Long expectedGen = toCommit.getExpectedDataGeneration(); + Long expectedGen = openKeyInfo.getExpectedDataGeneration(); auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index 5aa4fbeb36f4..0a8e08645649 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.helpers.OmFSOFile; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.WithMetadata; @@ -174,8 +175,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut writerClientId = Long.parseLong(clientId); } dbOpenFileKey = fsoFile.getOpenFileName(writerClientId); - omKeyInfo = OMFileRequest.getOmKeyInfoFromFileTable(true, - omMetadataManager, dbOpenFileKey, keyName); + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager + .getOpenKeyTable(getBucketLayout()).get(dbOpenFileKey); + omKeyInfo = omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; if (omKeyInfo == null) { String action = isRecovery ? "recovery" : isHSync ? "hsync" : "commit"; throw new OMException("Failed to " + action + " key, as " + @@ -208,7 +210,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .build(); openKeyToDelete.setModificationTime(Time.now()); OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, - dbOpenKeyToDeleteKey, openKeyToDelete, keyName, trxnLogIndex); + dbOpenKeyToDeleteKey, + new OmOpenKeyInfo.Builder().setKeyInfo(openKeyToDelete).build(), + keyName, trxnLogIndex); } omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime()); @@ -242,11 +246,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // creation after the knob turned on. Map oldKeyVersionsToDeleteMap = null; - validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); - // Optimistic locking validation has passed. Now set the rewrite fields to null so they are - // not persisted in the key table. - omKeyInfo.setExpectedDataGeneration(null); - omKeyInfo.setExpectedETag(null); + validateAtomicRewrite(keyToDelete, omOpenKeyInfo, auditMap); + // Optimistic locking validation has passed. long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace shouldn't check and increase. @@ -340,7 +341,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } else if (newOpenKeyInfo != null) { // isHSync is true and newOpenKeyInfo is set, update OpenKeyTable OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, - dbOpenFileKey, newOpenKeyInfo, keyName, trxnLogIndex); + dbOpenFileKey, + new OmOpenKeyInfo.Builder().setKeyInfo(newOpenKeyInfo).build(), + keyName, trxnLogIndex); } OMFileRequest.addFileTableCacheEntry(omMetadataManager, dbFileKey, @@ -348,9 +351,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omBucketInfo.incrUsedBytes(correctedSpace); + OmOpenKeyInfo newOmOpenKeyInfo = newOpenKeyInfo != null + ? new OmOpenKeyInfo.Builder().setKeyInfo(newOpenKeyInfo).build() : null; + OmOpenKeyInfo omOpenKeyToDelete = openKeyToDelete != null + ? new OmOpenKeyInfo.Builder().setKeyInfo(openKeyToDelete).build() : null; omClientResponse = new OMKeyCommitResponseWithFSO(omResponse.build(), omKeyInfo, dbFileKey, dbOpenFileKey, omBucketInfo.copyObject(), - oldKeyVersionsToDeleteMap, volumeId, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete); + oldKeyVersionsToDeleteMap, volumeId, isHSync, newOmOpenKeyInfo, dbOpenKeyToDeleteKey, omOpenKeyToDelete); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index b9cca39270c4..f8f494ac54ef 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.lock.OzoneLockStrategy; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; @@ -348,8 +349,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Add to cache entry can be done outside of lock for this openKey. // Even if bucket gets deleted, when commitKey we shall identify if // bucket gets deleted. + OmOpenKeyInfo.Builder openKeyInfoBuilder = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo); + if (keyArgs.hasExpectedDataGeneration()) { + openKeyInfoBuilder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); + } + if (keyArgs.hasExpectedETag()) { + openKeyInfoBuilder.setExpectedETag(keyArgs.getExpectedETag()); + } + OmOpenKeyInfo omOpenKeyInfo = openKeyInfoBuilder.build(); omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( - dbOpenKeyName, omKeyInfo, trxnLogIndex); + dbOpenKeyName, omOpenKeyInfo, trxnLogIndex); // Prepare response omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder() @@ -359,7 +369,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setOpenVersion(openVersion).build()) .setCmdType(Type.CreateKey); omClientResponse = new OMKeyCreateResponse(omResponse.build(), - omKeyInfo, missingParentInfos, clientID, bucketInfo.copyObject()); + omOpenKeyInfo, missingParentInfos, clientID, bucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java index 99fabb46de11..f44c79094114 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -187,11 +188,22 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() - quotaCheckStartTime); omBucketInfo.incrUsedNamespace(numKeysCreated); + // Build OmOpenKeyInfo with expectedDataGeneration/expectedETag + OmOpenKeyInfo.Builder openKeyInfoBuilder = new OmOpenKeyInfo.Builder() + .setKeyInfo(omFileInfo); + if (keyArgs.hasExpectedDataGeneration()) { + openKeyInfoBuilder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); + } + if (keyArgs.hasExpectedETag()) { + openKeyInfoBuilder.setExpectedETag(keyArgs.getExpectedETag()); + } + OmOpenKeyInfo omOpenKeyInfo = openKeyInfoBuilder.build(); + // Add to cache entry can be done outside of lock for this openKey. // Even if bucket gets deleted, when commitKey we shall identify if // bucket gets deleted. OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, - dbOpenFileName, omFileInfo, keyName, trxnLogIndex); + dbOpenFileName, omOpenKeyInfo, keyName, trxnLogIndex); // Add cache entries for the prefix directories. // Skip adding for the file key itself, until Key Commit. @@ -209,7 +221,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setOpenVersion(openVersion).build()) .setCmdType(Type.CreateKey); omClientResponse = new OMKeyCreateResponseWithFSO(omResponse.build(), - omFileInfo, missingParentInfos, clientID, + omOpenKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject(), volumeId); result = Result.SUCCESS; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 4726d4af2d5f..7213fc988526 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; @@ -165,20 +166,23 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut boolean isKeyNonEmpty = !OmKeyInfo.isKeyEmpty(omKeyInfo); omBucketInfo.decrUsedBytes(quotaReleased, isKeyNonEmpty); omBucketInfo.decrUsedNamespace(1L, isKeyNonEmpty); - OmKeyInfo deletedOpenKeyInfo = null; + OmOpenKeyInfo deletedOmOpenKeyInfo = null; // If omKeyInfo has hsync metadata, delete its corresponding open key as well String dbOpenKey = null; String hsyncClientId = omKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); if (hsyncClientId != null) { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId); - OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); - if (openKeyInfo != null) { - openKeyInfo = openKeyInfo.withMetadataMutations( + OmOpenKeyInfo omOpenKeyInfo = openKeyTable.get(dbOpenKey); + if (omOpenKeyInfo != null) { + OmKeyInfo openKeyInfo = omOpenKeyInfo.getKeyInfo().withMetadataMutations( metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); - openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); - deletedOpenKeyInfo = openKeyInfo; + OmOpenKeyInfo updatedOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(openKeyInfo) + .build(); + openKeyTable.addCacheEntry(dbOpenKey, updatedOpenKeyInfo, trxnLogIndex); + deletedOmOpenKeyInfo = updatedOpenKeyInfo; } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -187,7 +191,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omClientResponse = new OMKeyDeleteResponse( omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder()) .build(), omKeyInfo, - omBucketInfo.copyObject(), deletedOpenKeyInfo); + omBucketInfo.copyObject(), deletedOmOpenKeyInfo); if (omKeyInfo.isFile()) { auditMap.put(OzoneConsts.DATA_SIZE, String.valueOf(omKeyInfo.getDataSize())); auditMap.put(OzoneConsts.REPLICATION_CONFIG, omKeyInfo.getReplicationConfig().toString()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index 4737b85373db..aeacb79d8c2d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; @@ -135,7 +136,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut String ozonePathKey = omMetadataManager.getOzonePathKey(volumeId, bucketId, omKeyInfo.getParentObjectID(), omKeyInfo.getFileName()); - OmKeyInfo deletedOpenKeyInfo = null; + OmOpenKeyInfo deletedOmOpenKeyInfo = null; if (keyStatus.isDirectory()) { // Check if there are any sub path exists under the user requested path @@ -168,15 +169,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut String dbOpenKey = null; String hsyncClientId = omKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); if (hsyncClientId != null) { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); long parentId = omKeyInfo.getParentObjectID(); dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, fileName, hsyncClientId); - OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); - if (openKeyInfo != null) { - openKeyInfo = openKeyInfo.withMetadataMutations( + OmOpenKeyInfo omOpenKeyInfo = openKeyTable.get(dbOpenKey); + if (omOpenKeyInfo != null) { + OmKeyInfo openKeyInfo = omOpenKeyInfo.getKeyInfo().withMetadataMutations( metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); - openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); - deletedOpenKeyInfo = openKeyInfo; + OmOpenKeyInfo updatedOpenKeyInfo = new OmOpenKeyInfo.Builder().setKeyInfo(openKeyInfo).build(); + openKeyTable.addCacheEntry(dbOpenKey, updatedOpenKeyInfo, trxnLogIndex); + deletedOmOpenKeyInfo = updatedOpenKeyInfo; } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -190,7 +192,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omClientResponse = new OMKeyDeleteResponseWithFSO(omResponse .setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(), keyName, omKeyInfo, - omBucketInfo.copyObject(), keyStatus.isDirectory(), volumeId, deletedOpenKeyInfo); + omBucketInfo.copyObject(), keyStatus.isDirectory(), volumeId, deletedOmOpenKeyInfo); result = Result.SUCCESS; long endNanosDeleteKeySuccessLatencyNs = Time.monotonicNowNanos(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 2485c5855f4d..2f7943baf025 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -82,6 +82,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; @@ -795,14 +796,14 @@ protected void getFileEncryptionInfoForMpuKey(KeyArgs keyArgs, resolvedBucket.realBucket(), keyArgs.getKeyName(), keyArgs.getMultipartUploadID(), omMetadataManager); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(dbMultipartOpenKey); - if (omKeyInfo != null) { - if (omKeyInfo.getFileEncryptionInfo() != null) { + if (omOpenKeyInfo != null) { + if (omOpenKeyInfo.getFileEncryptionInfo() != null) { newKeyArgs.setFileEncryptionInfo( - OMPBHelper.convert(omKeyInfo.getFileEncryptionInfo())); + OMPBHelper.convert(omOpenKeyInfo.getFileEncryptionInfo())); } } else { LOG.warn("omKeyInfo not found. Key: " + dbMultipartOpenKey + @@ -990,13 +991,6 @@ protected OmKeyInfo prepareFileInfo( .setTags(KeyValueUtil.getFromProtobuf(keyArgs.getTagsList())) .setFileEncryptionInfo(encInfo); - if (keyArgs.hasExpectedDataGeneration()) { - builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); - } - if (keyArgs.hasExpectedETag()) { - builder.setExpectedETag(keyArgs.getExpectedETag()); - } - return builder.build(); } @@ -1043,9 +1037,6 @@ protected OmKeyInfo createFileInfo( .setUpdateID(transactionLogIndex) .setOwnerName(keyArgs.getOwnerName()) .setFile(true); - if (keyArgs.hasExpectedDataGeneration()) { - builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); - } if (omPathInfo instanceof OMFileRequest.OMPathInfoWithFSO) { // FileTable metadata format OMFileRequest.OMPathInfoWithFSO omPathInfoFSO @@ -1099,16 +1090,16 @@ private OmKeyInfo prepareMultipartFileInfo( .getMultipartKey(args.getVolumeName(), args.getBucketName(), args.getKeyName(), uploadID); } - OmKeyInfo partKeyInfo = + OmOpenKeyInfo partOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(multipartKey); - if (partKeyInfo == null) { + if (partOpenKeyInfo == null) { throw new OMException("No such Multipart upload is with specified " + "uploadId " + uploadID, OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } // For this upload part we don't need to check in KeyTable. As this // is not an actual key, it is a part of the key. - return createFileInfo(args, locations, partKeyInfo.getReplicationConfig(), + return createFileInfo(args, locations, partOpenKeyInfo.getReplicationConfig(), size, encInfo, prefixManager, omBucketInfo, omPathInfo, transactionLogIndex, objectID, configuration); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index e8a17d2e74fe..0e5e9de832b8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; @@ -190,7 +191,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - Map openKeyInfoMap = new HashMap<>(); + Map openKeyInfoMap = new HashMap<>(); // Mark all keys which can be deleted, in cache as deleted. Pair quotaReleasedEmptyKeys = markKeysAsDeletedInCache(ozoneManager, trxnLogIndex, omKeyInfoList, @@ -285,7 +286,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, Map keyToErrors, - boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { + boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { OMClientResponse omClientResponse; List deleteKeyErrors = new ArrayList<>(); for (Map.Entry key : keyToErrors.entrySet()) { @@ -304,7 +305,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, protected Pair markKeysAsDeletedInCache(OzoneManager ozoneManager, long trxnLogIndex, List omKeyInfoList, List dirList, - OMMetadataManager omMetadataManager, Map openKeyInfoMap) + OMMetadataManager omMetadataManager, Map openKeyInfoMap) throws IOException { int emptyKeys = 0; long quotaReleased = 0; @@ -325,15 +326,18 @@ protected Pair markKeysAsDeletedInCache(OzoneManager ozoneManager // If omKeyInfo has hsync metadata, delete its corresponding open key as well String hsyncClientId = omKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); if (hsyncClientId != null) { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId); - OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); - if (openKeyInfo != null) { - openKeyInfo = openKeyInfo.withMetadataMutations( + OmOpenKeyInfo omOpenKeyInfo = openKeyTable.get(dbOpenKey); + if (omOpenKeyInfo != null) { + OmKeyInfo openKeyInfo = omOpenKeyInfo.getKeyInfo().withMetadataMutations( metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); - openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); + OmOpenKeyInfo updatedOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(openKeyInfo) + .build(); + openKeyTable.addCacheEntry(dbOpenKey, updatedOpenKeyInfo, trxnLogIndex); // Add to the map of open keys to be deleted. - openKeyInfoMap.put(dbOpenKey, openKeyInfo); + openKeyInfoMap.put(dbOpenKey, updatedOpenKeyInfo); } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java index d2eb7784b9dc..9bf3ad2b7caa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMOpenKeysDeleteResponse; @@ -97,7 +98,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut OMClientResponse omClientResponse = null; Result result = null; // Map containing a pair of BucketId and delete key info. - Map> deletedOpenKeys = new HashMap<>(); + Map> deletedOpenKeys = new HashMap<>(); Map auditParams = new LinkedHashMap<>(); try { for (OpenKeyBucket openKeyBucket: submittedOpenKeyBuckets) { @@ -113,8 +114,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut result = Result.SUCCESS; List deletedOpenKeysLight = new ArrayList<>(deletedOpenKeys.size()); - for (Pair key : deletedOpenKeys.values()) { - OmKeyInfo keyInfo = key.getRight(); + for (Pair key : deletedOpenKeys.values()) { + OmKeyInfo keyInfo = key.getRight().getKeyInfo(); OzoneManagerProtocolProtos.KeyArgs keyArgs = OzoneManagerProtocolProtos.KeyArgs.newBuilder() .setVolumeName(keyInfo.getVolumeName()) .setBucketName(keyInfo.getBucketName()) @@ -166,7 +167,7 @@ private void processResults(OMMetrics omMetrics, long numSubmittedOpenKeys, protected void updateOpenKeyTableCache(OzoneManager ozoneManager, long trxnLogIndex, OpenKeyBucket keysPerBucket, - Map> deletedOpenKeys) throws IOException { + Map> deletedOpenKeys) throws IOException { boolean acquiredLock = false; String volumeName = keysPerBucket.getVolumeName(); @@ -184,10 +185,11 @@ protected void updateOpenKeyTableCache(OzoneManager ozoneManager, // If an open key is no longer present in the table, it was committed // and should not be deleted. - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(fullKeyName); - if (omKeyInfo != null) { + if (omOpenKeyInfo != null) { + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); if (trxnLogIndex < omKeyInfo.getUpdateID()) { LOG.warn("Transaction log index {} is smaller than " + "the current updateID {} of key {}, skipping deletion.", @@ -199,7 +201,10 @@ protected void updateOpenKeyTableCache(OzoneManager ozoneManager, omKeyInfo = omKeyInfo.toBuilder() .setUpdateID(trxnLogIndex) .build(); - deletedOpenKeys.put(fullKeyName, Pair.of(bucketId, omKeyInfo)); + OmOpenKeyInfo updatedOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo) + .build(); + deletedOpenKeys.put(fullKeyName, Pair.of(bucketId, updatedOpenKeyInfo)); // Update openKeyTable cache. omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index a501739d0c31..84d02ea74144 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -90,7 +91,7 @@ protected OzoneFileStatus getOzoneKeyStatus( @Override protected Pair markKeysAsDeletedInCache( OzoneManager ozoneManager, long trxnLogIndex, List omKeyInfoList, List dirList, - OMMetadataManager omMetadataManager, Map openKeyInfoMap) throws IOException { + OMMetadataManager omMetadataManager, Map openKeyInfoMap) throws IOException { long quotaReleased = 0L; int emptyKeys = 0; // Mark all keys which can be deleted, in cache as deleted. @@ -118,16 +119,18 @@ protected Pair markKeysAsDeletedInCache( // If omKeyInfo has hsync metadata, delete its corresponding open key as well final String hsyncClientId = updatedOmKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); if (hsyncClientId != null) { - final Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + final Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); final String dbOpenKey = omMetadataManager.getOpenFileName( volumeId, bucketId, parentId, fileName, hsyncClientId); - final OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); - if (openKeyInfo != null) { - final OmKeyInfo updatedOpenKeyInfo = openKeyInfo.withMetadataMutations( + final OmOpenKeyInfo omOpenKeyInfo = openKeyTable.get(dbOpenKey); + if (omOpenKeyInfo != null) { + final OmKeyInfo updatedOpenKeyInfo = omOpenKeyInfo.getKeyInfo().withMetadataMutations( metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); - openKeyTable.addCacheEntry(dbOpenKey, updatedOpenKeyInfo, trxnLogIndex); + OmOpenKeyInfo updatedOmOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(updatedOpenKeyInfo).build(); + openKeyTable.addCacheEntry(dbOpenKey, updatedOmOpenKeyInfo, trxnLogIndex); // Add to the map of open keys to be deleted. - openKeyInfoMap.put(dbOpenKey, updatedOpenKeyInfo); + openKeyInfoMap.put(dbOpenKey, updatedOmOpenKeyInfo); } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -165,7 +168,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, OzoneManagerProtocolProtos.OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, Map keyToErrors, - boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { + boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { OMClientResponse omClientResponse; List deleteKeyErrors = new ArrayList<>(); for (Map.Entry key : keyToErrors.entrySet()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 22f470c80c7b..be14677dad4d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; @@ -218,9 +219,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .build(); // Add to cache + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo) + .build(); omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( new CacheKey<>(multipartKey), - CacheValue.get(transactionLogIndex, omKeyInfo)); + CacheValue.get(transactionLogIndex, omOpenKeyInfo)); omMetadataManager.getMultipartInfoTable().addCacheEntry( new CacheKey<>(multipartKey), CacheValue.get(transactionLogIndex, multipartKeyInfo)); @@ -233,7 +237,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setBucketName(requestedBucket) .setKeyName(keyName) .setMultipartUploadID(keyArgs.getMultipartUploadID())) - .build(), multipartKeyInfo, omKeyInfo, getBucketLayout()); + .build(), multipartKeyInfo, omOpenKeyInfo, getBucketLayout()); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java index 919491d70499..a7d69a8f73d0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -203,9 +204,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut volumeId, bucketId, transactionLogIndex, missingParentInfos, null); + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo).build(); OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, - multipartOpenKey, omKeyInfo, keyName, - transactionLogIndex); + multipartOpenKey, omOpenKeyInfo, keyName, transactionLogIndex); // Add to cache omMetadataManager.getMultipartInfoTable().addCacheEntry( @@ -219,7 +221,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setBucketName(requestedBucket) .setKeyName(keyName) .setMultipartUploadID(keyArgs.getMultipartUploadID())) - .build(), multipartKeyInfo, omKeyInfo, multipartKey, + .build(), multipartKeyInfo, omOpenKeyInfo, multipartKey, missingParentInfos, getBucketLayout(), volumeId, bucketId, bucketInfo.copyObject()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index a9aeff0ac5d1..03c6c7ec8ba8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -32,8 +32,8 @@ import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; @@ -146,11 +146,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } - OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(multipartOpenKey); omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - if (omKeyInfo == null) { + if (omOpenKeyInfo == null) { // In old env, OpenKeycleanupservice may have deleted key from openKeyTable leaving behind // orphan parts in multipartInfoTable. LOG.warn("Entry doesn't exist in openKeyTable, bucket: {}, key: {}", bucketName, keyName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index ac123ff680ac..1a0a15dc1894 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; @@ -320,7 +321,8 @@ protected S3MultipartUploadCommitPartResponse getOmClientResponse( protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager, String openKey, String keyName) throws IOException { - return omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + return omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } protected String getOpenKey(String volumeName, String bucketName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index cadd7f80b62f..46d00b0b6711 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; @@ -551,8 +552,9 @@ protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneKey, protected OmKeyInfo getOmKeyInfoFromOpenKeyTable(String dbMultipartKey, String keyName, OMMetadataManager omMetadataManager) throws IOException { - return omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(dbMultipartKey); + return omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } protected void addKeyTableCacheEntry(OMMetadataManager omMetadataManager, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index a5c8b2703d68..038c374e07fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse; @@ -94,7 +95,8 @@ protected void addMultiPartToCache( // Add multi part to cache OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, - multipartOpenKey, omKeyInfo, + multipartOpenKey, + new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build(), keyName, transactionLogIndex); } @@ -170,7 +172,7 @@ protected OMClientResponse getOmClientResponse(String multipartKey, return new S3MultipartUploadCompleteResponseWithFSO(omResponse.build(), multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove, getBucketLayout(), omBucketInfo, volumeId, bucketId, - missingParentInfos, multipartKeyInfo); + missingParentInfos, multipartKeyInfo, null); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java index 428e90075b36..fa7cb22f0ef2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -36,10 +37,10 @@ public class OMFileCreateResponse extends OMKeyCreateResponse { public OMFileCreateResponse(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, @Nonnull List parentKeyInfos, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, @Nonnull List parentKeyInfos, long openKeySessionID, @Nonnull OmBucketInfo omBucketInfo) { - super(omResponse, omKeyInfo, parentKeyInfos, openKeySessionID, + super(omResponse, omOpenKeyInfo, parentKeyInfos, openKeySessionID, omBucketInfo); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java index 02c72110ad4d..1b5d5a2dab1e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java @@ -30,7 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -44,15 +44,17 @@ public class OMFileCreateResponseWithFSO extends OMFileCreateResponse { private List parentDirInfos; private long volumeId; + private OmOpenKeyInfo omOpenKeyInfo; public OMFileCreateResponseWithFSO(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, @Nonnull List parentDirInfos, long openKeySessionID, @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volumeId) { - super(omResponse, omKeyInfo, new ArrayList<>(), openKeySessionID, + super(omResponse, omOpenKeyInfo, new ArrayList<>(), openKeySessionID, omBucketInfo); this.parentDirInfos = parentDirInfos; this.volumeId = volumeId; + this.omOpenKeyInfo = omOpenKeyInfo; } /** @@ -93,7 +95,7 @@ public void addToDBBatch(OMMetadataManager omMetadataMgr, bucketKey, getOmBucketInfo()); } - OMFileRequest.addToOpenFileTable(omMetadataMgr, batchOp, getOmKeyInfo(), + OMFileRequest.addToOpenFileTable(omMetadataMgr, batchOp, omOpenKeyInfo, getOpenKeySessionID(), volumeId, getOmBucketInfo().getObjectID()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMRecoverLeaseResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMRecoverLeaseResponse.java index 4d66697d6918..ff90b6b218fc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMRecoverLeaseResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMRecoverLeaseResponse.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.key.OmKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -36,11 +36,11 @@ @CleanupTableInfo(cleanupTables = {FILE_TABLE, OPEN_FILE_TABLE}) public class OMRecoverLeaseResponse extends OmKeyResponse { - private OmKeyInfo openKeyInfo; + private OmOpenKeyInfo openKeyInfo; private String openKeyName; public OMRecoverLeaseResponse(@Nonnull OMResponse omResponse, - BucketLayout bucketLayout, String openKeyName, OmKeyInfo openKeyInfo) { + BucketLayout bucketLayout, String openKeyName, OmOpenKeyInfo openKeyInfo) { super(omResponse, bucketLayout); this.openKeyName = openKeyName; this.openKeyInfo = openKeyInfo; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java index eb8d7bd50591..5ec7a4e5e7b3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -35,14 +35,14 @@ @CleanupTableInfo(cleanupTables = {OPEN_KEY_TABLE, BUCKET_TABLE}) public class OMAllocateBlockResponse extends OmKeyResponse { - private OmKeyInfo omKeyInfo; + private OmOpenKeyInfo omOpenKeyInfo; private long clientID; public OMAllocateBlockResponse(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, long clientID, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, long clientID, @Nonnull BucketLayout bucketLayout) { super(omResponse, bucketLayout); - this.omKeyInfo = omKeyInfo; + this.omOpenKeyInfo = omOpenKeyInfo; this.clientID = clientID; } @@ -60,14 +60,14 @@ public OMAllocateBlockResponse(@Nonnull OMResponse omResponse, public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - String openKey = omMetadataManager.getOpenKey(omKeyInfo.getVolumeName(), - omKeyInfo.getBucketName(), omKeyInfo.getKeyName(), clientID); + String openKey = omMetadataManager.getOpenKey(omOpenKeyInfo.getVolumeName(), + omOpenKeyInfo.getBucketName(), omOpenKeyInfo.getKeyName(), clientID); omMetadataManager.getOpenKeyTable(getBucketLayout()) - .putWithBatch(batchOperation, openKey, omKeyInfo); + .putWithBatch(batchOperation, openKey, omOpenKeyInfo); } - protected OmKeyInfo getOmKeyInfo() { - return omKeyInfo; + protected OmOpenKeyInfo getOmOpenKeyInfo() { + return omOpenKeyInfo; } protected long getClientID() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseWithFSO.java index 2d174a172c61..5efe9b6301c7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseWithFSO.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -38,14 +38,16 @@ public class OMAllocateBlockResponseWithFSO extends OMAllocateBlockResponse { private long volumeId; private long bucketId; + private OmOpenKeyInfo omOpenKeyInfo; public OMAllocateBlockResponseWithFSO(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, long clientID, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, long clientID, @Nonnull BucketLayout bucketLayout, @Nonnull long volumeId, @Nonnull long bucketId) { - super(omResponse, omKeyInfo, clientID, bucketLayout); + super(omResponse, omOpenKeyInfo, clientID, bucketLayout); this.volumeId = volumeId; this.bucketId = bucketId; + this.omOpenKeyInfo = omOpenKeyInfo; } /** @@ -62,7 +64,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { OMFileRequest.addToOpenFileTable(omMetadataManager, batchOperation, - getOmKeyInfo(), getClientID(), volumeId, bucketId); + omOpenKeyInfo, getClientID(), volumeId, bucketId); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 127ec4ed2cbf..727af058d21c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; @@ -66,13 +67,13 @@ public class OMDirectoriesPurgeResponseWithFSO extends OmKeyResponse { private List paths; private Map, OmBucketInfo> volBucketInfoMap; private SnapshotInfo fromSnapshotInfo; - private Map openKeyInfoMap; + private Map openKeyInfoMap; public OMDirectoriesPurgeResponseWithFSO(@Nonnull OMResponse omResponse, @Nonnull List paths, @Nonnull BucketLayout bucketLayout, Map, OmBucketInfo> volBucketInfoMap, - SnapshotInfo fromSnapshotInfo, Map openKeyInfoMap) { + SnapshotInfo fromSnapshotInfo, Map openKeyInfoMap) { super(omResponse, bucketLayout); this.paths = paths; this.volBucketInfoMap = volBucketInfoMap; @@ -183,7 +184,7 @@ public void processPaths( } if (!openKeyInfoMap.isEmpty()) { - for (Map.Entry entry : openKeyInfoMap.entrySet()) { + for (Map.Entry entry : openKeyInfoMap.entrySet()) { keySpaceOmMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( keySpaceBatchOperation, entry.getKey(), entry.getValue()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java index 425c4f63ac5e..04b54465b08a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -48,8 +49,8 @@ public class OMKeyCommitResponse extends OmKeyResponse { private OmBucketInfo omBucketInfo; private Map keyToDeleteMap; private boolean isHSync; - private OmKeyInfo newOpenKeyInfo; - private OmKeyInfo openKeyToUpdate; + private OmOpenKeyInfo newOpenKeyInfo; + private OmOpenKeyInfo openKeyToUpdate; private String openKeyNameToUpdate; @SuppressWarnings("checkstyle:ParameterNumber") @@ -59,7 +60,7 @@ public OMKeyCommitResponse( @Nonnull OmBucketInfo omBucketInfo, Map keyToDeleteMap, boolean isHSync, - OmKeyInfo newOpenKeyInfo, String openKeyNameToUpdate, OmKeyInfo openKeyToUpdate) { + OmOpenKeyInfo newOpenKeyInfo, String openKeyNameToUpdate, OmOpenKeyInfo openKeyToUpdate) { super(omResponse, omBucketInfo.getBucketLayout()); this.omKeyInfo = omKeyInfo; this.ozoneKeyName = ozoneKeyName; @@ -151,7 +152,7 @@ protected boolean isHSync() { return isHSync; } - public OmKeyInfo getNewOpenKeyInfo() { + public OmOpenKeyInfo getNewOpenKeyInfo() { return newOpenKeyInfo; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseWithFSO.java index b210a8ce5d49..a6f0ded06af8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseWithFSO.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; @@ -43,6 +44,7 @@ public class OMKeyCommitResponseWithFSO extends OMKeyCommitResponse { private long volumeId; + private OmOpenKeyInfo newOmOpenKeyInfo; @SuppressWarnings("checkstyle:ParameterNumber") public OMKeyCommitResponseWithFSO( @@ -52,10 +54,13 @@ public OMKeyCommitResponseWithFSO( @Nonnull OmBucketInfo omBucketInfo, Map deleteKeyMap, long volumeId, boolean isHSync, - OmKeyInfo newOpenKeyInfo, String openKeyNameToUpdate, OmKeyInfo openKeyToUpdate) { + OmOpenKeyInfo newOpenKeyInfo, String openKeyNameToUpdate, OmOpenKeyInfo openKeyToUpdate) { super(omResponse, omKeyInfo, ozoneKeyName, openKeyName, - omBucketInfo, deleteKeyMap, isHSync, newOpenKeyInfo, openKeyNameToUpdate, openKeyToUpdate); + omBucketInfo, deleteKeyMap, isHSync, + newOpenKeyInfo, + openKeyNameToUpdate, openKeyToUpdate); this.volumeId = volumeId; + this.newOmOpenKeyInfo = newOpenKeyInfo; } /** @@ -76,9 +81,9 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, if (!this.isHSync()) { omMetadataManager.getOpenKeyTable(getBucketLayout()) .deleteWithBatch(batchOperation, getOpenKeyName()); - } else if (getNewOpenKeyInfo() != null) { + } else if (newOmOpenKeyInfo != null) { omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( - batchOperation, getOpenKeyName(), getNewOpenKeyInfo()); + batchOperation, getOpenKeyName(), newOmOpenKeyInfo); } OMFileRequest.addToFileTable(omMetadataManager, batchOperation, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java index 1bd03c4187be..5e6eb92fb881 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponse.java @@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.slf4j.Logger; @@ -42,16 +43,16 @@ public class OMKeyCreateResponse extends OmKeyResponse { protected static final Logger LOG = LoggerFactory.getLogger(OMKeyCreateResponse.class); - private OmKeyInfo omKeyInfo; + private OmOpenKeyInfo omOpenKeyInfo; private long openKeySessionID; private List parentKeyInfos; private OmBucketInfo omBucketInfo; public OMKeyCreateResponse(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, List parentKeyInfos, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, List parentKeyInfos, long openKeySessionID, @Nonnull OmBucketInfo omBucketInfo) { super(omResponse, omBucketInfo.getBucketLayout()); - this.omKeyInfo = omKeyInfo; + this.omOpenKeyInfo = omOpenKeyInfo; this.openKeySessionID = openKeySessionID; this.parentKeyInfos = parentKeyInfos; this.omBucketInfo = omBucketInfo; @@ -97,18 +98,18 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, bucketKey, getOmBucketInfo()); } - String openKey = omMetadataManager.getOpenKey(omKeyInfo.getVolumeName(), - omKeyInfo.getBucketName(), omKeyInfo.getKeyName(), openKeySessionID); + String openKey = omMetadataManager.getOpenKey(omOpenKeyInfo.getVolumeName(), + omOpenKeyInfo.getBucketName(), omOpenKeyInfo.getKeyName(), openKeySessionID); omMetadataManager.getOpenKeyTable(getBucketLayout()) - .putWithBatch(batchOperation, openKey, omKeyInfo); + .putWithBatch(batchOperation, openKey, omOpenKeyInfo); } protected long getOpenKeySessionID() { return openKeySessionID; } - protected OmKeyInfo getOmKeyInfo() { - return omKeyInfo; + protected OmOpenKeyInfo getOmOpenKeyInfo() { + return omOpenKeyInfo; } protected OmBucketInfo getOmBucketInfo() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java index a72ad370b940..6ce107e731c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCreateResponseWithFSO.java @@ -26,7 +26,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponseWithFSO; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -39,11 +39,11 @@ public class OMKeyCreateResponseWithFSO extends OMFileCreateResponseWithFSO { public OMKeyCreateResponseWithFSO(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, @Nonnull List parentDirInfos, long openKeySessionID, @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volumeId) { - super(omResponse, omKeyInfo, parentDirInfos, openKeySessionID, + super(omResponse, omOpenKeyInfo, parentDirInfos, openKeySessionID, omBucketInfo, volumeId); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java index be646c4ee289..0672a56e3d8a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -43,11 +44,11 @@ public class OMKeyDeleteResponse extends AbstractOMKeyDeleteResponse { private OmKeyInfo omKeyInfo; private OmBucketInfo omBucketInfo; // If not null, this key will be deleted from OpenKeyTable - private OmKeyInfo deletedOpenKeyInfo; + private OmOpenKeyInfo deletedOpenKeyInfo; public OMKeyDeleteResponse(@Nonnull OMResponse omResponse, @Nonnull OmKeyInfo omKeyInfo, - @Nonnull OmBucketInfo omBucketInfo, OmKeyInfo deletedOpenKeyInfo) { + @Nonnull OmBucketInfo omBucketInfo, OmOpenKeyInfo deletedOpenKeyInfo) { super(omResponse, omBucketInfo.getBucketLayout()); this.omKeyInfo = omKeyInfo; this.omBucketInfo = omBucketInfo; @@ -99,7 +100,7 @@ protected OmKeyInfo getOmKeyInfo() { return omKeyInfo; } - protected OmKeyInfo getDeletedOpenKeyInfo() { + protected OmOpenKeyInfo getDeletedOpenKeyInfo() { return deletedOpenKeyInfo; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java index 1b84bf51cb83..4ddb1c417d33 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -46,16 +47,18 @@ public class OMKeyDeleteResponseWithFSO extends OMKeyDeleteResponse { private boolean isDeleteDirectory; private String keyName; private long volumeId; + private OmOpenKeyInfo deletedOmOpenKeyInfo; @SuppressWarnings("parameternumber") public OMKeyDeleteResponseWithFSO(@Nonnull OMResponse omResponse, @Nonnull String keyName, @Nonnull OmKeyInfo omKeyInfo, @Nonnull OmBucketInfo omBucketInfo, - @Nonnull boolean isDeleteDirectory, @Nonnull long volumeId, OmKeyInfo deletedOpenKeyInfo) { + @Nonnull boolean isDeleteDirectory, @Nonnull long volumeId, OmOpenKeyInfo deletedOpenKeyInfo) { super(omResponse, omKeyInfo, omBucketInfo, deletedOpenKeyInfo); this.keyName = keyName; this.isDeleteDirectory = isDeleteDirectory; this.volumeId = volumeId; + this.deletedOmOpenKeyInfo = deletedOpenKeyInfo; } /** @@ -110,14 +113,13 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, getOmBucketInfo().getBucketName()), getOmBucketInfo()); // Update metadata which will be used to cleanup openKey in openKeyCleanupService - OmKeyInfo deletedOpenKeyInfo = getDeletedOpenKeyInfo(); - if (deletedOpenKeyInfo != null) { - String hsyncClientId = getDeletedOpenKeyInfo().getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); + if (deletedOmOpenKeyInfo != null) { + String hsyncClientId = deletedOmOpenKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); if (hsyncClientId != null) { - String dbOpenKey = omMetadataManager.getOpenKey(deletedOpenKeyInfo.getVolumeName(), - deletedOpenKeyInfo.getBucketName(), deletedOpenKeyInfo.getKeyName(), hsyncClientId); + String dbOpenKey = omMetadataManager.getOpenKey(deletedOmOpenKeyInfo.getVolumeName(), + deletedOmOpenKeyInfo.getBucketName(), deletedOmOpenKeyInfo.getKeyName(), hsyncClientId); omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( - batchOperation, dbOpenKey, deletedOpenKeyInfo); + batchOperation, dbOpenKey, deletedOmOpenKeyInfo); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java index 3cb1220b83ce..f008f70cd2cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -45,12 +46,12 @@ public class OMKeysDeleteResponse extends AbstractOMKeyDeleteResponse { private List omKeyInfoList; private OmBucketInfo omBucketInfo; - private Map openKeyInfoMap = new HashMap<>(); + private Map openKeyInfoMap = new HashMap<>(); public OMKeysDeleteResponse(@Nonnull OMResponse omResponse, @Nonnull List keyDeleteList, @Nonnull OmBucketInfo omBucketInfo, - @Nonnull Map openKeyInfoMap) { + @Nonnull Map openKeyInfoMap) { super(omResponse); this.omKeyInfoList = keyDeleteList; this.omBucketInfo = omBucketInfo; @@ -102,7 +103,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omBucketInfo.getBucketName()), omBucketInfo); if (!openKeyInfoMap.isEmpty()) { - for (Map.Entry entry : openKeyInfoMap.entrySet()) { + for (Map.Entry entry : openKeyInfoMap.entrySet()) { omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( batchOperation, entry.getKey(), entry.getValue()); } @@ -117,7 +118,7 @@ public OmBucketInfo getOmBucketInfo() { return omBucketInfo; } - protected Map getOpenKeyInfoMap() { + protected Map getOpenKeyInfoMap() { return openKeyInfoMap; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java index 0b283509354e..80721c4a9306 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -46,16 +47,18 @@ public class OMKeysDeleteResponseWithFSO extends OMKeysDeleteResponse { private List dirsList; private long volumeId; + private Map omOpenKeyInfoMap; public OMKeysDeleteResponseWithFSO( @Nonnull OzoneManagerProtocolProtos.OMResponse omResponse, @Nonnull List keyDeleteList, @Nonnull List dirDeleteList, @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volId, - @Nonnull Map openKeyInfoMap) { - super(omResponse, keyDeleteList, omBucketInfo, openKeyInfoMap); + @Nonnull Map openKeyInfoMap) { + super(omResponse, keyDeleteList, omBucketInfo, java.util.Collections.emptyMap()); this.dirsList = dirDeleteList; this.volumeId = volId; + this.omOpenKeyInfoMap = openKeyInfoMap; } @Override @@ -95,8 +98,8 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(getOmBucketInfo().getVolumeName(), getOmBucketInfo().getBucketName()), getOmBucketInfo()); - if (!getOpenKeyInfoMap().isEmpty()) { - for (Map.Entry entry : getOpenKeyInfoMap().entrySet()) { + if (!omOpenKeyInfoMap.isEmpty()) { + for (Map.Entry entry : omOpenKeyInfoMap.entrySet()) { omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( batchOperation, entry.getKey(), entry.getValue()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java index e46bd7db64ff..3767ec20d114 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMOpenKeysDeleteResponse.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -42,11 +42,11 @@ DELETED_TABLE, BUCKET_TABLE}) public class OMOpenKeysDeleteResponse extends AbstractOMKeyDeleteResponse { - private Map> keysToDelete; + private Map> keysToDelete; public OMOpenKeysDeleteResponse( @Nonnull OMResponse omResponse, - @Nonnull Map> keysToDelete, + @Nonnull Map> keysToDelete, @Nonnull BucketLayout bucketLayout) { super(omResponse, bucketLayout); @@ -69,12 +69,12 @@ public OMOpenKeysDeleteResponse( public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - Table openKeyTable = + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); - for (Map.Entry> keyInfoPair : keysToDelete.entrySet()) { + for (Map.Entry> keyInfoPair : keysToDelete.entrySet()) { addDeletionToBatch(omMetadataManager, batchOperation, openKeyTable, - keyInfoPair.getKey(), keyInfoPair.getValue().getValue(), keyInfoPair.getValue().getKey(), false); + keyInfoPair.getKey(), keyInfoPair.getValue().getValue().getKeyInfo(), keyInfoPair.getValue().getKey(), false); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponse.java index 0123ab11a949..93d0930c9c30 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponse.java @@ -26,8 +26,8 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.key.OmKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -38,15 +38,15 @@ @CleanupTableInfo(cleanupTables = {OPEN_KEY_TABLE, MULTIPART_INFO_TABLE}) public class S3InitiateMultipartUploadResponse extends OmKeyResponse { private OmMultipartKeyInfo omMultipartKeyInfo; - private OmKeyInfo omKeyInfo; + private OmOpenKeyInfo omOpenKeyInfo; public S3InitiateMultipartUploadResponse( @Nonnull OMResponse omResponse, @Nonnull OmMultipartKeyInfo omMultipartKeyInfo, - @Nonnull OmKeyInfo omKeyInfo, @Nonnull BucketLayout bucketLayout) { + @Nonnull OmOpenKeyInfo omOpenKeyInfo, @Nonnull BucketLayout bucketLayout) { super(omResponse, bucketLayout); this.omMultipartKeyInfo = omMultipartKeyInfo; - this.omKeyInfo = omKeyInfo; + this.omOpenKeyInfo = omOpenKeyInfo; } /** @@ -64,12 +64,12 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { String multipartKey = - omMetadataManager.getMultipartKey(omKeyInfo.getVolumeName(), - omKeyInfo.getBucketName(), omKeyInfo.getKeyName(), + omMetadataManager.getMultipartKey(omOpenKeyInfo.getVolumeName(), + omOpenKeyInfo.getBucketName(), omOpenKeyInfo.getKeyName(), omMultipartKeyInfo.getUploadID()); omMetadataManager.getOpenKeyTable(getBucketLayout()) - .putWithBatch(batchOperation, multipartKey, omKeyInfo); + .putWithBatch(batchOperation, multipartKey, omOpenKeyInfo); omMetadataManager.getMultipartInfoTable().putWithBatch(batchOperation, multipartKey, omMultipartKeyInfo); } @@ -80,7 +80,7 @@ public OmMultipartKeyInfo getOmMultipartKeyInfo() { } @VisibleForTesting - public OmKeyInfo getOmKeyInfo() { - return omKeyInfo; + public OmOpenKeyInfo getOmOpenKeyInfo() { + return omOpenKeyInfo; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponseWithFSO.java index c53720634931..052318a42765 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3InitiateMultipartUploadResponseWithFSO.java @@ -29,8 +29,8 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -47,21 +47,23 @@ public class S3InitiateMultipartUploadResponseWithFSO extends private long volumeId; private long bucketId; private OmBucketInfo omBucketInfo; + private OmOpenKeyInfo omOpenKeyInfo; @SuppressWarnings("parameternumber") public S3InitiateMultipartUploadResponseWithFSO( @Nonnull OMResponse omResponse, @Nonnull OmMultipartKeyInfo omMultipartKeyInfo, - @Nonnull OmKeyInfo omKeyInfo, @Nonnull String mpuDBKey, + @Nonnull OmOpenKeyInfo omOpenKeyInfo, @Nonnull String mpuDBKey, @Nonnull List parentDirInfos, @Nonnull BucketLayout bucketLayout, @Nonnull long volumeId, @Nonnull long bucketId, @Nonnull OmBucketInfo omBucketInfo) { - super(omResponse, omMultipartKeyInfo, omKeyInfo, bucketLayout); + super(omResponse, omMultipartKeyInfo, omOpenKeyInfo, bucketLayout); this.parentDirInfos = parentDirInfos; this.mpuDBKey = mpuDBKey; this.volumeId = volumeId; this.bucketId = bucketId; this.omBucketInfo = omBucketInfo; + this.omOpenKeyInfo = omOpenKeyInfo; } /** @@ -100,7 +102,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager, batchOperation, - getOmKeyInfo(), getOmMultipartKeyInfo().getUploadID(), volumeId, + omOpenKeyInfo, getOmMultipartKeyInfo().getUploadID(), volumeId, bucketId); omMetadataManager.getMultipartInfoTable().putWithBatch(batchOperation, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index 2147a039a531..70908610c701 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -56,6 +57,7 @@ public class S3MultipartUploadCompleteResponseWithFSO private List missingParentInfos; private OmMultipartKeyInfo multipartKeyInfo; + private OmOpenKeyInfo omOpenKeyInfo; @SuppressWarnings("checkstyle:ParameterNumber") public S3MultipartUploadCompleteResponseWithFSO( @@ -68,13 +70,15 @@ public S3MultipartUploadCompleteResponseWithFSO( OmBucketInfo omBucketInfo, @Nonnull long volumeId, @Nonnull long bucketId, List missingParentInfos, - OmMultipartKeyInfo multipartKeyInfo) { + OmMultipartKeyInfo multipartKeyInfo, + OmOpenKeyInfo omOpenKeyInfo) { super(omResponse, multipartKey, multipartOpenKey, omKeyInfo, allKeyInfoToRemove, bucketLayout, omBucketInfo, bucketId); this.volumeId = volumeId; this.bucketId = bucketId; this.missingParentInfos = missingParentInfos; this.multipartKeyInfo = multipartKeyInfo; + this.omOpenKeyInfo = omOpenKeyInfo; } /** @@ -110,13 +114,13 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, bucketKey, omBucketInfo); } - if (OMFileRequest.getOmKeyInfoFromFileTable(true, + if (omOpenKeyInfo != null && OMFileRequest.getOmKeyInfoFromFileTable(true, omMetadataManager, getMultiPartKey(), getOmKeyInfo().getKeyName()) != null) { // Add multi part to open key table. OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager, batchOperation, - getOmKeyInfo(), multipartKeyInfo.getUploadID(), volumeId, + omOpenKeyInfo, multipartKeyInfo.getUploadID(), volumeId, bucketId); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index c7e80f166ae9..15981e76b2c4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -335,6 +336,15 @@ public static void addKeyToTable(boolean openKeyTable, boolean addToCache, long trxnLogIndex, OMMetadataManager omMetadataManager) throws Exception { + addKeyToTable(openKeyTable, addToCache, omKeyInfo, clientID, trxnLogIndex, null, omMetadataManager); + } + + public static void addKeyToTable(boolean openKeyTable, boolean addToCache, + OmKeyInfo omKeyInfo, long clientID, + long trxnLogIndex, + Long expectedDataGeneration, + OMMetadataManager omMetadataManager) + throws Exception { String volumeName = omKeyInfo.getVolumeName(); String bucketName = omKeyInfo.getBucketName(); String keyName = omKeyInfo.getKeyName(); @@ -342,13 +352,17 @@ public static void addKeyToTable(boolean openKeyTable, boolean addToCache, if (openKeyTable) { String ozoneKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, clientID); + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo) + .setExpectedDataGeneration(expectedDataGeneration) + .build(); if (addToCache) { omMetadataManager.getOpenKeyTable(getDefaultBucketLayout()) .addCacheEntry(new CacheKey<>(ozoneKey), - CacheValue.get(trxnLogIndex, omKeyInfo)); + CacheValue.get(trxnLogIndex, omOpenKeyInfo)); } omMetadataManager.getOpenKeyTable(getDefaultBucketLayout()) - .put(ozoneKey, omKeyInfo); + .put(ozoneKey, omOpenKeyInfo); } else { String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); @@ -394,13 +408,15 @@ public static void addMultipartKeyToOpenKeyTable(boolean addToCache, omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), omKeyInfo.getKeyName(), uploadId); + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo).build(); if (addToCache) { omMetadataManager.getOpenKeyTable(BucketLayout.DEFAULT) .addCacheEntry(new CacheKey<>(multipartKey), - CacheValue.get(trxnLogIndex, omKeyInfo)); + CacheValue.get(trxnLogIndex, omOpenKeyInfo)); } omMetadataManager.getOpenKeyTable(BucketLayout.DEFAULT) - .put(multipartKey, omKeyInfo); + .put(multipartKey, omOpenKeyInfo); } /** @@ -1555,6 +1571,18 @@ public static String addFileToKeyTable(boolean openKeyTable, long clientID, long trxnLogIndex, OMMetadataManager omMetadataManager) throws Exception { + return addFileToKeyTable(openKeyTable, addToCache, fileName, omKeyInfo, + clientID, trxnLogIndex, null, omMetadataManager); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + public static String addFileToKeyTable(boolean openKeyTable, + boolean addToCache, String fileName, + OmKeyInfo omKeyInfo, + long clientID, long trxnLogIndex, + Long expectedDataGeneration, + OMMetadataManager omMetadataManager) + throws Exception { String ozoneDBKey; if (openKeyTable) { final long volumeId = omMetadataManager.getVolumeId( @@ -1564,13 +1592,17 @@ public static String addFileToKeyTable(boolean openKeyTable, ozoneDBKey = omMetadataManager.getOpenFileName( volumeId, bucketId, omKeyInfo.getParentObjectID(), fileName, clientID); + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo) + .setExpectedDataGeneration(expectedDataGeneration) + .build(); if (addToCache) { omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) .addCacheEntry(new CacheKey<>(ozoneDBKey), - CacheValue.get(trxnLogIndex, omKeyInfo)); + CacheValue.get(trxnLogIndex, omOpenKeyInfo)); } omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put(ozoneDBKey, omKeyInfo); + .put(ozoneDBKey, omOpenKeyInfo); } else { ozoneDBKey = omMetadataManager.getOzonePathKey( omMetadataManager.getVolumeId(omKeyInfo.getVolumeName()), @@ -1603,13 +1635,15 @@ public static void addMultipartKeyToOpenFileTable(boolean addToCache, String multipartKey = omMetadataManager.getMultipartKey(volumeId, bucketId, omKeyInfo.getParentObjectID(), fileName, uploadId); + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(omKeyInfo).build(); if (addToCache) { omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) .addCacheEntry(new CacheKey<>(multipartKey), - CacheValue.get(trxnLogIndex, omKeyInfo)); + CacheValue.get(trxnLogIndex, omOpenKeyInfo)); } omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put(multipartKey, omKeyInfo); + .put(multipartKey, omOpenKeyInfo); } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestWithFSO.java index c04fdf7dd11e..fe14669807d5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestWithFSO.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -197,13 +198,13 @@ protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, if (indx == pathComponents.length - 1) { String dbOpenFileName = omMetadataManager.getOpenFileName( volumeId, bucketId, parentId, pathElement, id); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(dbOpenFileName); if (doAssert) { - assertNotNull(omKeyInfo, "Invalid key!"); + assertNotNull(omOpenKeyInfo, "Invalid key!"); } - return omKeyInfo; + return omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } else { // directory String dbKey = omMetadataManager.getOzonePathKey(volumeId, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java index 590d12508195..8d54d21d5e1b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequestWithFSO; @@ -408,9 +409,9 @@ private void populateNamespace(boolean addKeyTable, boolean keyInfoWithHsyncFlag if (addOpenKeyTable) { String openKey = addToOpenFileTable(allocatedLocationList, openKeyInfoWithHsyncFlag); - omKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(openKey); - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); } // Set lease soft limit to 0 @@ -499,12 +500,12 @@ private void verifyTables(boolean hasKey, boolean hasOpenKey) } // Entry should be deleted from openKey Table. String openKey = getOpenFileName(); - omKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(openKey); if (hasOpenKey) { - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); } else { - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestWithFSO.java index 77bb8d1af494..e524bb7c98c0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestWithFSO.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.StringUtils; @@ -107,13 +108,13 @@ protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, String dbOpenFileName = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, pathElement, id); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(dbOpenFileName); if (doAssert) { - assertNotNull(omKeyInfo, "Invalid key!"); + assertNotNull(omOpenKeyInfo, "Invalid key!"); } - return omKeyInfo; + return omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } else { // directory String dbKey = omMetadataManager.getOzonePathKey(volumeId, bucketId, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 41b81471ff96..7deacf9a0fd8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -102,14 +103,14 @@ public void testValidateAndUpdateCacheWithUnknownBlockId() throws Exception { String openKey = addKeyToOpenKeyTable(allocatedLocationList); String ozoneKey = getOzonePathKey(); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable( omKeyCommitRequest.getBucketLayout()).get(openKey); - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); // Key should not be there in key table, as validateAndUpdateCache is // still not called. - omKeyInfo = + OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()) .get(ozoneKey); @@ -122,10 +123,10 @@ public void testValidateAndUpdateCacheWithUnknownBlockId() throws Exception { omClientResponse.getOMResponse().getStatus()); // Entry should be deleted from openKey Table. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()) .get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); // Now entry should be created in key Table. omKeyInfo = @@ -169,14 +170,14 @@ public void testValidateAndUpdateCache() throws Exception { String openKey = addKeyToOpenKeyTable(allocatedLocationList); String ozoneKey = getOzonePathKey(); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable( omKeyCommitRequest.getBucketLayout()).get(openKey); - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); // Key should not be there in key table, as validateAndUpdateCache is // still not called. - omKeyInfo = + OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()) .get(ozoneKey); @@ -189,10 +190,10 @@ public void testValidateAndUpdateCache() throws Exception { omClientResponse.getOMResponse().getStatus()); // Entry should be deleted from openKey Table. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()) .get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); // Now entry should be created in key Table. omKeyInfo = @@ -222,7 +223,7 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testAtomicRewrite() throws Exception { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); @@ -241,13 +242,13 @@ public void testAtomicRewrite() throws Exception { List acls = Collections.singletonList(OzoneAcl.parseAcl("user:foo:rw")); OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( volumeName, bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new ArrayList<>())); - omKeyInfoBuilder.setExpectedDataGeneration(1L); omKeyInfoBuilder.addAcl(acls.get(0)); - String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); - OmKeyInfo openKeyInfo = openKeyTable.get(openKey); - assertNotNull(openKeyInfo); - assertEquals(acls, openKeyInfo.getAcls()); + // Add key to open key table with expectedDataGeneration = 1L + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder, 1L); + OmOpenKeyInfo omOpenKeyInfo = openKeyTable.get(openKey); + assertNotNull(omOpenKeyInfo); + assertEquals(acls, omOpenKeyInfo.getKeyInfo().getAcls()); // At this stage, we have an openKey, with rewrite generation of 1. // However there is no closed key entry, so the commit should fail. OMClientResponse omClientResponse = @@ -255,11 +256,10 @@ public void testAtomicRewrite() throws Exception { assertEquals(ATOMIC_WRITE_CONFLICT, omClientResponse.getOMResponse().getStatus()); // Now add the key to the key table, and try again, but with different generation - omKeyInfoBuilder.setExpectedDataGeneration(null); omKeyInfoBuilder.setUpdateID(0L); OmKeyInfo invalidKeyInfo = omKeyInfoBuilder.build(); closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); - // This should fail as the updateID ia zero and the open key has rewrite generation of 1. + // This should fail as the updateID is zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); assertEquals(ATOMIC_WRITE_CONFLICT, omClientResponse.getOMResponse().getStatus()); @@ -274,7 +274,8 @@ public void testAtomicRewrite() throws Exception { assertEquals(OK, omClientResponse.getOMResponse().getStatus()); OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); - assertNull(committedKey.getExpectedDataGeneration()); + // After commit, the key should be in keyTable (not openKeyTable), so expectedDataGeneration + // is not applicable (it only exists on OmOpenKeyInfo for open keys) // Generation should be changed assertNotEquals(closedKeyInfo.getGeneration(), committedKey.getGeneration()); assertEquals(acls, committedKey.getAcls()); @@ -282,7 +283,7 @@ public void testAtomicRewrite() throws Exception { @Test public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); @@ -300,9 +301,10 @@ public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( volumeName, bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new ArrayList<>())); - omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); - String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + // Add key to open key table with EXPECTED_GEN_CREATE_IF_NOT_EXISTS + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); assertNotNull(openKeyTable.get(openKey)); assertNull(closedKeyTable.get(getOzonePathKey())); @@ -312,12 +314,12 @@ public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); assertNotNull(committedKey); - assertNull(committedKey.getExpectedDataGeneration()); + // After commit, the key is in keyTable (OmKeyInfo) which doesn't have expectedDataGeneration } @Test public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); @@ -335,9 +337,10 @@ public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( volumeName, bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new ArrayList<>())); - omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); - String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + // Add key to open key table with EXPECTED_GEN_CREATE_IF_NOT_EXISTS + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); assertNotNull(openKeyTable.get(openKey)); OmKeyInfo existingClosedKey = OMRequestTestUtils.createOmKeyInfo( @@ -376,14 +379,14 @@ public void testValidateAndUpdateCacheWithUncommittedBlocks() String openKey = addKeyToOpenKeyTable(allocatedBlockList); String ozoneKey = getOzonePathKey(); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable( omKeyCommitRequest.getBucketLayout()).get(openKey); - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); // Key should not be there in key table, as validateAndUpdateCache is // still not called. - omKeyInfo = + OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()) .get(ozoneKey); @@ -407,10 +410,10 @@ public void testValidateAndUpdateCacheWithUncommittedBlocks() .get(0).getLocationList().size()); // Entry should be deleted from openKey Table. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()) .get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); // Now entry should be created in key Table. omKeyInfo = @@ -534,16 +537,16 @@ private Map doKeyCommit(boolean isHSync, omClientResponse.getOMResponse().getStatus()); // Key should be present in both OpenKeyTable and KeyTable with HSync commit - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable( omKeyCommitRequest.getBucketLayout()).get(openKey); if (isHSync) { - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); } else { // Key should not exist in OpenKeyTable anymore with non-HSync commit - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); } - omKeyInfo = + OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()) .get(ozoneKey); assertNotNull(omKeyInfo); @@ -900,11 +903,11 @@ public void testPreAllocatedUncommittedBlockForEmptyFile() throws Exception { List allocatedLocationList = oneKeyLocations.stream().map(OmKeyLocationInfo::getFromProtobuf).collect(Collectors.toList()); String openKey = addKeyToOpenKeyTable(allocatedLocationList); - OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()).get(openKey); - assertNotNull(omKeyInfo); + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()).get(openKey); + assertNotNull(omOpenKeyInfo); String ozoneKey = getOzonePathKey(); - omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey); + OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey); assertNull(omKeyInfo); OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); @@ -913,8 +916,8 @@ public void testPreAllocatedUncommittedBlockForEmptyFile() throws Exception { Map toDeleteKeyList = ((OMKeyCommitResponse) omClientResponse).getKeysToDelete(); assertNull(toDeleteKeyList); - omKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()).get(openKey); + assertNull(omOpenKeyInfo); omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey); assertNotNull(omKeyInfo); @@ -1041,9 +1044,16 @@ protected String addKeyToOpenKeyTable(List locationList) @Nonnull protected String addKeyToOpenKeyTable(List locationList, OmKeyInfo.Builder keyInfoBuilder) throws Exception { + return addKeyToOpenKeyTable(locationList, keyInfoBuilder, null); + } + + @Nonnull + protected String addKeyToOpenKeyTable(List locationList, OmKeyInfo.Builder keyInfoBuilder, + Long expectedDataGeneration) + throws Exception { OmKeyInfo keyInfo = keyInfoBuilder.build(); keyInfo.appendNewBlocks(locationList, false); - OMRequestTestUtils.addKeyToTable(true, false, keyInfo, clientID, 0, omMetadataManager); + OMRequestTestUtils.addKeyToTable(true, false, keyInfo, clientID, 0, expectedDataGeneration, omMetadataManager); return omMetadataManager.getOpenKey(volumeName, bucketName, keyName, clientID); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java index 041090c01838..52467b7968e7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestWithFSO.java @@ -69,6 +69,13 @@ protected String getOzonePathKey() throws IOException { @Override protected String addKeyToOpenKeyTable(List locationList, OmKeyInfo.Builder keyInfoBuilder) throws Exception { + return addKeyToOpenKeyTable(locationList, keyInfoBuilder, null); + } + + @Override + protected String addKeyToOpenKeyTable(List locationList, OmKeyInfo.Builder keyInfoBuilder, + Long expectedDataGeneration) + throws Exception { // need to initialize parentID if (getParentDir() == null) { parentID = getBucketID(); @@ -83,8 +90,7 @@ protected String addKeyToOpenKeyTable(List locationList, OmKe String fileName = OzoneFSUtils.getFileName(keyName); return OMRequestTestUtils.addFileToKeyTable(true, false, - fileName, keyInfo, clientID, txnLogId, omMetadataManager); - + fileName, keyInfo, clientID, txnLogId, expectedDataGeneration, omMetadataManager); } @Override diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index d2b10b9df7b5..2fc7438d32fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -81,6 +81,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.lock.OzoneLockProvider; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -167,12 +168,12 @@ public void testCreateKeyExpectedGenCreateIfNotExistsKeyMissing( checkResponse(modifiedOmRequest, response, id, false, getBucketLayout()); - OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(getOpenKey(id)); - assertNotNull(openKeyInfo); + assertNotNull(omOpenKeyInfo); assertEquals(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - openKeyInfo.getExpectedDataGeneration()); - assertNull(openKeyInfo.getExpectedETag()); + omOpenKeyInfo.getExpectedDataGeneration()); + assertNull(omOpenKeyInfo.getExpectedETag()); } @ParameterizedTest @@ -201,9 +202,9 @@ public void testCreateKeyExpectedGenCreateIfNotExistsKeyAlreadyExists( assertEquals(KEY_ALREADY_EXISTS, response.getOMResponse().getStatus()); // As we got error, no entry should be created in openKeyTable. - OmKeyInfo openKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(openKeyInfo); + assertNull(omOpenKeyInfo); } @ParameterizedTest @@ -232,9 +233,9 @@ public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); // As we got error, no entry should be created in openKeyTable. - OmKeyInfo openKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(openKeyInfo); + assertNull(omOpenKeyInfo); } @ParameterizedTest @@ -348,15 +349,15 @@ public void testCreateWithExpectedETagSuccess( assertEquals(OK, response.getOMResponse().getStatus()); // Verify open key was normalized onto the atomic rewrite generation. - OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(getOpenKey(id)); - assertNotNull(openKeyInfo); + assertNotNull(omOpenKeyInfo); assertEquals(existingKeyInfo.getUpdateID(), - openKeyInfo.getExpectedDataGeneration()); - assertNull(openKeyInfo.getExpectedETag()); + omOpenKeyInfo.getExpectedDataGeneration()); + assertNull(omOpenKeyInfo.getExpectedETag()); // Creation time should remain the same on rewrite assertEquals(existingKeyInfo.getCreationTime(), - openKeyInfo.getCreationTime()); + omOpenKeyInfo.getKeyInfo().getCreationTime()); } @ParameterizedTest @@ -385,12 +386,12 @@ public void testValidateAndUpdateCache( String openKey = getOpenKey(id); // Before calling - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable( omKeyCreateRequest.getBucketLayout()) .get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); OMClientResponse omKeyCreateResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); @@ -403,7 +404,8 @@ public void testValidateAndUpdateCache( .getCreateKeyResponse().getKeyInfo().getKeyLocationListCount()); // Disk should have 1 version, as it is fresh key create. - OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()).get(openKey); + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()).get(openKey); + OmKeyInfo openKeyInfo = omOpenKeyInfo.getKeyInfo(); assertEquals(1, openKeyInfo.getKeyLocationVersions().size()); assertThat(openKeyInfo.getTags()).containsAllEntriesOf(tags); @@ -412,7 +414,7 @@ public void testValidateAndUpdateCache( omMetadataManager.getKeyTable(omKeyCreateRequest.getBucketLayout()) .put(getOzoneKey(), omMetadataManager .getOpenKeyTable(omKeyCreateRequest.getBucketLayout()) - .get(openKey)); + .get(openKey).getKeyInfo()); tags.remove("tag-key1"); tags.remove("tag-key2"); @@ -426,11 +428,11 @@ public void testValidateAndUpdateCache( openKey = getOpenKey(id); // Before calling - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable( omKeyCreateRequest.getBucketLayout()) .get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); @@ -449,8 +451,9 @@ public void testValidateAndUpdateCache( assertEquals(1, omMetadataManager.getOpenKeyTable( omKeyCreateRequest.getBucketLayout()) - .get(openKey).getKeyLocationVersions().size()); - openKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()).get(openKey); + .get(openKey).getKeyInfo().getKeyLocationVersions().size()); + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()).get(openKey); + openKeyInfo = omOpenKeyInfo.getKeyInfo(); assertEquals(1, openKeyInfo.getKeyLocationVersions().size()); assertThat(openKeyInfo.getTags()).containsAllEntriesOf(tags); @@ -494,10 +497,11 @@ private void checkResponse( // Check open table whether key is added or not. - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(bucketLayout).get(openKey); - assertNotNull(omKeyInfo); + assertNotNull(omOpenKeyInfo); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); assertNotNull(omKeyInfo.getLatestVersionLocations()); List omKeyLocationInfoList = @@ -558,10 +562,10 @@ public void testValidateAndUpdateCacheWithNoSuchMultipartUploadError( keyName, id); // Before calling - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); OMClientResponse omKeyCreateResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); @@ -572,10 +576,10 @@ public void testValidateAndUpdateCacheWithNoSuchMultipartUploadError( // As we got error, no entry should be created in openKeyTable. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); } @ParameterizedTest @@ -598,10 +602,10 @@ public void testValidateAndUpdateCacheWithVolumeNotFound( // Before calling - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); OMClientResponse omKeyCreateResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); @@ -611,10 +615,10 @@ public void testValidateAndUpdateCacheWithVolumeNotFound( // As We got an error, openKey Table should not have entry. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); } @@ -640,10 +644,10 @@ public void testValidateAndUpdateCacheWithBucketNotFound( omMetadataManager); // Before calling - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); OMClientResponse omKeyCreateResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); @@ -653,10 +657,10 @@ public void testValidateAndUpdateCacheWithBucketNotFound( // As We got an error, openKey Table should not have entry. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); } @@ -686,10 +690,10 @@ public void testValidateAndUpdateCacheWithInvalidPath( omMetadataManager, getBucketLayout()); // Before calling - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); OMClientResponse omKeyCreateResponse = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); @@ -700,10 +704,10 @@ public void testValidateAndUpdateCacheWithInvalidPath( // As We got an error, openKey Table should not have entry. - omKeyInfo = + omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - assertNull(omKeyInfo); + assertNull(omOpenKeyInfo); } @@ -814,7 +818,8 @@ public void testIgnoreClientACL(boolean ignoreClientACLs) throws Exception { checkResponse(modifiedOmRequest, omKeyCreateResponse, id, false, omKeyCreateRequest.getBucketLayout()); - OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); if (ignoreClientACLs) { assertFalse(omKeyInfo.getAcls().contains(OzoneAcl.parseAcl(ozoneAll))); } else { @@ -1229,10 +1234,10 @@ public void testKeyCreateInheritParentDefaultAcls( checkResponse(modifiedOmRequest, omKeyCreateResponse, id, false, omKeyCreateRequest.getBucketLayout()); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); - verifyKeyInheritAcls(omKeyInfo.getAcls(), bucketAcls); + verifyKeyInheritAcls(omOpenKeyInfo.getKeyInfo().getAcls(), bucketAcls); } @@ -1287,10 +1292,11 @@ public void testAtomicRewrite( response = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 105L); assertEquals(OK, response.getOMResponse().getStatus()); - OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(getOpenKey(omRequest.getCreateKeyRequest().getClientID())); + OmKeyInfo openKeyInfo = omOpenKeyInfo.getKeyInfo(); - assertEquals(existingKeyInfo.getGeneration(), openKeyInfo.getExpectedDataGeneration()); + assertEquals(existingKeyInfo.getGeneration(), omOpenKeyInfo.getExpectedDataGeneration()); // Creation time should remain the same on rewrite. assertEquals(existingKeyInfo.getCreationTime(), openKeyInfo.getCreationTime()); // Update ID should change @@ -1513,11 +1519,11 @@ protected OmKeyInfo checkCreatedPaths( // Check open key entry String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, omRequest.getCreateKeyRequest().getClientID()); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()) .get(openKey); - assertNotNull(omKeyInfo); - return omKeyInfo; + assertNotNull(omOpenKeyInfo); + return omOpenKeyInfo.getKeyInfo(); } protected long checkIntermediatePaths(Path keyPath) throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java index 236da45a64bb..2df78e30c3d3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.lock.OzoneLockProvider; @@ -135,11 +136,11 @@ protected OmKeyInfo checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, String fileName = keyPathFileName.toString(); String openKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentID, fileName, omRequest.getCreateKeyRequest().getClientID()); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()) .get(openKey); - assertNotNull(omKeyInfo); - return omKeyInfo; + assertNotNull(omOpenKeyInfo); + return omOpenKeyInfo.getKeyInfo(); } @Override diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 405cc706ef90..a70891d3f7a7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -77,6 +77,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -297,12 +298,12 @@ protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, throws Exception { String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, key, id); - OmKeyInfo omKeyInfo = + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); if (doAssert) { - assertNotNull(omKeyInfo, "Failed to find key in OpenKeyTable"); + assertNotNull(omOpenKeyInfo, "Failed to find key in OpenKeyTable"); } - return omKeyInfo; + return omOpenKeyInfo != null ? omOpenKeyInfo.getKeyInfo() : null; } public BucketLayout getBucketLayout() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java index 742d17a87b06..284c0c7d0fcd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -84,10 +85,11 @@ public void testValidateAndUpdateCache() throws Exception { modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); - OmKeyInfo openMPUKeyInfo = omMetadataManager + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager .getOpenKeyTable(s3InitiateMultipartUploadRequest.getBucketLayout()) .get(multipartKey); - assertNotNull(openMPUKeyInfo); + assertNotNull(omOpenKeyInfo); + OmKeyInfo openMPUKeyInfo = omOpenKeyInfo.getKeyInfo(); assertNotNull(openMPUKeyInfo.getLatestVersionLocations()); assertTrue(openMPUKeyInfo.getLatestVersionLocations().isMultipartKey()); assertNotNull(openMPUKeyInfo.getMetadata()); @@ -219,11 +221,11 @@ public void testMultipartUploadInheritParentDefaultAcls() modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); - OmKeyInfo omKeyInfo = omMetadataManager + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager .getOpenKeyTable(s3InitiateMultipartUploadRequest.getBucketLayout()) .get(multipartKey); - verifyKeyInheritAcls(omKeyInfo.getAcls(), bucketAcls); + verifyKeyInheritAcls(omOpenKeyInfo.getKeyInfo().getAcls(), bucketAcls); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java index d517be2bbba1..a910d709ad9e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -99,10 +100,11 @@ public void testValidateAndUpdateCache() throws Exception { modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); - OmKeyInfo omKeyInfo = omMetadataManager + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager .getOpenKeyTable(s3InitiateMultipartUploadReqFSO.getBucketLayout()) .get(multipartOpenFileKey); - assertNotNull(omKeyInfo, "Failed to find the fileInfo"); + assertNotNull(omOpenKeyInfo, "Failed to find the fileInfo"); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); assertNotNull(omKeyInfo.getLatestVersionLocations(), "Key Location is null!"); assertTrue( @@ -222,14 +224,14 @@ public void testMultipartUploadInheritParentDefaultAcls() bucketId, parentID, fileName, modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMultipartUploadID()); - OmKeyInfo omKeyInfo = omMetadataManager + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager .getOpenKeyTable(s3InitiateMultipartUploadReqFSO.getBucketLayout()) .get(multipartOpenFileKey); assertEquals(OzoneManagerProtocolProtos.Status.OK, omClientResponse.getOMResponse().getStatus()); - verifyKeyInheritAcls(dirs, omKeyInfo, volumeId, bucketId, bucketAcls); + verifyKeyInheritAcls(dirs, omOpenKeyInfo.getKeyInfo(), volumeId, bucketId, bucketAcls); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java index b5d78662507a..861e743a9c9b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -115,10 +116,11 @@ public void testValidateAndUpdateCacheSuccess() throws Exception { assertEquals(1, omMetadataManager.getMultipartInfoTable() .get(multipartKey).getPartKeyInfoMap().size()); - OmKeyInfo mpuOpenKeyInfo = omMetadataManager + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager .getOpenKeyTable(s3MultipartUploadCommitPartRequest.getBucketLayout()) .get(multipartOpenKey); - assertNotNull(mpuOpenKeyInfo); + assertNotNull(omOpenKeyInfo); + OmKeyInfo mpuOpenKeyInfo = omOpenKeyInfo.getKeyInfo(); assertNotNull(mpuOpenKeyInfo.getLatestVersionLocations()); assertTrue(mpuOpenKeyInfo.getLatestVersionLocations() .isMultipartKey()); @@ -383,9 +385,9 @@ public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { // Put the open key to simulate the part key upload using OMKeyCreateRequest String openMpuPartKey = addKeyToOpenKeyTable(volumeName, bucketName, keyName, clientID, allocatedBlockList); - OmKeyInfo openMpuPartKeyInfo = + OmOpenKeyInfo openMpuPartKeyOpenInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openMpuPartKey); - assertNotNull(openMpuPartKeyInfo); + assertNotNull(openMpuPartKeyOpenInfo); // Commit only the first 3 allocated blocks List committedKeyLocationList = allocatedKeyLocationList.subList(0, 3); @@ -422,10 +424,11 @@ public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { assertEquals(1, omMetadataManager.getMultipartInfoTable() .get(multipartKey).getPartKeyInfoMap().size()); - OmKeyInfo mpuOpenKeyInfo = omMetadataManager + OmOpenKeyInfo mpuOpenKeyOpenInfo = omMetadataManager .getOpenKeyTable(s3MultipartUploadCommitPartRequest.getBucketLayout()) .get(multipartOpenKey); - assertNotNull(mpuOpenKeyInfo); + assertNotNull(mpuOpenKeyOpenInfo); + OmKeyInfo mpuOpenKeyInfo = mpuOpenKeyOpenInfo.getKeyInfo(); assertNotNull(mpuOpenKeyInfo.getLatestVersionLocations()); assertTrue(mpuOpenKeyInfo.getLatestVersionLocations() .isMultipartKey()); @@ -505,9 +508,9 @@ public void testValidateAndUpdateCacheOnOverWriteWithUncommittedBlocks() throws String openMpuPartKey = addKeyToOpenKeyTable(volumeName, bucketName, keyName, clientID, overwriteAllocatedBlockList); - OmKeyInfo openMpuPartKeyInfo = + OmOpenKeyInfo openMpuPartKeyOpenInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openMpuPartKey); - assertNotNull(openMpuPartKeyInfo); + assertNotNull(openMpuPartKeyOpenInfo); // Commit only the first allocated blocks List overwriteCommittedKeyLocationList = overwriteAllocatedKeyLocationList.subList(0, 1); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java index bc11a00a49f0..b163216c2771 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseWithFSO.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.om.response.key.TestOMKeyCreateResponse; @@ -62,7 +63,8 @@ protected String getOpenKeyName() throws IOException { protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, OMResponse response) throws IOException { - return new OMFileCreateResponseWithFSO(response, keyInfo, + return new OMFileCreateResponseWithFSO(response, + new OmOpenKeyInfo.Builder().setKeyInfo(keyInfo).build(), new ArrayList<>(), clientID, bucketInfo, getVolumeId()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java index e4c3a64d66c4..20345f19550d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse; @@ -102,7 +103,8 @@ protected String getOpenKey() throws Exception { protected OMAllocateBlockResponse getOmAllocateBlockResponse( OmKeyInfo omKeyInfo, OmBucketInfo omBucketInfo, OMResponse omResponse) throws IOException { - return new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, + return new OMAllocateBlockResponse(omResponse, + new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build(), clientID, getBucketLayout()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseWithFSO.java index 95054d15b0a0..f28dfd186440 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseWithFSO.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -69,7 +70,8 @@ protected String getOpenKey() throws Exception { protected OMAllocateBlockResponse getOmAllocateBlockResponse( OmKeyInfo omKeyInfo, OmBucketInfo omBucketInfo, OMResponse omResponse) throws IOException { - return new OMAllocateBlockResponseWithFSO(omResponse, omKeyInfo, clientID, + return new OMAllocateBlockResponseWithFSO(omResponse, + new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build(), clientID, getBucketLayout(), omMetadataManager.getVolumeId(volumeName), omBucketInfo.getObjectID()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java index 1c2338f9d8a9..05bb473b64f4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -155,7 +156,9 @@ protected OMKeyCommitResponse getOmKeyCommitResponse(OmKeyInfo omKeyInfo, omMetadataManager.getOzoneDeletePathKey(e.getObjectID(), ozoneKey), new RepeatedOmKeyInfo(e, omBucketInfo.getObjectID()))); } + OmOpenKeyInfo omOpenKeyInfo = newOpenKeyInfo != null + ? new OmOpenKeyInfo.Builder().setKeyInfo(newOpenKeyInfo).build() : null; return new OMKeyCommitResponse(omResponse, omKeyInfo, ozoneKey, openKey, - omBucketInfo, deleteKeyMap, isHSync, newOpenKeyInfo, null, null); + omBucketInfo, deleteKeyMap, isHSync, omOpenKeyInfo, null, null); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseWithFSO.java index 0f7386c4389e..39318230048f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseWithFSO.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -53,8 +54,10 @@ protected OMKeyCommitResponse getOmKeyCommitResponse(OmKeyInfo omKeyInfo, omMetadataManager.getOzoneDeletePathKey(e.getObjectID(), deleteKey), new RepeatedOmKeyInfo(e, omBucketInfo.getObjectID()))); } + OmOpenKeyInfo omOpenKeyInfo = newOpenKeyInfo != null + ? new OmOpenKeyInfo.Builder().setKeyInfo(newOpenKeyInfo).build() : null; return new OMKeyCommitResponseWithFSO(omResponse, omKeyInfo, ozoneKey, - openKey, omBucketInfo, deleteKeyMap, volumeId, isHSync, newOpenKeyInfo, null, null); + openKey, omBucketInfo, deleteKeyMap, volumeId, isHSync, omOpenKeyInfo, null, null); } @Nonnull diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java index a44d955cb963..77a1260794db 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponse.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -97,7 +98,8 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, OMResponse response) throws IOException { - return new OMKeyCreateResponse(response, keyInfo, null, clientID, + return new OMKeyCreateResponse(response, + new OmOpenKeyInfo.Builder().setKeyInfo(keyInfo).build(), null, clientID, bucketInfo); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java index 169d0b33727a..b923cc7375c7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseWithFSO.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -63,7 +64,8 @@ protected OmKeyInfo getOmKeyInfo() { protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, OMResponse response) throws IOException { - return new OMKeyCreateResponseWithFSO(response, keyInfo, new ArrayList<>(), + return new OMKeyCreateResponseWithFSO(response, + new OmOpenKeyInfo.Builder().setKeyInfo(keyInfo).build(), new ArrayList<>(), clientID, bucketInfo, getVolumeId()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java index 9ee50336d19e..4ded1cd941d0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMOpenKeysDeleteResponse.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -68,8 +69,8 @@ public void testAddToDBBatchWithEmptyBlocks( this.bucketLayout = buckLayout; OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); - Map> keysToDelete = addOpenKeysToDB(volumeName, 3); - Map> keysToKeep = addOpenKeysToDB(volumeName, 3); + Map> keysToDelete = addOpenKeysToDB(volumeName, 3); + Map> keysToKeep = addOpenKeysToDB(volumeName, 3); createAndCommitResponse(keysToDelete, Status.OK); @@ -98,32 +99,32 @@ public void testAddToDBBatchWithNonEmptyBlocks( this.bucketLayout = buckLayout; OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); - Map> keysToDelete = addOpenKeysToDB(volumeName, 3, + Map> keysToDelete = addOpenKeysToDB(volumeName, 3, KEY_LENGTH); - Map> keysToKeep = addOpenKeysToDB(volumeName, 3, + Map> keysToKeep = addOpenKeysToDB(volumeName, 3, KEY_LENGTH); createAndCommitResponse(keysToDelete, Status.OK); - for (Map.Entry> entry: keysToDelete.entrySet()) { + for (Map.Entry> entry: keysToDelete.entrySet()) { // These keys should have been moved from the open key table to the // delete table. assertFalse( omMetadataManager.getOpenKeyTable(getBucketLayout()).isExist( entry.getKey())); String deleteKey = omMetadataManager.getOzoneDeletePathKey( - entry.getValue().getValue().getObjectID(), entry.getKey()); + entry.getValue().getValue().getKeyInfo().getObjectID(), entry.getKey()); assertTrue(omMetadataManager.getDeletedTable().isExist(deleteKey)); assertFalse(omMetadataManager.getDeletedTable().get(deleteKey).getOmKeyInfoList().get(0).isDeletedKeyCommitted()); } - for (Map.Entry> entry: keysToKeep.entrySet()) { + for (Map.Entry> entry: keysToKeep.entrySet()) { // These keys should not have been moved out of the open key table. assertTrue( omMetadataManager.getOpenKeyTable(getBucketLayout()).isExist( entry.getKey())); String deleteKey = omMetadataManager.getOzoneDeletePathKey( - entry.getValue().getValue().getObjectID(), entry.getKey()); + entry.getValue().getValue().getKeyInfo().getObjectID(), entry.getKey()); assertFalse(omMetadataManager.getDeletedTable().isExist(deleteKey)); } } @@ -140,7 +141,7 @@ public void testAddToDBBatchWithErrorResponse( this.bucketLayout = buckLayout; OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); - Map> keysToDelete = addOpenKeysToDB(volumeName, 3); + Map> keysToDelete = addOpenKeysToDB(volumeName, 3); createAndCommitResponse(keysToDelete, Status.INTERNAL_ERROR); @@ -159,7 +160,7 @@ public void testAddToDBBatchWithErrorResponse( * to a batch operation and committed to the database. * @throws Exception */ - private void createAndCommitResponse(Map> keysToDelete, + private void createAndCommitResponse(Map> keysToDelete, Status status) throws Exception { OMResponse omResponse = OMResponse.newBuilder() @@ -179,26 +180,26 @@ private void createAndCommitResponse(Map> keysToDe /** * Creates {@code numKeys} open keys with random names, maps each one to a - * new {@link OmKeyInfo} object, adds them to the open key table cache, and + * new {@link OmOpenKeyInfo} object, adds them to the open key table cache, and * returns them. These keys will have no associated block data. */ - private Map> addOpenKeysToDB(String volume, int numKeys) + private Map> addOpenKeysToDB(String volume, int numKeys) throws Exception { return addOpenKeysToDB(volume, numKeys, 0); } /** * Creates {@code numKeys} open keys with random names, maps each one to a - * new {@link OmKeyInfo} object, adds them to the open key table cache, and + * new {@link OmOpenKeyInfo} object, adds them to the open key table cache, and * returns them. * If {@code keyLength} is greater than 0, adds one block with that many * bytes of data for each key. * @throws Exception */ - private Map> addOpenKeysToDB(String volume, int numKeys, + private Map> addOpenKeysToDB(String volume, int numKeys, long keyLength) throws Exception { - Map> newOpenKeys = new HashMap<>(); + Map> newOpenKeys = new HashMap<>(); for (int i = 0; i < numKeys; i++) { String bucket = UUID.randomUUID().toString(); @@ -238,7 +239,8 @@ private Map> addOpenKeysToDB(String volume, int nu assertTrue(omMetadataManager.getOpenKeyTable(getBucketLayout()) .isExist(openKey)); - newOpenKeys.put(openKey, Pair.of(bucketInfo.getObjectID(), omKeyInfo)); + newOpenKeys.put(openKey, Pair.of(bucketInfo.getObjectID(), + new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build())); } return newOpenKeys; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponse.java index aa2bbc90e4bf..92b0cb3ae397 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponse.java @@ -23,6 +23,7 @@ import java.util.UUID; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.junit.jupiter.api.Test; /** @@ -53,9 +54,10 @@ public void testAddDBToBatch() throws Exception { String multipartKey = omMetadataManager.getMultipartKey(volumeName, bucketName, keyName, multipartUploadID); - OmKeyInfo openMPUKeyInfo = omMetadataManager.getOpenKeyTable( + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable( getBucketLayout()).get(multipartKey); - assertNotNull(openMPUKeyInfo); + assertNotNull(omOpenKeyInfo); + OmKeyInfo openMPUKeyInfo = omOpenKeyInfo.getKeyInfo(); assertNotNull(openMPUKeyInfo.getLatestVersionLocations()); assertTrue(openMPUKeyInfo.getLatestVersionLocations().isMultipartKey()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseWithFSO.java index 057092ff6bb2..b9a4be444435 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseWithFSO.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.junit.jupiter.api.Test; /** @@ -81,9 +82,10 @@ public void testAddDBToBatch() throws Exception { .getMultipartKey(volumeId, bucketId, parentID, fileName, multipartUploadID); - OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + OmOpenKeyInfo omOpenKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) .get(multipartOpenKey); - assertNotNull(omKeyInfo, "Failed to find the fileInfo"); + assertNotNull(omOpenKeyInfo, "Failed to find the fileInfo"); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); assertNotNull(omKeyInfo.getLatestVersionLocations(), "Key Location is null!"); assertTrue(omKeyInfo.getLatestVersionLocations().isMultipartKey(), "isMultipartKey is false!"); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index ac56273d628c..bbe59855f72d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -229,7 +230,8 @@ public S3InitiateMultipartUploadResponse createS3InitiateMPUResponseFSO( omMetadataManager.getBucketTable().get(buckDBKey); return new S3InitiateMultipartUploadResponseWithFSO(omResponse, - multipartKeyInfo, omKeyInfo, mpuKey, parentDirInfos, getBucketLayout(), + multipartKeyInfo, new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build(), + mpuKey, parentDirInfos, getBucketLayout(), volumeId, bucketId, omBucketInfo); } @@ -327,14 +329,14 @@ public S3MultipartUploadCompleteResponse createS3CompleteMPUResponseFSO( return new S3MultipartUploadCompleteResponseWithFSO(omResponse, multipartKey, multipartOpenKey, omKeyInfo, allKeyInfoToRemove, getBucketLayout(), omBucketInfo, volumeId, bucketId, null, - multipartKeyInfo); + multipartKeyInfo, null); } protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp( OmMultipartKeyInfo multipartKeyInfo, OmKeyInfo omKeyInfo, OMResponse omResponse, long volumeId, long bucketId) throws IOException { return new S3InitiateMultipartUploadResponse(omResponse, multipartKeyInfo, - omKeyInfo, getBucketLayout()); + new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build(), getBucketLayout()); } protected S3MultipartUploadAbortResponse getS3MultipartUploadAbortResp( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseWithFSO.java index 0aabd317f7b4..af29cd2a989c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartUploadAbortResponseWithFSO.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -76,7 +77,8 @@ protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp( omMetadataManager.getBucketTable().get(buckDBKey); return new S3InitiateMultipartUploadResponseWithFSO(omResponse, - multipartKeyInfo, omKeyInfo, mpuDBKey, new ArrayList<>(), + multipartKeyInfo, new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build(), + mpuDBKey, new ArrayList<>(), getBucketLayout(), volumeId, bucketId, omBucketInfo); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java index 4adfeb27d003..b49e8051b184 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java @@ -61,6 +61,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; @@ -435,22 +436,26 @@ private int getExpiredOpenKeys(boolean hsync, BucketLayout layout) { } private List getKeyInfo(BucketLayout bucketLayout, boolean openKey) { - List omKeyInfo = new ArrayList<>(); + List omKeyInfoList = new ArrayList<>(); - Table fileTable; if (openKey) { - fileTable = om.getMetadataManager().getOpenKeyTable(bucketLayout); + Table openKeyTable = om.getMetadataManager().getOpenKeyTable(bucketLayout); + try (Table.KeyValueIterator iterator = openKeyTable.iterator()) { + while (iterator.hasNext()) { + omKeyInfoList.add(iterator.next().getValue().getKeyInfo()); + } + } catch (Exception e) { + } } else { - fileTable = om.getMetadataManager().getKeyTable(bucketLayout); - } - try (Table.KeyValueIterator - iterator = fileTable.iterator()) { - while (iterator.hasNext()) { - omKeyInfo.add(iterator.next().getValue()); + Table keyTable = om.getMetadataManager().getKeyTable(bucketLayout); + try (Table.KeyValueIterator iterator = keyTable.iterator()) { + while (iterator.hasNext()) { + omKeyInfoList.add(iterator.next().getValue()); + } + } catch (Exception e) { } - } catch (Exception e) { } - return omKeyInfo; + return omKeyInfoList; } void waitForOpenKeyCleanup(boolean hsync, BucketLayout layout) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 5896266d7081..56ed4389ff56 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -78,6 +78,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; import org.apache.hadoop.ozone.recon.api.types.DUResponse; @@ -328,7 +329,7 @@ public static String convertToObjectPathForOpenKeySearch(String prevKeyPrefix, try { String[] names = EntityHandler.parseRequestPath(EntityHandler.normalizePath( prevKeyPrefix, BucketLayout.FILE_SYSTEM_OPTIMIZED)); - Table openFileTable = omMetadataManager.getOpenKeyTable( + Table openFileTable = omMetadataManager.getOpenKeyTable( BucketLayout.FILE_SYSTEM_OPTIMIZED); // Root-Level: Return the original path @@ -372,10 +373,10 @@ public static String convertToObjectPathForOpenKeySearch(String prevKeyPrefix, long dirID = handler.getDirObjectId(names, names.length); String keyKey = constructObjectPathWithPrefix(volumeId, bucketId, dirID) + OM_KEY_PREFIX + lastEntiry; - OmKeyInfo keyInfo = openFileTable.getSkipCache(keyKey); - if (keyInfo != null && keyInfo.getFileName().equals(lastEntiry)) { + OmOpenKeyInfo openKeyInfo = openFileTable.getSkipCache(keyKey); + if (openKeyInfo != null && openKeyInfo.getKeyInfo().getFileName().equals(lastEntiry)) { return constructObjectPathWithPrefix(volumeId, bucketId, - keyInfo.getParentObjectID()) + OM_KEY_PREFIX + lastEntiry; + openKeyInfo.getKeyInfo().getParentObjectID()) + OM_KEY_PREFIX + lastEntiry; } return prevKeyPrefix; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 9086f49723cd..f3eab0e4ef04 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.recon.ReconResponseUtils; import org.apache.hadoop.ozone.recon.ReconUtils; @@ -190,8 +191,8 @@ public Response getOpenKeyInfo( boolean skipPrevKeyDone = false; // Tracks if prevKey was used earlier boolean keysFound = false; // Flag to track if any keys are found String lastKey = null; - Map obsKeys = Collections.emptyMap(); - Map fsoKeys = Collections.emptyMap(); + Map obsKeys = Collections.emptyMap(); + Map fsoKeys = Collections.emptyMap(); // Validate startPrefix if it's provided if (isNotBlank(startPrefix) && !validateStartPrefix(startPrefix)) { @@ -201,15 +202,15 @@ public Response getOpenKeyInfo( // Use searchOpenKeys logic with adjustments for FSO and Non-FSO filtering if (includeNonFso) { // Search for non-FSO keys in KeyTable - Table openKeyTable = omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY); + Table openKeyTable = omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY); obsKeys = ReconUtils.extractKeysFromTable(openKeyTable, startPrefix, limit, prevKey); - for (Map.Entry entry : obsKeys.entrySet()) { + for (Map.Entry entry : obsKeys.entrySet()) { keysFound = true; skipPrevKeyDone = true; // Don't use the prevKey for the file table - KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue().getKeyInfo()); openKeyInsightInfo.getNonFSOKeyInfoList().add(keyEntityInfo); // Add to non-FSO list - replicatedTotal += entry.getValue().getReplicatedSize(); - unreplicatedTotal += entry.getValue().getDataSize(); + replicatedTotal += entry.getValue().getKeyInfo().getReplicatedSize(); + unreplicatedTotal += entry.getValue().getKeyInfo().getDataSize(); lastKey = entry.getKey(); // Update lastKey } } @@ -221,12 +222,12 @@ public Response getOpenKeyInfo( // If limit = -1 then we need to fetch all keys without limit int effectiveLimit = limit == -1 ? limit : limit - obsKeys.size(); fsoKeys = searchOpenKeysInFSO(startPrefix, effectiveLimit, effectivePrevKey); - for (Map.Entry entry : fsoKeys.entrySet()) { + for (Map.Entry entry : fsoKeys.entrySet()) { keysFound = true; - KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue().getKeyInfo()); openKeyInsightInfo.getFsoKeyInfoList().add(keyEntityInfo); // Add to FSO list - replicatedTotal += entry.getValue().getReplicatedSize(); - unreplicatedTotal += entry.getValue().getDataSize(); + replicatedTotal += entry.getValue().getKeyInfo().getReplicatedSize(); + unreplicatedTotal += entry.getValue().getKeyInfo().getDataSize(); lastKey = entry.getKey(); // Update lastKey } } @@ -252,15 +253,15 @@ public Response getOpenKeyInfo( } } - public Map searchOpenKeysInFSO(String startPrefix, + public Map searchOpenKeysInFSO(String startPrefix, int limit, String prevKey) throws IOException, IllegalArgumentException { - Map matchedKeys = new LinkedHashMap<>(); + Map matchedKeys = new LinkedHashMap<>(); // Convert the search prefix to an object path for FSO buckets String startPrefixObjectPath = ReconUtils.convertToObjectPathForOpenKeySearch( startPrefix, omMetadataManager, reconNamespaceSummaryManager, reconSCM); String[] names = parseRequestPath(startPrefixObjectPath); - Table openFileTable = + Table openFileTable = omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); // If names.length <= 2, then the search prefix is at the volume or bucket level hence diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java index b78e8cb1518f..08f69e6f2a29 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +49,8 @@ public void handlePutEvent(OMDBUpdateEvent event, Map replicatedSizeMap) { if (event.getValue() != null) { - OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue(); + OmOpenKeyInfo omOpenKeyInfo = (OmOpenKeyInfo) event.getValue(); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count + 1L); unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size + omKeyInfo.getDataSize()); @@ -72,7 +74,8 @@ public void handleDeleteEvent(OMDBUpdateEvent event, Map replicatedSizeMap) { if (event.getValue() != null) { - OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue(); + OmOpenKeyInfo omOpenKeyInfo = (OmOpenKeyInfo) event.getValue(); + OmKeyInfo omKeyInfo = omOpenKeyInfo.getKeyInfo(); objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count > 0 ? count - 1L : 0L); unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), @@ -107,8 +110,8 @@ public void handleUpdateEvent(OMDBUpdateEvent event, // In Update event the count for the open table will not change. So we // don't need to update the count. - OmKeyInfo oldKeyInfo = (OmKeyInfo) event.getOldValue(); - OmKeyInfo newKeyInfo = (OmKeyInfo) event.getValue(); + OmKeyInfo oldKeyInfo = ((OmOpenKeyInfo) event.getOldValue()).getKeyInfo(); + OmKeyInfo newKeyInfo = ((OmOpenKeyInfo) event.getValue()).getKeyInfo(); unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size - oldKeyInfo.getDataSize() + newKeyInfo.getDataSize()); @@ -134,12 +137,12 @@ public Triple getTableSizeAndCount(String tableName, long unReplicatedSize = 0; long replicatedSize = 0; - Table table = (Table) omMetadataManager.getTable(tableName); - try (TableIterator> iterator = table.iterator()) { + Table table = (Table) omMetadataManager.getTable(tableName); + try (TableIterator> iterator = table.iterator()) { while (iterator.hasNext()) { - Table.KeyValue kv = iterator.next(); + Table.KeyValue kv = iterator.next(); if (kv != null && kv.getValue() != null) { - OmKeyInfo omKeyInfo = kv.getValue(); + OmKeyInfo omKeyInfo = kv.getValue().getKeyInfo(); unReplicatedSize += omKeyInfo.getDataSize(); replicatedSize += omKeyInfo.getReplicatedSize(); count++; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index de8487de3293..7d5794cc68f9 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; @@ -352,7 +353,7 @@ public static void writeOpenFileToOm(OMMetadataManager omMetadataManager, .build(); omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put(openKey, omKeyInfo); + .put(openKey, new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build()); } /** @@ -380,7 +381,7 @@ public static void writeOpenKeyToOm(OMMetadataManager omMetadataManager, .build(); omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY) - .put(openKey, omKeyInfo); + .put(openKey, new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build()); } /** diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java index c4f419b6e634..e2a97399428f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java @@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -904,11 +905,11 @@ public void testGetOpenKeyInfo() throws Exception { getOmKeyInfo("sampleVol", "bucketOne", "key_one", true); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_one", omKeyInfo); - OmKeyInfo omKeyInfo1 = + .put("/sampleVol/bucketOne/key_one", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo).build()); + OmOpenKeyInfo omOpenKeyInfo1 = reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) .get("/sampleVol/bucketOne/key_one"); - assertEquals("key_one", omKeyInfo1.getKeyName()); + assertEquals("key_one", omOpenKeyInfo1.getKeyInfo().getKeyName()); Response openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(-1, "", "", true, true); KeyInsightInfoResponse keyInsightInfoResp = @@ -1052,11 +1053,11 @@ public void testGetOpenKeyInfoLimitParam() throws Exception { getOmKeyInfo("sampleVol", "bucketOne", "key_three", true); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_one", omKeyInfo1); + .put("/sampleVol/bucketOne/key_one", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo1).build()); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/bucketOne/key_two", omKeyInfo2); + .put("/sampleVol/bucketOne/key_two", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo2).build()); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_three", omKeyInfo3); + .put("/sampleVol/bucketOne/key_three", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo3).build()); Response openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(2, "", "", true, true); KeyInsightInfoResponse keyInsightInfoResp = @@ -1094,11 +1095,11 @@ public void testGetOpenKeyInfoWithIncludeFsoAndIncludeNonFsoParams() getOmKeyInfo("sampleVol", "non_fso_Bucket", "non_fso_key3", true); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/non_fso_Bucket/non_fso_key1", omKeyInfo1); + .put("/sampleVol/non_fso_Bucket/non_fso_key1", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo1).build()); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/non_fso_Bucket/non_fso_key2", omKeyInfo2); + .put("/sampleVol/non_fso_Bucket/non_fso_key2", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo2).build()); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/non_fso_Bucket/non_fso_key3", omKeyInfo3); + .put("/sampleVol/non_fso_Bucket/non_fso_key3", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo3).build()); omKeyInfo1 = getOmKeyInfo("sampleVol", "fso_Bucket", "fso_key1", false); @@ -1110,13 +1111,13 @@ public void testGetOpenKeyInfoWithIncludeFsoAndIncludeNonFsoParams() getOmKeyInfo("sampleVol", "fso_Bucket", "fso_key4", false); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/fso_Bucket/fso_key1", omKeyInfo1); + .put("/sampleVol/fso_Bucket/fso_key1", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo1).build()); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/fso_Bucket/fso_key2", omKeyInfo2); + .put("/sampleVol/fso_Bucket/fso_key2", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo2).build()); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/fso_Bucket/fso_key3", omKeyInfo3); + .put("/sampleVol/fso_Bucket/fso_key3", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo3).build()); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/fso_Bucket/fso_key4", omKeyInfo4); + .put("/sampleVol/fso_Bucket/fso_key4", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo4).build()); // CASE 1 :- Display only FSO keys in response // includeFsoKeys=true, includeNonFsoKeys=false @@ -1172,11 +1173,11 @@ public void testGetOpenKeyInfoPrevKeyParam() throws Exception { getOmKeyInfo("sampleVol", "bucketOne", "key_3", true); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_1", omKeyInfo1); + .put("/sampleVol/bucketOne/key_1", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo1).build()); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/bucketOne/key_2", omKeyInfo2); + .put("/sampleVol/bucketOne/key_2", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo2).build()); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_3", omKeyInfo3); + .put("/sampleVol/bucketOne/key_3", new OmOpenKeyInfo.Builder().setKeyInfo(omKeyInfo3).build()); Response openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(-1, "/sampleVol/bucketOne/key_1", "", true, true); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java index 92ee52652107..cd2b14466794 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java @@ -64,6 +64,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmOpenKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; @@ -401,6 +402,10 @@ public void testReprocessForCount() throws Exception { uploadID, 1, 100L); multipartKeyInfo.addPartKeyInfo(partKeyInfo); when(mockKeyValue.getValue()).thenReturn(multipartKeyInfo); + } else if (tableName.equals(OPEN_KEY_TABLE) || tableName.equals(OPEN_FILE_TABLE)) { + OmOpenKeyInfo omOpenKeyInfo = new OmOpenKeyInfo.Builder() + .setKeyInfo(mock(OmKeyInfo.class)).build(); + when(mockKeyValue.getValue()).thenReturn(omOpenKeyInfo); } else { when(mockKeyValue.getValue()).thenReturn(mock(OmKeyInfo.class)); } @@ -808,12 +813,25 @@ private OMDBUpdateEvent getOMUpdateEvent( String table, OMDBUpdateEvent.OMDBUpdateAction action, Object oldValue) { + // For open key tables, wrap OmKeyInfo in OmOpenKeyInfo + Object wrappedValue = value; + Object wrappedOldValue = oldValue; + if (OPEN_KEY_TABLE.equals(table) || OPEN_FILE_TABLE.equals(table)) { + if (value instanceof OmKeyInfo) { + wrappedValue = new OmOpenKeyInfo.Builder() + .setKeyInfo((OmKeyInfo) value).build(); + } + if (oldValue instanceof OmKeyInfo) { + wrappedOldValue = new OmOpenKeyInfo.Builder() + .setKeyInfo((OmKeyInfo) oldValue).build(); + } + } return new OMDBUpdateEvent.OMUpdateEventBuilder() .setAction(action) .setKey(name) - .setValue(value) + .setValue(wrappedValue) .setTable(table) - .setOldValue(oldValue) + .setOldValue(wrappedOldValue) .build(); }