From 2e174ff7f327e253e4e6ee95218a750232ae4c48 Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Sun, 3 May 2026 01:01:46 -0400 Subject: [PATCH 1/4] feat: introduce codec for filter OpenKeyInfo relate fields --- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 86 ++++++++++++++++--- .../ozone/om/helpers/RepeatedOmKeyInfo.java | 35 ++++++-- .../hadoop/ozone/om/codec/OMDBDefinition.java | 19 ++-- .../om/request/key/OMKeyCommitRequest.java | 5 -- .../key/OMKeyCommitRequestWithFSO.java | 4 - 5 files changed, 117 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index e388df27b4b3..e85e1e0ca0f0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -59,8 +59,13 @@ public final class OmKeyInfo extends WithParentObjectId implements CopyObject, WithTags { private static final Logger LOG = LoggerFactory.getLogger(OmKeyInfo.class); - private static final Codec CODEC_TRUE = newCodec(true); - private static final Codec CODEC_FALSE = newCodec(false); + // Codecs for openKeyTable + private static final Codec CODEC_TRUE = newCodec(true, true); + private static final Codec CODEC_FALSE = newCodec(false, true); + + // Codecs for keyTable + private static final Codec CODEC_KEY_TABLE_TRUE = newCodec(true, false); + private static final Codec CODEC_KEY_TABLE_FALSE = newCodec(false, false); /** * Metadata key flag to indicate whether a deleted key was a committed key. * The flag is set when a committed key is deleted from AOS but still held in @@ -133,19 +138,47 @@ private OmKeyInfo(Builder b) { this.expectedETag = b.expectedETag; } - private static Codec newCodec(boolean ignorePipeline) { + /** + * Creates a new codec for OmKeyInfo. + * + * @param ignorePipeline whether to ignore pipeline info during serialization + * @param includeOpenKeyFields whether to include fields only used in openKeyTable + * (expectedDataGeneration, expectedETag) in serialization. + * Use true for openKeyTable, false for keyTable. + * @return the codec + */ + private static Codec newCodec(boolean ignorePipeline, boolean includeOpenKeyFields) { return new DelegatedCodec<>( Proto2Codec.get(KeyInfo.getDefaultInstance()), OmKeyInfo::getFromProtobuf, - k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION), + k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION, includeOpenKeyFields), OmKeyInfo.class); } + /** + * Gets the codec for openKeyTable. This codec includes expectedDataGeneration + * and expectedETag fields during serialization. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for openKeyTable + */ public static Codec getCodec(boolean ignorePipeline) { LOG.debug("OmKeyInfo.getCodec ignorePipeline = {}", ignorePipeline); return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; } + /** + * Gets the codec for keyTable. This codec excludes fields are only + * meaningful for open keys. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for keyTable + */ + public static Codec getCodecForKeyTable(boolean ignorePipeline) { + LOG.debug("OmKeyInfo.getCodecForKeyTable ignorePipeline = {}", ignorePipeline); + return ignorePipeline ? CODEC_KEY_TABLE_TRUE : CODEC_KEY_TABLE_FALSE; + } + public String getVolumeName() { return volumeName; } @@ -755,7 +788,21 @@ public KeyInfo getNetworkProtobuf(String fullKeyName, int clientVersion, * @return KeyInfo */ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) { - return getProtobuf(ignorePipeline, null, clientVersion, false); + return getProtobuf(ignorePipeline, null, clientVersion, false, true); + } + + /** + * Gets KeyInfo for persistence with control over fields only used in openKeyTable. + * + * @param ignorePipeline true for persist to DB, false for network transmit. + * @param clientVersion the client version + * @param includeOpenKeyFields whether to include fields. + * Use true for openKeyTable, false for keyTable. + * @return KeyInfo + */ + public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion, + boolean includeOpenKeyFields) { + return getProtobuf(ignorePipeline, null, clientVersion, false, includeOpenKeyFields); } /** @@ -767,6 +814,22 @@ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) { */ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, int clientVersion, boolean latestVersionBlocks) { + return getProtobuf(ignorePipeline, fullKeyName, clientVersion, latestVersionBlocks, true); + } + + /** + * Gets KeyInfo with all parameters. + * + * @param ignorePipeline ignore pipeline flag + * @param fullKeyName user given key name + * @param clientVersion the client version + * @param latestVersionBlocks whether to include only latest version blocks + * @param includeOpenKeyFields whether to include fields only used in openKeyTable + * @return key info object + */ + private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, + int clientVersion, boolean latestVersionBlocks, + boolean includeOpenKeyFields) { long latestVersion = keyLocationVersions.isEmpty() ? -1 : keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion(); @@ -818,11 +881,14 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); } kb.setIsFile(isFile); - if (expectedDataGeneration != null) { - kb.setExpectedDataGeneration(expectedDataGeneration); - } - if (expectedETag != null) { - kb.setExpectedETag(expectedETag); + + if (includeOpenKeyFields) { + if (expectedDataGeneration != null) { + kb.setExpectedDataGeneration(expectedDataGeneration); + } + if (expectedETag != null) { + kb.setExpectedETag(expectedETag); + } } if (ownerName != null) { kb.setOwnerName(ownerName); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java index 0f10832114e3..439bac90d52d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java @@ -37,8 +37,13 @@ * admin wants to confirm if a given key is deleted from deletedTable metadata. */ public class RepeatedOmKeyInfo implements CopyObject { - private static final Codec CODEC_TRUE = newCodec(true); - private static final Codec CODEC_FALSE = newCodec(false); + // Codecs that include fields only used in openKeyTable (for backward compatibility) + private static final Codec CODEC_TRUE = newCodec(true, true); + private static final Codec CODEC_FALSE = newCodec(false, true); + + // Codecs for deletedTable - exclude fields only used in openKeyTable + private static final Codec CODEC_DELETED_TABLE_TRUE = newCodec(true, false); + private static final Codec CODEC_DELETED_TABLE_FALSE = newCodec(false, false); private final List omKeyInfoList; /** @@ -51,11 +56,11 @@ public class RepeatedOmKeyInfo implements CopyObject { */ private final long bucketId; - private static Codec newCodec(boolean ignorePipeline) { + private static Codec newCodec(boolean ignorePipeline, boolean includeOpenKeyFields) { return new DelegatedCodec<>( Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()), RepeatedOmKeyInfo::getFromProto, - k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION), + k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, includeOpenKeyFields), RepeatedOmKeyInfo.class); } @@ -63,6 +68,17 @@ public static Codec getCodec(boolean ignorePipeline) { return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; } + /** + * Gets the codec for deletedTable. This codec excludes fields only used in + * openKeyTable during serialization, as deleted keys are committed keys. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for deletedTable + */ + public static Codec getCodecForDeletedTable(boolean ignorePipeline) { + return ignorePipeline ? CODEC_DELETED_TABLE_TRUE : CODEC_DELETED_TABLE_FALSE; + } + public RepeatedOmKeyInfo(long bucketId) { this.omKeyInfoList = new ArrayList<>(); this.bucketId = bucketId; @@ -129,9 +145,18 @@ public static RepeatedOmKeyInfo getFromProto(RepeatedKeyInfo repeatedKeyInfo) { * @param compact true for persistence, false for network transmit */ public RepeatedKeyInfo getProto(boolean compact, int clientVersion) { + return getProto(compact, clientVersion, true); + } + + /** + * @param compact true for persistence, false for network transmit + * @param clientVersion the client version + * @param includeOpenKeyFields whether to include fields only used in openKeyTable + */ + public RepeatedKeyInfo getProto(boolean compact, int clientVersion, boolean includeOpenKeyFields) { List list = new ArrayList<>(); for (OmKeyInfo k : cloneOmKeyInfoList()) { - list.add(k.getProtobuf(compact, clientVersion)); + list.add(k.getProtobuf(compact, clientVersion, includeOpenKeyFields)); } RepeatedKeyInfo.Builder builder = RepeatedKeyInfo.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 323f11926af9..6114efef3fa6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -204,18 +204,18 @@ public final class OMDBDefinition extends DBDefinition.WithMap { //--------------------------------------------------------------------------- // Object Store (OBS) Tables: public static final String KEY_TABLE = "keyTable"; - /** keyTable: /volume/bucket/key :- KeyInfo. */ + /** keyTable: /volume/bucket/key :- KeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition KEY_TABLE_DEF = new DBColumnFamilyDefinition<>(KEY_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getCodecForKeyTable(true)); public static final String DELETED_TABLE = "deletedTable"; - /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo. */ + /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition DELETED_TABLE_DEF = new DBColumnFamilyDefinition<>(DELETED_TABLE, StringCodec.get(), - RepeatedOmKeyInfo.getCodec(true)); + RepeatedOmKeyInfo.getCodecForDeletedTable(true)); public static final String OPEN_KEY_TABLE = "openKeyTable"; /** openKeyTable: /volume/bucket/key/id :- KeyInfo. */ @@ -241,11 +241,11 @@ public final class OMDBDefinition extends DBDefinition.WithMap { //--------------------------------------------------------------------------- // File System Optimized (FSO) Tables: public static final String FILE_TABLE = "fileTable"; - /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo. */ + /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition FILE_TABLE_DEF = new DBColumnFamilyDefinition<>(FILE_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getCodecForKeyTable(true)); public static final String OPEN_FILE_TABLE = "openFileTable"; /** openFileTable: /volumeId/bucketId/parentId/fileName/id :- KeyInfo. */ @@ -262,11 +262,14 @@ public final class OMDBDefinition extends DBDefinition.WithMap { OmDirectoryInfo.getCodec()); public static final String DELETED_DIR_TABLE = "deletedDirectoryTable"; - /** deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo. */ + /** + * deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo + * (excludes fields only used in openKeyTable). + */ public static final DBColumnFamilyDefinition DELETED_DIR_TABLE_DEF = new DBColumnFamilyDefinition<>(DELETED_DIR_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getCodecForKeyTable(true)); //--------------------------------------------------------------------------- // S3 Multi-Tenancy Tables diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 28435bedf88a..512c6f80b6e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -303,13 +303,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); - // Optimistic locking validation has passed. Now set the rewrite fields to null so they are - // not persisted in the key table. - // Combination // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() - .setExpectedDataGeneration(null) - .setExpectedETag(null) .addAllMetadata(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())) .setUpdateID(trxnLogIndex) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index 5aa4fbeb36f4..2ab38865566a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -243,10 +243,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut Map oldKeyVersionsToDeleteMap = null; validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); - // Optimistic locking validation has passed. Now set the rewrite fields to null so they are - // not persisted in the key table. - omKeyInfo.setExpectedDataGeneration(null); - omKeyInfo.setExpectedETag(null); long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace shouldn't check and increase. From 3d6de91d510480aa3af3a1b16690886c840ba97b Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Sun, 3 May 2026 01:05:30 -0400 Subject: [PATCH 2/4] test: add tests --- .../ozone/om/helpers/TestOmKeyInfoCodec.java | 142 ++++++++++++++++++ .../helpers/TestRepeatedOmKeyInfoCodec.java | 102 +++++++++++++ 2 files changed, 244 insertions(+) diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java index a22f9992a1ed..b6cec33bbd42 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java @@ -18,8 +18,10 @@ package org.apache.hadoop.ozone.om.helpers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -35,6 +37,7 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.util.Time; import org.junit.jupiter.api.Test; @@ -92,6 +95,44 @@ private OmKeyInfo getKeyInfo(int chunkNum) { .build(); } + /** + * Creates an OmKeyInfo with fields only used in openKeyTable set. + * These fields (expectedDataGeneration, expectedETag) are only meaningful + * for keys in openKeyTable. + */ + private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) { + List omKeyLocationInfoList = new ArrayList<>(); + Pipeline pipeline = HddsTestUtils.getRandomPipeline(); + for (int i = 0; i < chunkNum; i++) { + BlockID blockID = new BlockID(i, i); + OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(blockID) + .setPipeline(pipeline) + .build(); + omKeyLocationInfoList.add(keyLocationInfo); + } + OmKeyLocationInfoGroup omKeyLocationInfoGroup = new + OmKeyLocationInfoGroup(0, omKeyLocationInfoList); + + return new OmKeyInfo.Builder() + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setReplicationConfig(RatisReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.THREE)) + .setVolumeName(VOLUME) + .setBucketName(BUCKET) + .setKeyName(KEYNAME) + .setObjectID(Time.now()) + .setUpdateID(Time.now()) + .setDataSize(100) + .setOmKeyLocationInfos( + Collections.singletonList(omKeyLocationInfoGroup)) + .setFileChecksum(checksum) + .setExpectedDataGeneration(12345L) + .setExpectedETag("test-etag-value") + .build(); + } + @Test public void test() throws IOException { testOmKeyInfoCodecWithoutPipeline(1); @@ -125,4 +166,105 @@ public void testOmKeyInfoCodecCompatibility(int chunkNum) throws IOException { assertNotNull(key.getLatestVersionLocations().getLocationList().get(0) .getPipeline()); } + + @Test + public void testOpenKeyTableCodecIncludesOpenKeyFields() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getCodec(true); + OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); + + assertEquals(12345L, originKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", originKey.getExpectedETag()); + + byte[] rawData = openKeyCodec.toPersistedFormat(originKey); + OmKeyInfo deserializedKey = openKeyCodec.fromPersistedFormat(rawData); + + assertEquals(12345L, deserializedKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", deserializedKey.getExpectedETag()); + + KeyInfo keyInfo = KeyInfo.parseFrom(rawData); + assertTrue(keyInfo.hasExpectedDataGeneration(), + "openKeyTable codec should include expectedDataGeneration in proto"); + assertTrue(keyInfo.hasExpectedETag(), + "openKeyTable codec should include expectedETag in proto"); + assertEquals(12345L, keyInfo.getExpectedDataGeneration()); + assertEquals("test-etag-value", keyInfo.getExpectedETag()); + } + + @Test + public void testKeyTableCodecExcludesOpenKeyFields() throws IOException { + final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); + assertEquals(12345L, originKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", originKey.getExpectedETag()); + + byte[] rawData = keyTableCodec.toPersistedFormat(originKey); + KeyInfo keyInfo = KeyInfo.parseFrom(rawData); + assertFalse(keyInfo.hasExpectedDataGeneration(), + "keyTable codec should NOT include expectedDataGeneration in proto"); + assertFalse(keyInfo.hasExpectedETag(), + "keyTable codec should NOT include expectedETag in proto"); + + OmKeyInfo deserializedKey = keyTableCodec.fromPersistedFormat(rawData); + assertEquals(VOLUME, deserializedKey.getVolumeName()); + assertEquals(BUCKET, deserializedKey.getBucketName()); + assertEquals(KEYNAME, deserializedKey.getKeyName()); + assertEquals(100, deserializedKey.getDataSize()); + + assertNull(deserializedKey.getExpectedDataGeneration(), + "Deserialized key from keyTable should have null expectedDataGeneration"); + assertNull(deserializedKey.getExpectedETag(), + "Deserialized key from keyTable should have null expectedETag"); + } + + @Test + public void testKeyTableCodecCanReadOpenKeyTableData() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getCodec(true); + final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + + OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); + byte[] rawData = openKeyCodec.toPersistedFormat(originKey); + OmKeyInfo deserializedKey = keyTableCodec.fromPersistedFormat(rawData); + + assertEquals(VOLUME, deserializedKey.getVolumeName()); + assertEquals(BUCKET, deserializedKey.getBucketName()); + assertEquals(12345L, deserializedKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", deserializedKey.getExpectedETag()); + } + + @Test + public void testCodecsWithKeyWithoutOpenKeyFields() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getCodec(true); + final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + + OmKeyInfo originKey = getKeyInfo(1); + assertNull(originKey.getExpectedDataGeneration()); + assertNull(originKey.getExpectedETag()); + + byte[] openKeyData = openKeyCodec.toPersistedFormat(originKey); + byte[] keyTableData = keyTableCodec.toPersistedFormat(originKey); + + OmKeyInfo fromOpenKeyCodec = openKeyCodec.fromPersistedFormat(openKeyData); + OmKeyInfo fromKeyTableCodec = keyTableCodec.fromPersistedFormat(keyTableData); + + assertEquals(VOLUME, fromOpenKeyCodec.getVolumeName()); + assertEquals(VOLUME, fromKeyTableCodec.getVolumeName()); + assertEquals(BUCKET, fromOpenKeyCodec.getBucketName()); + assertEquals(BUCKET, fromKeyTableCodec.getBucketName()); + assertNull(fromOpenKeyCodec.getExpectedDataGeneration()); + assertNull(fromKeyTableCodec.getExpectedDataGeneration()); + } + + @Test + public void testKeyTableCodecProducesSmallerOutput() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getCodec(true); + final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + + OmKeyInfo keyWithOpenFields = getKeyInfoWithOpenKeyFields(1); + + byte[] openKeyData = openKeyCodec.toPersistedFormat(keyWithOpenFields); + byte[] keyTableData = keyTableCodec.toPersistedFormat(keyWithOpenFields); + + assertTrue(keyTableData.length < openKeyData.length, + "keyTable codec should produce smaller serialized output when openKeyTable-only fields are set"); + } } diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java index 3227cfbe6a4b..e3c44982ef64 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; @@ -36,6 +37,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo; import org.apache.hadoop.util.Time; import org.junit.jupiter.api.Test; @@ -150,4 +152,104 @@ public void threadSafety() throws InterruptedException { } assertFalse(failed.get()); } + + private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) { + List omKeyLocationInfoList = new ArrayList<>(); + Pipeline pipeline = HddsTestUtils.getRandomPipeline(); + for (int i = 0; i < chunkNum; i++) { + BlockID blockID = new BlockID(i, i); + OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(blockID) + .setPipeline(pipeline) + .build(); + omKeyLocationInfoList.add(keyLocationInfo); + } + OmKeyLocationInfoGroup omKeyLocationInfoGroup = new + OmKeyLocationInfoGroup(0, omKeyLocationInfoList); + return new OmKeyInfo.Builder() + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setReplicationConfig( + RatisReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.THREE)) + .setVolumeName(VOLUME) + .setBucketName(BUCKET) + .setKeyName(KEYNAME) + .setObjectID(Time.now()) + .setUpdateID(Time.now()) + .setDataSize(100) + .setOmKeyLocationInfos( + Collections.singletonList(omKeyLocationInfoGroup)) + .setExpectedDataGeneration(12345L) + .setExpectedETag("test-etag-value") + .build(); + } + + @Test + void testRegularCodecIncludesOpenKeyFields() throws IOException { + final Codec codec = RepeatedOmKeyInfo.getCodec(true); + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo); + RepeatedKeyInfo proto = RepeatedKeyInfo.parseFrom(rawData); + + assertTrue(proto.getKeyInfo(0).hasExpectedDataGeneration()); + assertTrue(proto.getKeyInfo(0).hasExpectedETag()); + assertEquals(12345L, proto.getKeyInfo(0).getExpectedDataGeneration()); + assertEquals("test-etag-value", proto.getKeyInfo(0).getExpectedETag()); + } + + @Test + void testDeletedTableCodecExcludesOpenKeyFields() throws IOException { + final Codec codec = RepeatedOmKeyInfo.getCodecForDeletedTable(true); + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo); + RepeatedKeyInfo proto = RepeatedKeyInfo.parseFrom(rawData); + + assertFalse(proto.getKeyInfo(0).hasExpectedDataGeneration()); + assertFalse(proto.getKeyInfo(0).hasExpectedETag()); + + RepeatedOmKeyInfo deserialized = codec.fromPersistedFormat(rawData); + assertEquals(VOLUME, deserialized.getOmKeyInfoList().get(0).getVolumeName()); + assertEquals(BUCKET, deserialized.getOmKeyInfoList().get(0).getBucketName()); + assertEquals(bucketId, deserialized.getBucketId()); + } + + @Test + void testDeletedTableCodecCanReadRegularCodecData() throws IOException { + final Codec regularCodec = RepeatedOmKeyInfo.getCodec(true); + final Codec deletedTableCodec = RepeatedOmKeyInfo.getCodecForDeletedTable(true); + + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] rawData = regularCodec.toPersistedFormat(repeatedOmKeyInfo); + RepeatedOmKeyInfo deserialized = deletedTableCodec.fromPersistedFormat(rawData); + + assertEquals(VOLUME, deserialized.getOmKeyInfoList().get(0).getVolumeName()); + assertEquals(12345L, deserialized.getOmKeyInfoList().get(0).getExpectedDataGeneration()); + assertEquals("test-etag-value", deserialized.getOmKeyInfoList().get(0).getExpectedETag()); + } + + @Test + void testDeletedTableCodecProducesSmallerOutput() throws IOException { + final Codec regularCodec = RepeatedOmKeyInfo.getCodec(true); + final Codec deletedTableCodec = RepeatedOmKeyInfo.getCodecForDeletedTable(true); + + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] regularData = regularCodec.toPersistedFormat(repeatedOmKeyInfo); + byte[] deletedTableData = deletedTableCodec.toPersistedFormat(repeatedOmKeyInfo); + + assertTrue(deletedTableData.length < regularData.length, + "deletedTable codec should produce smaller output"); + } } From f03553710c2ac31c1fc4d12bde39acfd1e155caf Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Sun, 3 May 2026 03:17:55 -0400 Subject: [PATCH 3/4] test: remove unused check --- .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 41b81471ff96..265d701641a1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -274,7 +274,6 @@ public void testAtomicRewrite() throws Exception { assertEquals(OK, omClientResponse.getOMResponse().getStatus()); OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); - assertNull(committedKey.getExpectedDataGeneration()); // Generation should be changed assertNotEquals(closedKeyInfo.getGeneration(), committedKey.getGeneration()); assertEquals(acls, committedKey.getAcls()); @@ -312,7 +311,6 @@ public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); assertNotNull(committedKey); - assertNull(committedKey.getExpectedDataGeneration()); } @Test From 7d03b0f4042df00a71187c4becfc19717edcfbf0 Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Sun, 3 May 2026 21:55:11 -0400 Subject: [PATCH 4/4] chore: refine naming --- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 30 +++++++++---------- .../ozone/om/helpers/RepeatedOmKeyInfo.java | 23 +++++++++----- .../ozone/om/helpers/TestOmKeyInfoCodec.java | 24 +++++++-------- .../helpers/TestRepeatedOmKeyInfoCodec.java | 24 +++++++-------- .../hadoop/ozone/om/codec/OMDBDefinition.java | 12 ++++---- 5 files changed, 59 insertions(+), 54 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index e85e1e0ca0f0..aef83d349577 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -142,16 +142,15 @@ private OmKeyInfo(Builder b) { * Creates a new codec for OmKeyInfo. * * @param ignorePipeline whether to ignore pipeline info during serialization - * @param includeOpenKeyFields whether to include fields only used in openKeyTable - * (expectedDataGeneration, expectedETag) in serialization. - * Use true for openKeyTable, false for keyTable. + * @param isOpenKey true for openKeyTable (includes expectedDataGeneration/expectedETag), + * false for keyTable (excludes these fields) * @return the codec */ - private static Codec newCodec(boolean ignorePipeline, boolean includeOpenKeyFields) { + private static Codec newCodec(boolean ignorePipeline, boolean isOpenKey) { return new DelegatedCodec<>( Proto2Codec.get(KeyInfo.getDefaultInstance()), OmKeyInfo::getFromProtobuf, - k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION, includeOpenKeyFields), + k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey), OmKeyInfo.class); } @@ -162,8 +161,8 @@ private static Codec newCodec(boolean ignorePipeline, boolean include * @param ignorePipeline whether to ignore pipeline info * @return the codec for openKeyTable */ - public static Codec getCodec(boolean ignorePipeline) { - LOG.debug("OmKeyInfo.getCodec ignorePipeline = {}", ignorePipeline); + public static Codec getOpenKeyTableCodec(boolean ignorePipeline) { + LOG.debug("OmKeyInfo.getOpenKeyTableCodec ignorePipeline = {}", ignorePipeline); return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; } @@ -174,8 +173,8 @@ public static Codec getCodec(boolean ignorePipeline) { * @param ignorePipeline whether to ignore pipeline info * @return the codec for keyTable */ - public static Codec getCodecForKeyTable(boolean ignorePipeline) { - LOG.debug("OmKeyInfo.getCodecForKeyTable ignorePipeline = {}", ignorePipeline); + public static Codec getKeyTableCodec(boolean ignorePipeline) { + LOG.debug("OmKeyInfo.getKeyTableCodec ignorePipeline = {}", ignorePipeline); return ignorePipeline ? CODEC_KEY_TABLE_TRUE : CODEC_KEY_TABLE_FALSE; } @@ -796,13 +795,12 @@ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) { * * @param ignorePipeline true for persist to DB, false for network transmit. * @param clientVersion the client version - * @param includeOpenKeyFields whether to include fields. - * Use true for openKeyTable, false for keyTable. + * @param isOpenKey true for openKeyTable, false for keyTable * @return KeyInfo */ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion, - boolean includeOpenKeyFields) { - return getProtobuf(ignorePipeline, null, clientVersion, false, includeOpenKeyFields); + boolean isOpenKey) { + return getProtobuf(ignorePipeline, null, clientVersion, false, isOpenKey); } /** @@ -824,12 +822,12 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, * @param fullKeyName user given key name * @param clientVersion the client version * @param latestVersionBlocks whether to include only latest version blocks - * @param includeOpenKeyFields whether to include fields only used in openKeyTable + * @param isOpenKey true for openKeyTable, false for keyTable * @return key info object */ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, int clientVersion, boolean latestVersionBlocks, - boolean includeOpenKeyFields) { + boolean isOpenKey) { long latestVersion = keyLocationVersions.isEmpty() ? -1 : keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion(); @@ -882,7 +880,7 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, } kb.setIsFile(isFile); - if (includeOpenKeyFields) { + if (isOpenKey) { if (expectedDataGeneration != null) { kb.setExpectedDataGeneration(expectedDataGeneration); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java index 439bac90d52d..cf37681a793e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java @@ -37,7 +37,7 @@ * admin wants to confirm if a given key is deleted from deletedTable metadata. */ public class RepeatedOmKeyInfo implements CopyObject { - // Codecs that include fields only used in openKeyTable (for backward compatibility) + private static final Codec CODEC_TRUE = newCodec(true, true); private static final Codec CODEC_FALSE = newCodec(false, true); @@ -56,15 +56,22 @@ public class RepeatedOmKeyInfo implements CopyObject { */ private final long bucketId; - private static Codec newCodec(boolean ignorePipeline, boolean includeOpenKeyFields) { + private static Codec newCodec(boolean ignorePipeline, boolean isOpenKey) { return new DelegatedCodec<>( Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()), RepeatedOmKeyInfo::getFromProto, - k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, includeOpenKeyFields), + k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey), RepeatedOmKeyInfo.class); } - public static Codec getCodec(boolean ignorePipeline) { + /** + * Gets the codec for openKeyTable. This codec includes fields only used in + * openKeyTable during serialization. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for openKeyTable + */ + public static Codec getOpenKeyTableCodec(boolean ignorePipeline) { return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; } @@ -75,7 +82,7 @@ public static Codec getCodec(boolean ignorePipeline) { * @param ignorePipeline whether to ignore pipeline info * @return the codec for deletedTable */ - public static Codec getCodecForDeletedTable(boolean ignorePipeline) { + public static Codec getDeletedTableCodec(boolean ignorePipeline) { return ignorePipeline ? CODEC_DELETED_TABLE_TRUE : CODEC_DELETED_TABLE_FALSE; } @@ -151,12 +158,12 @@ public RepeatedKeyInfo getProto(boolean compact, int clientVersion) { /** * @param compact true for persistence, false for network transmit * @param clientVersion the client version - * @param includeOpenKeyFields whether to include fields only used in openKeyTable + * @param isOpenKey true for openKeyTable, false for keyTable/deletedTable */ - public RepeatedKeyInfo getProto(boolean compact, int clientVersion, boolean includeOpenKeyFields) { + public RepeatedKeyInfo getProto(boolean compact, int clientVersion, boolean isOpenKey) { List list = new ArrayList<>(); for (OmKeyInfo k : cloneOmKeyInfoList()) { - list.add(k.getProtobuf(compact, clientVersion, includeOpenKeyFields)); + list.add(k.getProtobuf(compact, clientVersion, isOpenKey)); } RepeatedKeyInfo.Builder builder = RepeatedKeyInfo.newBuilder() diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java index b6cec33bbd42..78b17c22cc10 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java @@ -54,7 +54,7 @@ public class TestOmKeyInfoCodec extends Proto2CodecTestBase { @Override public Codec getCodec() { - return OmKeyInfo.getCodec(false); + return OmKeyInfo.getOpenKeyTableCodec(false); } private static FileChecksum createEmptyChecksum() { @@ -143,7 +143,7 @@ public void test() throws IOException { public void testOmKeyInfoCodecWithoutPipeline(int chunkNum) throws IOException { - final Codec codec = OmKeyInfo.getCodec(true); + final Codec codec = OmKeyInfo.getOpenKeyTableCodec(true); OmKeyInfo originKey = getKeyInfo(chunkNum); byte[] rawData = codec.toPersistedFormat(originKey); OmKeyInfo key = codec.fromPersistedFormat(rawData); @@ -156,8 +156,8 @@ public void testOmKeyInfoCodecWithoutPipeline(int chunkNum) } public void testOmKeyInfoCodecCompatibility(int chunkNum) throws IOException { - final Codec codecWithoutPipeline = OmKeyInfo.getCodec(true); - final Codec codecWithPipeline = OmKeyInfo.getCodec(false); + final Codec codecWithoutPipeline = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec codecWithPipeline = OmKeyInfo.getOpenKeyTableCodec(false); OmKeyInfo originKey = getKeyInfo(chunkNum); byte[] rawData = codecWithPipeline.toPersistedFormat(originKey); OmKeyInfo key = codecWithoutPipeline.fromPersistedFormat(rawData); @@ -169,7 +169,7 @@ public void testOmKeyInfoCodecCompatibility(int chunkNum) throws IOException { @Test public void testOpenKeyTableCodecIncludesOpenKeyFields() throws IOException { - final Codec openKeyCodec = OmKeyInfo.getCodec(true); + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); assertEquals(12345L, originKey.getExpectedDataGeneration()); @@ -192,7 +192,7 @@ public void testOpenKeyTableCodecIncludesOpenKeyFields() throws IOException { @Test public void testKeyTableCodecExcludesOpenKeyFields() throws IOException { - final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); assertEquals(12345L, originKey.getExpectedDataGeneration()); assertEquals("test-etag-value", originKey.getExpectedETag()); @@ -218,8 +218,8 @@ public void testKeyTableCodecExcludesOpenKeyFields() throws IOException { @Test public void testKeyTableCodecCanReadOpenKeyTableData() throws IOException { - final Codec openKeyCodec = OmKeyInfo.getCodec(true); - final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); byte[] rawData = openKeyCodec.toPersistedFormat(originKey); @@ -233,8 +233,8 @@ public void testKeyTableCodecCanReadOpenKeyTableData() throws IOException { @Test public void testCodecsWithKeyWithoutOpenKeyFields() throws IOException { - final Codec openKeyCodec = OmKeyInfo.getCodec(true); - final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); OmKeyInfo originKey = getKeyInfo(1); assertNull(originKey.getExpectedDataGeneration()); @@ -256,8 +256,8 @@ public void testCodecsWithKeyWithoutOpenKeyFields() throws IOException { @Test public void testKeyTableCodecProducesSmallerOutput() throws IOException { - final Codec openKeyCodec = OmKeyInfo.getCodec(true); - final Codec keyTableCodec = OmKeyInfo.getCodecForKeyTable(true); + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); OmKeyInfo keyWithOpenFields = getKeyInfoWithOpenKeyFields(1); diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java index e3c44982ef64..810d5bb78760 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java @@ -42,7 +42,7 @@ import org.junit.jupiter.api.Test; /** - * Test {@link RepeatedOmKeyInfo#getCodec(boolean)}. + * Test {@link RepeatedOmKeyInfo#getOpenKeyTableCodec(boolean)}. */ public class TestRepeatedOmKeyInfoCodec extends Proto2CodecTestBase { @@ -53,7 +53,7 @@ public class TestRepeatedOmKeyInfoCodec @Override public Codec getCodec() { - return RepeatedOmKeyInfo.getCodec(true); + return RepeatedOmKeyInfo.getOpenKeyTableCodec(true); } private OmKeyInfo getKeyInfo(int chunkNum) { @@ -96,7 +96,7 @@ void test() throws Exception { } public void testWithoutPipeline(int chunkNum) throws IOException { - final Codec codec = RepeatedOmKeyInfo.getCodec(true); + final Codec codec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); OmKeyInfo originKey = getKeyInfo(chunkNum); long bucketId = Time.now(); RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey, bucketId); @@ -110,9 +110,9 @@ public void testWithoutPipeline(int chunkNum) throws IOException { public void testCompatibility(int chunkNum) throws IOException { final Codec codecWithoutPipeline - = RepeatedOmKeyInfo.getCodec(true); + = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); final Codec codecWithPipeline - = RepeatedOmKeyInfo.getCodec(false); + = RepeatedOmKeyInfo.getOpenKeyTableCodec(false); OmKeyInfo originKey = getKeyInfo(chunkNum); long bucketId = Time.now(); RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey, bucketId); @@ -127,7 +127,7 @@ public void threadSafety() throws InterruptedException { final OmKeyInfo key = getKeyInfo(1); long bucketId = Time.now(); final RepeatedOmKeyInfo subject = new RepeatedOmKeyInfo(key, bucketId); - final Codec codec = RepeatedOmKeyInfo.getCodec(true); + final Codec codec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); final AtomicBoolean failed = new AtomicBoolean(); ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true) .build(); @@ -187,7 +187,7 @@ private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) { @Test void testRegularCodecIncludesOpenKeyFields() throws IOException { - final Codec codec = RepeatedOmKeyInfo.getCodec(true); + final Codec codec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); long bucketId = Time.now(); RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); @@ -203,7 +203,7 @@ void testRegularCodecIncludesOpenKeyFields() throws IOException { @Test void testDeletedTableCodecExcludesOpenKeyFields() throws IOException { - final Codec codec = RepeatedOmKeyInfo.getCodecForDeletedTable(true); + final Codec codec = RepeatedOmKeyInfo.getDeletedTableCodec(true); OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); long bucketId = Time.now(); RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); @@ -222,8 +222,8 @@ void testDeletedTableCodecExcludesOpenKeyFields() throws IOException { @Test void testDeletedTableCodecCanReadRegularCodecData() throws IOException { - final Codec regularCodec = RepeatedOmKeyInfo.getCodec(true); - final Codec deletedTableCodec = RepeatedOmKeyInfo.getCodecForDeletedTable(true); + final Codec regularCodec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); + final Codec deletedTableCodec = RepeatedOmKeyInfo.getDeletedTableCodec(true); OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); long bucketId = Time.now(); @@ -239,8 +239,8 @@ void testDeletedTableCodecCanReadRegularCodecData() throws IOException { @Test void testDeletedTableCodecProducesSmallerOutput() throws IOException { - final Codec regularCodec = RepeatedOmKeyInfo.getCodec(true); - final Codec deletedTableCodec = RepeatedOmKeyInfo.getCodecForDeletedTable(true); + final Codec regularCodec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); + final Codec deletedTableCodec = RepeatedOmKeyInfo.getDeletedTableCodec(true); OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); long bucketId = Time.now(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 6114efef3fa6..2d50efc210a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -208,21 +208,21 @@ public final class OMDBDefinition extends DBDefinition.WithMap { public static final DBColumnFamilyDefinition KEY_TABLE_DEF = new DBColumnFamilyDefinition<>(KEY_TABLE, StringCodec.get(), - OmKeyInfo.getCodecForKeyTable(true)); + OmKeyInfo.getKeyTableCodec(true)); public static final String DELETED_TABLE = "deletedTable"; /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition DELETED_TABLE_DEF = new DBColumnFamilyDefinition<>(DELETED_TABLE, StringCodec.get(), - RepeatedOmKeyInfo.getCodecForDeletedTable(true)); + RepeatedOmKeyInfo.getDeletedTableCodec(true)); public static final String OPEN_KEY_TABLE = "openKeyTable"; /** openKeyTable: /volume/bucket/key/id :- KeyInfo. */ public static final DBColumnFamilyDefinition OPEN_KEY_TABLE_DEF = new DBColumnFamilyDefinition<>(OPEN_KEY_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getOpenKeyTableCodec(true)); public static final String MULTIPART_INFO_TABLE = "multipartInfoTable"; /** multipartInfoTable: /volume/bucket/key/uploadId :- parts. */ @@ -245,14 +245,14 @@ public final class OMDBDefinition extends DBDefinition.WithMap { public static final DBColumnFamilyDefinition FILE_TABLE_DEF = new DBColumnFamilyDefinition<>(FILE_TABLE, StringCodec.get(), - OmKeyInfo.getCodecForKeyTable(true)); + OmKeyInfo.getKeyTableCodec(true)); public static final String OPEN_FILE_TABLE = "openFileTable"; /** openFileTable: /volumeId/bucketId/parentId/fileName/id :- KeyInfo. */ public static final DBColumnFamilyDefinition OPEN_FILE_TABLE_DEF = new DBColumnFamilyDefinition<>(OPEN_FILE_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getOpenKeyTableCodec(true)); public static final String DIRECTORY_TABLE = "directoryTable"; /** directoryTable: /volumeId/bucketId/parentId/dirName :- DirInfo. */ @@ -269,7 +269,7 @@ public final class OMDBDefinition extends DBDefinition.WithMap { public static final DBColumnFamilyDefinition DELETED_DIR_TABLE_DEF = new DBColumnFamilyDefinition<>(DELETED_DIR_TABLE, StringCodec.get(), - OmKeyInfo.getCodecForKeyTable(true)); + OmKeyInfo.getKeyTableCodec(true)); //--------------------------------------------------------------------------- // S3 Multi-Tenancy Tables