diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index da6c46f9b6c0..b118266d7f11 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -59,8 +59,13 @@ public final class OmKeyInfo extends WithParentObjectId implements CopyObject, WithTags { private static final Logger LOG = LoggerFactory.getLogger(OmKeyInfo.class); - private static final Codec CODEC_TRUE = newCodec(true); - private static final Codec CODEC_FALSE = newCodec(false); + // Codecs for openKeyTable + private static final Codec CODEC_TRUE = newCodec(true, true); + private static final Codec CODEC_FALSE = newCodec(false, true); + + // Codecs for keyTable + private static final Codec CODEC_KEY_TABLE_TRUE = newCodec(true, false); + private static final Codec CODEC_KEY_TABLE_FALSE = newCodec(false, false); /** * Metadata key flag to indicate whether a deleted key was a committed key. * The flag is set when a committed key is deleted from AOS but still held in @@ -131,19 +136,46 @@ private OmKeyInfo(Builder b) { this.expectedDataGeneration = b.expectedDataGeneration; } - private static Codec newCodec(boolean ignorePipeline) { + /** + * Creates a new codec for OmKeyInfo. + * + * @param ignorePipeline whether to ignore pipeline info during serialization + * @param isOpenKey true for openKeyTable (includes expectedDataGeneration/expectedETag), + * false for keyTable (excludes these fields) + * @return the codec + */ + private static Codec newCodec(boolean ignorePipeline, boolean isOpenKey) { return new DelegatedCodec<>( Proto2Codec.get(KeyInfo.getDefaultInstance()), OmKeyInfo::getFromProtobuf, - k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION), + k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey), OmKeyInfo.class); } - public static Codec getCodec(boolean ignorePipeline) { - LOG.debug("OmKeyInfo.getCodec ignorePipeline = {}", ignorePipeline); + /** + * Gets the codec for openKeyTable. This codec includes expectedDataGeneration + * and expectedETag fields during serialization. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for openKeyTable + */ + public static Codec getOpenKeyTableCodec(boolean ignorePipeline) { + LOG.debug("OmKeyInfo.getOpenKeyTableCodec ignorePipeline = {}", ignorePipeline); return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; } + /** + * Gets the codec for keyTable. This codec excludes fields are only + * meaningful for open keys. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for keyTable + */ + public static Codec getKeyTableCodec(boolean ignorePipeline) { + LOG.debug("OmKeyInfo.getKeyTableCodec ignorePipeline = {}", ignorePipeline); + return ignorePipeline ? CODEC_KEY_TABLE_TRUE : CODEC_KEY_TABLE_FALSE; + } + public String getVolumeName() { return volumeName; } @@ -738,7 +770,20 @@ public KeyInfo getNetworkProtobuf(String fullKeyName, int clientVersion, * @return KeyInfo */ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) { - return getProtobuf(ignorePipeline, null, clientVersion, false); + return getProtobuf(ignorePipeline, null, clientVersion, false, true); + } + + /** + * Gets KeyInfo for persistence with control over fields only used in openKeyTable. + * + * @param ignorePipeline true for persist to DB, false for network transmit. + * @param clientVersion the client version + * @param isOpenKey true for openKeyTable, false for keyTable + * @return KeyInfo + */ + public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion, + boolean isOpenKey) { + return getProtobuf(ignorePipeline, null, clientVersion, false, isOpenKey); } /** @@ -750,6 +795,22 @@ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) { */ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, int clientVersion, boolean latestVersionBlocks) { + return getProtobuf(ignorePipeline, fullKeyName, clientVersion, latestVersionBlocks, true); + } + + /** + * Gets KeyInfo with all parameters. + * + * @param ignorePipeline ignore pipeline flag + * @param fullKeyName user given key name + * @param clientVersion the client version + * @param latestVersionBlocks whether to include only latest version blocks + * @param isOpenKey true for openKeyTable, false for keyTable + * @return key info object + */ + private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, + int clientVersion, boolean latestVersionBlocks, + boolean isOpenKey) { long latestVersion = keyLocationVersions.isEmpty() ? -1 : keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion(); @@ -801,8 +862,11 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); } kb.setIsFile(isFile); - if (expectedDataGeneration != null) { - kb.setExpectedDataGeneration(expectedDataGeneration); + + if (isOpenKey) { + if (expectedDataGeneration != null) { + kb.setExpectedDataGeneration(expectedDataGeneration); + } } if (ownerName != null) { kb.setOwnerName(ownerName); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java index 0f10832114e3..cf37681a793e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java @@ -37,8 +37,13 @@ * admin wants to confirm if a given key is deleted from deletedTable metadata. */ public class RepeatedOmKeyInfo implements CopyObject { - private static final Codec CODEC_TRUE = newCodec(true); - private static final Codec CODEC_FALSE = newCodec(false); + + private static final Codec CODEC_TRUE = newCodec(true, true); + private static final Codec CODEC_FALSE = newCodec(false, true); + + // Codecs for deletedTable - exclude fields only used in openKeyTable + private static final Codec CODEC_DELETED_TABLE_TRUE = newCodec(true, false); + private static final Codec CODEC_DELETED_TABLE_FALSE = newCodec(false, false); private final List omKeyInfoList; /** @@ -51,18 +56,36 @@ public class RepeatedOmKeyInfo implements CopyObject { */ private final long bucketId; - private static Codec newCodec(boolean ignorePipeline) { + private static Codec newCodec(boolean ignorePipeline, boolean isOpenKey) { return new DelegatedCodec<>( Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()), RepeatedOmKeyInfo::getFromProto, - k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION), + k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey), RepeatedOmKeyInfo.class); } - public static Codec getCodec(boolean ignorePipeline) { + /** + * Gets the codec for openKeyTable. This codec includes fields only used in + * openKeyTable during serialization. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for openKeyTable + */ + public static Codec getOpenKeyTableCodec(boolean ignorePipeline) { return ignorePipeline ? CODEC_TRUE : CODEC_FALSE; } + /** + * Gets the codec for deletedTable. This codec excludes fields only used in + * openKeyTable during serialization, as deleted keys are committed keys. + * + * @param ignorePipeline whether to ignore pipeline info + * @return the codec for deletedTable + */ + public static Codec getDeletedTableCodec(boolean ignorePipeline) { + return ignorePipeline ? CODEC_DELETED_TABLE_TRUE : CODEC_DELETED_TABLE_FALSE; + } + public RepeatedOmKeyInfo(long bucketId) { this.omKeyInfoList = new ArrayList<>(); this.bucketId = bucketId; @@ -129,9 +152,18 @@ public static RepeatedOmKeyInfo getFromProto(RepeatedKeyInfo repeatedKeyInfo) { * @param compact true for persistence, false for network transmit */ public RepeatedKeyInfo getProto(boolean compact, int clientVersion) { + return getProto(compact, clientVersion, true); + } + + /** + * @param compact true for persistence, false for network transmit + * @param clientVersion the client version + * @param isOpenKey true for openKeyTable, false for keyTable/deletedTable + */ + public RepeatedKeyInfo getProto(boolean compact, int clientVersion, boolean isOpenKey) { List list = new ArrayList<>(); for (OmKeyInfo k : cloneOmKeyInfoList()) { - list.add(k.getProtobuf(compact, clientVersion)); + list.add(k.getProtobuf(compact, clientVersion, isOpenKey)); } RepeatedKeyInfo.Builder builder = RepeatedKeyInfo.newBuilder() diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java index a22f9992a1ed..78b17c22cc10 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java @@ -18,8 +18,10 @@ package org.apache.hadoop.ozone.om.helpers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -35,6 +37,7 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.util.Time; import org.junit.jupiter.api.Test; @@ -51,7 +54,7 @@ public class TestOmKeyInfoCodec extends Proto2CodecTestBase { @Override public Codec getCodec() { - return OmKeyInfo.getCodec(false); + return OmKeyInfo.getOpenKeyTableCodec(false); } private static FileChecksum createEmptyChecksum() { @@ -92,6 +95,44 @@ private OmKeyInfo getKeyInfo(int chunkNum) { .build(); } + /** + * Creates an OmKeyInfo with fields only used in openKeyTable set. + * These fields (expectedDataGeneration, expectedETag) are only meaningful + * for keys in openKeyTable. + */ + private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) { + List omKeyLocationInfoList = new ArrayList<>(); + Pipeline pipeline = HddsTestUtils.getRandomPipeline(); + for (int i = 0; i < chunkNum; i++) { + BlockID blockID = new BlockID(i, i); + OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(blockID) + .setPipeline(pipeline) + .build(); + omKeyLocationInfoList.add(keyLocationInfo); + } + OmKeyLocationInfoGroup omKeyLocationInfoGroup = new + OmKeyLocationInfoGroup(0, omKeyLocationInfoList); + + return new OmKeyInfo.Builder() + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setReplicationConfig(RatisReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.THREE)) + .setVolumeName(VOLUME) + .setBucketName(BUCKET) + .setKeyName(KEYNAME) + .setObjectID(Time.now()) + .setUpdateID(Time.now()) + .setDataSize(100) + .setOmKeyLocationInfos( + Collections.singletonList(omKeyLocationInfoGroup)) + .setFileChecksum(checksum) + .setExpectedDataGeneration(12345L) + .setExpectedETag("test-etag-value") + .build(); + } + @Test public void test() throws IOException { testOmKeyInfoCodecWithoutPipeline(1); @@ -102,7 +143,7 @@ public void test() throws IOException { public void testOmKeyInfoCodecWithoutPipeline(int chunkNum) throws IOException { - final Codec codec = OmKeyInfo.getCodec(true); + final Codec codec = OmKeyInfo.getOpenKeyTableCodec(true); OmKeyInfo originKey = getKeyInfo(chunkNum); byte[] rawData = codec.toPersistedFormat(originKey); OmKeyInfo key = codec.fromPersistedFormat(rawData); @@ -115,8 +156,8 @@ public void testOmKeyInfoCodecWithoutPipeline(int chunkNum) } public void testOmKeyInfoCodecCompatibility(int chunkNum) throws IOException { - final Codec codecWithoutPipeline = OmKeyInfo.getCodec(true); - final Codec codecWithPipeline = OmKeyInfo.getCodec(false); + final Codec codecWithoutPipeline = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec codecWithPipeline = OmKeyInfo.getOpenKeyTableCodec(false); OmKeyInfo originKey = getKeyInfo(chunkNum); byte[] rawData = codecWithPipeline.toPersistedFormat(originKey); OmKeyInfo key = codecWithoutPipeline.fromPersistedFormat(rawData); @@ -125,4 +166,105 @@ public void testOmKeyInfoCodecCompatibility(int chunkNum) throws IOException { assertNotNull(key.getLatestVersionLocations().getLocationList().get(0) .getPipeline()); } + + @Test + public void testOpenKeyTableCodecIncludesOpenKeyFields() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); + + assertEquals(12345L, originKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", originKey.getExpectedETag()); + + byte[] rawData = openKeyCodec.toPersistedFormat(originKey); + OmKeyInfo deserializedKey = openKeyCodec.fromPersistedFormat(rawData); + + assertEquals(12345L, deserializedKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", deserializedKey.getExpectedETag()); + + KeyInfo keyInfo = KeyInfo.parseFrom(rawData); + assertTrue(keyInfo.hasExpectedDataGeneration(), + "openKeyTable codec should include expectedDataGeneration in proto"); + assertTrue(keyInfo.hasExpectedETag(), + "openKeyTable codec should include expectedETag in proto"); + assertEquals(12345L, keyInfo.getExpectedDataGeneration()); + assertEquals("test-etag-value", keyInfo.getExpectedETag()); + } + + @Test + public void testKeyTableCodecExcludesOpenKeyFields() throws IOException { + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); + OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); + assertEquals(12345L, originKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", originKey.getExpectedETag()); + + byte[] rawData = keyTableCodec.toPersistedFormat(originKey); + KeyInfo keyInfo = KeyInfo.parseFrom(rawData); + assertFalse(keyInfo.hasExpectedDataGeneration(), + "keyTable codec should NOT include expectedDataGeneration in proto"); + assertFalse(keyInfo.hasExpectedETag(), + "keyTable codec should NOT include expectedETag in proto"); + + OmKeyInfo deserializedKey = keyTableCodec.fromPersistedFormat(rawData); + assertEquals(VOLUME, deserializedKey.getVolumeName()); + assertEquals(BUCKET, deserializedKey.getBucketName()); + assertEquals(KEYNAME, deserializedKey.getKeyName()); + assertEquals(100, deserializedKey.getDataSize()); + + assertNull(deserializedKey.getExpectedDataGeneration(), + "Deserialized key from keyTable should have null expectedDataGeneration"); + assertNull(deserializedKey.getExpectedETag(), + "Deserialized key from keyTable should have null expectedETag"); + } + + @Test + public void testKeyTableCodecCanReadOpenKeyTableData() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); + + OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1); + byte[] rawData = openKeyCodec.toPersistedFormat(originKey); + OmKeyInfo deserializedKey = keyTableCodec.fromPersistedFormat(rawData); + + assertEquals(VOLUME, deserializedKey.getVolumeName()); + assertEquals(BUCKET, deserializedKey.getBucketName()); + assertEquals(12345L, deserializedKey.getExpectedDataGeneration()); + assertEquals("test-etag-value", deserializedKey.getExpectedETag()); + } + + @Test + public void testCodecsWithKeyWithoutOpenKeyFields() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); + + OmKeyInfo originKey = getKeyInfo(1); + assertNull(originKey.getExpectedDataGeneration()); + assertNull(originKey.getExpectedETag()); + + byte[] openKeyData = openKeyCodec.toPersistedFormat(originKey); + byte[] keyTableData = keyTableCodec.toPersistedFormat(originKey); + + OmKeyInfo fromOpenKeyCodec = openKeyCodec.fromPersistedFormat(openKeyData); + OmKeyInfo fromKeyTableCodec = keyTableCodec.fromPersistedFormat(keyTableData); + + assertEquals(VOLUME, fromOpenKeyCodec.getVolumeName()); + assertEquals(VOLUME, fromKeyTableCodec.getVolumeName()); + assertEquals(BUCKET, fromOpenKeyCodec.getBucketName()); + assertEquals(BUCKET, fromKeyTableCodec.getBucketName()); + assertNull(fromOpenKeyCodec.getExpectedDataGeneration()); + assertNull(fromKeyTableCodec.getExpectedDataGeneration()); + } + + @Test + public void testKeyTableCodecProducesSmallerOutput() throws IOException { + final Codec openKeyCodec = OmKeyInfo.getOpenKeyTableCodec(true); + final Codec keyTableCodec = OmKeyInfo.getKeyTableCodec(true); + + OmKeyInfo keyWithOpenFields = getKeyInfoWithOpenKeyFields(1); + + byte[] openKeyData = openKeyCodec.toPersistedFormat(keyWithOpenFields); + byte[] keyTableData = keyTableCodec.toPersistedFormat(keyWithOpenFields); + + assertTrue(keyTableData.length < openKeyData.length, + "keyTable codec should produce smaller serialized output when openKeyTable-only fields are set"); + } } diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java index 3227cfbe6a4b..810d5bb78760 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; @@ -36,11 +37,12 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo; import org.apache.hadoop.util.Time; import org.junit.jupiter.api.Test; /** - * Test {@link RepeatedOmKeyInfo#getCodec(boolean)}. + * Test {@link RepeatedOmKeyInfo#getOpenKeyTableCodec(boolean)}. */ public class TestRepeatedOmKeyInfoCodec extends Proto2CodecTestBase { @@ -51,7 +53,7 @@ public class TestRepeatedOmKeyInfoCodec @Override public Codec getCodec() { - return RepeatedOmKeyInfo.getCodec(true); + return RepeatedOmKeyInfo.getOpenKeyTableCodec(true); } private OmKeyInfo getKeyInfo(int chunkNum) { @@ -94,7 +96,7 @@ void test() throws Exception { } public void testWithoutPipeline(int chunkNum) throws IOException { - final Codec codec = RepeatedOmKeyInfo.getCodec(true); + final Codec codec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); OmKeyInfo originKey = getKeyInfo(chunkNum); long bucketId = Time.now(); RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey, bucketId); @@ -108,9 +110,9 @@ public void testWithoutPipeline(int chunkNum) throws IOException { public void testCompatibility(int chunkNum) throws IOException { final Codec codecWithoutPipeline - = RepeatedOmKeyInfo.getCodec(true); + = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); final Codec codecWithPipeline - = RepeatedOmKeyInfo.getCodec(false); + = RepeatedOmKeyInfo.getOpenKeyTableCodec(false); OmKeyInfo originKey = getKeyInfo(chunkNum); long bucketId = Time.now(); RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey, bucketId); @@ -125,7 +127,7 @@ public void threadSafety() throws InterruptedException { final OmKeyInfo key = getKeyInfo(1); long bucketId = Time.now(); final RepeatedOmKeyInfo subject = new RepeatedOmKeyInfo(key, bucketId); - final Codec codec = RepeatedOmKeyInfo.getCodec(true); + final Codec codec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); final AtomicBoolean failed = new AtomicBoolean(); ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true) .build(); @@ -150,4 +152,104 @@ public void threadSafety() throws InterruptedException { } assertFalse(failed.get()); } + + private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) { + List omKeyLocationInfoList = new ArrayList<>(); + Pipeline pipeline = HddsTestUtils.getRandomPipeline(); + for (int i = 0; i < chunkNum; i++) { + BlockID blockID = new BlockID(i, i); + OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(blockID) + .setPipeline(pipeline) + .build(); + omKeyLocationInfoList.add(keyLocationInfo); + } + OmKeyLocationInfoGroup omKeyLocationInfoGroup = new + OmKeyLocationInfoGroup(0, omKeyLocationInfoList); + return new OmKeyInfo.Builder() + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setReplicationConfig( + RatisReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.THREE)) + .setVolumeName(VOLUME) + .setBucketName(BUCKET) + .setKeyName(KEYNAME) + .setObjectID(Time.now()) + .setUpdateID(Time.now()) + .setDataSize(100) + .setOmKeyLocationInfos( + Collections.singletonList(omKeyLocationInfoGroup)) + .setExpectedDataGeneration(12345L) + .setExpectedETag("test-etag-value") + .build(); + } + + @Test + void testRegularCodecIncludesOpenKeyFields() throws IOException { + final Codec codec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo); + RepeatedKeyInfo proto = RepeatedKeyInfo.parseFrom(rawData); + + assertTrue(proto.getKeyInfo(0).hasExpectedDataGeneration()); + assertTrue(proto.getKeyInfo(0).hasExpectedETag()); + assertEquals(12345L, proto.getKeyInfo(0).getExpectedDataGeneration()); + assertEquals("test-etag-value", proto.getKeyInfo(0).getExpectedETag()); + } + + @Test + void testDeletedTableCodecExcludesOpenKeyFields() throws IOException { + final Codec codec = RepeatedOmKeyInfo.getDeletedTableCodec(true); + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo); + RepeatedKeyInfo proto = RepeatedKeyInfo.parseFrom(rawData); + + assertFalse(proto.getKeyInfo(0).hasExpectedDataGeneration()); + assertFalse(proto.getKeyInfo(0).hasExpectedETag()); + + RepeatedOmKeyInfo deserialized = codec.fromPersistedFormat(rawData); + assertEquals(VOLUME, deserialized.getOmKeyInfoList().get(0).getVolumeName()); + assertEquals(BUCKET, deserialized.getOmKeyInfoList().get(0).getBucketName()); + assertEquals(bucketId, deserialized.getBucketId()); + } + + @Test + void testDeletedTableCodecCanReadRegularCodecData() throws IOException { + final Codec regularCodec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); + final Codec deletedTableCodec = RepeatedOmKeyInfo.getDeletedTableCodec(true); + + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] rawData = regularCodec.toPersistedFormat(repeatedOmKeyInfo); + RepeatedOmKeyInfo deserialized = deletedTableCodec.fromPersistedFormat(rawData); + + assertEquals(VOLUME, deserialized.getOmKeyInfoList().get(0).getVolumeName()); + assertEquals(12345L, deserialized.getOmKeyInfoList().get(0).getExpectedDataGeneration()); + assertEquals("test-etag-value", deserialized.getOmKeyInfoList().get(0).getExpectedETag()); + } + + @Test + void testDeletedTableCodecProducesSmallerOutput() throws IOException { + final Codec regularCodec = RepeatedOmKeyInfo.getOpenKeyTableCodec(true); + final Codec deletedTableCodec = RepeatedOmKeyInfo.getDeletedTableCodec(true); + + OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1); + long bucketId = Time.now(); + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields, bucketId); + + byte[] regularData = regularCodec.toPersistedFormat(repeatedOmKeyInfo); + byte[] deletedTableData = deletedTableCodec.toPersistedFormat(repeatedOmKeyInfo); + + assertTrue(deletedTableData.length < regularData.length, + "deletedTable codec should produce smaller output"); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 323f11926af9..2d50efc210a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -204,25 +204,25 @@ public final class OMDBDefinition extends DBDefinition.WithMap { //--------------------------------------------------------------------------- // Object Store (OBS) Tables: public static final String KEY_TABLE = "keyTable"; - /** keyTable: /volume/bucket/key :- KeyInfo. */ + /** keyTable: /volume/bucket/key :- KeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition KEY_TABLE_DEF = new DBColumnFamilyDefinition<>(KEY_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getKeyTableCodec(true)); public static final String DELETED_TABLE = "deletedTable"; - /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo. */ + /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition DELETED_TABLE_DEF = new DBColumnFamilyDefinition<>(DELETED_TABLE, StringCodec.get(), - RepeatedOmKeyInfo.getCodec(true)); + RepeatedOmKeyInfo.getDeletedTableCodec(true)); public static final String OPEN_KEY_TABLE = "openKeyTable"; /** openKeyTable: /volume/bucket/key/id :- KeyInfo. */ public static final DBColumnFamilyDefinition OPEN_KEY_TABLE_DEF = new DBColumnFamilyDefinition<>(OPEN_KEY_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getOpenKeyTableCodec(true)); public static final String MULTIPART_INFO_TABLE = "multipartInfoTable"; /** multipartInfoTable: /volume/bucket/key/uploadId :- parts. */ @@ -241,18 +241,18 @@ public final class OMDBDefinition extends DBDefinition.WithMap { //--------------------------------------------------------------------------- // File System Optimized (FSO) Tables: public static final String FILE_TABLE = "fileTable"; - /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo. */ + /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo (excludes fields only used in openKeyTable). */ public static final DBColumnFamilyDefinition FILE_TABLE_DEF = new DBColumnFamilyDefinition<>(FILE_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getKeyTableCodec(true)); public static final String OPEN_FILE_TABLE = "openFileTable"; /** openFileTable: /volumeId/bucketId/parentId/fileName/id :- KeyInfo. */ public static final DBColumnFamilyDefinition OPEN_FILE_TABLE_DEF = new DBColumnFamilyDefinition<>(OPEN_FILE_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getOpenKeyTableCodec(true)); public static final String DIRECTORY_TABLE = "directoryTable"; /** directoryTable: /volumeId/bucketId/parentId/dirName :- DirInfo. */ @@ -262,11 +262,14 @@ public final class OMDBDefinition extends DBDefinition.WithMap { OmDirectoryInfo.getCodec()); public static final String DELETED_DIR_TABLE = "deletedDirectoryTable"; - /** deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo. */ + /** + * deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo + * (excludes fields only used in openKeyTable). + */ public static final DBColumnFamilyDefinition DELETED_DIR_TABLE_DEF = new DBColumnFamilyDefinition<>(DELETED_DIR_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + OmKeyInfo.getKeyTableCodec(true)); //--------------------------------------------------------------------------- // S3 Multi-Tenancy Tables diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 752397efc7d2..512c6f80b6e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -303,12 +303,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); - // Optimistic locking validation has passed. Now set the rewrite fields to null so they are - // not persisted in the key table. - // Combination // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() - .setExpectedDataGeneration(null) .addAllMetadata(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())) .setUpdateID(trxnLogIndex) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index 25b5a4b15d41..2ab38865566a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -243,9 +243,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut Map oldKeyVersionsToDeleteMap = null; validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); - // Optimistic locking validation has passed. Now set the rewrite fields to null so they are - // not persisted in the key table. - omKeyInfo.setExpectedDataGeneration(null); long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace shouldn't check and increase. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 41b81471ff96..265d701641a1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -274,7 +274,6 @@ public void testAtomicRewrite() throws Exception { assertEquals(OK, omClientResponse.getOMResponse().getStatus()); OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); - assertNull(committedKey.getExpectedDataGeneration()); // Generation should be changed assertNotEquals(closedKeyInfo.getGeneration(), committedKey.getGeneration()); assertEquals(acls, committedKey.getAcls()); @@ -312,7 +311,6 @@ public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); assertNotNull(committedKey); - assertNull(committedKey.getExpectedDataGeneration()); } @Test