Skip to content

Commit 48dcfae

Browse files
author
Colm Dougan
committed
HDDS-14004. EventNotification: Capture data to the completed operation ledger table
1 parent 01b2b2a commit 48dcfae

30 files changed

Lines changed: 656 additions & 20 deletions

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,9 @@
4444
import org.apache.hadoop.ozone.om.OMMetadataManager;
4545
import org.apache.hadoop.ozone.om.S3SecretManager;
4646
import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
47+
import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo;
4748
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
49+
import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo;
4850
import org.apache.hadoop.ozone.om.response.OMClientResponse;
4951
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
5052
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -409,6 +411,46 @@ private String addToBatch(Queue<Entry> buffer, BatchOperation batchOperation) {
409411
try {
410412
addToBatchWithTrace(omResponse,
411413
() -> response.checkAndUpdateDB(omMetadataManager, batchOperation));
414+
415+
// This is a strawman approach and requires some discussion
416+
// with the community on approach.
417+
//
418+
// At the moment any response type which we want to capture a
419+
// OmCompletedRequestInfo for is required to implement the
420+
// interface HasCompletedRequestInfo and the method
421+
// getCompletedRequestInfo() and we then have this extra step
422+
// here in the double buffer to capture the rows.
423+
//
424+
// It would seem ideal that the double buffer shouldn't have to
425+
// know/care that there is the concept of capturing this
426+
// OmCompletedRequestInfo row for certain response times but the
427+
// approach described above seemed like the least invasive
428+
// approach overall. I am open to other views.
429+
//
430+
// Other approaches I considered:
431+
// - adding a similar getCompletedRequestInfo method to each
432+
// *request* type which want to capture a row for. The downside
433+
// of this was that since requests are not part of the
434+
// addToBatch flow the OmCompletedRequestInfo instance then had
435+
// to be passed through from the request to the relevant
436+
// response constructors and this created quite a bit of code
437+
// churn which I perceived as messy
438+
//
439+
// * in terms of the actual data capture, rather than this
440+
// "instanceof" approach in this class I toyed with
441+
// having each response type which we want to capture a row for
442+
// capturing it itself in its on implementation of addDBBatch
443+
// (i.e. no need for any new code in this class). This
444+
// seemed to be a bit messier to me in terms of code duplication
445+
//
446+
if (response instanceof HasCompletedRequestInfo) {
447+
OmCompletedRequestInfo completedRequestInfo = ((HasCompletedRequestInfo) response).getCompletedRequestInfo(
448+
entry.getTermIndex().getIndex());
449+
450+
omMetadataManager.getCompletedRequestInfoTable().putWithBatch(
451+
batchOperation, completedRequestInfo.getTrxLogIndex(), completedRequestInfo);
452+
}
453+
412454
} catch (IOException ex) {
413455
// During Adding to RocksDB batch entry got an exception.
414456
// We should terminate the OM.

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
295295
.setOpenVersion(openVersion).build())
296296
.setCmdType(CreateFile);
297297
omClientResponse = new OMFileCreateResponse(omResponse.build(),
298-
omKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject());
298+
omKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject(),
299+
isRecursive, isOverWrite);
299300

300301
result = Result.SUCCESS;
301302
} catch (IOException | InvalidPathException ex) {

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestWithFSO.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
216216
.setCmdType(Type.CreateFile);
217217
omClientResponse = new OMFileCreateResponseWithFSO(omResponse.build(),
218218
omFileInfo, missingParentInfos, clientID,
219-
omBucketInfo.copyObject(), volumeId);
219+
omBucketInfo.copyObject(), volumeId,
220+
isRecursive, isOverWrite);
220221

221222
result = Result.SUCCESS;
222223
} catch (IOException | InvalidPathException ex) {

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,8 @@ private OMClientResponse renameKey(OmKeyInfo toKeyParent, String toKeyName,
342342
OMClientResponse omClientResponse = new OMKeyRenameResponseWithFSO(
343343
omResponse.setRenameKeyResponse(RenameKeyResponse.newBuilder()).build(),
344344
dbFromKey, dbToKey, fromKeyParent, toKeyParent, fromKeyValue,
345-
omBucketInfo, isRenameDirectory, getBucketLayout());
345+
omBucketInfo, isRenameDirectory, getBucketLayout(),
346+
fromKeyName, toKeyName);
346347
return omClientResponse;
347348
}
348349

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/HasCompletedRequestInfo.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.om.response;
19+
20+
import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo;
21+
22+
/**
23+
* Interface to define that a response class provides a
24+
* OmCompletedRequestInfo implementation.
25+
*/
26+
public interface HasCompletedRequestInfo {
27+
28+
OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex);
29+
}

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketCreateResponse.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,19 @@
2626
import org.apache.hadoop.hdds.utils.db.BatchOperation;
2727
import org.apache.hadoop.ozone.om.OMMetadataManager;
2828
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
29+
import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo;
2930
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
3031
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
32+
import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo;
3133
import org.apache.hadoop.ozone.om.response.OMClientResponse;
3234
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
35+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
3336

3437
/**
3538
* Response for CreateBucket request.
3639
*/
3740
@CleanupTableInfo(cleanupTables = {BUCKET_TABLE, VOLUME_TABLE})
38-
public final class OMBucketCreateResponse extends OMClientResponse {
41+
public final class OMBucketCreateResponse extends OMClientResponse implements HasCompletedRequestInfo {
3942

4043
private final OmBucketInfo omBucketInfo;
4144
private final OmVolumeArgs omVolumeArgs;
@@ -87,5 +90,16 @@ public OmBucketInfo getOmBucketInfo() {
8790
return omBucketInfo;
8891
}
8992

93+
@Override
94+
public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) {
95+
return OmCompletedRequestInfo.newBuilder()
96+
.setTrxLogIndex(trxnLogIndex)
97+
.setCmdType(Type.CreateBucket)
98+
.setCreationTime(System.currentTimeMillis())
99+
.setVolumeName(omBucketInfo.getVolumeName())
100+
.setBucketName(omBucketInfo.getBucketName())
101+
.setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs())
102+
.build();
103+
}
90104
}
91105

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,19 @@
2424
import java.io.IOException;
2525
import org.apache.hadoop.hdds.utils.db.BatchOperation;
2626
import org.apache.hadoop.ozone.om.OMMetadataManager;
27+
import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo;
2728
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
2829
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
30+
import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo;
2931
import org.apache.hadoop.ozone.om.response.OMClientResponse;
3032
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
33+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
3134

3235
/**
3336
* Response for DeleteBucket request.
3437
*/
3538
@CleanupTableInfo(cleanupTables = {BUCKET_TABLE, VOLUME_TABLE})
36-
public final class OMBucketDeleteResponse extends OMClientResponse {
39+
public final class OMBucketDeleteResponse extends OMClientResponse implements HasCompletedRequestInfo {
3740

3841
private String volumeName;
3942
private String bucketName;
@@ -89,5 +92,15 @@ public String getBucketName() {
8992
return bucketName;
9093
}
9194

95+
@Override
96+
public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) {
97+
return OmCompletedRequestInfo.newBuilder()
98+
.setTrxLogIndex(trxnLogIndex)
99+
.setCmdType(Type.DeleteBucket)
100+
.setCreationTime(System.currentTimeMillis())
101+
.setVolumeName(volumeName)
102+
.setBucketName(bucketName)
103+
.setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs())
104+
.build();
105+
}
92106
}
93-

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,22 @@
2626
import org.apache.hadoop.ozone.om.OMMetadataManager;
2727
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
2828
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
29+
import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo;
2930
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
3031
import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequest.Result;
3132
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
33+
import org.apache.hadoop.ozone.om.response.HasCompletedRequestInfo;
3234
import org.apache.hadoop.ozone.om.response.key.OmKeyResponse;
3335
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
36+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
3437
import org.slf4j.Logger;
3538
import org.slf4j.LoggerFactory;
3639

3740
/**
3841
* Response for create directory request.
3942
*/
4043
@CleanupTableInfo(cleanupTables = {KEY_TABLE})
41-
public class OMDirectoryCreateResponse extends OmKeyResponse {
44+
public class OMDirectoryCreateResponse extends OmKeyResponse implements HasCompletedRequestInfo {
4245

4346
private static final Logger LOG =
4447
LoggerFactory.getLogger(OMDirectoryCreateResponse.class);
@@ -98,4 +101,17 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager,
98101
LOG.debug("Directory already exists. addToDBBatch is a no-op");
99102
}
100103
}
104+
105+
@Override
106+
public OmCompletedRequestInfo getCompletedRequestInfo(long trxnLogIndex) {
107+
return OmCompletedRequestInfo.newBuilder()
108+
.setTrxLogIndex(trxnLogIndex)
109+
.setCmdType(Type.CreateDirectory)
110+
.setCreationTime(System.currentTimeMillis())
111+
.setVolumeName(dirKeyInfo.getVolumeName())
112+
.setBucketName(dirKeyInfo.getBucketName())
113+
.setKeyName(dirKeyInfo.getKeyName())
114+
.setOpArgs(new OmCompletedRequestInfo.OperationArgs.NoArgs())
115+
.build();
116+
}
101117
}

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,32 @@
2424
import java.util.List;
2525
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
2626
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
27+
import org.apache.hadoop.ozone.om.helpers.OmCompletedRequestInfo;
2728
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
2829
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
2930
import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse;
3031
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
32+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
3133

3234
/**
3335
* Response for crate file request.
3436
*/
3537
@CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE})
3638
public class OMFileCreateResponse extends OMKeyCreateResponse {
3739

40+
private boolean isRecursive;
41+
private boolean isOverWrite;
42+
3843
public OMFileCreateResponse(@Nonnull OMResponse omResponse,
3944
@Nonnull OmKeyInfo omKeyInfo, @Nonnull List<OmKeyInfo> parentKeyInfos,
4045
long openKeySessionID,
41-
@Nonnull OmBucketInfo omBucketInfo) {
46+
@Nonnull OmBucketInfo omBucketInfo,
47+
boolean isRecursive, boolean isOverWrite) {
4248
super(omResponse, omKeyInfo, parentKeyInfos, openKeySessionID,
4349
omBucketInfo);
50+
51+
this.isRecursive = isRecursive;
52+
this.isOverWrite = isOverWrite;
4453
}
4554

4655
/**
@@ -53,4 +62,13 @@ public OMFileCreateResponse(@Nonnull OMResponse omResponse, @Nonnull
5362
checkStatusNotOK();
5463
}
5564

65+
@Override
66+
public Type getOperationType() {
67+
return Type.CreateFile;
68+
}
69+
70+
@Override
71+
public OmCompletedRequestInfo.OperationArgs getCompletedRequestInfoArgs() {
72+
return new OmCompletedRequestInfo.OperationArgs.CreateFileArgs(isRecursive, isOverWrite);
73+
}
5674
}

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseWithFSO.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,29 @@
4242
BUCKET_TABLE})
4343
public class OMFileCreateResponseWithFSO extends OMFileCreateResponse {
4444

45+
private static final boolean DEFAULT_IS_RECURSIVE = false;
46+
private static final boolean DEFAULT_IS_OVERWRITE = false;
47+
4548
private List<OmDirectoryInfo> parentDirInfos;
4649
private long volumeId;
4750

4851
public OMFileCreateResponseWithFSO(@Nonnull OMResponse omResponse,
4952
@Nonnull OmKeyInfo omKeyInfo,
5053
@Nonnull List<OmDirectoryInfo> parentDirInfos, long openKeySessionID,
5154
@Nonnull OmBucketInfo omBucketInfo, @Nonnull long volumeId) {
55+
this(omResponse, omKeyInfo, parentDirInfos, openKeySessionID,
56+
omBucketInfo, volumeId, DEFAULT_IS_RECURSIVE,
57+
DEFAULT_IS_OVERWRITE);
58+
}
59+
60+
@SuppressWarnings("checkstyle:ParameterNumber")
61+
public OMFileCreateResponseWithFSO(@Nonnull OMResponse omResponse,
62+
@Nonnull OmKeyInfo omKeyInfo,
63+
@Nonnull List<OmDirectoryInfo> parentDirInfos, long openKeySessionID,
64+
@Nonnull OmBucketInfo omBucketInfo, @Nonnull long volumeId,
65+
boolean isRecursive, boolean isOverWrite) {
5266
super(omResponse, omKeyInfo, new ArrayList<>(), openKeySessionID,
53-
omBucketInfo);
67+
omBucketInfo, isRecursive, isOverWrite);
5468
this.parentDirInfos = parentDirInfos;
5569
this.volumeId = volumeId;
5670
}

0 commit comments

Comments
 (0)