-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfinal.patch
More file actions
120 lines (117 loc) · 6.39 KB
/
final.patch
File metadata and controls
120 lines (117 loc) · 6.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
Subject: [PATCH] test
---
Index: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java (revision 21d30a840ad081696b788bc43eac9bc8b52bc3ff)
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java (date 1763360811269)
@@ -32,6 +32,7 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartAbortInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload;
+import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
@@ -275,13 +276,11 @@
// When abort uploaded key, we need to subtract the PartKey length
// from the volume usedBytes.
long quotaReleased = 0;
- int keyFactor = omMultipartKeyInfo.getReplicationConfig()
- .getRequiredNodes();
- for (PartKeyInfo iterPartKeyInfo : omMultipartKeyInfo.
- getPartKeyInfoMap()) {
- quotaReleased +=
- iterPartKeyInfo.getPartKeyInfo().getDataSize() * keyFactor;
- }
+ for (PartKeyInfo iterPartKeyInfo : omMultipartKeyInfo.getPartKeyInfoMap()) {
+ quotaReleased += QuotaUtil.getReplicatedSize(
+ iterPartKeyInfo.getPartKeyInfo().getDataSize(),
+ omMultipartKeyInfo.getReplicationConfig());
+ }
omBucketInfo.incrUsedBytes(-quotaReleased);
OmMultipartAbortInfo omMultipartAbortInfo =
Index: hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3ExpiredMultipartUploadsAbortRequest.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3ExpiredMultipartUploadsAbortRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3ExpiredMultipartUploadsAbortRequest.java
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3ExpiredMultipartUploadsAbortRequest.java (revision 21d30a840ad081696b788bc43eac9bc8b52bc3ff)
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3ExpiredMultipartUploadsAbortRequest.java (date 1763360876685)
@@ -35,6 +35,7 @@
.Status;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
@@ -310,6 +311,48 @@
metrics.getNumExpiredMPUPartsAborted());
}
+ /**
+ * Verify that aborting expired MPUs does not make bucket usedBytes negative.
+ */
+ @ParameterizedTest
+ @MethodSource("bucketLayouts")
+ public void testAbortExpiredMPUsDoesNotMakeUsedBytesNegative(
+ BucketLayout buckLayout) throws Exception {
+ this.bucketLayout = buckLayout;
+
+ final String volumeName = UUID.randomUUID().toString();
+ final String bucketName = UUID.randomUUID().toString();
+ final String keyName = UUID.randomUUID().toString();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, getBucketLayout());
+
+ final int numMPUs = 1;
+ final int numParts = 3;
+
+ List<String> mpuKeys = createMPUs(volumeName, bucketName, keyName,
+ numMPUs, numParts, getBucketLayout());
+
+ String bucketKey =
+ omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo bucketInfoBefore =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ long usedBytesBefore = bucketInfoBefore.getUsedBytes();
+
+ Assertions.assertTrue(usedBytesBefore > 0,
+ "Expected bucket usedBytes to be > 0 before aborting expired MPUs");
+
+ abortExpiredMPUsFromCache(volumeName, bucketName, mpuKeys);
+ OmBucketInfo bucketInfoAfter =
+ omMetadataManager.getBucketTable().get(bucketKey);
+ long usedBytesAfter = bucketInfoAfter.getUsedBytes();
+
+ Assertions.assertEquals(0L, usedBytesAfter,
+ "Bucket usedBytes should be 0 after aborting expired MPUs");
+ Assertions.assertTrue(usedBytesAfter >= 0,
+ "Bucket usedBytes must not be negative after aborting expired MPUs");
+ }
+
/**
* Constructs a new {@link S3ExpiredMultipartUploadsAbortRequest} objects,
* and calls its {@link S3ExpiredMultipartUploadsAbortRequest#preExecute}
@@ -481,7 +524,7 @@
// Add key to open key table to be used in MPU commit processing
OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo(volume,
bucket, keyName, HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.ONE, parentID + j, parentID,
+ HddsProtos.ReplicationFactor.THREE, parentID + j, parentID,
trxnLogIndex, Time.now(), true);
String fileName = OzoneFSUtils.getFileName(keyName);
OMRequestTestUtils.addFileToKeyTable(true, false,
@@ -563,7 +606,7 @@
OMRequestTestUtils.addKeyToTable(
true, true,
volume, bucket, keyName, clientID, HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.ONE, omMetadataManager);
+ HddsProtos.ReplicationFactor.THREE, omMetadataManager);
OMClientResponse commitResponse =
s3MultipartUploadCommitPartRequest.validateAndUpdateCache(