Skip to content

HDDS-14611. Rename committed bytes to finalized key bytes and align committed space terminology #9871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
priyeshkaratha wants to merge 1 commit into
base: master
Choose a base branch
from
+39 −39
Open

HDDS-14611. Rename committed bytes to finalized key bytes and align committed space terminology #9871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...st-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ private boolean verifyStorageDistributionAfterKeyCreation() {
assertEquals(60, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getTotalOpenKeyBytes());
assertEquals(30, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getMultipartOpenKeyBytes());
assertEquals(30, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getOpenKeyAndFileBytes());
assertEquals(60, storageResponse.getUsedSpaceBreakDown().getCommittedKeyBytes());
assertEquals(60, storageResponse.getUsedSpaceBreakDown().getFinalizedKeyBytes());
assertEquals(3, storageResponse.getDataNodeUsage().size());
List<DatanodeStorageReport> reports = storageResponse.getDataNodeUsage();
List<HddsProtos.DatanodeUsageInfoProto> scmReports =
Expand Down Expand Up @@ -289,7 +289,7 @@ private boolean verifyStorageDistributionAfterKeyCreation() {
}
assertEquals(totalReserved, storageResponse.getGlobalStorage().getTotalReservedSpace());
assertEquals(totalMinFreeSpace, storageResponse.getGlobalStorage().getTotalMinimumFreeSpace());
assertEquals(totalPreAllocated, storageResponse.getGlobalStorage().getTotalOzonePreAllocatedContainerSpace());
assertEquals(totalPreAllocated, storageResponse.getGlobalStorage().getTotalOzoneCommittedSpace());
assertEquals(totalOzoneCapacity, storageResponse.getGlobalStorage().getTotalOzoneCapacity());
assertEquals(totalOzoneUsedSpace, storageResponse.getGlobalStorage().getTotalOzoneUsedSpace());
assertEquals(totalFileSystemCapacity, storageResponse.getGlobalStorage().getTotalFileSystemCapacity());
Expand Down
18 changes: 9 additions & 9 deletions ...ne/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ public Response getStorageDistribution() {
LOG.error("Error calculating namespace metrics", e);
// Initialize with default values
namespaceMetrics.put("totalUsedNamespace", 0L);
namespaceMetrics.put("totalCommittedSize", 0L);
namespaceMetrics.put("totalFinalizedKeyBytes", 0L);
namespaceMetrics.put("pendingDirectorySize", 0L);
namespaceMetrics.put("pendingKeySize", 0L);
namespaceMetrics.put("totalKeys", 0L);
Expand Down Expand Up @@ -235,7 +235,7 @@ private GlobalStorageReport calculateGlobalStorageReport() {
.setTotalReservedSpace(stats.getReserved() != null ? stats.getReserved().get() : 0L)
.setTotalOzoneFreeSpace(stats.getRemaining() != null ? stats.getRemaining().get() : 0L)
.setTotalOzoneUsedSpace(stats.getScmUsed() != null ? stats.getScmUsed().get() : 0L)
.setTotalOzonePreAllocatedContainerSpace(stats.getCommitted() != null ? stats.getCommitted().get() : 0L)
.setTotalOzoneCommittedSpace(stats.getCommitted() != null ? stats.getCommitted().get() : 0L)
.setTotalMinimumFreeSpace(stats.getFreeSpaceToSpare() != null ? stats.getFreeSpaceToSpare().get() : 0L)
.build();

Expand All @@ -248,11 +248,11 @@ private GlobalStorageReport calculateGlobalStorageReport() {
private Map<String, Long> calculateNamespaceMetrics(OpenKeyBytesInfo totalOpenKeySize) throws IOException {
Map<String, Long> metrics = new HashMap<>();
Map<String, Long> totalPendingAtOmSide = reconGlobalMetricsService.calculatePendingSizes();
long totalCommittedSize = calculateCommittedSize();
long totalFinalizedKeyBytes = calculateFinalizedKeyBytesSize();
long pendingDirectorySize = totalPendingAtOmSide.getOrDefault("pendingDirectorySize", 0L);
long pendingKeySize = totalPendingAtOmSide.getOrDefault("pendingKeySize", 0L);
long totalUsedNamespace = pendingDirectorySize + pendingKeySize +
totalOpenKeySize.getTotalOpenKeyBytes() + totalCommittedSize;
totalOpenKeySize.getTotalOpenKeyBytes() + totalFinalizedKeyBytes;

long totalKeys = 0L;
// Keys from OBJECT_STORE buckets.
Expand All @@ -267,7 +267,7 @@ private Map<String, Long> calculateNamespaceMetrics(OpenKeyBytesInfo totalOpenKe
if (fileRecord != null) {
totalKeys += fileRecord.getValue();
}
metrics.put("totalCommittedSize", totalCommittedSize);
metrics.put("totalFinalizedKeyBytes", totalFinalizedKeyBytes);
metrics.put("totalUsedNamespace", totalUsedNamespace);
metrics.put("totalKeys", totalKeys);
return metrics;
Expand All @@ -281,7 +281,7 @@ private StorageCapacityDistributionResponse buildStorageDistributionResponse(

// Safely get values from namespaceMetrics with null checks
Long totalUsedNamespace = namespaceMetrics.get("totalUsedNamespace");
Long totalCommittedSize = namespaceMetrics.get("totalCommittedSize");
Long totalFinalizedKeyBytes = namespaceMetrics.get("totalFinalizedKeyBytes");
Long totalKeys = namespaceMetrics.get("totalKeys");

return StorageCapacityDistributionResponse.newBuilder()
Expand All @@ -291,7 +291,7 @@ private StorageCapacityDistributionResponse buildStorageDistributionResponse(
totalUsedNamespace != null ? totalUsedNamespace : 0L,
totalKeys != null ? totalKeys : 0L))
.setUsedSpaceBreakDown(new UsedSpaceBreakDown(
totalOpenKeySize, totalCommittedSize != null ? totalCommittedSize : 0L))
totalOpenKeySize, totalFinalizedKeyBytes != null ? totalFinalizedKeyBytes : 0L))
.build();
}

Expand All @@ -310,7 +310,7 @@ private OpenKeyBytesInfo calculateOpenKeySizes() {
return new OpenKeyBytesInfo(openKeyDataSize, totalMPUKeySize);
}

private long calculateCommittedSize() {
private long calculateFinalizedKeyBytesSize() {
try {
Response rootResponse = nsSummaryEndpoint.getDiskUsage("/", false, true, false);
if (rootResponse.getStatus() != Response.Status.OK.getStatusCode()) {
Expand All @@ -320,7 +320,7 @@ private long calculateCommittedSize() {
DUResponse duRootRes = (DUResponse) rootResponse.getEntity();
return duRootRes != null ? duRootRes.getSizeWithReplica() : 0L;
} catch (IOException e) {
LOG.error("IOException while calculating committed size", e);
LOG.error("IOException while calculating finalized keys size", e);
return 0L;
}
}
Expand Down
20 changes: 10 additions & 10 deletions ...zone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/GlobalStorageReport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
* <li><b>totalOzoneFreeSpace</b>:
* Remaining allocatable space within Ozone capacity.</li>
*
* <li><b>totalOzonePreAllocatedContainerSpace</b>:
* <li><b>totalOzoneCommittedSpace</b>:
* Space pre-allocated for containers but not yet fully utilized.</li>
*
* <li><b>totalMinimumFreeSpace</b>:
Expand Down Expand Up @@ -92,8 +92,8 @@ public class GlobalStorageReport {
@JsonProperty("totalOzoneFreeSpace")
private long totalOzoneFreeSpace;

@JsonProperty("totalOzonePreAllocatedContainerSpace")
private long totalOzonePreAllocatedContainerSpace;
@JsonProperty("totalOzoneCommittedSpace")
private long totalOzoneCommittedSpace;

@JsonProperty("totalMinimumFreeSpace")
private long totalMinimumFreeSpace;
Expand All @@ -118,8 +118,8 @@ public long getTotalOzoneFreeSpace() {
return totalOzoneFreeSpace;
}

public long getTotalOzonePreAllocatedContainerSpace() {
return totalOzonePreAllocatedContainerSpace;
public long getTotalOzoneCommittedSpace() {
return totalOzoneCommittedSpace;
}

public long getTotalMinimumFreeSpace() {
Expand All @@ -135,7 +135,7 @@ public GlobalStorageReport(Builder builder) {
this.totalOzoneCapacity = builder.totalOzoneCapacity;
this.totalOzoneUsedSpace = builder.totalOzoneUsedSpace;
this.totalOzoneFreeSpace = builder.totalOzoneFreeSpace;
this.totalOzonePreAllocatedContainerSpace = builder.totalOzonePreAllocatedContainerSpace;
this.totalOzoneCommittedSpace = builder.totalOzoneCommittedSpace;
this.totalMinimumFreeSpace = builder.totalMinimumFreeSpace;
}

Expand All @@ -152,15 +152,15 @@ public static final class Builder {
private long totalOzoneCapacity;
private long totalOzoneUsedSpace;
private long totalOzoneFreeSpace;
private long totalOzonePreAllocatedContainerSpace;
private long totalOzoneCommittedSpace;
private long totalMinimumFreeSpace;

public Builder() {
totalReservedSpace = 0;
totalOzoneCapacity = 0;
totalOzoneUsedSpace = 0;
totalOzoneFreeSpace = 0;
totalOzonePreAllocatedContainerSpace = 0;
totalOzoneCommittedSpace = 0;
totalMinimumFreeSpace = 0;
}

Expand All @@ -184,8 +184,8 @@ public Builder setTotalOzoneFreeSpace(long totalOzoneFreeSpace) {
return this;
}

public Builder setTotalOzonePreAllocatedContainerSpace(long totalOzonePreAllocatedContainerSpace) {
this.totalOzonePreAllocatedContainerSpace = totalOzonePreAllocatedContainerSpace;
public Builder setTotalOzoneCommittedSpace(long totalOzoneCommittedSpace) {
this.totalOzoneCommittedSpace = totalOzoneCommittedSpace;
return this;
}

Expand Down
12 changes: 6 additions & 6 deletions ...ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,22 +47,22 @@ public class UsedSpaceBreakDown {
@JsonProperty("openKeyBytes")
private OpenKeyBytesInfo openKeyBytes;

@JsonProperty("committedKeyBytes")
private long committedKeyBytes;
@JsonProperty("finalizedKeyBytes")
private long finalizedKeyBytes;

public UsedSpaceBreakDown() {
}

public UsedSpaceBreakDown(OpenKeyBytesInfo openKeyBytes, long committedKeyBytes) {
public UsedSpaceBreakDown(OpenKeyBytesInfo openKeyBytes, long finalizedKeyBytes) {
this.openKeyBytes = openKeyBytes;
this.committedKeyBytes = committedKeyBytes;
this.finalizedKeyBytes = finalizedKeyBytes;
}

public OpenKeyBytesInfo getOpenKeyBytes() {
return openKeyBytes;
}

public long getCommittedKeyBytes() {
return committedKeyBytes;
public long getFinalizedKeyBytes() {
return finalizedKeyBytes;
}
}
4 changes: 2 additions & 2 deletions hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6874,7 +6874,7 @@
"totalOzoneCapacity": 879519597390,
"totalOzoneFreeSpace": 539776978944,
"totalOzoneUsedSpace": 30679040,
"totalOzonePreAllocatedContainerSpace":1022024
"totalOzoneCommittedSpace":1022024
},
"globalNamespace": {
"totalUsedSpace": 12349932,
Expand All @@ -6886,7 +6886,7 @@
"openKeyAndFileBytes": 19255266,
"multipartOpenKeyBytes": 0
},
"committedKeyBytes": 1249923
"finalizedKeyBytes": 1249923
},
"dataNodeUsage": [
{
Expand Down
4 changes: 2 additions & 2 deletions .../webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export const StorageDistribution = {
totalOzoneFreeSpace: 4096,
totalOzoneCapacity: 10240,
totalReservedSpace: 1024,
totalOzonePreAllocatedContainerSpace: 1024
totalOzoneCommittedSpace: 1024
},
globalNamespace: {
totalUsedSpace: 4096,
Expand All @@ -35,7 +35,7 @@ export const StorageDistribution = {
openKeyAndFileBytes: 512,
multipartOpenKeyBytes: 512
},
committedKeyBytes: 2048
finalizedKeyBytes: 2048
},
dataNodeUsage: [
{
Expand Down
4 changes: 2 additions & 2 deletions .../src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export const DEFAULT_CAPACITY_UTILIZATION: UtilizationResponse = {
totalOzoneCapacity: 0,
totalOzoneFreeSpace: 0,
totalOzoneUsedSpace: 0,
totalOzonePreAllocatedContainerSpace: 0,
totalOzoneCommittedSpace: 0,
totalMinimumFreeSpace: 0
},
globalNamespace: {
Expand All @@ -38,7 +38,7 @@ export const DEFAULT_CAPACITY_UTILIZATION: UtilizationResponse = {
openKeyAndFileBytes: 0,
multipartOpenKeyBytes: 0
},
committedKeyBytes: 0
finalizedKeyBytes: 0
},
dataNodeUsage: []
};
Expand Down
4 changes: 2 additions & 2 deletions ...recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ const Capacity: React.FC<object> = () => {
color: '#11073a'
}, {
title: 'CONTAINER PRE-ALLOCATED',
value: storageDistribution.data.globalStorage.totalOzonePreAllocatedContainerSpace,
value: storageDistribution.data.globalStorage.totalOzoneCommittedSpace,
color: '#f47b2d'
}, {
title: 'REMAINING SPACE',
Expand All @@ -366,7 +366,7 @@ const Capacity: React.FC<object> = () => {
color: '#f47c2d'
}, {
title: 'COMMITTED KEYS',
value: storageDistribution.data.usedSpaceBreakdown.committedKeyBytes,
value: storageDistribution.data.usedSpaceBreakdown.finalizedKeyBytes,
color: '#f4a233'
}, {
title: (
Expand Down
4 changes: 2 additions & 2 deletions ...one/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type GlobalStorage = {
totalOzoneCapacity: number;
totalOzoneUsedSpace: number;
totalOzoneFreeSpace: number;
totalOzonePreAllocatedContainerSpace: number;
totalOzoneCommittedSpace: number;
totalMinimumFreeSpace: number;
};

Expand All @@ -33,7 +33,7 @@ type GlobalNamespace = {

type UsedSpaceBreakdown = {
openKeyBytes: OpenKeyBytesInfo;
committedKeyBytes: number;
finalizedKeyBytes: number;
};

type OpenKeyBytesInfo = {
Expand Down
4 changes: 2 additions & 2 deletions ...econ/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,9 @@ public void testStorageDistributionApiReturnsSuccess() throws Exception {
assertEquals(OPEN_KEY_BYTES + OPEN_MPU_KEY_BYTES,
distributionResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getTotalOpenKeyBytes());
assertEquals(EXPECTED_COMMITTED_KEY_BYTES,
distributionResponse.getUsedSpaceBreakDown().getCommittedKeyBytes());
distributionResponse.getUsedSpaceBreakDown().getFinalizedKeyBytes());
assertEquals(COMMITTED * 3,
distributionResponse.getGlobalStorage().getTotalOzonePreAllocatedContainerSpace());
distributionResponse.getGlobalStorage().getTotalOzoneCommittedSpace());
for (int i = 0; i < 3; i++) {
DatanodeStorageReport report = distributionResponse.getDataNodeUsage().get(i);
assertEquals(OZONE_CAPACITY, report.getCapacity());
Expand Down