From cc88f807990a2c4233a2cf14a044a53111903e61 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Tue, 3 Feb 2026 22:27:42 +0530 Subject: [PATCH] HDDS-14555. Clarify Open Key Bytes breakdown in Cluster Capacity page tooltip --- .../TestStorageDistributionEndpoint.java | 4 +- .../api/StorageDistributionEndpoint.java | 26 +++++----- .../recon/api/types/OpenKeyBytesInfo.java | 48 +++++++++++++++++++ .../recon/api/types/UsedSpaceBreakDown.java | 6 +-- .../webapps/recon/ozone-recon-web/api/db.json | 6 ++- .../capacityMocks/capacityResponseMocks.ts | 6 ++- .../src/v2/constants/capacity.constants.tsx | 6 ++- .../src/v2/pages/capacity/capacity.less | 11 +++++ .../src/v2/pages/capacity/capacity.tsx | 33 +++++++++++-- .../src/v2/types/capacity.types.ts | 8 +++- 10 files changed, 128 insertions(+), 26 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OpenKeyBytesInfo.java diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java index ea6894831a72..92ab44da9508 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java @@ -214,7 +214,9 @@ private boolean verifyStorageDistributionAfterKeyCreation() { assertEquals(20, storageResponse.getGlobalNamespace().getTotalKeys()); assertEquals(60, storageResponse.getGlobalNamespace().getTotalUsedSpace()); - assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes()); + assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getTotalOpenKeyBytes()); + assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getMultipartOpenKeyBytes()); + assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getOpenKeyAndFileBytes()); assertEquals(60, storageResponse.getUsedSpaceBreakDown().getCommittedKeyBytes()); assertEquals(3, storageResponse.getDataNodeUsage().size()); List reports = storageResponse.getDataNodeUsage(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java index 96cf36fff31a..1b26fcb5e9dd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.recon.api.types.DatanodeStorageReport; import org.apache.hadoop.ozone.recon.api.types.GlobalNamespaceReport; import org.apache.hadoop.ozone.recon.api.types.GlobalStorageReport; +import org.apache.hadoop.ozone.recon.api.types.OpenKeyBytesInfo; import org.apache.hadoop.ozone.recon.api.types.StorageCapacityDistributionResponse; import org.apache.hadoop.ozone.recon.api.types.UsedSpaceBreakDown; import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; @@ -87,15 +88,15 @@ public Response getStorageDistribution() { try { List nodeStorageReports = collectDatanodeReports(); GlobalStorageReport globalStorageReport = calculateGlobalStorageReport(); + OpenKeyBytesInfo totalOpenKeySize = calculateOpenKeySizes(); Map namespaceMetrics = new HashMap<>(); try { - namespaceMetrics = calculateNamespaceMetrics(); + namespaceMetrics = calculateNamespaceMetrics(totalOpenKeySize); } catch (Exception e) { LOG.error("Error calculating namespace metrics", e); // Initialize with default values namespaceMetrics.put("totalUsedNamespace", 0L); - namespaceMetrics.put("totalOpenKeySize", 0L); namespaceMetrics.put("totalCommittedSize", 0L); namespaceMetrics.put("pendingDirectorySize", 0L); namespaceMetrics.put("pendingKeySize", 0L); @@ -103,7 +104,7 @@ public Response getStorageDistribution() { } StorageCapacityDistributionResponse response = buildStorageDistributionResponse( - nodeStorageReports, globalStorageReport, namespaceMetrics); + nodeStorageReports, globalStorageReport, namespaceMetrics, totalOpenKeySize); return Response.ok(response).build(); } catch (Exception e) { LOG.error("Error getting storage distribution", e); @@ -132,14 +133,14 @@ private GlobalStorageReport calculateGlobalStorageReport() { } } - private Map calculateNamespaceMetrics() throws IOException { + private Map calculateNamespaceMetrics(OpenKeyBytesInfo totalOpenKeySize) throws IOException { Map metrics = new HashMap<>(); Map totalPendingAtOmSide = reconGlobalMetricsService.calculatePendingSizes(); - long totalOpenKeySize = calculateOpenKeySizes(); long totalCommittedSize = calculateCommittedSize(); long pendingDirectorySize = totalPendingAtOmSide.getOrDefault("pendingDirectorySize", 0L); long pendingKeySize = totalPendingAtOmSide.getOrDefault("pendingKeySize", 0L); - long totalUsedNamespace = pendingDirectorySize + pendingKeySize + totalOpenKeySize + totalCommittedSize; + long totalUsedNamespace = pendingDirectorySize + pendingKeySize + + totalOpenKeySize.getTotalOpenKeyBytes() + totalCommittedSize; long totalKeys = 0L; // Keys from OBJECT_STORE buckets. @@ -154,8 +155,6 @@ private Map calculateNamespaceMetrics() throws IOException { if (fileRecord != null) { totalKeys += fileRecord.getValue(); } - - metrics.put("totalOpenKeySize", totalOpenKeySize); metrics.put("totalCommittedSize", totalCommittedSize); metrics.put("totalUsedNamespace", totalUsedNamespace); metrics.put("totalKeys", totalKeys); @@ -165,11 +164,11 @@ private Map calculateNamespaceMetrics() throws IOException { private StorageCapacityDistributionResponse buildStorageDistributionResponse( List nodeStorageReports, GlobalStorageReport storageMetrics, - Map namespaceMetrics) { + Map namespaceMetrics, + OpenKeyBytesInfo totalOpenKeySize) { // Safely get values from namespaceMetrics with null checks Long totalUsedNamespace = namespaceMetrics.get("totalUsedNamespace"); - Long totalOpenKeySize = namespaceMetrics.get("totalOpenKeySize"); Long totalCommittedSize = namespaceMetrics.get("totalCommittedSize"); Long totalKeys = namespaceMetrics.get("totalKeys"); Long totalContainerPreAllocated = nodeStorageReports.stream() @@ -183,8 +182,7 @@ private StorageCapacityDistributionResponse buildStorageDistributionResponse( totalUsedNamespace != null ? totalUsedNamespace : 0L, totalKeys != null ? totalKeys : 0L)) .setUsedSpaceBreakDown(new UsedSpaceBreakDown( - totalOpenKeySize != null ? totalOpenKeySize : 0L, - totalCommittedSize != null ? totalCommittedSize : 0L, totalContainerPreAllocated)) + totalOpenKeySize, totalCommittedSize != null ? totalCommittedSize : 0L, totalContainerPreAllocated)) .build(); } @@ -195,12 +193,12 @@ private List collectDatanodeReports() { .collect(Collectors.toList()); } - private long calculateOpenKeySizes() { + private OpenKeyBytesInfo calculateOpenKeySizes() { Map openKeySummary = reconGlobalMetricsService.getOpenKeySummary(); Map openKeyMPUSummary = reconGlobalMetricsService.getMPUKeySummary(); long openKeyDataSize = openKeySummary.getOrDefault("totalReplicatedDataSize", 0L); long totalMPUKeySize = openKeyMPUSummary.getOrDefault("totalReplicatedDataSize", 0L); - return openKeyDataSize + totalMPUKeySize; + return new OpenKeyBytesInfo(openKeyDataSize, totalMPUKeySize); } private long calculateCommittedSize() { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OpenKeyBytesInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OpenKeyBytesInfo.java new file mode 100644 index 000000000000..329bf7018a0c --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OpenKeyBytesInfo.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +/** + * Represents information about the open keys in a storage system. + * This class provides details regarding different types of open key bytes + * and calculates the total size of open key bytes. + */ +public class OpenKeyBytesInfo { + + private long totalOpenKeyBytes; + private long openKeyAndFileBytes; + private long multipartOpenKeyBytes; + + public OpenKeyBytesInfo(long openKeyAndFileBytes, long multipartOpenKeyBytes) { + this.openKeyAndFileBytes = openKeyAndFileBytes; + this.multipartOpenKeyBytes = multipartOpenKeyBytes; + this.totalOpenKeyBytes = openKeyAndFileBytes + multipartOpenKeyBytes; + } + + public long getTotalOpenKeyBytes() { + return totalOpenKeyBytes; + } + + public long getOpenKeyAndFileBytes() { + return openKeyAndFileBytes; + } + + public long getMultipartOpenKeyBytes() { + return multipartOpenKeyBytes; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java index 8d82d89058ad..c7c6d2aa7f87 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java @@ -45,7 +45,7 @@ public class UsedSpaceBreakDown { @JsonProperty("openKeyBytes") - private long openKeyBytes; + private OpenKeyBytesInfo openKeyBytes; @JsonProperty("committedKeyBytes") private long committedKeyBytes; @@ -56,13 +56,13 @@ public class UsedSpaceBreakDown { public UsedSpaceBreakDown() { } - public UsedSpaceBreakDown(long openKeyBytes, long committedKeyBytes, long preAllocatedContainerBytes) { + public UsedSpaceBreakDown(OpenKeyBytesInfo openKeyBytes, long committedKeyBytes, long preAllocatedContainerBytes) { this.openKeyBytes = openKeyBytes; this.committedKeyBytes = committedKeyBytes; this.preAllocatedContainerBytes = preAllocatedContainerBytes; } - public long getOpenKeyBytes() { + public OpenKeyBytesInfo getOpenKeyBytes() { return openKeyBytes; } diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json index 37663e3704e3..c7cb60792865 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json @@ -6878,7 +6878,11 @@ "totalKeys": 1576 }, "usedSpaceBreakdown": { - "openKeyBytes": 19255266, + "openKeyBytes": { + "totalOpenKeyBytes": 19255266, + "openKeyAndFileBytes": 19255266, + "multipartOpenKeyBytes": 0 + }, "committedKeyBytes": 1249923, "preAllocatedContainerBytes": 1022024 }, diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts index 970521e962d5..2459fdbd69ec 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts @@ -27,7 +27,11 @@ export const StorageDistribution = { totalKeys: 12 }, usedSpaceBreakdown: { - openKeyBytes: 1024, + openKeyBytes: { + totalOpenKeyBytes: 1024, + openKeyAndFileBytes: 512, + multipartOpenKeyBytes: 512 + }, committedKeyBytes: 2048, preAllocatedContainerBytes: 1024 }, diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx index 1699d7ffeb40..3f51c0879c50 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx @@ -29,7 +29,11 @@ export const DEFAULT_CAPACITY_UTILIZATION: UtilizationResponse = { totalKeys: 0 }, usedSpaceBreakdown: { - openKeyBytes: 0, + openKeyBytes: { + totalOpenKeyBytes : 0, + openKeyAndFileBytes: 0, + multipartOpenKeyBytes: 0 + }, committedKeyBytes: 0, preAllocatedContainerBytes: 0 }, diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.less index e8fa61cf3638..2d3ec71ed5ad 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.less +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.less @@ -142,6 +142,17 @@ } } +.openkeys-space-breakdown { + display: grid; + grid-template-columns: 150px auto; + grid-column-gap: 20px; + grid-row-gap: 4px; + + .ant-tag { + text-align: center; + } +} + .ant-statistic-title { font-size: 12px; } diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx index ec9ad436e592..96d10a076620 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx @@ -61,7 +61,12 @@ const Capacity: React.FC = () => { CONSTANTS.DEFAULT_CAPACITY_UTILIZATION, { retryAttempts: 2, - onError: (error) => showDataFetchError(error) + onError: (error) => showDataFetchError(error), + onSuccess: (data) => { + console.log('Storage Distribution API Response:', data); + console.log('usedSpaceBreakdown:', data.usedSpaceBreakdown); + console.log('openKeyBytes:', data.usedSpaceBreakdown?.openKeyBytes); + } } ); @@ -218,7 +223,7 @@ const Capacity: React.FC = () => { }> - Datanodes + Datanodes ); @@ -248,6 +253,26 @@ const Capacity: React.FC = () => { ); +const openKeyUsageBreakdown = ( + + OPEN KEYS + + Open Key/File Bytes + {filesize(storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.openKeyAndFileBytes ?? 0, {round: 1})} + Multipart Key Bytes + {filesize(storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.multipartOpenKeyBytes ?? 0, {round: 1})} + + } + > + + + + ); + return ( <>
@@ -311,8 +336,8 @@ const Capacity: React.FC = () => { title: 'TOTAL', value: storageDistribution.data.globalStorage.totalUsedSpace }, { - title: 'OPEN KEYS', - value: storageDistribution.data.usedSpaceBreakdown.openKeyBytes, + title: openKeyUsageBreakdown, + value: storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.totalOpenKeyBytes ?? 0, color: '#f47c2d' }, { title: 'COMMITTED KEYS', diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts index 5257c17b8196..b399b36e8b8c 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts @@ -28,11 +28,17 @@ type GlobalNamespace = { }; type UsedSpaceBreakdown = { - openKeyBytes: number; + openKeyBytes: OpenKeyBytesInfo; committedKeyBytes: number; preAllocatedContainerBytes: number; }; +type OpenKeyBytesInfo = { + totalOpenKeyBytes: number; + openKeyAndFileBytes: number; + multipartOpenKeyBytes: number; +}; + type DNPendingDeleteStat = { hostName: string; datanodeUuid: string;