HDDS-14555. [Recon] Clarify Open Key Bytes breakdown in Cluster Capacity page

hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java

@@ -214,7 +214,9 @@ private boolean verifyStorageDistributionAfterKeyCreation() {
 
       assertEquals(20, storageResponse.getGlobalNamespace().getTotalKeys());
       assertEquals(60, storageResponse.getGlobalNamespace().getTotalUsedSpace());
-      assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes());
+      assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getTotalOpenKeyBytes());
+      assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getMultipartOpenKeyBytes());
+      assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getOpenKeyAndFileBytes());
       assertEquals(60, storageResponse.getUsedSpaceBreakDown().getCommittedKeyBytes());
       assertEquals(3, storageResponse.getDataNodeUsage().size());
       List<DatanodeStorageReport> reports = storageResponse.getDataNodeUsage();

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java

@@ -39,6 +39,7 @@
 import org.apache.hadoop.ozone.recon.api.types.DatanodeStorageReport;
 import org.apache.hadoop.ozone.recon.api.types.GlobalNamespaceReport;
 import org.apache.hadoop.ozone.recon.api.types.GlobalStorageReport;
+import org.apache.hadoop.ozone.recon.api.types.OpenKeyBytesInfo;
 import org.apache.hadoop.ozone.recon.api.types.StorageCapacityDistributionResponse;
 import org.apache.hadoop.ozone.recon.api.types.UsedSpaceBreakDown;
 import org.apache.hadoop.ozone.recon.scm.ReconNodeManager;
@@ -87,23 +88,23 @@ public Response getStorageDistribution() {
     try {
       List<DatanodeStorageReport> nodeStorageReports = collectDatanodeReports();
       GlobalStorageReport globalStorageReport = calculateGlobalStorageReport();
+      OpenKeyBytesInfo totalOpenKeySize = calculateOpenKeySizes();
 
       Map<String, Long> namespaceMetrics = new HashMap<>();
       try {
-        namespaceMetrics = calculateNamespaceMetrics();
+        namespaceMetrics = calculateNamespaceMetrics(totalOpenKeySize);
       } catch (Exception e) {
         LOG.error("Error calculating namespace metrics", e);
         // Initialize with default values
         namespaceMetrics.put("totalUsedNamespace", 0L);
-        namespaceMetrics.put("totalOpenKeySize", 0L);
         namespaceMetrics.put("totalCommittedSize", 0L);
         namespaceMetrics.put("pendingDirectorySize", 0L);
         namespaceMetrics.put("pendingKeySize", 0L);
         namespaceMetrics.put("totalKeys", 0L);
       }
 
       StorageCapacityDistributionResponse response = buildStorageDistributionResponse(
-              nodeStorageReports, globalStorageReport, namespaceMetrics);
+              nodeStorageReports, globalStorageReport, namespaceMetrics, totalOpenKeySize);
       return Response.ok(response).build();
     } catch (Exception e) {
       LOG.error("Error getting storage distribution", e);
@@ -132,14 +133,14 @@ private GlobalStorageReport calculateGlobalStorageReport() {
     }
   }
 
-  private Map<String, Long> calculateNamespaceMetrics() throws IOException {
+  private Map<String, Long> calculateNamespaceMetrics(OpenKeyBytesInfo totalOpenKeySize) throws IOException {
     Map<String, Long> metrics = new HashMap<>();
     Map<String, Long> totalPendingAtOmSide = reconGlobalMetricsService.calculatePendingSizes();
-    long totalOpenKeySize = calculateOpenKeySizes();
     long totalCommittedSize = calculateCommittedSize();
     long pendingDirectorySize = totalPendingAtOmSide.getOrDefault("pendingDirectorySize", 0L);
     long pendingKeySize = totalPendingAtOmSide.getOrDefault("pendingKeySize", 0L);
-    long totalUsedNamespace = pendingDirectorySize + pendingKeySize + totalOpenKeySize + totalCommittedSize;
+    long totalUsedNamespace = pendingDirectorySize + pendingKeySize +
+        totalOpenKeySize.getTotalOpenKeyBytes() + totalCommittedSize;
 
     long totalKeys = 0L;
     // Keys from OBJECT_STORE buckets.
@@ -154,8 +155,6 @@ private Map<String, Long> calculateNamespaceMetrics() throws IOException {
     if (fileRecord != null) {
       totalKeys += fileRecord.getValue();
     }
-
-    metrics.put("totalOpenKeySize", totalOpenKeySize);
     metrics.put("totalCommittedSize", totalCommittedSize);
     metrics.put("totalUsedNamespace", totalUsedNamespace);
     metrics.put("totalKeys", totalKeys);
@@ -165,11 +164,11 @@ private Map<String, Long> calculateNamespaceMetrics() throws IOException {
   private StorageCapacityDistributionResponse buildStorageDistributionResponse(
           List<DatanodeStorageReport> nodeStorageReports,
           GlobalStorageReport storageMetrics,
-          Map<String, Long> namespaceMetrics) {
+          Map<String, Long> namespaceMetrics,
+          OpenKeyBytesInfo totalOpenKeySize) {
 
     // Safely get values from namespaceMetrics with null checks
     Long totalUsedNamespace = namespaceMetrics.get("totalUsedNamespace");
-    Long totalOpenKeySize = namespaceMetrics.get("totalOpenKeySize");
     Long totalCommittedSize = namespaceMetrics.get("totalCommittedSize");
     Long totalKeys = namespaceMetrics.get("totalKeys");
     Long totalContainerPreAllocated = nodeStorageReports.stream()
@@ -183,8 +182,7 @@ private StorageCapacityDistributionResponse buildStorageDistributionResponse(
                     totalUsedNamespace != null ? totalUsedNamespace : 0L,
                     totalKeys != null ? totalKeys : 0L))
             .setUsedSpaceBreakDown(new UsedSpaceBreakDown(
-                    totalOpenKeySize != null ? totalOpenKeySize : 0L,
-                    totalCommittedSize != null ? totalCommittedSize : 0L, totalContainerPreAllocated))
+                    totalOpenKeySize, totalCommittedSize != null ? totalCommittedSize : 0L, totalContainerPreAllocated))
             .build();
   }
 
@@ -195,12 +193,12 @@ private List<DatanodeStorageReport> collectDatanodeReports() {
         .collect(Collectors.toList());
   }
 
-  private long calculateOpenKeySizes() {
+  private OpenKeyBytesInfo calculateOpenKeySizes() {
     Map<String, Long> openKeySummary = reconGlobalMetricsService.getOpenKeySummary();
     Map<String, Long> openKeyMPUSummary = reconGlobalMetricsService.getMPUKeySummary();
     long openKeyDataSize = openKeySummary.getOrDefault("totalReplicatedDataSize", 0L);
     long totalMPUKeySize = openKeyMPUSummary.getOrDefault("totalReplicatedDataSize", 0L);
-    return openKeyDataSize + totalMPUKeySize;
+    return new OpenKeyBytesInfo(openKeyDataSize, totalMPUKeySize);
   }
 
   private long calculateCommittedSize() {

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OpenKeyBytesInfo.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api.types;
+
+/**
+ * Represents information about the open keys in a storage system.
+ * This class provides details regarding different types of open key bytes
+ * and calculates the total size of open key bytes.
+ */
+public class OpenKeyBytesInfo {
+
+  private long totalOpenKeyBytes;
+  private long openKeyAndFileBytes;
+  private long multipartOpenKeyBytes;
+
+  public OpenKeyBytesInfo(long openKeyAndFileBytes, long multipartOpenKeyBytes) {
+    this.openKeyAndFileBytes = openKeyAndFileBytes;
+    this.multipartOpenKeyBytes = multipartOpenKeyBytes;
+    this.totalOpenKeyBytes = openKeyAndFileBytes + multipartOpenKeyBytes;
+  }
+
+  public long getTotalOpenKeyBytes() {
+    return totalOpenKeyBytes;
+  }
+
+  public long getOpenKeyAndFileBytes() {
+    return openKeyAndFileBytes;
+  }
+
+  public long getMultipartOpenKeyBytes() {
+    return multipartOpenKeyBytes;
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java

@@ -45,7 +45,7 @@
 public class UsedSpaceBreakDown {
 
   @JsonProperty("openKeyBytes")
-  private long openKeyBytes;
+  private OpenKeyBytesInfo openKeyBytes;
 
   @JsonProperty("committedKeyBytes")
   private long committedKeyBytes;
@@ -56,13 +56,13 @@ public class UsedSpaceBreakDown {
   public UsedSpaceBreakDown() {
   }
 
-  public UsedSpaceBreakDown(long openKeyBytes, long committedKeyBytes, long preAllocatedContainerBytes) {
+  public UsedSpaceBreakDown(OpenKeyBytesInfo openKeyBytes, long committedKeyBytes, long preAllocatedContainerBytes) {
     this.openKeyBytes = openKeyBytes;
     this.committedKeyBytes = committedKeyBytes;
     this.preAllocatedContainerBytes = preAllocatedContainerBytes;
   }
 
-  public long getOpenKeyBytes() {
+  public OpenKeyBytesInfo getOpenKeyBytes() {
     return openKeyBytes;
   }
 

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json

@@ -6878,7 +6878,11 @@
         "totalKeys": 1576
     },
     "usedSpaceBreakdown": {
-        "openKeyBytes": 19255266,
+        "openKeyBytes": {
+        "totalOpenKeyBytes": 19255266,
+        "openKeyAndFileBytes": 19255266,
+        "multipartOpenKeyBytes": 0
+        },
         "committedKeyBytes": 1249923,
         "preAllocatedContainerBytes": 1022024
     },

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts

@@ -27,7 +27,11 @@ export const StorageDistribution = {
     totalKeys: 12
   },
   usedSpaceBreakdown: {
-    openKeyBytes: 1024,
+    openKeyBytes: {
+      totalOpenKeyBytes: 1024,
+      openKeyAndFileBytes: 512,
+      multipartOpenKeyBytes: 512
+    },
     committedKeyBytes: 2048,
     preAllocatedContainerBytes: 1024
   },

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx

@@ -29,7 +29,11 @@ export const DEFAULT_CAPACITY_UTILIZATION: UtilizationResponse = {
     totalKeys: 0
   },
   usedSpaceBreakdown: {
-    openKeyBytes: 0,
+    openKeyBytes: {
+      totalOpenKeyBytes : 0,
+      openKeyAndFileBytes: 0,
+      multipartOpenKeyBytes: 0
+    },
     committedKeyBytes: 0,
     preAllocatedContainerBytes: 0
   },

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.less

@@ -142,6 +142,17 @@
   }
 }
 
+.openkeys-space-breakdown {
+  display: grid;
+  grid-template-columns: 150px auto;
+  grid-column-gap: 20px;
+  grid-row-gap: 4px;
+
+  .ant-tag {
+    text-align: center;
+  }
+}
+
 .ant-statistic-title {
   font-size: 12px;
 }

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx

@@ -61,7 +61,12 @@ const Capacity: React.FC<object> = () => {
     CONSTANTS.DEFAULT_CAPACITY_UTILIZATION,
     {
       retryAttempts: 2,
-      onError: (error) => showDataFetchError(error)
+      onError: (error) => showDataFetchError(error),
+      onSuccess: (data) => {
+        console.log('Storage Distribution API Response:', data);
+        console.log('usedSpaceBreakdown:', data.usedSpaceBreakdown);
+        console.log('openKeyBytes:', data.usedSpaceBreakdown?.openKeyBytes);
+      }
     }
   );
 
@@ -218,7 +223,7 @@ const Capacity: React.FC<object> = () => {
       </>
     }>
       <CheckCircleFilled style={{ color: '#1ea57a', marginRight: 8, fontSize: 14 }} />
-        Datanodes  
+        Datanodes
     </Popover>
   );
 
@@ -248,6 +253,26 @@ const Capacity: React.FC<object> = () => {
     </span>
   );
 
+const openKeyUsageBreakdown = (
+    <span>
+      OPEN KEYS
+      <Popover
+        title="Open Key Breakdown"
+        placement='topLeft'
+        content={
+          <div className='openkeys-space-breakdown'>
+            Open Key/File Bytes
+            <Tag color='red'>{filesize(storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.openKeyAndFileBytes ?? 0, {round: 1})}</Tag>
+            Multipart Key Bytes
+            <Tag color='green'>{filesize(storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.multipartOpenKeyBytes ?? 0, {round: 1})}</Tag>
+          </div>
+        }
+      >
+        <InfoCircleOutlined style={{ color: '#2f84d8', fontSize: 12, marginLeft: 2 }} />
+      </Popover>
+    </span>
+   );
+
   return (
     <>
       <div className='page-header-v2'>
@@ -311,8 +336,8 @@ const Capacity: React.FC<object> = () => {
             title: 'TOTAL',
             value: storageDistribution.data.globalStorage.totalUsedSpace
           }, {
-            title: 'OPEN KEYS',
-            value: storageDistribution.data.usedSpaceBreakdown.openKeyBytes,
+            title: openKeyUsageBreakdown,
+            value: storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.totalOpenKeyBytes ?? 0,
             color: '#f47c2d'
           }, {
             title: 'COMMITTED KEYS',

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts

@@ -28,11 +28,17 @@ type GlobalNamespace = {
 };
 
 type UsedSpaceBreakdown = {
-  openKeyBytes: number;
+  openKeyBytes: OpenKeyBytesInfo;
   committedKeyBytes: number;
   preAllocatedContainerBytes: number;
 };
 
+type OpenKeyBytesInfo = {
+  totalOpenKeyBytes: number;
+  openKeyAndFileBytes: number;
+  multipartOpenKeyBytes: number;
+};
+
 type DNPendingDeleteStat = {
   hostName: string;
   datanodeUuid: string;