HDDS-9534. Support namespace summaries (du, dist & counts) for LEGACY buckets with file system disabled (apache#5517)

Author: ArafatKhan2198

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -743,6 +743,47 @@ public static String normalizeKey(String keyName,
     return keyName;
   }
 
+  /**
+   * Normalizes a given path up to the bucket level.
+   *
+   * This method takes a path as input and normalises uptil the bucket level.
+   * It handles empty, removes leading slashes, and splits the path into
+   * segments. It then extracts the volume and bucket names, forming a
+   * normalized path with a single slash. Finally, any remaining segments are
+   * joined as the key name, returning the complete standardized path.
+   *
+   * @param path The path string to be normalized.
+   * @return The normalized path string.
+   */
+  public static String normalizePathUptoBucket(String path) {
+    if (path == null || path.isEmpty()) {
+      return OM_KEY_PREFIX; // Handle empty path
+    }
+
+    // Remove leading slashes
+    path = path.replaceAll("^/*", "");
+
+    String[] segments = path.split(OM_KEY_PREFIX, -1);
+
+    String volumeName = segments[0];
+    String bucketName = segments.length > 1 ? segments[1] : "";
+
+    // Combine volume and bucket.
+    StringBuilder normalizedPath = new StringBuilder(volumeName);
+    if (!bucketName.isEmpty()) {
+      normalizedPath.append(OM_KEY_PREFIX).append(bucketName);
+    }
+
+    // Add remaining segments as the key
+    if (segments.length > 2) {
+      normalizedPath.append(OM_KEY_PREFIX).append(
+          String.join(OM_KEY_PREFIX,
+              Arrays.copyOfRange(segments, 2, segments.length)));
+    }
+
+    return normalizedPath.toString();
+  }
+
 
   /**
    * For a given service ID, return list of configured OM hosts.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java

@@ -17,9 +17,11 @@
  */
 package org.apache.hadoop.ozone.recon.api.handlers;
 
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
 import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -163,6 +165,8 @@ public static BucketHandler getBucketHandler(
                 ReconOMMetadataManager omMetadataManager,
                 OzoneStorageContainerManager reconSCM,
                 OmBucketInfo bucketInfo) throws IOException {
+    // Check if enableFileSystemPaths flag is set to true.
+    boolean enableFileSystemPaths = isEnableFileSystemPaths(omMetadataManager);
 
     // If bucketInfo is null then entity type is UNKNOWN
     if (Objects.isNull(bucketInfo)) {
@@ -172,10 +176,17 @@ public static BucketHandler getBucketHandler(
           .equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
         return new FSOBucketHandler(reconNamespaceSummaryManager,
             omMetadataManager, reconSCM, bucketInfo);
-      } else if (bucketInfo.getBucketLayout()
-          .equals(BucketLayout.LEGACY)) {
-        return new LegacyBucketHandler(reconNamespaceSummaryManager,
-            omMetadataManager, reconSCM, bucketInfo);
+      } else if (bucketInfo.getBucketLayout().equals(BucketLayout.LEGACY)) {
+        // Choose handler based on enableFileSystemPaths flag for legacy layout.
+        // If enableFileSystemPaths is false, then the legacy bucket is treated
+        // as an OBS bucket.
+        if (enableFileSystemPaths) {
+          return new LegacyBucketHandler(reconNamespaceSummaryManager,
+              omMetadataManager, reconSCM, bucketInfo);
+        } else {
+          return new OBSBucketHandler(reconNamespaceSummaryManager,
+              omMetadataManager, reconSCM, bucketInfo);
+        }
       } else if (bucketInfo.getBucketLayout()
           .equals(BucketLayout.OBJECT_STORE)) {
         return new OBSBucketHandler(reconNamespaceSummaryManager,
@@ -188,6 +199,22 @@ public static BucketHandler getBucketHandler(
     }
   }
 
+  /**
+   * Determines whether FileSystemPaths are enabled for Legacy Buckets
+   * based on the Ozone configuration.
+   *
+   * @param ReconOMMetadataManager Instance
+   * @return True if FileSystemPaths are enabled, false otherwise.
+   */
+  private static boolean isEnableFileSystemPaths(ReconOMMetadataManager omMetadataManager) {
+    OzoneConfiguration configuration = omMetadataManager.getOzoneConfiguration();
+    if (configuration == null) {
+      configuration = new OzoneConfiguration();
+    }
+    return configuration.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
+        OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
+  }
+
   public static BucketHandler getBucketHandler(
       ReconNamespaceSummaryManager reconNamespaceSummaryManager,
       ReconOMMetadataManager omMetadataManager,

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
 import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.recon.ReconConstants;
 import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
 import org.apache.hadoop.ozone.recon.api.types.DUResponse;
@@ -60,9 +61,18 @@ public EntityHandler(
     this.omMetadataManager = omMetadataManager;
     this.reconSCM = reconSCM;
     this.bucketHandler = bucketHandler;
-    normalizedPath = normalizePath(path);
-    names = parseRequestPath(normalizedPath);
 
+    // Defaulting to FILE_SYSTEM_OPTIMIZED if bucketHandler is null
+    BucketLayout layout =
+        (bucketHandler != null) ? bucketHandler.getBucketLayout() :
+            BucketLayout.FILE_SYSTEM_OPTIMIZED;
+
+    // Normalize the path based on the determined layout
+    normalizedPath = normalizePath(path, layout);
+
+    // Choose the parsing method based on the bucket layout
+    names = (layout == BucketLayout.OBJECT_STORE) ?
+        parseObjectStorePath(normalizedPath) : parseRequestPath(normalizedPath);
   }
 
   public abstract NamespaceSummaryResponse getSummaryResponse()
@@ -118,7 +128,8 @@ public static EntityHandler getEntityHandler(
           String path) throws IOException {
     BucketHandler bucketHandler;
 
-    String normalizedPath = normalizePath(path);
+    String normalizedPath =
+        normalizePath(path, BucketLayout.FILE_SYSTEM_OPTIMIZED);
     String[] names = parseRequestPath(normalizedPath);
     if (path.equals(OM_KEY_PREFIX)) {
       return EntityType.ROOT.create(reconNamespaceSummaryManager,
@@ -156,23 +167,36 @@ public static EntityHandler getEntityHandler(
       String volName = names[0];
       String bucketName = names[1];
 
-      String keyName = BucketHandler.getKeyName(names);
-
+      // Assuming getBucketHandler already validates volume and bucket existence
       bucketHandler = BucketHandler.getBucketHandler(
-              reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM,
-              volName, bucketName);
+          reconNamespaceSummaryManager, omMetadataManager, reconSCM, volName,
+          bucketName);
 
-      // check if either volume or bucket doesn't exist
-      if (bucketHandler == null
-          || !omMetadataManager.volumeExists(volName)
-          || !bucketHandler.bucketExists(volName, bucketName)) {
+      if (bucketHandler == null) {
         return EntityType.UNKNOWN.create(reconNamespaceSummaryManager,
-                omMetadataManager, reconSCM, null, path);
+            omMetadataManager, reconSCM, null, path);
+      }
+
+      // Directly handle path normalization and parsing based on the layout
+      if (bucketHandler.getBucketLayout() == BucketLayout.OBJECT_STORE) {
+        String[] parsedObjectLayoutPath = parseObjectStorePath(
+            normalizePath(path, bucketHandler.getBucketLayout()));
+        if (parsedObjectLayoutPath == null) {
+          return EntityType.UNKNOWN.create(reconNamespaceSummaryManager,
+              omMetadataManager, reconSCM, null, path);
+        }
+        // Use the key part directly from the parsed path
+        return bucketHandler.determineKeyPath(parsedObjectLayoutPath[2])
+            .create(reconNamespaceSummaryManager, omMetadataManager, reconSCM,
+                bucketHandler, path);
+      } else {
+        // Use the existing names array for non-OBJECT_STORE layouts to derive
+        // the keyName
+        String keyName = BucketHandler.getKeyName(names);
+        return bucketHandler.determineKeyPath(keyName)
+            .create(reconNamespaceSummaryManager, omMetadataManager, reconSCM,
+                bucketHandler, path);
       }
-      return bucketHandler.determineKeyPath(keyName)
-          .create(reconNamespaceSummaryManager,
-          omMetadataManager, reconSCM, bucketHandler, path);
     }
   }
 
@@ -256,7 +280,52 @@ public static String[] parseRequestPath(String path) {
     return names;
   }
 
-  private static String normalizePath(String path) {
+  /**
+   * Splits an object store path into volume, bucket, and key name components.
+   *
+   * This method parses a path of the format "/volumeName/bucketName/keyName",
+   * including paths with additional '/' characters within the key name. It's
+   * designed for object store paths where the first three '/' characters
+   * separate the root, volume and bucket names from the key name.
+   *
+   * @param path The object store path to parse, starting with a slash.
+   * @return A String array with three elements: volume name, bucket name, and
+   * key name, or {null} if the path format is invalid.
+   */
+  public static String[] parseObjectStorePath(String path) {
+    // Removing the leading slash for correct splitting
+    path = path.substring(1);
+
+    // Splitting the modified path by "/", limiting to 3 parts
+    String[] parts = path.split("/", 3);
+
+    // Checking if we correctly obtained 3 parts after removing the leading slash
+    if (parts.length <= 3) {
+      return parts;
+    } else {
+      return null;
+    }
+  }
+
+  /**
+   * Normalizes a given path based on the specified bucket layout.
+   *
+   * This method adjusts the path according to the bucket layout.
+   * For {OBJECT_STORE Layout}, it normalizes the path up to the bucket level
+   * using OmUtils.normalizePathUptoBucket. For other layouts, it
+   * normalizes the entire path, including the key, using
+   * OmUtils.normalizeKey, and does not preserve any trailing slashes.
+   * The normalized path will always be prefixed with OM_KEY_PREFIX to ensure it
+   * is consistent with the expected format for object storage paths in Ozone.
+   *
+   * @param path
+   * @param bucketLayout
+   * @return A normalized path
+   */
+  private static String normalizePath(String path, BucketLayout bucketLayout) {
+    if (bucketLayout == BucketLayout.OBJECT_STORE) {
+      return OM_KEY_PREFIX + OmUtils.normalizePathUptoBucket(path);
+    }
     return OM_KEY_PREFIX + OmUtils.normalizeKey(path, false);
   }
 }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOMMetadataManager.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
@@ -105,4 +106,11 @@ List<OmBucketInfo> listBucketsUnderVolume(String volumeName,
    */
   List<OmBucketInfo> listBucketsUnderVolume(
       String volumeName) throws IOException;
+
+  /**
+   * Return the OzoneConfiguration instance used by Recon.
+   * @return
+   */
+  OzoneConfiguration getOzoneConfiguration();
+
 }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java

@@ -291,6 +291,11 @@ public List<OmBucketInfo> listBucketsUnderVolume(final String volumeName)
         Integer.MAX_VALUE);
   }
 
+  @Override
+  public OzoneConfiguration getOzoneConfiguration() {
+    return ozoneConfiguration;
+  }
+
   private List<OmBucketInfo> listAllBuckets(final int maxNumberOfBuckets)
       throws IOException {
     List<OmBucketInfo> result = new ArrayList<>();

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java

@@ -63,7 +63,7 @@
  */
 public class NSSummaryTask implements ReconOmTask {
   private static final Logger LOG =
-          LoggerFactory.getLogger(NSSummaryTask.class);
+      LoggerFactory.getLogger(NSSummaryTask.class);
 
   private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
   private final ReconOMMetadataManager reconOMMetadataManager;
@@ -173,4 +173,3 @@ public Pair<String, Boolean> reprocess(OMMetadataManager omMetadataManager) {
   }
 
 }
-