From a08f9751d662d2fbdaa7ad8303ae29514f2e2c9e Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 3 Mar 2025 16:28:35 -0800 Subject: [PATCH 1/7] compact_on_deletion WIP Change-Id: I5f14a8fafa6a66c2f8530ce1592bd8c9daf10f73 WIP Change-Id: Icffb8f02d38836d45a6af4a620754c0619333b9e WIP Change-Id: I68774824a062ced73257d4769cf9f53fe6f1cfa8 Checkstyle Change-Id: If90067ce9bc046a2a87308a0ebaae1590ebaee9d Fix counter bug Change-Id: I20a70336a8b4063e2fb6c727e0d624419536841b Compact DeletedTable as well. Change-Id: I1ca99a7f687e137581f7a161bf14ef2e84670acb Fix tests Change-Id: Ia03024fc364be05c031e4d3dc3fb7eb230a53b08 Added test in TestOMKeyPurgeRequestAndResponse. Change-Id: I2cfe621d938e9ff74622c3952d9dd939b2c0763a Fix test Change-Id: I714cc6f3270776eb13f3ce2b6cba0aba012a47a8 Checkstyle Change-Id: I497f44abc2ef39547d5601f52303f8b47ef3863c Fix test Change-Id: I65a48d777b2ea0fe590aa2934a99351a6bd920cb Close ManagedCompactRangeOptions properly Change-Id: I14e7ce2eb4765cc2cfbf2e1caeb8910ce601b590 Shorten compaction threshold and interval Change-Id: I7be18c12a23469da6362be16c1938489fe9f597c Refactor, move counters to CompactionService. Change-Id: I80fba6fe68b5d9f918e9da05eb2d10e7990d5741 (cherry picked from commit 823109590b4c6120b71e4dae93b5243eed559b2a) Conflicts: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java Change-Id: I0c7f07bf9b893b9bac2bf6b918d32214f7dab1a2 Update Change-Id: Ie58d0b44f0dfef8a6555d3c286104239d938d24a --- .../src/main/resources/ozone-default.xml | 30 +++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 14 ++ .../apache/hadoop/ozone/om/KeyManager.java | 7 + .../hadoop/ozone/om/KeyManagerImpl.java | 38 ++++ .../OMDirectoriesPurgeResponseWithFSO.java | 12 +- .../om/response/key/OMKeyPurgeResponse.java | 11 +- .../ozone/om/service/CompactionService.java | 201 ++++++++++++++++++ ...tOMDirectoriesPurgeRequestAndResponse.java | 10 +- .../key/TestOMKeyPurgeRequestAndResponse.java | 11 +- .../om/service/TestCompactionService.java | 109 ++++++++++ 10 files changed, 434 insertions(+), 9 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 502a5a9ccc70..c067512b74fb 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1518,6 +1518,36 @@ to the OM. + + + ozone.compaction.service.enabled + true + OZONE, OM, PERFORMANCE + + Enable or disable a background job that periodically compacts rocksdb tables flagged for compaction. + + + + ozone.om.compaction.service.run.interval + 5m + OZONE, OM, PERFORMANCE + + A background job that periodically compacts rocksdb tables flagged for compaction. + Unit could be defined with postfix (ns,ms,s,m,h,d) + + + + + ozone.om.compaction.service.timeout + 10m + OZONE, OM, PERFORMANCE + A timeout value of compaction service. If this is set + greater than 0, the service will stop waiting for compaction + completion after this time. + Unit could be defined with postfix (ns,ms,s,m,h,d) + + + hdds.rest.http-address 0.0.0.0:9880 diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 0a1fd9f681c5..fb9d2735ecaf 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -624,4 +624,18 @@ private OMConfigKeys() { */ public static final String OZONE_OM_SERVER_LIST_MAX_SIZE = "ozone.om.server.list.max.size"; public static final int OZONE_OM_SERVER_LIST_MAX_SIZE_DEFAULT = 1000; + /** + * Configuration properties for Compaction Service. + */ + public static final String OZONE_COMPACTION_SERVICE_ENABLED = "ozone.compaction.service.enabled"; + public static final boolean OZONE_COMPACTION_SERVICE_ENABLED_DEFAULT = true; + public static final String OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL = + "ozone.om.compaction.service.run.interval"; + public static final long + OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL_DEFAULT + = TimeUnit.MINUTES.toMillis(5); + + public static final String OZONE_OM_COMPACTION_SERVICE_TIMEOUT + = "ozone.om.compaction.service.timeout"; + public static final String OZONE_COMPACTION_SERVICE_TIMEOUT_DEFAULT = "10m"; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index db3d47dfcdb9..cce6a6d90b7a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; @@ -329,4 +330,10 @@ DeleteKeysResult getPendingDeletionSubFiles(long volumeId, * @return Background service. */ SnapshotDirectoryCleaningService getSnapshotDirectoryService(); + + /** + * Returns the instance of Compaction service. + * @return Background service. + */ + CompactionService getCompactionService(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 735dfca55900..54ef3f2b6336 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -97,6 +97,7 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; +import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.MultipartUploadCleanupService; @@ -135,8 +136,14 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_ENABLED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_CLEANUP_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_CLEANUP_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_CLEANUP_SERVICE_TIMEOUT; @@ -200,6 +207,7 @@ public class KeyManagerImpl implements KeyManager { private BackgroundService multipartUploadCleanupService; private SnapshotDirectoryCleaningService snapshotDirectoryCleaningService; private DNSToSwitchMapping dnsToSwitchMapping; + private CompactionService compactionService; public KeyManagerImpl(OzoneManager om, ScmClient scmClient, OzoneConfiguration conf, OMPerformanceMetrics metrics) { @@ -232,6 +240,9 @@ public KeyManagerImpl(OzoneManager om, ScmClient scmClient, @Override public void start(OzoneConfiguration configuration) { + boolean isCompactionServiceEnabled = configuration.getBoolean(OZONE_COMPACTION_SERVICE_ENABLED, + OZONE_COMPACTION_SERVICE_ENABLED_DEFAULT); + startCompactionService(configuration, isCompactionServiceEnabled); boolean isSnapshotDeepCleaningEnabled = configuration.getBoolean(OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED, OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT); if (keyDeletingService == null) { @@ -368,6 +379,24 @@ public void start(OzoneConfiguration configuration) { : new CachedDNSToSwitchMapping(newInstance)); } + private void startCompactionService(OzoneConfiguration configuration, + boolean isCompactionServiceEnabled) { + if (compactionService == null && isCompactionServiceEnabled) { + long compactionInterval = configuration.getTimeDuration( + OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL, + OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + long serviceTimeout = configuration.getTimeDuration( + OZONE_OM_COMPACTION_SERVICE_TIMEOUT, + OZONE_COMPACTION_SERVICE_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); + compactionService = new CompactionService(ozoneManager, TimeUnit.MILLISECONDS, + compactionInterval, + serviceTimeout, configuration); + compactionService.start(); + } + } + KeyProviderCryptoExtension getKMSProvider() { return kmsProvider; } @@ -402,6 +431,10 @@ public void stop() throws IOException { snapshotDirectoryCleaningService.shutdown(); snapshotDirectoryCleaningService = null; } + if (compactionService != null) { + compactionService.shutdown(); + compactionService = null; + } } private OmBucketInfo getBucketInfo(String volumeName, String bucketName) @@ -810,6 +843,11 @@ public SnapshotDirectoryCleaningService getSnapshotDirectoryService() { return snapshotDirectoryCleaningService; } + @Override + public CompactionService getCompactionService() { + return compactionService; + } + public boolean isSstFilteringSvcEnabled() { long serviceInterval = ozoneManager.getConfiguration() .getTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 547b2db04105..4b6c41776646 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.key.OMDirectoriesPurgeRequestWithFSO; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -83,6 +84,8 @@ public OMDirectoriesPurgeResponseWithFSO(OMResponse omResponse) { @Override public void addToDBBatch(OMMetadataManager metadataManager, BatchOperation batchOp) throws IOException { + CompactionService compactionService = ((OmMetadataManagerImpl) metadataManager) + .getOzoneManager().getKeyManager().getCompactionService(); if (fromSnapshotInfo != null) { OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) metadataManager) @@ -96,13 +99,13 @@ public void addToDBBatch(OMMetadataManager metadataManager, // Init Batch Operation for snapshot db. try (BatchOperation writeBatch = fromSnapshotStore.initBatchOperation()) { - processPaths(fromSnapshot.getMetadataManager(), writeBatch); + processPaths(fromSnapshot.getMetadataManager(), writeBatch, compactionService); fromSnapshotStore.commitBatchOperation(writeBatch); } } metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo); } else { - processPaths(metadataManager, batchOp); + processPaths(metadataManager, batchOp, compactionService); } // update bucket quota in active db @@ -114,7 +117,7 @@ public void addToDBBatch(OMMetadataManager metadataManager, } public void processPaths(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { + BatchOperation batchOperation, CompactionService compactionService) throws IOException { for (OzoneManagerProtocolProtos.PurgePathRequest path : paths) { final long volumeId = path.getVolumeId(); final long bucketId = path.getBucketId(); @@ -143,6 +146,8 @@ public void processPaths(OMMetadataManager omMetadataManager, } } + compactionService.compactDirectoryTableIfNeeded(markDeletedSubDirsList.size()); + for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId, @@ -167,6 +172,7 @@ public void processPaths(OMMetadataManager omMetadataManager, omMetadataManager.getDeletedTable().putWithBatch(batchOperation, deletedKey, repeatedOmKeyInfo); } + compactionService.compactFileTableIfNeeded(deletedSubFilesList.size()); if (!openKeyInfoMap.isEmpty()) { for (Map.Entry entry : openKeyInfoMap.entrySet()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index cd2f7d190f45..07465e73ccbd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest; +import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -36,6 +37,7 @@ import java.io.IOException; import java.util.List; + import jakarta.annotation.Nonnull; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; @@ -73,6 +75,8 @@ public OMKeyPurgeResponse(@Nonnull OMResponse omResponse) { @Override public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { + CompactionService compactionService = ((OmMetadataManagerImpl) omMetadataManager).getOzoneManager(). + getKeyManager().getCompactionService(); if (fromSnapshot != null) { OmSnapshotManager omSnapshotManager = @@ -86,14 +90,14 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, // Init Batch Operation for snapshot db. try (BatchOperation writeBatch = fromSnapshotStore.initBatchOperation()) { - processKeys(writeBatch, fromOmSnapshot.getMetadataManager()); + processKeys(writeBatch, fromOmSnapshot.getMetadataManager(), compactionService); processKeysToUpdate(writeBatch, fromOmSnapshot.getMetadataManager()); fromSnapshotStore.commitBatchOperation(writeBatch); } } omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); } else { - processKeys(batchOperation, omMetadataManager); + processKeys(batchOperation, omMetadataManager, compactionService); processKeysToUpdate(batchOperation, omMetadataManager); } } @@ -114,11 +118,12 @@ private void processKeysToUpdate(BatchOperation batchOp, } private void processKeys(BatchOperation batchOp, - OMMetadataManager metadataManager) throws IOException { + OMMetadataManager metadataManager, CompactionService compactionService) throws IOException { for (String key : purgeKeyList) { metadataManager.getDeletedTable().deleteWithBatch(batchOp, key); } + compactionService.compactDeletedTableIfNeeded(purgeKeyList.size()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java new file mode 100644 index 000000000000..4fc80d50c78c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.service; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * This is the background service to compact OM rocksdb tables. + */ +public class CompactionService extends BackgroundService { + private static final Logger LOG = + LoggerFactory.getLogger(CompactionService.class); + + // Use only a single thread for Compaction. + private static final int COMPACTOR_THREAD_POOL_SIZE = 1; + + private final OzoneManager ozoneManager; + private final OMMetadataManager omMetadataManager; + private final AtomicLong numCompactions; + private final AtomicBoolean suspended; + private Set tablesToCompact = new HashSet<>(); + // this counter is used by OMDirectoriesPurgeResponseWithFSO to track the number of directories deleted + private final AtomicLong uncompactedDirDeletes = new AtomicLong(0); + private final AtomicLong uncompactedFileDeletes = new AtomicLong(0); + private final AtomicLong uncompactedDeletes = new AtomicLong(0); + + public CompactionService(OzoneManager ozoneManager, TimeUnit unit, long interval, long timeout, + ConfigurationSource conf) { + super("CompactionService", interval, unit, + COMPACTOR_THREAD_POOL_SIZE, timeout, + ozoneManager.getThreadNamePrefix()); + this.ozoneManager = ozoneManager; + this.omMetadataManager = this.ozoneManager.getMetadataManager(); + + this.numCompactions = new AtomicLong(0); + this.suspended = new AtomicBoolean(false); + } + + /** + * Suspend the service (for testing). + */ + @VisibleForTesting + public void suspend() { + suspended.set(true); + } + + /** + * Resume the service if suspended (for testing). + */ + @VisibleForTesting + public void resume() { + suspended.set(false); + } + + /** + * Returns the number of manual compactions performed. + * + * @return long count. + */ + @VisibleForTesting + public long getNumCompactions() { + return numCompactions.get(); + } + + + public AtomicLong getUncompactedDirDeletes() { + return uncompactedDirDeletes; + } + + public AtomicLong getUncompactedDeletes() { + return uncompactedDeletes; + } + + public AtomicLong getUncompactedFileDeletes() { + return uncompactedFileDeletes; + } + + public void compactDirectoryTableIfNeeded(long batchSize) throws IOException { + + String columnFamilyName = "directoryTable"; + long compactionThreshold = 10 * 1000; + if (uncompactedDirDeletes.addAndGet(batchSize) < compactionThreshold) { + return; + } + addTask(columnFamilyName); + uncompactedDirDeletes.set(0); + } + + public void compactFileTableIfNeeded(long batchSize) { + String columnFamilyName = "fileTable"; + long compactionThreshold = 10 * 1000; + if (uncompactedFileDeletes.addAndGet(batchSize) < compactionThreshold) { + return; + } + addTask(columnFamilyName); + uncompactedFileDeletes.set(0); + } + + public void compactDeletedTableIfNeeded(long batchSize) { + String columnFamilyName = "deletedTable"; + long compactionThreshold = 10 * 1000; + if (uncompactedDeletes.addAndGet(batchSize) < compactionThreshold) { + return; + } + addTask(columnFamilyName); + uncompactedDeletes.set(0); + } + + @Override + public synchronized BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + for (String table : tablesToCompact) { + queue.add(new CompactTask(table)); + } + tablesToCompact.clear(); + return queue; + } + + public synchronized void addTask(String tableName) { + tablesToCompact.add(tableName); + } + + private boolean shouldRun() { + return !suspended.get(); + } + + private class CompactTask implements BackgroundTask { + private final String tableName; + + CompactTask(String tableName) { + this.tableName = tableName; + } + + @Override + public int getPriority() { + return 0; + } + + @Override + public BackgroundTaskResult call() throws Exception { + // trigger full compaction for the specified table. + if (!shouldRun()) { + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + LOG.debug("Running CompactTask"); + long startTime = Time.monotonicNow(); + LOG.info("Compacting column family: {}", tableName); + try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { + options.setBottommostLevelCompaction( + ManagedCompactRangeOptions.BottommostLevelCompaction.kForce); + // Find CF Handler + RocksDatabase.ColumnFamily columnFamily = + ((RDBStore) omMetadataManager.getStore()).getDb() + .getColumnFamily(tableName); + ((RDBStore) omMetadataManager.getStore()).getDb().compactRange( + columnFamily, null, null, options); + LOG.info("Compaction of column family: {} completed in {} ms", + tableName, Time.monotonicNow() - startTime); + numCompactions.incrementAndGet(); + } + + return () -> 1; + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java index 9eb8738b9d48..a403bd9fbecb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -42,6 +43,7 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse; +import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -52,7 +54,6 @@ * Tests {@link OMKeyPurgeRequest} and {@link OMKeyPurgeResponse}. */ public class TestOMDirectoriesPurgeRequestAndResponse extends TestOMKeyRequest { - private int numKeys = 10; /** @@ -185,6 +186,7 @@ private OMRequest preExecute(OMRequest originalOmRequest) throws IOException { @Test public void testValidateAndUpdateCacheCheckQuota() throws Exception { + keyManager.start(getOzoneConfiguration()); // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyInfos = createAndDeleteKeys(1, null); // The keys should be present in the DeletedKeys table before purging @@ -216,6 +218,7 @@ public void testValidateAndUpdateCacheCheckQuota() throws Exception { @Test public void testValidateAndUpdateCacheSnapshotLastTransactionInfoUpdated() throws Exception { + keyManager.start(getOzoneConfiguration()); // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyInfos = createAndDeleteKeys(1, null); // The keys should be present in the DeletedKeys table before purging @@ -269,6 +272,7 @@ public void testValidateAndUpdateCacheSnapshotLastTransactionInfoUpdated() throw @Test public void testValidateAndUpdateCacheQuotaBucketRecreated() throws Exception { + keyManager.start(getOzoneConfiguration()); // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyInfos = createAndDeleteKeys(1, null); // The keys should be present in the DeletedKeys table before purging @@ -347,5 +351,9 @@ private void validateDeletedKeys(OMMetadataManager omMetadataManager, assertTrue(omMetadataManager.getDeletedTable().isExist( deletedKey)); } + CompactionService compactionService = ((OmMetadataManagerImpl) this.omMetadataManager) + .getOzoneManager().getKeyManager().getCompactionService(); + assertEquals(deletedKeyNames.size(), + compactionService.getUncompactedFileDeletes().get()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index c323fecd5015..3ab86de05430 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -22,11 +22,12 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; - import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.junit.jupiter.api.Test; @@ -125,6 +126,7 @@ private OMRequest preExecute(OMRequest originalOmRequest) throws IOException { @Test public void testValidateAndUpdateCache() throws Exception { + keyManager.start(getOzoneConfiguration()); // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyNames = createAndDeleteKeys(1, null); @@ -168,6 +170,7 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testKeyPurgeInSnapshot() throws Exception { + keyManager.start(getOzoneConfiguration()); // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyNames = createAndDeleteKeys(1, null); @@ -214,8 +217,12 @@ public void testKeyPurgeInSnapshot() throws Exception { try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { - OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse(omResponse, deletedKeyNames, snapInfo, null); + OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse( + omResponse, deletedKeyNames, snapInfo, null); omKeyPurgeResponse.addToDBBatch(omMetadataManager, batchOperation); + CompactionService compactionService = ((OmMetadataManagerImpl) omMetadataManager) + .getOzoneManager().getKeyManager().getCompactionService(); + assertEquals(deletedKeyNames.size(), compactionService.getUncompactedDeletes().get()); // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java new file mode 100644 index 000000000000..f424be1de85b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.service; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; +import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OmTestManagers; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.ozone.test.GenericTestUtils; +import org.apache.ratis.util.ExitUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(OrderAnnotation.class) +class TestCompactionService { + private OzoneManager om; + private static final Logger LOG = + LoggerFactory.getLogger(TestCompactionService.class); + + private static final int SERVICE_INTERVAL = 1; + private static final int WAIT_TIME = (int) Duration.ofSeconds(10).toMillis(); + private KeyManager keyManager; + + @BeforeAll + void setup(@TempDir Path tempDir) throws Exception { + ExitUtils.disableSystemExit(); + + OzoneConfiguration conf = new OzoneConfiguration(); + System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); + ServerUtils.setOzoneMetaDirPath(conf, tempDir.toString()); + conf.setTimeDuration(OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL, + SERVICE_INTERVAL, TimeUnit.MILLISECONDS); + conf.setQuietMode(false); + OmTestManagers omTestManagers = new OmTestManagers(conf); + keyManager = omTestManagers.getKeyManager(); + om = omTestManagers.getOzoneManager(); + } + + @AfterAll + void cleanup() { + if (om.stop()) { + om.join(); + } + } + + /** + * Add a compaction request and verify that it is processed. + * + * @throws IOException - on Failure. + */ + @Timeout(300) + @Test + public void testCompact() throws Exception { + + CompactionService compactionService = keyManager.getCompactionService(); + + compactionService.suspend(); + // wait for submitted tasks to complete + Thread.sleep(SERVICE_INTERVAL); + final long oldkeyCount = compactionService.getNumCompactions(); + LOG.info("oldkeyCount={}", oldkeyCount); + + final int keyCount = 1; + compactionService.addTask(FILE_TABLE); + + compactionService.resume(); + + GenericTestUtils.waitFor( + () -> compactionService.getNumCompactions() >= oldkeyCount + keyCount, + SERVICE_INTERVAL, WAIT_TIME); + } + +} From 983471d87621157a3532fe8196c5600c819e4002 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 7 Mar 2025 14:51:09 -0800 Subject: [PATCH 2/7] Rewrite. Moved the deletion counter into TypedTable and have CompactionService check uncompacted deletes periodically. Change-Id: I606abc7240cb1ff81b041b37eeff9800c6ea96a9 --- .../src/main/resources/ozone-default.xml | 10 +- .../hadoop/hdds/utils/db/TypedTable.java | 13 ++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 3 + .../hadoop/ozone/om/KeyManagerImpl.java | 9 +- .../OMDirectoriesPurgeResponseWithFSO.java | 12 +- .../om/response/key/OMKeyPurgeResponse.java | 11 +- .../ozone/om/service/CompactionService.java | 114 ++++++++---------- ...tOMDirectoriesPurgeRequestAndResponse.java | 4 + .../om/service/TestCompactionService.java | 46 ++++--- 9 files changed, 118 insertions(+), 104 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c067512b74fb..29942b5585ba 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1536,7 +1536,6 @@ Unit could be defined with postfix (ns,ms,s,m,h,d) - ozone.om.compaction.service.timeout 10m @@ -1547,6 +1546,15 @@ Unit could be defined with postfix (ns,ms,s,m,h,d) + + ozone.om.compaction.service.threshold + 10000 + OZONE, OM, PERFORMANCE + + Compact rocksdb column families when the number of deleted keys + exceeds this number. + + hdds.rest.http-address diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java index f144e2c03c58..2e66a4e99e61 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.utils.IOUtils; @@ -68,6 +69,7 @@ public class TypedTable implements Table { private final CodecBuffer.Capacity bufferCapacity = new CodecBuffer.Capacity(this, BUFFER_SIZE_DEFAULT); private final TableCache cache; + private final AtomicLong uncompactedDeletes = new AtomicLong(0); /** * The same as this(rawTable, codecRegistry, keyType, valueType, @@ -400,12 +402,14 @@ public void delete(KEY key) throws IOException { } else { rawTable.delete(encodeKey(key)); } + uncompactedDeletes.addAndGet(1); } @Override public void deleteWithBatch(BatchOperation batch, KEY key) throws IOException { rawTable.deleteWithBatch(batch, encodeKey(key)); + uncompactedDeletes.addAndGet(1); } @Override @@ -413,6 +417,15 @@ public void deleteRange(KEY beginKey, KEY endKey) throws IOException { rawTable.deleteRange(encodeKey(beginKey), encodeKey(endKey)); } + public AtomicLong getUncompactedDeletes() { + return uncompactedDeletes; + } + + + public void resetUncompactedDeletes() { + uncompactedDeletes.set(0); + } + @Override public Table.KeyValueIterator iterator() throws IOException { return iterator(null); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index fb9d2735ecaf..0cc988695ec3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -638,4 +638,7 @@ private OMConfigKeys() { public static final String OZONE_OM_COMPACTION_SERVICE_TIMEOUT = "ozone.om.compaction.service.timeout"; public static final String OZONE_COMPACTION_SERVICE_TIMEOUT_DEFAULT = "10m"; + public static final String OZONE_OM_COMPACTION_SERVICE_THRESHOLD + = "ozone.om.compaction.service.threshold"; + public static final String OZONE_COMPACTION_SERVICE_THRESHOLD_DEFAULT = "10000"; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 54ef3f2b6336..e3203193ae9c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -138,10 +138,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_THRESHOLD_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_COMPACTION_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_THRESHOLD; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_CLEANUP_SERVICE_INTERVAL; @@ -390,9 +392,12 @@ private void startCompactionService(OzoneConfiguration configuration, OZONE_OM_COMPACTION_SERVICE_TIMEOUT, OZONE_COMPACTION_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); + long compactionThreshold = configuration.getTimeDuration( + OZONE_OM_COMPACTION_SERVICE_THRESHOLD, + OZONE_COMPACTION_SERVICE_THRESHOLD_DEFAULT, + TimeUnit.MILLISECONDS); compactionService = new CompactionService(ozoneManager, TimeUnit.MILLISECONDS, - compactionInterval, - serviceTimeout, configuration); + compactionInterval, serviceTimeout, compactionThreshold); compactionService.start(); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 4b6c41776646..547b2db04105 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -34,7 +34,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.key.OMDirectoriesPurgeRequestWithFSO; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; -import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -84,8 +83,6 @@ public OMDirectoriesPurgeResponseWithFSO(OMResponse omResponse) { @Override public void addToDBBatch(OMMetadataManager metadataManager, BatchOperation batchOp) throws IOException { - CompactionService compactionService = ((OmMetadataManagerImpl) metadataManager) - .getOzoneManager().getKeyManager().getCompactionService(); if (fromSnapshotInfo != null) { OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) metadataManager) @@ -99,13 +96,13 @@ public void addToDBBatch(OMMetadataManager metadataManager, // Init Batch Operation for snapshot db. try (BatchOperation writeBatch = fromSnapshotStore.initBatchOperation()) { - processPaths(fromSnapshot.getMetadataManager(), writeBatch, compactionService); + processPaths(fromSnapshot.getMetadataManager(), writeBatch); fromSnapshotStore.commitBatchOperation(writeBatch); } } metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo); } else { - processPaths(metadataManager, batchOp, compactionService); + processPaths(metadataManager, batchOp); } // update bucket quota in active db @@ -117,7 +114,7 @@ public void addToDBBatch(OMMetadataManager metadataManager, } public void processPaths(OMMetadataManager omMetadataManager, - BatchOperation batchOperation, CompactionService compactionService) throws IOException { + BatchOperation batchOperation) throws IOException { for (OzoneManagerProtocolProtos.PurgePathRequest path : paths) { final long volumeId = path.getVolumeId(); final long bucketId = path.getBucketId(); @@ -146,8 +143,6 @@ public void processPaths(OMMetadataManager omMetadataManager, } } - compactionService.compactDirectoryTableIfNeeded(markDeletedSubDirsList.size()); - for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId, @@ -172,7 +167,6 @@ public void processPaths(OMMetadataManager omMetadataManager, omMetadataManager.getDeletedTable().putWithBatch(batchOperation, deletedKey, repeatedOmKeyInfo); } - compactionService.compactFileTableIfNeeded(deletedSubFilesList.size()); if (!openKeyInfoMap.isEmpty()) { for (Map.Entry entry : openKeyInfoMap.entrySet()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index 07465e73ccbd..cd2f7d190f45 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -27,7 +27,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest; -import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -37,7 +36,6 @@ import java.io.IOException; import java.util.List; - import jakarta.annotation.Nonnull; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; @@ -75,8 +73,6 @@ public OMKeyPurgeResponse(@Nonnull OMResponse omResponse) { @Override public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - CompactionService compactionService = ((OmMetadataManagerImpl) omMetadataManager).getOzoneManager(). - getKeyManager().getCompactionService(); if (fromSnapshot != null) { OmSnapshotManager omSnapshotManager = @@ -90,14 +86,14 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, // Init Batch Operation for snapshot db. try (BatchOperation writeBatch = fromSnapshotStore.initBatchOperation()) { - processKeys(writeBatch, fromOmSnapshot.getMetadataManager(), compactionService); + processKeys(writeBatch, fromOmSnapshot.getMetadataManager()); processKeysToUpdate(writeBatch, fromOmSnapshot.getMetadataManager()); fromSnapshotStore.commitBatchOperation(writeBatch); } } omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); } else { - processKeys(batchOperation, omMetadataManager, compactionService); + processKeys(batchOperation, omMetadataManager); processKeysToUpdate(batchOperation, omMetadataManager); } } @@ -118,12 +114,11 @@ private void processKeysToUpdate(BatchOperation batchOp, } private void processKeys(BatchOperation batchOp, - OMMetadataManager metadataManager, CompactionService compactionService) throws IOException { + OMMetadataManager metadataManager) throws IOException { for (String key : purgeKeyList) { metadataManager.getDeletedTable().deleteWithBatch(batchOp, key); } - compactionService.compactDeletedTableIfNeeded(purgeKeyList.size()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index 4fc80d50c78c..155a1e831ea8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -19,13 +19,13 @@ package org.apache.hadoop.ozone.om.service; import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.hdds.utils.db.TypedTable; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -34,12 +34,18 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.HashSet; -import java.util.Set; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; + /** * This is the background service to compact OM rocksdb tables. */ @@ -54,14 +60,13 @@ public class CompactionService extends BackgroundService { private final OMMetadataManager omMetadataManager; private final AtomicLong numCompactions; private final AtomicBoolean suspended; - private Set tablesToCompact = new HashSet<>(); - // this counter is used by OMDirectoriesPurgeResponseWithFSO to track the number of directories deleted - private final AtomicLong uncompactedDirDeletes = new AtomicLong(0); - private final AtomicLong uncompactedFileDeletes = new AtomicLong(0); - private final AtomicLong uncompactedDeletes = new AtomicLong(0); + private final long compactionThreshold; + // list of tables that can be compacted + private static final List COMPACTABLE_TABLES = + Arrays.asList(DELETED_DIR_TABLE, DELETED_TABLE, DIRECTORY_TABLE, FILE_TABLE, KEY_TABLE); public CompactionService(OzoneManager ozoneManager, TimeUnit unit, long interval, long timeout, - ConfigurationSource conf) { + long compactionThreshold) { super("CompactionService", interval, unit, COMPACTOR_THREAD_POOL_SIZE, timeout, ozoneManager.getThreadNamePrefix()); @@ -70,6 +75,7 @@ public CompactionService(OzoneManager ozoneManager, TimeUnit unit, long interval this.numCompactions = new AtomicLong(0); this.suspended = new AtomicBoolean(false); + this.compactionThreshold = compactionThreshold; } /** @@ -100,66 +106,58 @@ public long getNumCompactions() { public AtomicLong getUncompactedDirDeletes() { - return uncompactedDirDeletes; + TypedTable table = (TypedTable)omMetadataManager.getTable(DELETED_DIR_TABLE); + return table.getUncompactedDeletes(); } public AtomicLong getUncompactedDeletes() { - return uncompactedDeletes; - } - - public AtomicLong getUncompactedFileDeletes() { - return uncompactedFileDeletes; - } - - public void compactDirectoryTableIfNeeded(long batchSize) throws IOException { - - String columnFamilyName = "directoryTable"; - long compactionThreshold = 10 * 1000; - if (uncompactedDirDeletes.addAndGet(batchSize) < compactionThreshold) { - return; - } - addTask(columnFamilyName); - uncompactedDirDeletes.set(0); + TypedTable table = (TypedTable)omMetadataManager.getTable(DELETED_TABLE); + return table.getUncompactedDeletes(); } - public void compactFileTableIfNeeded(long batchSize) { - String columnFamilyName = "fileTable"; - long compactionThreshold = 10 * 1000; - if (uncompactedFileDeletes.addAndGet(batchSize) < compactionThreshold) { - return; - } - addTask(columnFamilyName); - uncompactedFileDeletes.set(0); + public AtomicLong getUncompactedDeletedDirs() { + TypedTable table = (TypedTable)omMetadataManager.getTable(DIRECTORY_TABLE); + return table.getUncompactedDeletes(); } - public void compactDeletedTableIfNeeded(long batchSize) { - String columnFamilyName = "deletedTable"; - long compactionThreshold = 10 * 1000; - if (uncompactedDeletes.addAndGet(batchSize) < compactionThreshold) { - return; - } - addTask(columnFamilyName); - uncompactedDeletes.set(0); + public AtomicLong getUncompactedFileDeletes() { + TypedTable table = (TypedTable)omMetadataManager.getTable(FILE_TABLE); + return table.getUncompactedDeletes(); } @Override public synchronized BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); - for (String table : tablesToCompact) { - queue.add(new CompactTask(table)); + + for (String tableName : COMPACTABLE_TABLES) { + TypedTable table = (TypedTable)omMetadataManager.getTable(tableName); + if (table.getUncompactedDeletes().get() > compactionThreshold) { + queue.add(new CompactTask(tableName)); + table.resetUncompactedDeletes(); + } } - tablesToCompact.clear(); return queue; } - public synchronized void addTask(String tableName) { - tablesToCompact.add(tableName); - } - private boolean shouldRun() { return !suspended.get(); } + protected void compactFully(String tableName) throws IOException { + long startTime = Time.monotonicNow(); + LOG.info("Compacting column family: {}", tableName); + try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { + options.setBottommostLevelCompaction( + ManagedCompactRangeOptions.BottommostLevelCompaction.kForce); + // Find CF Handler + RocksDatabase rocksDatabase = ((RDBStore) omMetadataManager.getStore()).getDb(); + RocksDatabase.ColumnFamily columnFamily = rocksDatabase.getColumnFamily(tableName); + rocksDatabase.compactRange(columnFamily, null, null, options); + LOG.info("Compaction of column family: {} completed in {} ms", + tableName, Time.monotonicNow() - startTime); + } + } + private class CompactTask implements BackgroundTask { private final String tableName; @@ -179,23 +177,11 @@ public BackgroundTaskResult call() throws Exception { return BackgroundTaskResult.EmptyTaskResult.newResult(); } LOG.debug("Running CompactTask"); - long startTime = Time.monotonicNow(); - LOG.info("Compacting column family: {}", tableName); - try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { - options.setBottommostLevelCompaction( - ManagedCompactRangeOptions.BottommostLevelCompaction.kForce); - // Find CF Handler - RocksDatabase.ColumnFamily columnFamily = - ((RDBStore) omMetadataManager.getStore()).getDb() - .getColumnFamily(tableName); - ((RDBStore) omMetadataManager.getStore()).getDb().compactRange( - columnFamily, null, null, options); - LOG.info("Compaction of column family: {} completed in {} ms", - tableName, Time.monotonicNow() - startTime); - numCompactions.incrementAndGet(); - } + compactFully(tableName); + numCompactions.incrementAndGet(); return () -> 1; } } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java index a403bd9fbecb..f0caf1cf2d6a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java @@ -355,5 +355,9 @@ private void validateDeletedKeys(OMMetadataManager omMetadataManager, .getOzoneManager().getKeyManager().getCompactionService(); assertEquals(deletedKeyNames.size(), compactionService.getUncompactedFileDeletes().get()); + assertEquals(0, + compactionService.getUncompactedDirDeletes().get()); + assertEquals(0, + compactionService.getUncompactedDeletedDirs().get()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java index f424be1de85b..e0cd456fc68f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; -import org.apache.hadoop.ozone.om.KeyManager; -import org.apache.hadoop.ozone.om.OmTestManagers; +import org.apache.hadoop.hdds.utils.db.TypedTable; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; import org.junit.jupiter.api.Test; @@ -42,20 +41,21 @@ import java.nio.file.Path; import java.time.Duration; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(OrderAnnotation.class) class TestCompactionService { - private OzoneManager om; private static final Logger LOG = LoggerFactory.getLogger(TestCompactionService.class); private static final int SERVICE_INTERVAL = 1; private static final int WAIT_TIME = (int) Duration.ofSeconds(10).toMillis(); - private KeyManager keyManager; @BeforeAll void setup(@TempDir Path tempDir) throws Exception { @@ -67,16 +67,6 @@ void setup(@TempDir Path tempDir) throws Exception { conf.setTimeDuration(OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL, SERVICE_INTERVAL, TimeUnit.MILLISECONDS); conf.setQuietMode(false); - OmTestManagers omTestManagers = new OmTestManagers(conf); - keyManager = omTestManagers.getKeyManager(); - om = omTestManagers.getOzoneManager(); - } - - @AfterAll - void cleanup() { - if (om.stop()) { - om.join(); - } } /** @@ -87,8 +77,25 @@ void cleanup() { @Timeout(300) @Test public void testCompact() throws Exception { + OzoneManager ozoneManager = mock(OzoneManager.class); + OMMetadataManager metadataManager = mock(OMMetadataManager.class); + + TypedTable table = mock(TypedTable.class); + AtomicLong numDeletes = new AtomicLong(2); + + when(ozoneManager.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getTable(anyString())).thenReturn(table); + when(table.getUncompactedDeletes()).thenReturn(numDeletes); + + CompactionService compactionService = new CompactionService(ozoneManager, TimeUnit.MILLISECONDS, + TimeUnit.SECONDS.toMillis(SERVICE_INTERVAL), TimeUnit.SECONDS.toMillis(60), 1) { - CompactionService compactionService = keyManager.getCompactionService(); + @Override + public void compactFully(String tableName) throws IOException { + LOG.info("Compacting column family: {}", tableName); + } + }; + compactionService.start(); compactionService.suspend(); // wait for submitted tasks to complete @@ -96,13 +103,12 @@ public void testCompact() throws Exception { final long oldkeyCount = compactionService.getNumCompactions(); LOG.info("oldkeyCount={}", oldkeyCount); - final int keyCount = 1; - compactionService.addTask(FILE_TABLE); + final int compactionTriggered = 1; compactionService.resume(); GenericTestUtils.waitFor( - () -> compactionService.getNumCompactions() >= oldkeyCount + keyCount, + () -> compactionService.getNumCompactions() >= oldkeyCount + compactionTriggered, SERVICE_INTERVAL, WAIT_TIME); } From 09037a978a2d89b85b7228aa6ec06925ab44c9af Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 7 Mar 2025 15:31:49 -0800 Subject: [PATCH 3/7] Fix bug Change-Id: Ieb39c536c06bb90dbea0f67d3a7a69b8d38bde68 --- .../ozone/om/service/CompactionService.java | 21 ------------------- ...tOMDirectoriesPurgeRequestAndResponse.java | 21 ++++++++++++++----- .../key/TestOMKeyPurgeRequestAndResponse.java | 8 ++++--- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index 155a1e831ea8..cfaacc7dcf2e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -104,27 +104,6 @@ public long getNumCompactions() { return numCompactions.get(); } - - public AtomicLong getUncompactedDirDeletes() { - TypedTable table = (TypedTable)omMetadataManager.getTable(DELETED_DIR_TABLE); - return table.getUncompactedDeletes(); - } - - public AtomicLong getUncompactedDeletes() { - TypedTable table = (TypedTable)omMetadataManager.getTable(DELETED_TABLE); - return table.getUncompactedDeletes(); - } - - public AtomicLong getUncompactedDeletedDirs() { - TypedTable table = (TypedTable)omMetadataManager.getTable(DIRECTORY_TABLE); - return table.getUncompactedDeletes(); - } - - public AtomicLong getUncompactedFileDeletes() { - TypedTable table = (TypedTable)omMetadataManager.getTable(FILE_TABLE); - return table.getUncompactedDeletes(); - } - @Override public synchronized BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java index f0caf1cf2d6a..f8e6b83e346c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java @@ -18,6 +18,9 @@ package org.apache.hadoop.ozone.om.request.key; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -28,6 +31,7 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.TypedTable; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.ClientVersion; @@ -263,7 +267,7 @@ public void testValidateAndUpdateCacheSnapshotLastTransactionInfoUpdated() throw performBatchOperationCommit(omClientResponse); // The keys should exist in the DeletedKeys table after dir delete - validateDeletedKeys(rcOmSnapshot.get().getMetadataManager(), deletedKeyNames); + validateDeletedKeys(rcOmSnapshot.get().getMetadataManager(), deletedKeyNames, true); snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapshotInfo.getTableKey()); assertEquals(snapshotInfo, snapshotInfoOnDisk); rcOmSnapshot.close(); @@ -347,17 +351,24 @@ private List validateDeletedKeysTable(OMMetadataManager omMetadataManage private void validateDeletedKeys(OMMetadataManager omMetadataManager, List deletedKeyNames) throws IOException { + validateDeletedKeys(omMetadataManager, deletedKeyNames, false); + } + + private void validateDeletedKeys(OMMetadataManager omMetadataManager, + List deletedKeyNames, boolean isSnapshot) throws IOException { for (String deletedKey : deletedKeyNames) { assertTrue(omMetadataManager.getDeletedTable().isExist( deletedKey)); } CompactionService compactionService = ((OmMetadataManagerImpl) this.omMetadataManager) .getOzoneManager().getKeyManager().getCompactionService(); - assertEquals(deletedKeyNames.size(), - compactionService.getUncompactedFileDeletes().get()); + // If it's deleted from a snapshot, it doesn't affect fileTable. + long expectedUncompactedDeletes = isSnapshot ? 0 : deletedKeyNames.size(); + assertEquals(expectedUncompactedDeletes, + ((TypedTable)this.omMetadataManager.getTable(FILE_TABLE)).getUncompactedDeletes().get()); assertEquals(0, - compactionService.getUncompactedDirDeletes().get()); + ((TypedTable)this.omMetadataManager.getTable(DELETED_DIR_TABLE)).getUncompactedDeletes().get()); assertEquals(0, - compactionService.getUncompactedDeletedDirs().get()); + ((TypedTable)this.omMetadataManager.getTable(DIRECTORY_TABLE)).getUncompactedDeletes().get()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index 3ab86de05430..767b6004e7d6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.UUID; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.hdds.utils.db.TypedTable; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -41,6 +42,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -220,9 +223,8 @@ public void testKeyPurgeInSnapshot() throws Exception { OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse( omResponse, deletedKeyNames, snapInfo, null); omKeyPurgeResponse.addToDBBatch(omMetadataManager, batchOperation); - CompactionService compactionService = ((OmMetadataManagerImpl) omMetadataManager) - .getOzoneManager().getKeyManager().getCompactionService(); - assertEquals(deletedKeyNames.size(), compactionService.getUncompactedDeletes().get()); + assertEquals(deletedKeyNames.size(), + ((TypedTable)this.omMetadataManager.getTable(DELETED_TABLE)).getUncompactedDeletes().get()); // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); From 7ce31f98aa308e2e8a25fc4d81a61b0833cccec9 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 7 Mar 2025 16:09:47 -0800 Subject: [PATCH 4/7] Checkstyle Change-Id: I6bded13066ae8e503c273cecb7f821be5d1d80a5 --- .../ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index 767b6004e7d6..fce500f379a4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -27,8 +27,6 @@ import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.junit.jupiter.api.Test; @@ -43,7 +41,6 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; From fd5eb1e5f544e15006086486fecc3d3eb5f88133 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 7 Mar 2025 16:11:23 -0800 Subject: [PATCH 5/7] Checkstyle and findbugs Change-Id: Ie67717d22bddc4f8012ad700c9c7b311cfe8445e --- .../request/key/TestOMDirectoriesPurgeRequestAndResponse.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java index f8e6b83e346c..6689c0a60e25 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -47,7 +46,6 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse; -import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -360,8 +358,6 @@ private void validateDeletedKeys(OMMetadataManager omMetadataManager, assertTrue(omMetadataManager.getDeletedTable().isExist( deletedKey)); } - CompactionService compactionService = ((OmMetadataManagerImpl) this.omMetadataManager) - .getOzoneManager().getKeyManager().getCompactionService(); // If it's deleted from a snapshot, it doesn't affect fileTable. long expectedUncompactedDeletes = isSnapshot ? 0 : deletedKeyNames.size(); assertEquals(expectedUncompactedDeletes, From b8352bb647b195f85deab13a47af1e91e756db2f Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Sat, 8 Mar 2025 17:36:03 +0530 Subject: [PATCH 6/7] Checkstyle issues fixed. --- .../ozone/om/service/CompactionService.java | 46 +++++++++---------- .../om/service/TestCompactionService.java | 35 +++++++------- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index cfaacc7dcf2e..69979eb309df 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -1,14 +1,13 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +17,19 @@ package org.apache.hadoop.ozone.om.service; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; + import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; @@ -33,19 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; - /** * This is the background service to compact OM rocksdb tables. */ diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java index e0cd456fc68f..0f30f0d88611 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -1,11 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -14,11 +13,20 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package org.apache.hadoop.ozone.om.service; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; @@ -37,17 +45,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(OrderAnnotation.class) class TestCompactionService { From f58b94c2ba9bd848d2ceb836f19c84cb34d9a1f3 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 8 Mar 2025 18:09:47 +0100 Subject: [PATCH 7/7] fix TestOzoneConfigurationFields (see HDDS-12294) --- .../main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 0e6f80620021..6db9961d567b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -622,11 +622,6 @@ private OMConfigKeys() { public static final String OZONE_OM_MAX_BUCKET = "ozone.om.max.buckets"; public static final int OZONE_OM_MAX_BUCKET_DEFAULT = 100000; - /** - * Configuration property to configure the max server side response size for list calls. - */ - public static final String OZONE_OM_SERVER_LIST_MAX_SIZE = "ozone.om.server.list.max.size"; - public static final int OZONE_OM_SERVER_LIST_MAX_SIZE_DEFAULT = 1000; /** * Configuration properties for Compaction Service. */