diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index b4e08e885f12..c6be74a9cb34 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -232,6 +232,16 @@ public final class OzoneConfigKeys { OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT = "300s"; // 300s for default + public static final String OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL = + "ozone.snapshot.deleting.service.interval"; + public static final String + OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT = "30s"; + + public static final String OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT = + "ozone.snapshot.deleting.service.timeout"; + public static final String + OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT_DEFAULT = "300s"; + public static final String OZONE_BLOCK_DELETING_SERVICE_WORKERS = "ozone.block.deleting.service.workers"; public static final int OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 13a3319a527f..3acc47bc09e4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -237,6 +237,11 @@ public enum Units { TB, GB, MB, KB, B } public static final int INVALID_PORT = -1; + /** + * Object ID to identify reclaimable uncommitted blocks. + */ + public static final long OBJECT_ID_RECLAIM_BLOCKS = 0L; + /** * Default SCM Datanode ID file name. diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f093bf7303bf..855e21c0111f 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3132,6 +3132,14 @@ sst filtering service per time interval. + + ozone.snapshot.deleting.limit.per.task + 10 + OZONE, PERFORMANCE, OM + The maximum number of snapshots that would be reclaimed by + Snapshot Deleting Service per run. + + ozone.snapshot.filtering.service.interval 1m @@ -3147,6 +3155,25 @@ + + ozone.snapshot.deleting.service.timeout + 300s + OZONE, PERFORMANCE, OM + + Timeout value for SnapshotDeletingService. + + + + + ozone.snapshot.deleting.service.interval + 30s + OZONE, PERFORMANCE, OM + + The time interval between successive SnapshotDeletingService + thread run. + + + ozone.scm.event.ContainerReport.thread.pool.size 10 diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 586ad1d48d10..249bc48e844c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -308,6 +308,7 @@ public static boolean isReadOnly( case SetRangerServiceVersion: case CreateSnapshot: case DeleteSnapshot: + case SnapshotMoveDeletedKeys: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index 30a587cf3e49..223de3ebcec2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -93,8 +93,9 @@ public enum OMAction implements AuditAction { TENANT_LIST_USER, CREATE_SNAPSHOT, + LIST_SNAPSHOT, DELETE_SNAPSHOT, - LIST_SNAPSHOT; + SNAPSHOT_MOVE_DELETED_KEYS; @Override public String getAction() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 9ba5485da276..eddfd389258e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -324,6 +324,10 @@ private OMConfigKeys() { "ozone.snapshot.filtering.limit.per.task"; public static final int SNAPSHOT_SST_DELETING_LIMIT_PER_TASK_DEFAULT = 2; + public static final String SNAPSHOT_DELETING_LIMIT_PER_TASK = + "ozone.snapshot.deleting.limit.per.task"; + public static final int SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT = 10; + public static final String OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL = "ozone.snapshot.filtering.service.interval"; public static final String diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index f15908dcd805..bc297a92fd49 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -368,10 +368,17 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { .setBucketName(bucketName) .setSnapshotStatus(snapshotStatus.toProto()) .setCreationTime(creationTime) - .setDeletionTime(deletionTime) - .setPathPreviousSnapshotID(pathPreviousSnapshotID) - .setGlobalPreviousSnapshotID(globalPreviousSnapshotID) - .setSnapshotPath(snapshotPath) + .setDeletionTime(deletionTime); + + if (pathPreviousSnapshotID != null) { + sib.setPathPreviousSnapshotID(pathPreviousSnapshotID); + } + + if (globalPreviousSnapshotID != null) { + sib.setGlobalPreviousSnapshotID(globalPreviousSnapshotID); + } + + sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) .setDbTxSequenceNumber(dbTxSequenceNumber); return sib.build(); @@ -392,12 +399,19 @@ public static SnapshotInfo getFromProtobuf( .setSnapshotStatus(SnapshotStatus.valueOf(snapshotInfoProto .getSnapshotStatus())) .setCreationTime(snapshotInfoProto.getCreationTime()) - .setDeletionTime(snapshotInfoProto.getDeletionTime()) - .setPathPreviousSnapshotID(snapshotInfoProto. - getPathPreviousSnapshotID()) - .setGlobalPreviousSnapshotID(snapshotInfoProto. - getGlobalPreviousSnapshotID()) - .setSnapshotPath(snapshotInfoProto.getSnapshotPath()) + .setDeletionTime(snapshotInfoProto.getDeletionTime()); + + if (snapshotInfoProto.hasPathPreviousSnapshotID()) { + osib.setPathPreviousSnapshotID(snapshotInfoProto. + getPathPreviousSnapshotID()); + } + + if (snapshotInfoProto.hasGlobalPreviousSnapshotID()) { + osib.setGlobalPreviousSnapshotID(snapshotInfoProto. + getGlobalPreviousSnapshotID()); + } + + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); @@ -499,8 +513,9 @@ public boolean equals(Object o) { name.equals(that.name) && volumeName.equals(that.volumeName) && bucketName.equals(that.bucketName) && snapshotStatus == that.snapshotStatus && - pathPreviousSnapshotID.equals(that.pathPreviousSnapshotID) && - globalPreviousSnapshotID.equals(that.globalPreviousSnapshotID) && + Objects.equals(pathPreviousSnapshotID, that.pathPreviousSnapshotID) && + Objects.equals( + globalPreviousSnapshotID, that.globalPreviousSnapshotID) && snapshotPath.equals(that.snapshotPath) && checkpointDir.equals(that.checkpointDir); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java index eebb4d87517c..0ea1a1c0e6a7 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.ozone.om.helpers; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; + /** * Mixin class to handle ObjectID and UpdateID. */ @@ -52,7 +54,7 @@ public long getUpdateID() { } /** - * Set the Object ID. If this value is already set then this function throws. + * Set the Object ID. * There is a reason why we cannot use the final here. The object * ({@link OmVolumeArgs}/ {@link OmBucketInfo}/ {@link OmKeyInfo}) is * deserialized from the protobuf in many places in code. We need to set @@ -61,7 +63,7 @@ public long getUpdateID() { * @param obId - long */ public void setObjectID(long obId) { - if (this.objectID != 0) { + if (this.objectID != 0 && obId != OBJECT_ID_RECLAIM_BLOCKS) { throw new UnsupportedOperationException("Attempt to modify object ID " + "which is not zero. Current Object ID is " + this.objectID); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 5f028e939f00..8a70f64361eb 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -130,8 +130,9 @@ enum Type { ListSnapshot = 113; SnapshotDiff = 114; DeleteSnapshot = 115; + SnapshotMoveDeletedKeys = 116; - TransferLeadership = 116; + TransferLeadership = 117; } message OMRequest { @@ -244,8 +245,9 @@ message OMRequest { optional ListSnapshotRequest ListSnapshotRequest = 113; optional SnapshotDiffRequest snapshotDiffRequest = 114; optional DeleteSnapshotRequest DeleteSnapshotRequest = 115; + optional SnapshotMoveDeletedKeysRequest SnapshotMoveDeletedKeysRequest = 116; - optional hdds.TransferLeadershipRequestProto TransferOmLeadershipRequest = 116; + optional hdds.TransferLeadershipRequestProto TransferOmLeadershipRequest = 117; } @@ -352,8 +354,9 @@ message OMResponse { optional ListSnapshotResponse ListSnapshotResponse = 113; optional SnapshotDiffResponse snapshotDiffResponse = 114; optional DeleteSnapshotResponse DeleteSnapshotResponse = 115; + optional SnapshotMoveDeletedKeysResponse SnapshotMoveDeletedKeysResponse = 116; - optional hdds.TransferLeadershipResponseProto TransferOmLeadershipResponse = 116; + optional hdds.TransferLeadershipResponseProto TransferOmLeadershipResponse = 117; } enum Status { @@ -1703,6 +1706,17 @@ message DeleteSnapshotRequest { optional uint64 deletionTime = 4; } +message SnapshotMoveDeletedKeysRequest { + optional SnapshotInfo fromSnapshot = 1; + repeated SnapshotMoveKeyInfos nextDBKeys = 2; + repeated SnapshotMoveKeyInfos reclaimKeys = 3; +} + +message SnapshotMoveKeyInfos { + optional string key = 1; + repeated KeyInfo keyInfos = 2; +} + message DeleteTenantRequest { optional string tenantId = 1; } @@ -1765,6 +1779,10 @@ message DeleteSnapshotResponse { } +message SnapshotMoveDeletedKeysResponse { + +} + message SnapshotDiffReportProto { optional string volumeName = 1; optional string bucketName = 2; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 6b7eb26ee10f..14f7ca72d2f7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -247,4 +247,10 @@ List getPendingDeletionSubFiles(long volumeId, * @return Background service. */ BackgroundService getSnapshotSstFilteringService(); + + /** + * Returns the instance of Snapshot Deleting service. + * @return Background service. + */ + BackgroundService getSnapshotDeletingService(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 4ff5ad64d116..0cd707d484c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -84,6 +84,7 @@ import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; +import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -110,6 +111,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -159,6 +164,7 @@ public class KeyManagerImpl implements KeyManager { private BackgroundService keyDeletingService; private BackgroundService snapshotSstFilteringService; + private BackgroundService snapshotDeletingService; private final KeyProviderCryptoExtension kmsProvider; private final boolean enableFileSystemPaths; @@ -261,6 +267,24 @@ public void start(OzoneConfiguration configuration) { snapshotSstFilteringService.start(); } } + + if (snapshotDeletingService == null) { + long snapshotServiceInterval = configuration.getTimeDuration( + OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, + OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + long snapshotServiceTimeout = configuration.getTimeDuration( + OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT, + OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); + try { + snapshotDeletingService = new SnapshotDeletingService( + snapshotServiceInterval, snapshotServiceTimeout, ozoneManager); + snapshotDeletingService.start(); + } catch (IOException e) { + LOG.error("Error starting Snapshot Deleting Service", e); + } + } } KeyProviderCryptoExtension getKMSProvider() { @@ -285,6 +309,10 @@ public void stop() throws IOException { snapshotSstFilteringService.shutdown(); snapshotSstFilteringService = null; } + if (snapshotDeletingService != null) { + snapshotDeletingService.shutdown(); + snapshotDeletingService = null; + } } private OmBucketInfo getBucketInfo(String volumeName, String bucketName) @@ -621,6 +649,10 @@ public BackgroundService getSnapshotSstFilteringService() { return snapshotSstFilteringService; } + public BackgroundService getSnapshotDeletingService() { + return snapshotDeletingService; + } + @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index bbb784cfe482..2c5a6c9b6129 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -230,7 +230,7 @@ public SnapshotInfo getSnapshotInfo(String volumeName, bucketName, snapshotName)); } - private SnapshotInfo getSnapshotInfo(String key) throws IOException { + public SnapshotInfo getSnapshotInfo(String key) throws IOException { SnapshotInfo snapshotInfo; try { snapshotInfo = ozoneManager.getMetadataManager() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ac4c71657949..35361ef4bcdd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -452,6 +452,7 @@ private enum State { // This metadata reader points to the active filesystem private OmMetadataReader omMetadataReader; private OmSnapshotManager omSnapshotManager; + private SnapshotChainManager snapshotChainManager; /** A list of property that are reconfigurable at runtime. */ private final SortedSet reconfigurableProperties = @@ -769,6 +770,7 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { omMetadataReader = new OmMetadataReader(keyManager, prefixManager, this, LOG, AUDIT, metrics); omSnapshotManager = new OmSnapshotManager(this); + snapshotChainManager = new SnapshotChainManager(metadataManager); // Snapshot metrics updateActiveSnapshotMetrics(); @@ -1508,6 +1510,15 @@ public OmSnapshotManager getOmSnapshotManager() { return omSnapshotManager; } + /** + * Get Snapshot Chain Manager. + * + * @return SnapshotChainManager. + */ + public SnapshotChainManager getSnapshotChainManager() { + return snapshotChainManager; + } + /** * Get metadata manager. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index a8fb867401ac..0df80d299271 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -49,6 +49,7 @@ public class SnapshotChainManager { snapshotChainPath; private Map latestPathSnapshotID; private String latestGlobalSnapshotID; + private Map snapshotPathToTableKey; private static final Logger LOG = LoggerFactory.getLogger(SnapshotChainManager.class); @@ -57,10 +58,14 @@ public SnapshotChainManager(OMMetadataManager metadataManager) snapshotChainGlobal = new LinkedHashMap<>(); snapshotChainPath = new HashMap<>(); latestPathSnapshotID = new HashMap<>(); + snapshotPathToTableKey = new HashMap<>(); latestGlobalSnapshotID = null; loadFromSnapshotInfoTable(metadataManager); } + /** + * Add snapshot to global snapshot chain. + */ private void addSnapshotGlobal(String snapshotID, String prevGlobalID) throws IOException { // set previous snapshotID to null if it is "" for @@ -90,9 +95,13 @@ private void addSnapshotGlobal(String snapshotID, latestGlobalSnapshotID = snapshotID; }; + /** + * Add snapshot to bucket snapshot chain(path based). + */ private void addSnapshotPath(String snapshotPath, String snapshotID, - String prevPathID) throws IOException { + String prevPathID, + String snapTableKey) throws IOException { // set previous snapshotID to null if it is "" for // internal in-mem structure if (prevPathID != null && prevPathID.isEmpty()) { @@ -130,6 +139,8 @@ private void addSnapshotPath(String snapshotPath, .put(snapshotID, new SnapshotChainInfo(snapshotID, prevPathID, null)); + // store snapshot ID to snapshot DB table key in the map + snapshotPathToTableKey.put(snapshotID, snapTableKey); // set state variable latestPath snapshot entry to this snapshotID latestPathSnapshotID.put(snapshotPath, snapshotID); }; @@ -246,6 +257,10 @@ private boolean deleteSnapshotPath(String snapshotPath, return status; } + /** + * Loads the snapshot chain from SnapshotInfo table. + * @param metadataManager OMMetadataManager + */ private void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) throws IOException { // read from snapshotInfo table to populate @@ -256,6 +271,8 @@ private void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) Table.KeyValue< String, SnapshotInfo > kv; snapshotChainGlobal.clear(); snapshotChainPath.clear(); + latestPathSnapshotID.clear(); + snapshotPathToTableKey.clear(); while (keyIter.hasNext()) { kv = keyIter.next(); @@ -275,7 +292,8 @@ public void addSnapshot(SnapshotInfo sinfo) throws IOException { sinfo.getGlobalPreviousSnapshotID()); addSnapshotPath(sinfo.getSnapshotPath(), sinfo.getSnapshotID(), - sinfo.getPathPreviousSnapshotID()); + sinfo.getPathPreviousSnapshotID(), + sinfo.getTableKey()); } /** @@ -503,6 +521,10 @@ public String previousPathSnapshot(String snapshotPath, String snapshotID) .getPreviousSnapshotID(); } + public String getTableKey(String snapshotPath) { + return snapshotPathToTableKey.get(snapshotPath); + } + @VisibleForTesting public void loadSnapshotInfo(OMMetadataManager metadataManager) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index e556fb506504..bc08a9059811 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -72,6 +72,7 @@ import org.apache.hadoop.ozone.om.request.security.OMRenewDelegationTokenRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest; @@ -215,6 +216,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotCreateRequest(omRequest); case DeleteSnapshot: return new OMSnapshotDeleteRequest(omRequest); + case SnapshotMoveDeletedKeys: + return new OMSnapshotMoveDeletedKeysRequest(omRequest); case DeleteOpenKeys: BucketLayout bktLayout = BucketLayout.DEFAULT; if (omRequest.getDeleteOpenKeysRequest().hasBucketLayout()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 830e60600dd8..8febf9fcf5dc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -86,6 +86,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.WRITE; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes .BUCKET_NOT_FOUND; @@ -837,6 +838,9 @@ protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey( LOG.info("Detect allocated but uncommitted blocks {} in key {}.", uncommitted, omKeyInfo.getKeyName()); OmKeyInfo pseudoKeyInfo = omKeyInfo.copyObject(); + // This is a special marker to indicate that SnapshotDeletingService + // can reclaim this key's blocks unconditionally. + pseudoKeyInfo.setObjectID(OBJECT_ID_RECLAIM_BLOCKS); // TODO dataSize of pseudoKey is not real here List uncommittedGroups = new ArrayList<>(); // version not matters in the current logic of keyDeletingService, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index d8cbdad0cdaa..2ee67692ff9a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; import com.google.common.base.Optional; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -28,6 +29,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -114,7 +116,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, boolean acquiredBucketLock = false, acquiredSnapshotLock = false; IOException exception = null; OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); - + SnapshotChainManager snapshotChainManager = + ozoneManager.getSnapshotChainManager(); + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); OMClientResponse omClientResponse = null; @@ -146,6 +150,26 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, .getLatestSequenceNumber(); snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); + // Set previous path and global snapshot + String latestPathSnapshot = + snapshotChainManager.getLatestPathSnapshot(snapshotPath); + String latestGlobalSnapshot = + snapshotChainManager.getLatestGlobalSnapshot(); + + if (StringUtils.isEmpty(latestPathSnapshot)) { + snapshotInfo.setPathPreviousSnapshotID(null); + } else { + snapshotInfo.setPathPreviousSnapshotID(latestPathSnapshot); + } + + if (StringUtils.isEmpty(latestGlobalSnapshot)) { + snapshotInfo.setGlobalPreviousSnapshotID(null); + } else { + snapshotInfo.setGlobalPreviousSnapshotID(latestGlobalSnapshot); + } + + snapshotChainManager.addSnapshot(snapshotInfo); + omMetadataManager.getSnapshotInfoTable() .addCacheEntry(new CacheKey<>(key), new CacheValue<>(Optional.of(snapshotInfo), transactionLogIndex)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java new file mode 100644 index 000000000000..f1db67846f88 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; + +/** + * Handles OMSnapshotMoveDeletedKeys Request. + */ +public class OMSnapshotMoveDeletedKeysRequest extends OMClientRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(OMSnapshotMoveDeletedKeysRequest.class); + + public OMSnapshotMoveDeletedKeysRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); + SnapshotChainManager snapshotChainManager = + ozoneManager.getSnapshotChainManager(); + + SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = + getOmRequest().getSnapshotMoveDeletedKeysRequest(); + SnapshotInfo fromSnapshot = SnapshotInfo.getFromProtobuf( + moveDeletedKeysRequest.getFromSnapshot()); + + // If there is no Non-Deleted Snapshot move the + // keys to Active Object Store. + SnapshotInfo nextSnapshot = null; + OMClientResponse omClientResponse = null; + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); + try { + OmSnapshot omFromSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(fromSnapshot.getVolumeName(), + fromSnapshot.getBucketName(), + getSnapshotPrefix(fromSnapshot.getName())); + + nextSnapshot = getNextActiveSnapshot(fromSnapshot, + snapshotChainManager, omSnapshotManager); + + // Get next non-deleted snapshot. + List nextDBKeysList = + moveDeletedKeysRequest.getNextDBKeysList(); + List reclaimKeysList = + moveDeletedKeysRequest.getReclaimKeysList(); + + OmSnapshot omNextSnapshot = null; + + if (nextSnapshot != null) { + omNextSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(nextSnapshot.getVolumeName(), + nextSnapshot.getBucketName(), + getSnapshotPrefix(nextSnapshot.getName())); + } + + omClientResponse = new OMSnapshotMoveDeletedKeysResponse( + omResponse.build(), omFromSnapshot, omNextSnapshot, + nextDBKeysList, reclaimKeysList); + + } catch (IOException ex) { + omClientResponse = new OMSnapshotMoveDeletedKeysResponse( + createErrorOMResponse(omResponse, ex)); + } finally { + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + } + + return omClientResponse; + } + + /** + * Get the next non deleted snapshot in the snapshot chain. + */ + private SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo, + SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) + throws IOException { + while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + + String nextPathSnapshot = + chainManager.nextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + + String tableKey = chainManager.getTableKey(nextPathSnapshot); + SnapshotInfo nextSnapshotInfo = + omSnapshotManager.getSnapshotInfo(tableKey); + + if (nextSnapshotInfo.getSnapshotStatus().equals( + SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { + return nextSnapshotInfo; + } + } + return null; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java new file mode 100644 index 000000000000..95d7bc0bafa2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; + +/** + * Response for OMSnapshotMoveDeletedKeysRequest. + */ +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { + + private OmSnapshot fromSnapshot; + private OmSnapshot nextSnapshot; + private List nextDBKeysList; + private List reclaimKeysList; + + public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, + @Nonnull OmSnapshot omFromSnapshot, OmSnapshot omNextSnapshot, + List nextDBKeysList, + List reclaimKeysList) { + super(omResponse); + this.fromSnapshot = omFromSnapshot; + this.nextSnapshot = omNextSnapshot; + this.nextDBKeysList = nextDBKeysList; + this.reclaimKeysList = reclaimKeysList; + } + + /** + * For when the request is not successful. + * For a successful request, the other constructor should be used. + */ + public OMSnapshotMoveDeletedKeysResponse(@Nonnull OMResponse omResponse) { + super(omResponse); + checkStatusNotOK(); + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + if (nextSnapshot != null) { + DBStore nextSnapshotStore = nextSnapshot.getMetadataManager().getStore(); + // Init Batch Operation for snapshot db. + try (BatchOperation writeBatch = nextSnapshotStore.initBatchOperation()) { + processKeys(writeBatch, nextSnapshot.getMetadataManager(), + nextDBKeysList); + nextSnapshotStore.commitBatchOperation(writeBatch); + } + } else { + // Handle the case where there is no next Snapshot. + processKeys(batchOperation, omMetadataManager, nextDBKeysList); + } + + // Update From Snapshot Deleted Table. + DBStore fromSnapshotStore = fromSnapshot.getMetadataManager().getStore(); + try (BatchOperation fromSnapshotBatchOp = + fromSnapshotStore.initBatchOperation()) { + processKeys(fromSnapshotBatchOp, fromSnapshot.getMetadataManager(), + reclaimKeysList); + fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); + } + } + + private void processKeys(BatchOperation batchOp, + OMMetadataManager metadataManager, + List keyList) throws IOException { + for (SnapshotMoveKeyInfos dBKey : keyList) { + RepeatedOmKeyInfo omKeyInfos = + createRepeatedOmKeyInfo(dBKey.getKeyInfosList()); + if (omKeyInfos == null) { + continue; + } + metadataManager.getDeletedTable().putWithBatch(batchOp, + dBKey.getKey(), omKeyInfos); + } + } + + private RepeatedOmKeyInfo createRepeatedOmKeyInfo(List keyInfoList) + throws IOException { + RepeatedOmKeyInfo result = null; + + for (KeyInfo keyInfo: keyInfoList) { + if (result == null) { + result = new RepeatedOmKeyInfo(OmKeyInfo.getFromProtobuf(keyInfo)); + } else { + result.addOmKeyInfo(OmKeyInfo.getFromProtobuf(keyInfo)); + } + } + + return result; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java new file mode 100644 index 000000000000..d98acd448616 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.service; + +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.Message; +import org.apache.ratis.protocol.RaftClientRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; + +/** + * Background Service to clean-up deleted snapshot and reclaim space. + */ +public class SnapshotDeletingService extends BackgroundService { + private static final Logger LOG = + LoggerFactory.getLogger(SnapshotDeletingService.class); + + // Use only a single thread for Snapshot Deletion. Multiple threads would read + // from the same table and can send deletion requests for same snapshot + // multiple times. + private static final int SNAPSHOT_DELETING_CORE_POOL_SIZE = 1; + private final ClientId clientId = ClientId.randomId(); + private final AtomicLong runCount; + + private final OzoneManager ozoneManager; + private final OmSnapshotManager omSnapshotManager; + private final SnapshotChainManager chainManager; + private final AtomicBoolean suspended; + private final OzoneConfiguration conf; + private final AtomicLong successRunCount; + private final long snapshotDeletionPerTask; + + public SnapshotDeletingService(long interval, long serviceTimeout, + OzoneManager ozoneManager) throws IOException { + super(SnapshotDeletingService.class.getSimpleName(), interval, + TimeUnit.MILLISECONDS, SNAPSHOT_DELETING_CORE_POOL_SIZE, + serviceTimeout); + this.ozoneManager = ozoneManager; + this.omSnapshotManager = ozoneManager.getOmSnapshotManager(); + this.chainManager = ozoneManager.getSnapshotChainManager(); + this.runCount = new AtomicLong(0); + this.successRunCount = new AtomicLong(0); + this.suspended = new AtomicBoolean(false); + this.conf = ozoneManager.getConfiguration(); + this.snapshotDeletionPerTask = conf + .getLong(SNAPSHOT_DELETING_LIMIT_PER_TASK, + SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT); + } + + private class SnapshotDeletingTask implements BackgroundTask { + + @Override + public BackgroundTaskResult call() throws Exception { + if (!shouldRun()) { + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + + runCount.incrementAndGet(); + + Table snapshotInfoTable = + ozoneManager.getMetadataManager().getSnapshotInfoTable(); + try (TableIterator> iterator = snapshotInfoTable.iterator()) { + + long snapshotLimit = snapshotDeletionPerTask; + + while (iterator.hasNext() && snapshotLimit > 0) { + SnapshotInfo snapInfo = iterator.next().getValue(); + SnapshotInfo.SnapshotStatus snapshotStatus = + snapInfo.getSnapshotStatus(); + + // Only Iterate in deleted snapshot + if (!snapshotStatus.equals( + SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED)) { + continue; + } + + OmSnapshot omSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(snapInfo.getVolumeName(), + snapInfo.getBucketName(), + getSnapshotPrefix(snapInfo.getName())); + + Table snapshotDeletedTable = + omSnapshot.getMetadataManager().getDeletedTable(); + + if (snapshotDeletedTable.isEmpty()) { + continue; + } + + // Get bucketInfo for the snapshot bucket to get bucket layout. + String dbBucketKey = ozoneManager.getMetadataManager().getBucketKey( + snapInfo.getVolumeName(), snapInfo.getBucketName()); + OmBucketInfo bucketInfo = ozoneManager.getMetadataManager() + .getBucketTable().get(dbBucketKey); + + if (bucketInfo == null) { + throw new OMException("Bucket " + snapInfo.getBucketName() + + " is not found", BUCKET_NOT_FOUND); + } + + //TODO: [SNAPSHOT] Add lock to deletedTable and Active DB. + SnapshotInfo previousSnapshot = getPreviousSnapshot(snapInfo); + Table previousKeyTable = null; + OmSnapshot omPreviousSnapshot = null; + + // Split RepeatedOmKeyInfo and update current snapshot deletedKeyTable + // and next snapshot deletedKeyTable. + if (previousSnapshot != null) { + omPreviousSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(previousSnapshot.getVolumeName(), + previousSnapshot.getBucketName(), + getSnapshotPrefix(previousSnapshot.getName())); + + previousKeyTable = omPreviousSnapshot + .getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); + } + + // Move key to either next non deleted snapshot's deletedTable + // or keep it in current snapshot deleted table. + List toReclaimList = new ArrayList<>(); + List toNextDBList = new ArrayList<>(); + + try (TableIterator> deletedIterator = snapshotDeletedTable + .iterator()) { + + String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; + iterator.seek(snapshotBucketKey); + + while (deletedIterator.hasNext()) { + Table.KeyValue + deletedKeyValue = deletedIterator.next(); + String deletedKey = deletedKeyValue.getKey(); + + // Exit if it is out of the bucket scope. + if (!deletedKey.startsWith(snapshotBucketKey)) { + break; + } + + RepeatedOmKeyInfo repeatedOmKeyInfo = deletedKeyValue.getValue(); + + SnapshotMoveKeyInfos.Builder toReclaim = SnapshotMoveKeyInfos + .newBuilder() + .setKey(deletedKey); + SnapshotMoveKeyInfos.Builder toNextDb = SnapshotMoveKeyInfos + .newBuilder() + .setKey(deletedKey); + + for (OmKeyInfo keyInfo: repeatedOmKeyInfo.getOmKeyInfoList()) { + splitRepeatedOmKeyInfo(toReclaim, toNextDb, + keyInfo, previousKeyTable); + } + + // If all the KeyInfos are reclaimable in RepeatedOmKeyInfo + // then no need to update current snapshot deletedKeyTable. + if (!(toReclaim.getKeyInfosCount() == + repeatedOmKeyInfo.getOmKeyInfoList().size())) { + toReclaimList.add(toReclaim.build()); + } + toNextDBList.add(toNextDb.build()); + + } + // Submit Move request to OM. + submitSnapshotMoveDeletedKeys(snapInfo, toReclaimList, + toNextDBList); + snapshotLimit--; + successRunCount.incrementAndGet(); + } catch (IOException ex) { + LOG.error("Error while running Snapshot Deleting Service", ex); + } + } + } catch (IOException e) { + LOG.error("Error while running Snapshot Deleting Service", e); + } + + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + + private void splitRepeatedOmKeyInfo(SnapshotMoveKeyInfos.Builder toReclaim, + SnapshotMoveKeyInfos.Builder toNextDb, OmKeyInfo keyInfo, + Table previousKeyTable) throws IOException { + if (checkKeyReclaimable(previousKeyTable, keyInfo)) { + // Move to next non deleted snapshot's deleted table + toNextDb.addKeyInfos(keyInfo.getProtobuf( + ClientVersion.CURRENT_VERSION)); + } else { + // Update in current db's deletedKeyTable + toReclaim.addKeyInfos(keyInfo + .getProtobuf(ClientVersion.CURRENT_VERSION)); + } + } + + private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, + List toReclaimList, + List toNextDBList) { + + SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = + SnapshotMoveDeletedKeysRequest.newBuilder() + .setFromSnapshot(snapInfo.getProtobuf()); + + SnapshotMoveDeletedKeysRequest moveDeletedKeys = + moveDeletedKeysBuilder.addAllReclaimKeys(toReclaimList) + .addAllNextDBKeys(toNextDBList).build(); + + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(Type.SnapshotMoveDeletedKeys) + .setSnapshotMoveDeletedKeysRequest(moveDeletedKeys) + .setClientId(clientId.toString()) + .build(); + + submitRequest(omRequest); + } + + private boolean checkKeyReclaimable( + Table previousKeyTable, OmKeyInfo deletedKeyInfo) + throws IOException { + + // Handle case when the deleted snapshot is the first snapshot. + if (previousKeyTable == null) { + return false; + } + + // These are uncommitted blocks wrapped into a pseudo KeyInfo + if (deletedKeyInfo.getObjectID() == OBJECT_ID_RECLAIM_BLOCKS) { + return false; + } + + //TODO: [SNAPSHOT] Handle Renamed Keys + String dbKey = ozoneManager.getMetadataManager() + .getOzoneKey(deletedKeyInfo.getVolumeName(), + deletedKeyInfo.getBucketName(), deletedKeyInfo.getKeyName()); + + OmKeyInfo prevKeyInfo = previousKeyTable.get(dbKey); + if (prevKeyInfo != null && + prevKeyInfo.getObjectID() == deletedKeyInfo.getObjectID()) { + return true; + } + return false; + } + + private SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo) + throws IOException { + if (chainManager.hasPreviousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + String previousPathSnapshot = chainManager.previousPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + String tableKey = chainManager.getTableKey(previousPathSnapshot); + return omSnapshotManager.getSnapshotInfo(tableKey); + } + return null; + } + + private void submitRequest(OMRequest omRequest) { + try { + if (isRatisEnabled()) { + OzoneManagerRatisServer server = ozoneManager.getOmRatisServer(); + + RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder() + .setClientId(clientId) + .setServerId(server.getRaftPeerId()) + .setGroupId(server.getRaftGroupId()) + .setCallId(runCount.get()) + .setMessage(Message.valueOf( + OMRatisHelper.convertRequestToByteString(omRequest))) + .setType(RaftClientRequest.writeRequestType()) + .build(); + + server.submitRequest(omRequest, raftClientRequest); + } else { + ozoneManager.getOmServerProtocol().submitRequest(null, omRequest); + } + } catch (ServiceException e) { + LOG.error("Snapshot Deleting request failed. " + + "Will retry at next run.", e); + } + } + + private boolean isRatisEnabled() { + return ozoneManager.isRatisEnabled(); + } + + } + + @Override + public BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + queue.add(new SnapshotDeletingTask()); + return queue; + } + + private boolean shouldRun() { + return !suspended.get() && ozoneManager.isLeaderReady(); + } + + /** + * Suspend the service. + */ + @VisibleForTesting + void suspend() { + suspended.set(true); + } + + /** + * Resume the service if suspended. + */ + @VisibleForTesting + void resume() { + suspended.set(false); + } + + public long getRunCount() { + return runCount.get(); + } + + public long getSuccessfulRunCount() { + return successRunCount.get(); + } + + @VisibleForTesting + public void setSuccessRunCount(long num) { + successRunCount.getAndSet(num); + } +} + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 94e551a33526..e530f712cb11 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -210,7 +211,11 @@ public void testPreExecuteNameLength() throws Exception { @Test public void testValidateAndUpdateCache() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); @@ -249,7 +254,11 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testEmptyRenamedKeyTable() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); OmKeyInfo toKeyInfo = addKey("key1"); OmKeyInfo fromKeyInfo = addKey("key2"); @@ -293,7 +302,11 @@ public void testEmptyRenamedKeyTable() throws Exception { @Test public void testEntryExists() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index 14b7955cbe1f..b8d7c74173b0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -258,6 +259,10 @@ public void testEntryNotExist() throws Exception { */ @Test public void testEntryExists() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); when(ozoneManager.isAdmin(any())).thenReturn(true); String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java new file mode 100644 index 000000000000..e1da3128c66e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.service; + +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; +import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmTestManagers; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.test.PathUtils; +import org.apache.ozone.test.GenericTestUtils; +import org.apache.ratis.util.ExitUtils; +import org.junit.Rule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT; + +/** + * Test Snapshot Deleting Service. + */ +public class TestSnapshotDeletingService { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private OzoneManagerProtocol writeClient; + private OzoneManager om; + + private KeyManager keyManager; + private OMMetadataManager omMetadataManager; + private OzoneConfiguration conf; + private OmTestManagers omTestManagers; + private static final String VOLUME_NAME = "vol1"; + private static final String BUCKET_NAME_ONE = "bucket1"; + private static final String BUCKET_NAME_TWO = "bucket2"; + + + @BeforeAll + public static void setup() { + ExitUtils.disableSystemExit(); + } + + @BeforeEach + public void createConfAndInitValues() throws Exception { + conf = new OzoneConfiguration(); + File testDir = PathUtils.getTestDir(TestSnapshotDeletingService.class); + System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); + ServerUtils.setOzoneMetaDirPath(conf, testDir.getPath()); + conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, + 1000, TimeUnit.MILLISECONDS); + conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT, + 100000, TimeUnit.MILLISECONDS); + conf.setQuietMode(false); + omTestManagers = new OmTestManagers(conf); + keyManager = omTestManagers.getKeyManager(); + omMetadataManager = omTestManagers.getMetadataManager(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + } + + @AfterEach + public void cleanup() throws Exception { + if (om != null) { + om.stop(); + } + } + + @Test + @Disabled("HDDS-7974") + public void testSnapshotKeySpaceReclaim() throws Exception { + SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) + keyManager.getSnapshotDeletingService(); + KeyDeletingService deletingService = (KeyDeletingService) + keyManager.getDeletingService(); + + // Suspending SnapshotDeletingService + snapshotDeletingService.suspend(); + createSnapshotDataForBucket1(); + snapshotDeletingService.resume(); + + deletingService.start(); + GenericTestUtils.waitFor(() -> + deletingService.getRunCount().get() >= 1, + 1000, 10000); + + GenericTestUtils.waitFor(() -> + snapshotDeletingService.getSuccessfulRunCount() >= 1, + 1000, 10000); + + OmSnapshot bucket1snap3 = (OmSnapshot) om.getOmSnapshotManager() + .checkForSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, + getSnapshotPrefix("bucket1snap3")); + + // Check bucket1key1 added to next non deleted snapshot db. + RepeatedOmKeyInfo omKeyInfo = + bucket1snap3.getMetadataManager() + .getDeletedTable().get("/vol1/bucket1/bucket1key1"); + Assertions.assertNotNull(omKeyInfo); + + // Check bucket1key2 not in active DB. As the key is updated + // in bucket1snap2 + RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager + .getDeletedTable().get("/vol1/bucket1/bucket1key2"); + Assertions.assertNull(omKeyInfo1); + deletingService.shutdown(); + } + + @Test + @Disabled("HDDS-7974") + public void testMultipleSnapshotKeyReclaim() throws Exception { + + SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) + keyManager.getSnapshotDeletingService(); + KeyDeletingService deletingService = (KeyDeletingService) + keyManager.getDeletingService(); + + // Suspending SnapshotDeletingService + snapshotDeletingService.suspend(); + int snapshotCount = createSnapshotDataForBucket1(); + + OmKeyArgs bucket2key1 = createVolumeBucketKey(VOLUME_NAME, BUCKET_NAME_TWO, + BucketLayout.DEFAULT, "bucket2key1"); + + OmKeyArgs bucket2key2 = createKey(VOLUME_NAME, BUCKET_NAME_TWO, + "bucket2key2"); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, "bucket2snap1", + ++snapshotCount); + + // Both key 1 and key 2 can be reclaimed when Snapshot 1 is deleted. + writeClient.deleteKey(bucket2key1); + writeClient.deleteKey(bucket2key2); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, "bucket2snap2", + ++snapshotCount); + + String snapshotKey2 = "/vol1/bucket2/bucket2snap1"; + SnapshotInfo snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + + snapshotInfo + .setSnapshotStatus(SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + om.getMetadataManager() + .getSnapshotInfoTable().put(snapshotKey2, snapshotInfo); + snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + Assertions.assertEquals(snapshotInfo.getSnapshotStatus(), + SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + + snapshotDeletingService.resume(); + + deletingService.start(); + GenericTestUtils.waitFor(() -> + deletingService.getRunCount().get() >= 1, + 1000, 10000); + + GenericTestUtils.waitFor(() -> + snapshotDeletingService.getSuccessfulRunCount() >= 1, + 1000, 10000); + + // Check bucket2key1 added active db as it can be reclaimed. + RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager + .getDeletedTable().get("/vol1/bucket2/bucket2key1"); + + // Check bucket2key2 added active db as it can be reclaimed. + RepeatedOmKeyInfo omKeyInfo2 = omMetadataManager + .getDeletedTable().get("/vol1/bucket2/bucket2key2"); + + //TODO: [SNAPSHOT] Check this shouldn't be null when KeyDeletingService + // is modified for Snapshot + Assertions.assertNull(omKeyInfo1); + Assertions.assertNull(omKeyInfo2); + deletingService.shutdown(); + } + + private OmKeyArgs createVolumeBucketKey(String volumeName, String bucketName, + BucketLayout bucketLayout, String keyName) throws IOException { + // cheat here, just create a volume and bucket entry so that we can + // create the keys, we put the same data for key and value since the + // system does not decode the object + OMRequestTestUtils.addVolumeToOM(omMetadataManager, + OmVolumeArgs.newBuilder() + .setOwnerName("owner") + .setAdminName("admin") + .setVolume(volumeName) + .build()); + + OMRequestTestUtils.addBucketToOM(omMetadataManager, + OmBucketInfo.newBuilder().setVolumeName(volumeName) + .setBucketName(bucketName) + .setBucketLayout(bucketLayout) + .build()); + + return createKey(volumeName, bucketName, keyName); + } + + + private int createSnapshotDataForBucket1() throws Exception { + int snapshotCount = 0; + OmKeyArgs bucket1key1 = createVolumeBucketKey(VOLUME_NAME, BUCKET_NAME_ONE, + BucketLayout.DEFAULT, "bucket1key1"); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap1", + ++snapshotCount); + + OmKeyArgs bucket1key2 = createKey(VOLUME_NAME, BUCKET_NAME_ONE, + "bucket1key2"); + + // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. + writeClient.deleteKey(bucket1key1); + // Key 2 is deleted here, which means we can reclaim + // it when snapshot 2 is deleted. + writeClient.deleteKey(bucket1key2); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap2", + ++snapshotCount); + createKey(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1key4"); + OmKeyArgs bucket1key5 = createKey(VOLUME_NAME, BUCKET_NAME_ONE, + "bucket1key5"); + writeClient.deleteKey(bucket1key5); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap3", + ++snapshotCount); + + String snapshotKey2 = "/vol1/bucket1/bucket1snap2"; + SnapshotInfo snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + + snapshotInfo + .setSnapshotStatus(SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + om.getMetadataManager() + .getSnapshotInfoTable().put(snapshotKey2, snapshotInfo); + snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + Assertions.assertEquals(snapshotInfo.getSnapshotStatus(), + SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + return snapshotCount; + } + + private OmKeyArgs createKey(String volumeName, String bucketName, + String keyName) throws IOException { + OmKeyArgs keyArg = + new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setAcls(Collections.emptyList()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setLocationInfoList(new ArrayList<>()) + .build(); + + // Open and write the key. + OpenKeySession session = writeClient.openKey(keyArg); + writeClient.commitKey(keyArg, session.getId()); + + return keyArg; + } + + private void createSnapshot(String volName, String bucketName, + String snapName, int count) throws Exception { + writeClient.createSnapshot(volName, bucketName, snapName); + + GenericTestUtils.waitFor(() -> { + try { + return omMetadataManager.countRowsInTable( + omMetadataManager.getSnapshotInfoTable()) >= count; + } catch (IOException e) { + e.printStackTrace(); + } + return false; + }, 1000, 10000); + } +} +