Skip to content

Commit c1b27a8

Browse files
SaketaChalamchalaSaketaChalamchala
authored
HDDS-10630. Add missing parent directories deleted between initiate and complete MPU (apache#6496)
1 parent 0c59c18 commit c1b27a8

5 files changed

Lines changed: 190 additions & 11 deletions

File tree

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java‎

Lines changed: 97 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import java.io.IOException;
2525
import java.nio.file.InvalidPathException;
26+
import java.nio.file.Paths;
2627
import java.util.ArrayList;
2728
import java.util.Collections;
2829
import java.util.List;
@@ -31,6 +32,10 @@
3132
import java.util.function.BiFunction;
3233

3334
import org.apache.hadoop.hdds.client.ReplicationConfig;
35+
import org.apache.hadoop.ozone.om.OzoneConfigUtil;
36+
import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestWithFSO;
37+
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
38+
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
3439
import org.apache.ratis.server.protocol.TermIndex;
3540
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
3641
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
@@ -41,6 +46,7 @@
4146
import org.apache.hadoop.ozone.om.exceptions.OMException;
4247
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
4348
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
49+
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
4450
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4551
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
4652
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -176,11 +182,72 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
176182
OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
177183
volumeName, bucketName);
178184

179-
String ozoneKey = omMetadataManager.getOzoneKey(
180-
volumeName, bucketName, keyName);
181-
182-
String dbOzoneKey =
183-
getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName);
185+
List<OmDirectoryInfo> missingParentInfos;
186+
OMFileRequest.OMPathInfoWithFSO pathInfoFSO = OMFileRequest
187+
.verifyDirectoryKeysInPath(omMetadataManager, volumeName, bucketName,
188+
keyName, Paths.get(keyName));
189+
missingParentInfos = OMDirectoryCreateRequestWithFSO
190+
.getAllMissingParentDirInfo(ozoneManager, keyArgs, omBucketInfo,
191+
pathInfoFSO, trxnLogIndex);
192+
193+
if (missingParentInfos != null) {
194+
final long volumeId = omMetadataManager.getVolumeId(volumeName);
195+
final long bucketId = omMetadataManager.getBucketId(volumeName,
196+
bucketName);
197+
198+
// add all missing parents to directory table
199+
addMissingParentsToCache(omBucketInfo, missingParentInfos,
200+
omMetadataManager, volumeId, bucketId, trxnLogIndex);
201+
202+
String multipartOpenKey = omMetadataManager
203+
.getMultipartKey(volumeId, bucketId,
204+
pathInfoFSO.getLastKnownParentId(),
205+
pathInfoFSO.getLeafNodeName(),
206+
keyArgs.getMultipartUploadID());
207+
208+
if (getOmKeyInfoFromOpenKeyTable(multipartOpenKey,
209+
keyName, omMetadataManager) == null) {
210+
211+
final ReplicationConfig replicationConfig = OzoneConfigUtil
212+
.resolveReplicationConfigPreference(keyArgs.getType(),
213+
keyArgs.getFactor(), keyArgs.getEcReplicationConfig(),
214+
omBucketInfo != null ?
215+
omBucketInfo.getDefaultReplicationConfig() :
216+
null, ozoneManager);
217+
218+
OmMultipartKeyInfo multipartKeyInfoFromArgs =
219+
new OmMultipartKeyInfo.Builder()
220+
.setUploadID(keyArgs.getMultipartUploadID())
221+
.setCreationTime(keyArgs.getModificationTime())
222+
.setReplicationConfig(replicationConfig)
223+
.setObjectID(pathInfoFSO.getLeafNodeObjectId())
224+
.setUpdateID(trxnLogIndex)
225+
.setParentID(pathInfoFSO.getLastKnownParentId())
226+
.build();
227+
228+
OmKeyInfo keyInfoFromArgs = new OmKeyInfo.Builder()
229+
.setVolumeName(volumeName)
230+
.setBucketName(bucketName)
231+
.setKeyName(keyName)
232+
.setCreationTime(keyArgs.getModificationTime())
233+
.setModificationTime(keyArgs.getModificationTime())
234+
.setReplicationConfig(replicationConfig)
235+
.setOmKeyLocationInfos(Collections.singletonList(
236+
new OmKeyLocationInfoGroup(0, new ArrayList<>(), true)))
237+
.setAcls(getAclsForKey(keyArgs, omBucketInfo, pathInfoFSO,
238+
ozoneManager.getPrefixManager()))
239+
.setObjectID(pathInfoFSO.getLeafNodeObjectId())
240+
.setUpdateID(trxnLogIndex)
241+
.setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ?
242+
OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null)
243+
.setParentObjectID(pathInfoFSO.getLastKnownParentId())
244+
.build();
245+
246+
// Add missing multi part info to open key table
247+
addMultiPartToCache(omMetadataManager, multipartOpenKey,
248+
pathInfoFSO, keyInfoFromArgs, trxnLogIndex);
249+
}
250+
}
184251

185252
String dbMultipartOpenKey =
186253
getDBMultipartOpenKey(volumeName, bucketName, keyName, uploadID,
@@ -189,6 +256,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
189256
OmMultipartKeyInfo multipartKeyInfo = omMetadataManager
190257
.getMultipartInfoTable().get(multipartKey);
191258

259+
String ozoneKey = omMetadataManager.getOzoneKey(
260+
volumeName, bucketName, keyName);
261+
262+
String dbOzoneKey =
263+
getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName);
264+
192265
// Check for directory exists with same name for the LEGACY_FS,
193266
// if it exists throw error.
194267
checkDirectoryAlreadyExists(ozoneManager, omBucketInfo, keyName,
@@ -284,7 +357,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
284357
omClientResponse =
285358
getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey,
286359
omKeyInfo, allKeyInfoToRemove, omBucketInfo,
287-
volumeId, bucketId);
360+
volumeId, bucketId, missingParentInfos, multipartKeyInfo);
288361

289362
result = Result.SUCCESS;
290363
} else {
@@ -325,7 +398,8 @@ protected OMClientResponse getOmClientResponse(String multipartKey,
325398
OMResponse.Builder omResponse, String dbMultipartOpenKey,
326399
OmKeyInfo omKeyInfo, List<OmKeyInfo> allKeyInfoToRemove,
327400
OmBucketInfo omBucketInfo,
328-
long volumeId, long bucketId) {
401+
long volumeId, long bucketId, List<OmDirectoryInfo> missingParentInfos,
402+
OmMultipartKeyInfo multipartKeyInfo) {
329403

330404
return new S3MultipartUploadCompleteResponse(omResponse.build(),
331405
multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove,
@@ -464,6 +538,22 @@ protected String getDBOzoneKey(OMMetadataManager omMetadataManager,
464538
return omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
465539
}
466540

541+
protected void addMissingParentsToCache(OmBucketInfo omBucketInfo,
542+
List<OmDirectoryInfo> missingParentInfos,
543+
OMMetadataManager omMetadataManager,
544+
long volumeId, long bucketId, long transactionLogIndex
545+
) throws IOException {
546+
// FSO is disabled. Do nothing.
547+
}
548+
549+
protected void addMultiPartToCache(
550+
OMMetadataManager omMetadataManager, String multipartOpenKey,
551+
OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo,
552+
long transactionLogIndex
553+
) throws IOException {
554+
// FSO is disabled. Do nothing.
555+
}
556+
467557
protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneKey,
468558
String keyName, OMMetadataManager omMetadataManager) throws IOException {
469559
return omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
import org.apache.hadoop.ozone.om.exceptions.OMException;
2424
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
2525
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
26+
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
2627
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
28+
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
2729
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
2830
import org.apache.hadoop.ozone.om.response.OMClientResponse;
2931
import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse;
@@ -74,6 +76,37 @@ protected void checkDirectoryAlreadyExists(OzoneManager ozoneManager,
7476
}
7577
}
7678

79+
@Override
80+
protected void addMissingParentsToCache(OmBucketInfo omBucketInfo,
81+
List<OmDirectoryInfo> missingParentInfos,
82+
OMMetadataManager omMetadataManager, long volumeId, long bucketId,
83+
long transactionLogIndex) throws IOException {
84+
85+
// validate and update namespace for missing parent directory.
86+
checkBucketQuotaInNamespace(omBucketInfo, missingParentInfos.size());
87+
omBucketInfo.incrUsedNamespace(missingParentInfos.size());
88+
89+
// Add cache entries for the missing parent directories.
90+
OMFileRequest.addDirectoryTableCacheEntries(omMetadataManager,
91+
volumeId, bucketId, transactionLogIndex,
92+
missingParentInfos, null);
93+
}
94+
95+
@Override
96+
protected void addMultiPartToCache(
97+
OMMetadataManager omMetadataManager, String multipartOpenKey,
98+
OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo,
99+
long transactionLogIndex
100+
) throws IOException {
101+
102+
// Add multi part to cache
103+
OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager,
104+
multipartOpenKey, omKeyInfo, pathInfoFSO.getLeafNodeName(),
105+
transactionLogIndex);
106+
107+
}
108+
109+
77110
@Override
78111
protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneFileKey,
79112
String keyName, OMMetadataManager omMetadataManager) throws IOException {
@@ -147,11 +180,13 @@ protected OMClientResponse getOmClientResponse(String multipartKey,
147180
OzoneManagerProtocolProtos.OMResponse.Builder omResponse,
148181
String dbMultipartOpenKey, OmKeyInfo omKeyInfo,
149182
List<OmKeyInfo> allKeyInfoToRemove, OmBucketInfo omBucketInfo,
150-
long volumeId, long bucketId) {
183+
long volumeId, long bucketId, List<OmDirectoryInfo> missingParentInfos,
184+
OmMultipartKeyInfo multipartKeyInfo) {
151185

152186
return new S3MultipartUploadCompleteResponseWithFSO(omResponse.build(),
153187
multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove,
154-
getBucketLayout(), omBucketInfo, volumeId, bucketId);
188+
getBucketLayout(), omBucketInfo, volumeId, bucketId,
189+
missingParentInfos, multipartKeyInfo);
155190
}
156191

157192
@Override

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,12 @@ protected String addToKeyTable(OMMetadataManager omMetadataManager,
129129
protected OmKeyInfo getOmKeyInfo() {
130130
return omKeyInfo;
131131
}
132+
133+
protected OmBucketInfo getOmBucketInfo() {
134+
return omBucketInfo;
135+
}
136+
137+
protected String getMultiPartKey() {
138+
return multipartKey;
139+
}
132140
}

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
import org.apache.hadoop.ozone.om.OMMetadataManager;
2323
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
2424
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
25+
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
2526
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
27+
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
2628
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
2729
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
2830
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -52,6 +54,10 @@ public class S3MultipartUploadCompleteResponseWithFSO
5254
private long volumeId;
5355
private long bucketId;
5456

57+
private List<OmDirectoryInfo> missingParentInfos;
58+
59+
private OmMultipartKeyInfo multipartKeyInfo;
60+
5561
@SuppressWarnings("checkstyle:ParameterNumber")
5662
public S3MultipartUploadCompleteResponseWithFSO(
5763
@Nonnull OMResponse omResponse,
@@ -61,11 +67,15 @@ public S3MultipartUploadCompleteResponseWithFSO(
6167
@Nonnull List<OmKeyInfo> allKeyInfoToRemove,
6268
@Nonnull BucketLayout bucketLayout,
6369
OmBucketInfo omBucketInfo,
64-
@Nonnull long volumeId, @Nonnull long bucketId) {
70+
@Nonnull long volumeId, @Nonnull long bucketId,
71+
List<OmDirectoryInfo> missingParentInfos,
72+
OmMultipartKeyInfo multipartKeyInfo) {
6573
super(omResponse, multipartKey, multipartOpenKey, omKeyInfo,
6674
allKeyInfoToRemove, bucketLayout, omBucketInfo);
6775
this.volumeId = volumeId;
6876
this.bucketId = bucketId;
77+
this.missingParentInfos = missingParentInfos;
78+
this.multipartKeyInfo = multipartKeyInfo;
6979
}
7080

7181
/**
@@ -78,6 +88,39 @@ public S3MultipartUploadCompleteResponseWithFSO(
7888
checkStatusNotOK();
7989
}
8090

91+
@Override
92+
public void addToDBBatch(OMMetadataManager omMetadataManager,
93+
BatchOperation batchOperation) throws IOException {
94+
if (missingParentInfos != null) {
95+
// Create missing parent directory entries.
96+
for (OmDirectoryInfo parentDirInfo : missingParentInfos) {
97+
final String parentKey = omMetadataManager.getOzonePathKey(
98+
volumeId, bucketId, parentDirInfo.getParentObjectID(),
99+
parentDirInfo.getName());
100+
omMetadataManager.getDirectoryTable().putWithBatch(batchOperation,
101+
parentKey, parentDirInfo);
102+
}
103+
104+
// namespace quota changes for parent directory
105+
String bucketKey = omMetadataManager.getBucketKey(
106+
getOmBucketInfo().getVolumeName(),
107+
getOmBucketInfo().getBucketName());
108+
omMetadataManager.getBucketTable().putWithBatch(batchOperation,
109+
bucketKey, getOmBucketInfo());
110+
111+
if (OMFileRequest.getOmKeyInfoFromFileTable(true,
112+
omMetadataManager, getMultiPartKey(), getOmKeyInfo().getKeyName())
113+
!= null) {
114+
// Add multi part to open key table.
115+
OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager,
116+
batchOperation,
117+
getOmKeyInfo(), multipartKeyInfo.getUploadID(), volumeId,
118+
bucketId);
119+
}
120+
}
121+
super.addToDBBatch(omMetadataManager, batchOperation);
122+
}
123+
81124
@Override
82125
protected String addToKeyTable(OMMetadataManager omMetadataManager,
83126
BatchOperation batchOperation) throws IOException {

‎hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,8 @@ public S3MultipartUploadCompleteResponse createS3CompleteMPUResponseFSO(
306306

307307
String multipartKey = omMetadataManager
308308
.getMultipartKey(volumeName, bucketName, keyName, multipartUploadID);
309+
OmMultipartKeyInfo multipartKeyInfo = omMetadataManager
310+
.getMultipartInfoTable().get(multipartKey);
309311

310312
final long volumeId = omMetadataManager.getVolumeId(volumeName);
311313
final long bucketId = omMetadataManager.getBucketId(volumeName,
@@ -324,7 +326,8 @@ public S3MultipartUploadCompleteResponse createS3CompleteMPUResponseFSO(
324326

325327
return new S3MultipartUploadCompleteResponseWithFSO(omResponse,
326328
multipartKey, multipartOpenKey, omKeyInfo, allKeyInfoToRemove,
327-
getBucketLayout(), omBucketInfo, volumeId, bucketId);
329+
getBucketLayout(), omBucketInfo, volumeId, bucketId, null,
330+
multipartKeyInfo);
328331
}
329332

330333
protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp(

0 commit comments

Comments
 (0)