From 1a4c98da3e605fe981a70b748b765e535899c3f1 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Wed, 26 Jun 2024 15:51:08 -0400 Subject: [PATCH 1/2] Retrieve MD5 from object metadata --- .../score/server/config/S3Config.java | 3 ++ .../repository/s3/S3DownloadService.java | 37 ++++++++++++++++++- .../src/main/resources/application.yml | 3 ++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/score-server/src/main/java/bio/overture/score/server/config/S3Config.java b/score-server/src/main/java/bio/overture/score/server/config/S3Config.java index 282266ca..927e19ab 100644 --- a/score-server/src/main/java/bio/overture/score/server/config/S3Config.java +++ b/score-server/src/main/java/bio/overture/score/server/config/S3Config.java @@ -39,6 +39,9 @@ public class S3Config { private boolean sigV4Enabled; private String masterEncryptionKeyId; + @Value("md5chksum") + private String customMd5Property; + @Value("${upload.connection.timeout}") private int connectionTimeout; diff --git a/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java b/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java index 5fa0904e..0b65f987 100644 --- a/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java +++ b/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java @@ -23,8 +23,10 @@ import bio.overture.score.core.model.ObjectKey; import bio.overture.score.core.model.ObjectSpecification; import bio.overture.score.core.model.Part; +import bio.overture.score.core.util.MD5s; import bio.overture.score.core.util.ObjectKeys; import bio.overture.score.core.util.PartCalculator; +import bio.overture.score.server.config.S3Config; import bio.overture.score.server.exception.IdNotFoundException; import bio.overture.score.server.exception.InternalUnrecoverableError; import bio.overture.score.server.exception.NotRetryableException; @@ -89,6 +91,7 @@ public class S3DownloadService implements DownloadService { @Autowired private URLGenerator urlGenerator; @Autowired private PartCalculator partCalculator; @Autowired private MetadataService metadataService; + @Autowired private S3Config s3config; @Override public ObjectSpecification download( @@ -118,6 +121,9 @@ public ObjectSpecification download( parts = partCalculator.divide(offset, length); } fillPartUrls(objectKey, parts, false, forExternalUse); + + val md5 = getObjectMd5(metadata); + objectSpec = new ObjectSpecification( objectKey.getKey(), @@ -125,7 +131,7 @@ public ObjectSpecification download( objectId, parts, metadata.getContentLength(), - metadata.getETag(), + md5, false); } @@ -189,6 +195,35 @@ public ObjectSpecification download( } } + /** + * Looks for MD5 hash in the object metadata. This is used as part of the fallback behaviour when + * the .meta file cannot be found. To find the MD5, we will look for a value using the built in S3 + * getContendMD5(), and if no value is found there we will check in a configurable user meta data + * property. The name of this property is configurable via the S3 Config. If no MD5 value can be + * found in either of these locations then it will be returned null. + * + *

A user can still download files with the MD5 set to null, but they will always fail to + * validate through the CLI. To complete a download of a file in this state, the user should add + * the argument to their CLI download command: --validate false + * + * @param metadata + * @return + */ + private String getObjectMd5(ObjectMetadata metadata) { + val contentMd5 = metadata.getContentMD5(); + if (contentMd5 != null) { + return contentMd5; + } + val userMetadataMd5 = + metadata.getUserMetaDataOf(s3config.getCustomMd5Property()); // get literal from config + if (userMetadataMd5 != null) { + return MD5s.toHex(userMetadataMd5); + } + + // No value found, returning null. + return null; + } + private static ObjectSpecification removeUrls(ObjectSpecification spec) { spec.getParts().forEach(x -> x.setUrl(null)); return spec; diff --git a/score-server/src/main/resources/application.yml b/score-server/src/main/resources/application.yml index a17920bd..f9a86c25 100644 --- a/score-server/src/main/resources/application.yml +++ b/score-server/src/main/resources/application.yml @@ -34,6 +34,9 @@ s3: secured: true sigV4Enabled: true + # custom meta property with md5 hash, unused when upload state files are available (default behaviour) + # customMd5Property: md5chksum + #amazon endpoint: s3-external-1.amazonaws.com From 82dda61793d38b8795441c6165c9b61765fd4c11 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Wed, 26 Jun 2024 15:55:32 -0400 Subject: [PATCH 2/2] Decode Base64 formatted contentMD5 value --- .../overture/score/server/repository/s3/S3DownloadService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java b/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java index 0b65f987..bf8ceb7e 100644 --- a/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java +++ b/score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java @@ -212,7 +212,7 @@ public ObjectSpecification download( private String getObjectMd5(ObjectMetadata metadata) { val contentMd5 = metadata.getContentMD5(); if (contentMd5 != null) { - return contentMd5; + return MD5s.toHex(contentMd5); } val userMetadataMd5 = metadata.getUserMetaDataOf(s3config.getCustomMd5Property()); // get literal from config