diff --git a/score-client/src/main/java/bio/overture/score/client/command/MountCommand.java b/score-client/src/main/java/bio/overture/score/client/command/MountCommand.java index f1fa66e8..03469d34 100644 --- a/score-client/src/main/java/bio/overture/score/client/command/MountCommand.java +++ b/score-client/src/main/java/bio/overture/score/client/command/MountCommand.java @@ -59,7 +59,6 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Slf4j @@ -67,7 +66,6 @@ @Parameters( separators = "=", commandDescription = "Mount a read-only FUSE file system view of the remote storage repository") -@Profile("!kf") public class MountCommand extends RepositoryAccessCommand { /** Constants. */ diff --git a/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java b/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java index 1e35e00d..9b17ede9 100644 --- a/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java +++ b/score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java @@ -36,7 +36,6 @@ import lombok.val; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Slf4j @@ -44,7 +43,6 @@ @Parameters( separators = "=", commandDescription = "Upload file object(s) to the remote storage repository") -@Profile("!kf") public class UploadCommand extends RepositoryAccessCommand { /** Options. */ diff --git a/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java b/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java index 26c42dcc..216b5b12 100644 --- a/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java +++ b/score-client/src/main/java/bio/overture/score/client/command/ViewCommand.java @@ -57,7 +57,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationArguments; -import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Slf4j @@ -65,7 +64,6 @@ @Parameters( separators = "=", commandDescription = "Locally store/display some or all of a remote SAM/BAM file object") -@Profile("!kf") public class ViewCommand extends RepositoryAccessCommand { public enum OutputFormat { diff --git a/score-client/src/main/java/bio/overture/score/client/config/StorageConfig.java b/score-client/src/main/java/bio/overture/score/client/config/StorageConfig.java index 6d8388ac..9b8e6c20 100644 --- a/score-client/src/main/java/bio/overture/score/client/config/StorageConfig.java +++ b/score-client/src/main/java/bio/overture/score/client/config/StorageConfig.java @@ -18,7 +18,7 @@ public class StorageConfig { @Value("${storage.url}") private String endpoint; - @Value("${kf.download.partSize}") + @Value("${download.partSize}") private int partSize; @Autowired private DownloadStateStore downloadStateStore; diff --git a/score-client/src/main/java/bio/overture/score/client/metadata/legacy/LegacyMetadataService.java b/score-client/src/main/java/bio/overture/score/client/metadata/legacy/LegacyMetadataService.java index 47944ab9..dbc946ea 100644 --- a/score-client/src/main/java/bio/overture/score/client/metadata/legacy/LegacyMetadataService.java +++ b/score-client/src/main/java/bio/overture/score/client/metadata/legacy/LegacyMetadataService.java @@ -24,11 +24,9 @@ import lombok.NonNull; import lombok.val; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; @Service -@Profile("!kf") public class LegacyMetadataService implements MetadataService { @Autowired private LegacyMetadataClient legacyMetadataClient; diff --git a/score-client/src/main/java/bio/overture/score/client/storage/score/ScoreStorageService.java b/score-client/src/main/java/bio/overture/score/client/storage/ScoreStorageService.java similarity index 99% rename from score-client/src/main/java/bio/overture/score/client/storage/score/ScoreStorageService.java rename to score-client/src/main/java/bio/overture/score/client/storage/ScoreStorageService.java index 3551a1be..d0a6793d 100644 --- a/score-client/src/main/java/bio/overture/score/client/storage/score/ScoreStorageService.java +++ b/score-client/src/main/java/bio/overture/score/client/storage/ScoreStorageService.java @@ -15,7 +15,7 @@ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package bio.overture.score.client.storage.score; +package bio.overture.score.client.storage; import static org.springframework.http.HttpMethod.DELETE; import static org.springframework.http.HttpMethod.GET; @@ -27,7 +27,6 @@ import bio.overture.score.client.exception.NotResumableException; import bio.overture.score.client.exception.NotRetryableException; import bio.overture.score.client.exception.RetryableException; -import bio.overture.score.client.storage.AbstractStorageService; import bio.overture.score.core.model.DataChannel; import bio.overture.score.core.model.ObjectInfo; import bio.overture.score.core.model.ObjectSpecification; diff --git a/score-client/src/main/java/bio/overture/score/client/storage/kf/KFStorageClient.java b/score-client/src/main/java/bio/overture/score/client/storage/kf/KFStorageClient.java deleted file mode 100644 index ca073c22..00000000 --- a/score-client/src/main/java/bio/overture/score/client/storage/kf/KFStorageClient.java +++ /dev/null @@ -1,129 +0,0 @@ -package bio.overture.score.client.storage.kf; - -import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; -import static java.util.Objects.isNull; -import static org.springframework.http.HttpHeaders.AUTHORIZATION; -import static org.springframework.http.HttpHeaders.CONTENT_LENGTH; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import lombok.Builder; -import lombok.Data; -import lombok.NonNull; -import lombok.SneakyThrows; -import lombok.val; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Profile; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; - -@Builder -@Component -@Profile("kf") -public class KFStorageClient { - - private static final long MIN_EXPIRATION = 3600; - private final String jwt; - private final String tokenUrl; - private final String apiUrl; - private final long urlExpiration; - private final RetryTemplate retry; - private final RestTemplate restTemplate = new RestTemplate(); - - public KFStorageClient( - @Value("${client.accessToken}") @NonNull String jwt, - @Value("${kf.token.url}") @NonNull String tokenUrl, - @Value("${kf.download.url}") @NonNull String apiUrl, - @Value("${kf.download.expiration}") long urlExpiration, - @Autowired RetryTemplate retry) { - this.jwt = jwt; - this.apiUrl = apiUrl; - this.urlExpiration = Math.max(urlExpiration, MIN_EXPIRATION); - this.tokenUrl = tokenUrl; - this.retry = retry; - } - - public String generateAccessToken() { - val accessTokenResponse = getObject(AccessTokenResponse.class, jwt, tokenUrl); - return accessTokenResponse.getAccessToken(); - } - - public PresignedUrl generatePresignedUrl(@NonNull String objectId) { - val accessToken = generateAccessToken(); - val response = getResponse(UrlResponse.class, accessToken, getKFDownloadEndpoint(objectId)); - val url = response.getBody().getUrl(); - val size = peekResponseContentSize(url); - return PresignedUrl.builder().size(size).url(url).build(); - } - - private String getKFDownloadEndpoint(String objectId) { - return format("%s/user/data/download/%s?expires_in=%s", apiUrl, objectId, urlExpiration); - } - - @SneakyThrows - private ResponseEntity getResponse(Class responseType, String accessToken, String url) { - val entity = new HttpEntity(null, buildAuthHeader(accessToken)); - return retry.execute( - ctx -> restTemplate.exchange(new URI(url), HttpMethod.GET, entity, responseType)); - } - - @SneakyThrows - private T getObject(Class responseType, String accessToken, String url) { - return getResponse(responseType, accessToken, url).getBody(); - } - - @SneakyThrows - private static long peekResponseContentSize(String url) { - val u = new URL(url); - val conn = (HttpURLConnection) u.openConnection(); - val contentLength = conn.getContentLengthLong(); - checkState( - contentLength != -1, - "The following response does not contain the '%s' header: %s", - CONTENT_LENGTH, - conn.getHeaderFields()); - checkState( - contentLength > 0, - "The following response contains a content length of 0: %s", - conn.getHeaderFields()); - return contentLength; - } - - private static HttpHeaders buildAuthHeader(String accessToken) { - val headers = new HttpHeaders(); - if (!isNull(accessToken)) { - headers.set(AUTHORIZATION, "Bearer " + accessToken); - } - return headers; - } - - @lombok.Value - @Builder - public static class PresignedUrl { - @NonNull private final String url; - private final long size; - } - - @Data - public static class AccessTokenResponse { - @JsonProperty("access_token") - private String accessToken; - - @JsonProperty("refresh_token") - private String refreshToken; - } - - @Data - public static class UrlResponse { - private String url; - } -} diff --git a/score-client/src/main/java/bio/overture/score/client/storage/kf/KFStorageService.java b/score-client/src/main/java/bio/overture/score/client/storage/kf/KFStorageService.java deleted file mode 100644 index f5f49d32..00000000 --- a/score-client/src/main/java/bio/overture/score/client/storage/kf/KFStorageService.java +++ /dev/null @@ -1,170 +0,0 @@ -package bio.overture.score.client.storage.kf; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; - -import bio.overture.score.client.download.DownloadStateStore; -import bio.overture.score.client.storage.AbstractStorageService; -import bio.overture.score.client.storage.kf.KFStorageClient.PresignedUrl; -import bio.overture.score.core.model.DataChannel; -import bio.overture.score.core.model.ObjectInfo; -import bio.overture.score.core.model.ObjectSpecification; -import bio.overture.score.core.model.Part; -import bio.overture.score.core.model.UploadProgress; -import bio.overture.score.core.util.SimplePartCalculator; -import com.google.common.collect.ImmutableList; -import java.io.IOException; -import java.util.List; -import java.util.Optional; -import lombok.NonNull; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Profile; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -@Slf4j -@Service -@Profile("kf") -public class KFStorageService extends AbstractStorageService { - - /** Dependencies. */ - private final KFStorageClient kfStorageClient; - - private final SimplePartCalculator partCalculator; - - @Autowired - public KFStorageService( - @NonNull DownloadStateStore downloadStateStore, - @Qualifier("pingTemplate") @NonNull RestTemplate dataTemplate, - @NonNull RetryTemplate retry, - SimplePartCalculator partCalculator, - @NonNull KFStorageClient kfStorageClient) { - super(downloadStateStore, dataTemplate, retry); - this.kfStorageClient = kfStorageClient; - this.partCalculator = partCalculator; - } - - @Override - public ObjectSpecification getDownloadSpecification(String objectId, long offset, long length) - throws IOException { - val presignedUrl = kfStorageClient.generatePresignedUrl(objectId); - val parts = generateParts(presignedUrl, offset, length); - return ObjectSpecification.builder() - .objectId(objectId) - .objectMd5(null) // useless, since we do not have the md5 values of the initial part uploads - .objectSize(presignedUrl.getSize()) - .parts(parts) - .objectKey(null) - .relocated(false) - .build(); - } - - @Override - public ObjectSpecification getExternalDownloadSpecification( - String objectId, long offset, long length) throws IOException { - return getDownloadSpecification(objectId); - } - - @Override - protected Optional getEncryptedAccessToken() { - return Optional.empty(); - } - - @SneakyThrows - private List generateParts(PresignedUrl presignedUrl, long offset, long length) { - val url = presignedUrl.getUrl(); - val totalSize = presignedUrl.getSize(); - checkArgument( - offset + length < totalSize, - "The sum of: offset(%s) + length(%s) = %s must be less than the total size (%s)", - offset, - length, - offset + length, - totalSize); - List parts; - if (offset == 0 && length < 0) { - parts = partCalculator.divide(totalSize); - } else { - parts = partCalculator.divide(offset, length); - } - parts.forEach( - x -> { - x.setMd5(null); // Unobtainable - x.setSourceMd5( - null); // useless, since we do not have the md5 values of the initial part uploads - x.setUrl(url); - }); - return ImmutableList.copyOf(parts); - } - - private static String getNonImplementedMessage() { - val calledMethod = Thread.currentThread().getStackTrace()[2].getMethodName(); - return format("The method '%s' is not implemented for KF", calledMethod); - } - - /** Not implemented for KF */ - @Override - public String ping() { - log.warn("The 'ping' method is not implemented properly in KF. This is a bypass"); - return "bypass ping"; - } - - @Override - public boolean isUploadDataRecoverable(String objectId, long fileSize) throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public void deleteUploadPart(String objectId, String uploadId, Part part) throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public void finalizeUpload(String objectId, String uploadId) throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public void finalizeUploadPart( - String objectId, - String uploadId, - int partNumber, - String md5, - String etag, - boolean disableChecksum) - throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public boolean isObjectExist(String objectId) throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public void uploadPart(DataChannel channel, Part part, String objectId, String uploadId) - throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public List listObjects() { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public UploadProgress getProgress(String objectId, long fileSize) throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } - - @Override - public ObjectSpecification initiateUpload( - String objectId, long length, boolean overwrite, String md5) throws IOException { - throw new IllegalStateException(getNonImplementedMessage()); - } -} diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index a2f3558d..cad66423 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -36,16 +36,9 @@ client: ssl: custom: false -kfportal: - url: https://arranger.kids-first.io/june_13/graphql -kf: - token: - url: https://v2i1r42t6d.execute-api.us-east-1.amazonaws.com/beta/token - download: - url: https://data.kidsfirstdrc.org - partSize: 1073741824 - expiration: 86400 +download: + partSize: 1073741824 transport: fileFrom: memory diff --git a/score-client/src/test/resources/application.yml b/score-client/src/test/resources/application.yml index 2b3fd078..537c3611 100644 --- a/score-client/src/test/resources/application.yml +++ b/score-client/src/test/resources/application.yml @@ -36,16 +36,8 @@ client: ssl: custom: false -kfportal: - url: https://arranger.kids-first.io/june_13/graphql - -kf: - token: - url: https://v2i1r42t6d.execute-api.us-east-1.amazonaws.com/beta/token - download: - url: https://data.kidsfirstdrc.org - partSize: 1073741824 - expiration: 86400 +download: + partSize: 1073741824 transport: fileFrom: memory