From 7f0e4c37920fe451803e5cda911e4f88b776af6c Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 18 Jun 2019 16:41:24 +0100 Subject: [PATCH 01/22] Added initial work for cosmos indexing strategy --- Microsoft.Health.Dicom.sln | 18 +- .../Features/Storage/DicomBlobDataStore.cs | 1 + .../Features/Persistence/DicomDataStore.cs | 66 +++++ .../Features/Persistence/DicomIdentity.cs | 49 ++++ .../Exceptions/IndexDataStoreException.cs | 47 ++++ .../Persistence}/IDicomBlobDataStore.cs | 2 +- .../Features/Persistence/IDicomDataStore.cs | 16 ++ .../Persistence/IDicomIndexDataStore.cs | 25 ++ .../Validation/DicomIdentifierValidator.cs | 4 +- .../Microsoft.Health.Dicom.Core.csproj | 1 + .../Documents/QuerySeriesDocumentTests.cs | 61 +++++ .../DicomCollectionUpgradeManagerTests.cs | 102 ++++++++ ...oft.Health.Dicom.CosmosDb.UnitTests.csproj | 21 ++ .../AssemblyInfo.cs | 11 + .../Config/DicomCosmosConfiguration.cs | 15 ++ .../Constants.cs | 12 + .../Features/Health/DicomCosmosHealthCheck.cs | 42 ++++ .../Features/Storage/DicomCosmosDataStore.cs | 230 ++++++++++++++++++ .../Storage/DicomDocumentClientInitializer.cs | 145 +++++++++++ .../Storage/DocumentClientExtensions.cs | 112 +++++++++ .../Storage/Documents/QueryInstance.cs | 64 +++++ .../Storage/Documents/QuerySeriesDocument.cs | 92 +++++++ .../DicomCollectionUpgradeManager.cs | 34 +++ .../Versioning/IDicomCollectionUpdater.cs | 13 + .../Microsoft.Health.Dicom.CosmosDb.csproj | 18 ++ .../DicomCosmosDbRegistrationExtensions.cs | 74 ++++++ ...soft.Health.Dicom.Tests.Integration.csproj | 1 + .../Persistence/DicomBlobStorageTests.cs | 4 +- .../DicomBlobStorageTestsFixture.cs | 3 +- .../DicomCosmosDataStoreTestsFixture.cs | 101 ++++++++ .../Persistence/DicomIndexDataStoreTests.cs | 59 +++++ ...icrosoft.Health.Dicom.Web.Tests.E2E.csproj | 2 + 32 files changed, 1438 insertions(+), 7 deletions(-) create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs rename src/{Microsoft.Health.Dicom.Blob/Features/Storage => Microsoft.Health.Dicom.Core/Features/Persistence}/IDicomBlobDataStore.cs (93%) create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Constants.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs create mode 100644 test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs create mode 100644 test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs diff --git a/Microsoft.Health.Dicom.sln b/Microsoft.Health.Dicom.sln index 93f8d8963e..2923d5fc31 100644 --- a/Microsoft.Health.Dicom.sln +++ b/Microsoft.Health.Dicom.sln @@ -28,9 +28,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Core EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Api", "src\Microsoft.Health.Dicom.Api\Microsoft.Health.Dicom.Api.csproj", "{B0570D75-E376-44AC-870B-87ECB54F0AE3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.Api.UnitTests", "src\Microsoft.Health.Dicom.Api.UnitTests\Microsoft.Health.Dicom.Api.UnitTests.csproj", "{D7B538E5-8B3B-487C-8F6A-475F80C50DFE}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Api.UnitTests", "src\Microsoft.Health.Dicom.Api.UnitTests\Microsoft.Health.Dicom.Api.UnitTests.csproj", "{D7B538E5-8B3B-487C-8F6A-475F80C50DFE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.Core.UnitTests", "src\Microsoft.Health.Dicom.Core.UnitTests\Microsoft.Health.Dicom.Core.UnitTests.csproj", "{FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Core.UnitTests", "src\Microsoft.Health.Dicom.Core.UnitTests\Microsoft.Health.Dicom.Core.UnitTests.csproj", "{FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.CosmosDb", "src\Microsoft.Health.Dicom.CosmosDb\Microsoft.Health.Dicom.CosmosDb.csproj", "{E78AB378-6CF4-442A-BC34-81A1B7431ECD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.CosmosDb.UnitTests", "src\Microsoft.Health.Dicom.CosmosDb.UnitTests\Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj", "{FA0B32A4-E623-4B24-902D-BFDC6E42E612}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -78,6 +82,14 @@ Global {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}.Release|Any CPU.Build.0 = Release|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Release|Any CPU.Build.0 = Release|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -93,6 +105,8 @@ Global {B0570D75-E376-44AC-870B-87ECB54F0AE3} = {176641B3-297C-4E04-A83D-8F80F80485E8} {D7B538E5-8B3B-487C-8F6A-475F80C50DFE} = {176641B3-297C-4E04-A83D-8F80F80485E8} {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D} = {176641B3-297C-4E04-A83D-8F80F80485E8} + {E78AB378-6CF4-442A-BC34-81A1B7431ECD} = {176641B3-297C-4E04-A83D-8F80F80485E8} + {FA0B32A4-E623-4B24-902D-BFDC6E42E612} = {176641B3-297C-4E04-A83D-8F80F80485E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution RESX_SortFileContentOnSave = True diff --git a/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs b/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs index 8f3cdf1c51..d9b3c26cea 100644 --- a/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs +++ b/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Blob.Configs; +using Microsoft.Health.Dicom.Core.Features.Persistence; namespace Microsoft.Health.Dicom.Blob.Features.Storage { diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs new file mode 100644 index 0000000000..49a4c0ca03 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs @@ -0,0 +1,66 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Dicom; +using EnsureThat; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomDataStore : IDicomDataStore + { + // This value should not be changed without taking into consideration concurrency. + private const bool OverwriteBlobsIfExists = false; + private readonly IDicomBlobDataStore _dicomBlobDataStore; + private readonly IDicomIndexDataStore _dicomIndexDataStore; + + public DicomDataStore( + IDicomBlobDataStore dicomBlobDataStore, + IDicomIndexDataStore dicomIndexDataStore) + { + EnsureArg.IsNotNull(dicomBlobDataStore, nameof(dicomBlobDataStore)); + EnsureArg.IsNotNull(dicomIndexDataStore, nameof(dicomIndexDataStore)); + + _dicomBlobDataStore = dicomBlobDataStore; + _dicomIndexDataStore = dicomIndexDataStore; + } + + public async Task StoreAsync(DicomFile dicomFile, CancellationToken cancellationToken = default) + { + var dicomIdentity = DicomIdentity.Create(dicomFile.Dataset); + var instanceBlobName = GetInstanceBlobName(dicomIdentity); + + // Step 1. Store the blob file + using (var memoryStream = new MemoryStream()) + { + await dicomFile.SaveAsync(memoryStream); + await _dicomBlobDataStore.AddFileAsStreamAsync(instanceBlobName, memoryStream, overwriteIfExists: OverwriteBlobsIfExists, cancellationToken); + } + + // Step 2. Attempt to index the file; this will fail if it exists, which should never happen. + await _dicomIndexDataStore.IndexInstanceAsync(dicomFile.Dataset, cancellationToken); + } + + public async Task DeleteInstanceAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default) + { + var instanceBlobName = GetInstanceBlobName(studyInstanceUID, seriesInstanceUID, sopInstanceUID); + + // Reverse process of store. + // 1. Delete the instance index + await _dicomIndexDataStore.DeleteInstanceIndexAsync(studyInstanceUID, seriesInstanceUID, sopInstanceUID, cancellationToken); + + // 2. Delete the instance file. + await _dicomBlobDataStore.DeleteFileIfExistsAsync(instanceBlobName, cancellationToken); + } + + private static string GetInstanceBlobName(DicomIdentity dicomIdentity) + => GetInstanceBlobName(dicomIdentity.StudyInstanceUID, dicomIdentity.SeriesInstanceUID, dicomIdentity.SopInstanceUID); + + private static string GetInstanceBlobName(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) + => $"{studyInstanceUID}\\{seriesInstanceUID}\\{sopInstanceUID}"; + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs new file mode 100644 index 0000000000..219e18c4ec --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Text.RegularExpressions; +using Dicom; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomIdentity + { + [JsonConstructor] + public DicomIdentity(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + + // Run the instance identifiers through the regular expression check. + EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + + StudyInstanceUID = studyInstanceUID; + SeriesInstanceUID = seriesInstanceUID; + SopInstanceUID = sopInstanceUID; + } + + public string StudyInstanceUID { get; } + + public string SeriesInstanceUID { get; } + + public string SopInstanceUID { get; } + + public static DicomIdentity Create(DicomDataset dicomDataset) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + + return new DicomIdentity( + dicomDataset.GetSingleValue(DicomTag.StudyInstanceUID), + dicomDataset.GetSingleValue(DicomTag.SeriesInstanceUID), + dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID)); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs new file mode 100644 index 0000000000..89e8ed94ce --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Net; +using System.Runtime.Serialization; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions +{ + [Serializable] + public class IndexDataStoreException : Exception + { + public IndexDataStoreException(HttpStatusCode httpStatusCode) + { + StatusCode = httpStatusCode; + } + + public IndexDataStoreException(HttpStatusCode? httpStatusCode, Exception innerException) + : base(innerException.Message, innerException) + { + StatusCode = httpStatusCode ?? HttpStatusCode.InternalServerError; + } + + public IndexDataStoreException() + { + } + + public IndexDataStoreException(string message) + : base(message) + { + } + + public IndexDataStoreException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected IndexDataStoreException(SerializationInfo serializationInfo, StreamingContext streamingContext) + { + throw new NotImplementedException(); + } + + public HttpStatusCode StatusCode { get; } + } +} diff --git a/src/Microsoft.Health.Dicom.Blob/Features/Storage/IDicomBlobDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomBlobDataStore.cs similarity index 93% rename from src/Microsoft.Health.Dicom.Blob/Features/Storage/IDicomBlobDataStore.cs rename to src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomBlobDataStore.cs index 68408ba268..3ba2f2bf41 100644 --- a/src/Microsoft.Health.Dicom.Blob/Features/Storage/IDicomBlobDataStore.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomBlobDataStore.cs @@ -8,7 +8,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Health.Dicom.Blob.Features.Storage +namespace Microsoft.Health.Dicom.Core.Features.Persistence { public interface IDicomBlobDataStore { diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs new file mode 100644 index 0000000000..10af4300a5 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Dicom; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public interface IDicomDataStore + { + Task StoreAsync(DicomFile dicomFile, CancellationToken cancellationToken = default); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs new file mode 100644 index 0000000000..4a2c4d8f1e --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Dicom; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public interface IDicomIndexDataStore + { + Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default); + + Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default); + + Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); + + Task> DeleteSeriesIndexAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); + + Task DeleteInstanceIndexAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs index ee735095f6..1635a99683 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs @@ -14,8 +14,10 @@ namespace Microsoft.Health.Dicom.Core.Features.Validation /// public class DicomIdentifierValidator : RegularExpressionValidator { + internal const string IdentifierRegex = "^[A-Za-z0-9\\-\\.]{1,64}$"; + public DicomIdentifierValidator() - : base("^[A-Za-z0-9\\-\\.]{1,64}$", RegexOptions.Singleline | RegexOptions.Compiled) + : base(IdentifierRegex, RegexOptions.Singleline | RegexOptions.Compiled) { } } diff --git a/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj b/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj index f6637217d5..6a1e4d99b1 100644 --- a/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj +++ b/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj @@ -7,6 +7,7 @@ + diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs new file mode 100644 index 0000000000..46dd046685 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using Dicom; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class QuerySeriesDocumentTests + { + [Fact] + public void GivenInvalidInstanceIdentifers_WhenCreatingQuerySeriesDocument_ExceptionsThrown() + { + Assert.Throws(() => new QuerySeriesDocument(null, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), null)); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => new QuerySeriesDocument("sameid", "sameid")); + + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(null, Guid.NewGuid().ToString())); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(Guid.NewGuid().ToString(), null)); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId("sameid", "sameid")); + + Assert.Throws(() => QuerySeriesDocument.GetPartitionKey(null)); + Assert.Throws(() => QuerySeriesDocument.GetPartitionKey(string.Empty)); + } + + [Fact] + public void GivenExistingInstance_WhenAddingToInstancesHashSet_IsNotAdded() + { + var document = new QuerySeriesDocument(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + + var dataset = new DicomDataset(); + var sopInstanceUID = Guid.NewGuid().ToString(); + var testPatientName = Guid.NewGuid().ToString(); + dataset.Add(DicomTag.SOPInstanceUID, sopInstanceUID); + dataset.Add(DicomTag.PatientName, testPatientName); + + var instanceDocument1 = QueryInstance.Create(dataset, new[] { DicomTag.PatientName }); + var instanceDocument2 = QueryInstance.Create(dataset, new[] { DicomTag.PatientName }); + + Assert.Throws(() => document.AddInstance(null)); + Assert.True(document.AddInstance(instanceDocument1)); + Assert.False(document.AddInstance(instanceDocument2)); + + Assert.Equal(testPatientName, document.DistinctIndexedAttributes[DicomTag.PatientName].First()); + + Assert.Throws(() => document.RemoveInstance(null)); + Assert.Throws(() => document.RemoveInstance(string.Empty)); + Assert.True(document.RemoveInstance(sopInstanceUID)); + Assert.False(document.RemoveInstance(sopInstanceUID)); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs new file mode 100644 index 0000000000..eed14282c0 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs @@ -0,0 +1,102 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Versioning +{ + public class DicomCollectionUpgradeManagerTests + { + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration = new CosmosDataStoreConfiguration + { + AllowDatabaseCreation = false, + ConnectionMode = ConnectionMode.Direct, + ConnectionProtocol = Protocol.Https, + DatabaseId = "testdatabaseid", + Host = "https://fakehost", + Key = "ZmFrZWtleQ==", // "fakekey" + PreferredLocations = null, + }; + + private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration = new CosmosCollectionConfiguration + { + CollectionId = "testcollectionid", + }; + + private readonly DicomCollectionUpgradeManager _manager; + private readonly IDocumentClient _client; + + public DicomCollectionUpgradeManagerTests() + { + ICosmosDbDistributedLockFactory factory = Substitute.For(); + ICosmosDbDistributedLock cosmosDbDistributedLock = Substitute.For(); + IQueryable collectionVersionWrappers = Substitute.For, IDocumentQuery>(); + var optionsMonitor = Substitute.For>(); + + optionsMonitor.Get(Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); + + factory.Create(Arg.Any(), Arg.Any(), Arg.Any()).Returns(cosmosDbDistributedLock); + cosmosDbDistributedLock.TryAcquireLock().Returns(true); + + _client = Substitute.For(); + _client.CreateDocumentQuery(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(collectionVersionWrappers); + + collectionVersionWrappers.AsDocumentQuery().ExecuteNextAsync().Returns(new FeedResponse(new List())); + + var updaters = new IDicomCollectionUpdater[] { }; + _manager = new DicomCollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionIndexIsUpdated() + { + var documentCollection = new DocumentCollection(); + + await UpdateCollectionAsync(documentCollection); + + await _client.Received(1).ReplaceDocumentCollectionAsync(Arg.Is(documentCollection)); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionVersionWrapperIsSaved() + { + var documentCollection = new DocumentCollection(); + + await UpdateCollectionAsync(documentCollection); + + await _client.Received(1).UpsertDocumentAsync(Arg.Is(_cosmosDataStoreConfiguration.GetRelativeCollectionUri(_cosmosCollectionConfiguration.CollectionId)), Arg.Is(x => x.Version == _manager.CollectionSettingsVersion)); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionTTLIsSetToNeg1() + { + var documentCollection = new DocumentCollection(); + + await UpdateCollectionAsync(documentCollection); + + Assert.Equal(-1, documentCollection.DefaultTimeToLive); + } + + private async Task UpdateCollectionAsync(DocumentCollection documentCollection) + { + await _manager.SetupCollectionAsync(_client, documentCollection); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj new file mode 100644 index 0000000000..efa36dc435 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp2.2 + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs b/src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs new file mode 100644 index 0000000000..6b736f9d32 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Resources; +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Health.Dicom.CosmosDb.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Dicom.Tests.Integration")] +[assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs new file mode 100644 index 0000000000..f9bb476cf3 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Dicom; + +namespace Microsoft.Health.Dicom.CosmosDb.Config +{ + public class DicomCosmosConfiguration + { + public IEnumerable QueryAttributes { get; } = new[] { DicomTag.PatientName, DicomTag.ReferringPhysicianName, DicomTag.Modality }; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Constants.cs b/src/Microsoft.Health.Dicom.CosmosDb/Constants.cs new file mode 100644 index 0000000000..5342d23aee --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Constants.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Dicom.CosmosDb +{ + internal static class Constants + { + public const string CollectionConfigurationName = "dicomCosmosDb"; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs new file mode 100644 index 0000000000..c124779aae --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Azure.Documents; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Health; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Extensions.DependencyInjection; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Health +{ + public class DicomCosmosHealthCheck : CosmosHealthCheck + { + /// + /// Initializes a new instance of the class. + /// + /// The document client factory/ + /// The CosmosDB configuration. + /// The IOptions accessor to get a named version. + /// The test provider + /// The logger. + public DicomCosmosHealthCheck( + IScoped documentClient, + CosmosDataStoreConfiguration configuration, + IOptionsSnapshot namedCosmosCollectionConfigurationAccessor, + IDocumentClientTestProvider testProvider, + ILogger logger) + : base( + documentClient, + configuration, + namedCosmosCollectionConfigurationAccessor, + Constants.CollectionConfigurationName, + testProvider, + logger) + { + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs new file mode 100644 index 0000000000..49d9d1117f --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -0,0 +1,230 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Dicom; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Microsoft.Health.Extensions.DependencyInjection; +using Polly; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + internal class DicomCosmosDataStore : IDicomIndexDataStore + { + private const string PreConditionFailedExceptionMessage = "one of the specified pre-condition is not met"; + private readonly IDocumentClient _documentClient; + private readonly Uri _collectionUri; + private readonly DicomCosmosConfiguration _dicomConfiguration; + private readonly string _databaseId; + private readonly string _collectionId; + + public DicomCosmosDataStore( + IScoped documentClient, + CosmosDataStoreConfiguration configuration, + IOptionsMonitor namedCosmosCollectionConfigurationAccessor, + DicomCosmosConfiguration dicomConfiguration) + { + EnsureArg.IsNotNull(documentClient?.Value, nameof(documentClient)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); + EnsureArg.IsNotNull(dicomConfiguration, nameof(dicomConfiguration)); + + CosmosCollectionConfiguration cosmosCollectionConfiguration = namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName); + + _documentClient = documentClient.Value; + _collectionUri = configuration.GetRelativeCollectionUri(cosmosCollectionConfiguration.CollectionId); + _databaseId = configuration.DatabaseId; + _collectionId = cosmosCollectionConfiguration.CollectionId; + _dicomConfiguration = dicomConfiguration; + } + + /// + public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + + var dicomIdentity = DicomIdentity.Create(dicomDataset); + var defaultDocument = new QuerySeriesDocument(dicomIdentity.StudyInstanceUID, dicomIdentity.SeriesInstanceUID); + + RequestOptions requestOptions = CreateRequestOptions(defaultDocument.PartitionKey); + + // Retry when the pre-condition fails on replace. + IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); + await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + QuerySeriesDocument document = await _documentClient.GetorCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); + + var instance = QueryInstance.Create(dicomDataset, _dicomConfiguration.QueryAttributes); + + if (!document.AddInstance(instance)) + { + throw new IndexDataStoreException(HttpStatusCode.Conflict); + } + + // Note, we do a replace in-case the document was deleted. The entire read and replace should be retried on ETag check failed. + requestOptions.AccessCondition = new AccessCondition() { Condition = document.ETag, Type = AccessConditionType.IfMatch }; + + var documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, document.Id); + await documentClient.ReplaceDocumentAsync(documentUri, document, requestOptions, cancellationToken: cancellationToken); + }, + retryPolicy); + } + + /// + public async Task> DeleteSeriesIndexAsync( + string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + + Uri documentUri = UriFactory.CreateDocumentUri( + _databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); + RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); + + // Retry when the pre-condition fails on delete. + IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); + return await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + DocumentResponse response; + + try + { + // Read document to work-out which instances are about to be removed. + response = await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + } + catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Do nothing when the series document does not exist; must be deleted. + return Array.Empty(); + } + + // Update the ETag check on the request options. + requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; + + // Delete the entire series document; if this fails the entire read and delete should be retried. + await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); + + return response.Document.Instances + .Select(x => new DicomIdentity(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)) + .ToArray(); + }, + retryPolicy); + } + + /// + public async Task DeleteInstanceIndexAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + + Uri documentUri = UriFactory.CreateDocumentUri( + _databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); + RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); + + DocumentResponse response; + + IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); + await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + response = await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + + // If the instance does not exist, throw not found exception. + if (!response.Document.RemoveInstance(sopInstanceUID)) + { + throw new IndexDataStoreException(HttpStatusCode.NotFound); + } + + requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; + + // On delete or replace we should retry the entire read and delete. + if (response.Document.Instances.Count == 0) + { + await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); + } + else + { + await documentClient.ReplaceDocumentAsync(documentUri, response.Document, requestOptions, cancellationToken: cancellationToken); + } + }, + retryPolicy); + } + + /// + public async Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + + FeedOptions feedOptions = CreateFeedOptions(studyInstanceUID); + var identityQuery = _documentClient.CreateDocumentQuery(_collectionUri, feedOptions) + .SelectMany(x => x.Instances.Select(y => new { x.StudyInstanceUID, x.SeriesInstanceUID, y.SopInstanceUID })) + .AsDocumentQuery(); + + return await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + var results = new List(); + + while (identityQuery.HasMoreResults) + { + FeedResponse nextResults = await identityQuery.ExecuteNextAsync(cancellationToken); + results.AddRange(nextResults); + } + + return results; + }); + } + + /// + public async Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + + string documentId = QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID); + RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); + + return await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + DocumentResponse response = await documentClient.ReadDocumentAsync( + UriFactory.CreateDocumentUri(_databaseId, _collectionId, documentId), + requestOptions, + cancellationToken: cancellationToken); + + return response.Document.Instances.Select( + x => new DicomIdentity(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)); + }); + } + + private RequestOptions CreateRequestOptions(string partitionKey) + => new RequestOptions() { PartitionKey = new PartitionKey(partitionKey) }; + + private FeedOptions CreateFeedOptions(string partitionKey) + => new FeedOptions() { PartitionKey = new PartitionKey(partitionKey) }; + + private static IAsyncPolicy CreatePreConditionFailedRetryPolicy() + => Policy + .Handle(ex => ex.StatusCode == HttpStatusCode.BadRequest && ex.Message.Contains(PreConditionFailedExceptionMessage, StringComparison.InvariantCultureIgnoreCase)) + .RetryForeverAsync(); + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs new file mode 100644 index 0000000000..d8c0fe155f --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs @@ -0,0 +1,145 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Extensions.Logging; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + public class DicomDocumentClientInitializer : IDocumentClientInitializer + { + private readonly IDocumentClientTestProvider _testProvider; + private readonly ILogger _logger; + + public DicomDocumentClientInitializer(IDocumentClientTestProvider testProvider, ILogger logger) + { + EnsureArg.IsNotNull(logger, nameof(logger)); + EnsureArg.IsNotNull(testProvider, nameof(testProvider)); + + _testProvider = testProvider; + _logger = logger; + } + + /// + public IDocumentClient CreateDocumentClient(CosmosDataStoreConfiguration configuration) + { + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + _logger.LogInformation("Creating DocumentClient instance for {DatabaseUrl}", configuration.RelativeDatabaseUri); + + var connectionPolicy = new ConnectionPolicy + { + ConnectionMode = configuration.ConnectionMode, + ConnectionProtocol = configuration.ConnectionProtocol, + RetryOptions = new RetryOptions + { + MaxRetryAttemptsOnThrottledRequests = configuration.RetryOptions.MaxNumberOfRetries, + MaxRetryWaitTimeInSeconds = configuration.RetryOptions.MaxWaitTimeInSeconds, + }, + }; + + if (configuration.PreferredLocations != null && configuration.PreferredLocations.Any()) + { + _logger.LogInformation("Setting DocumentClient PreferredLocations to {PreferredLocations}", string.Join(";", configuration.PreferredLocations)); + + foreach (var preferredLocation in configuration.PreferredLocations) + { + connectionPolicy.PreferredLocations.Add(preferredLocation); + } + } + + // Setting TypeNameHandling to any value other than 'None' will be flagged + // as causing potential security issues + var serializerSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + DateFormatHandling = DateFormatHandling.IsoDateFormat, + DateParseHandling = DateParseHandling.DateTimeOffset, + DateTimeZoneHandling = DateTimeZoneHandling.RoundtripKind, + TypeNameHandling = TypeNameHandling.None, + }; + + serializerSettings.Converters.Add(new StringEnumConverter()); + + // By default, the Json.NET serializer uses 'F' instead of 'f' for fractions. + // 'F' will omit the trailing digits if they are 0. You might end up getting something like '2018-02-07T20:04:49.97114+00:00' + // where the fraction is actually 971140. Because the ordering is done as string, + // if the values don't always have complete 7 digits, the comparison might not work properly. + serializerSettings.Converters.Add(new IsoDateTimeConverter { DateTimeFormat = "o" }); + + return new DocumentClient(new Uri(configuration.Host), configuration.Key, serializerSettings, connectionPolicy, configuration.DefaultConsistencyLevel); + } + + /// + public async Task OpenDocumentClient(IDocumentClient client, CosmosDataStoreConfiguration configuration, CosmosCollectionConfiguration cosmosCollectionConfiguration) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + Uri absoluteCollectionUri = configuration.GetAbsoluteCollectionUri(cosmosCollectionConfiguration.CollectionId); + + _logger.LogInformation("Opening DocumentClient connection to {CollectionUri}", absoluteCollectionUri); + try + { + await _testProvider.PerformTest(client, configuration, cosmosCollectionConfiguration); + + _logger.LogInformation("Established DocumentClient connection to {CollectionUri}", absoluteCollectionUri); + } + catch (Exception e) + { + _logger.LogCritical(e, "Failed to connect to DocumentClient collection {CollectionUri}", absoluteCollectionUri); + throw; + } + } + + /// + public async Task InitializeDataStore(IDocumentClient documentClient, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable collectionInitializers) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers)); + + try + { + _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); + + if (cosmosDataStoreConfiguration.AllowDatabaseCreation) + { + _logger.LogDebug("CreateDatabaseIfNotExists {DatabaseId})", cosmosDataStoreConfiguration.DatabaseId); + + var options = new RequestOptions { OfferThroughput = cosmosDataStoreConfiguration.InitialDatabaseThroughput }; + await documentClient.CreateDatabaseIfNotExistsAsync( + new Database + { + Id = cosmosDataStoreConfiguration.DatabaseId, + }, + options); + } + + foreach (var collectionInitializer in collectionInitializers) + { + await collectionInitializer.InitializeCollection(documentClient); + } + + _logger.LogInformation("Cosmos DB Database {DatabaseId} and collections successfully initialized", cosmosDataStoreConfiguration.DatabaseId); + } + catch (Exception ex) + { + _logger.LogCritical(ex, "Cosmos DB Database {DatabaseId} and collections initialization failed", cosmosDataStoreConfiguration.DatabaseId); + throw; + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs new file mode 100644 index 0000000000..30b61aeff1 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -0,0 +1,112 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Newtonsoft.Json; +using Polly; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + public static class DocumentClientExtensions + { + public static async Task ThrowIndexDataStoreException(this IDocumentClient documentClient, Func> action, IAsyncPolicy retryPolicy = null) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNull(action, nameof(action)); + + try + { + if (retryPolicy != null) + { + return await retryPolicy.ExecuteAsync(() => action(documentClient)); + } + + return await action(documentClient); + } + catch (DocumentClientException e) + { + throw new IndexDataStoreException(e.StatusCode, e); + } + } + + public static async Task ThrowIndexDataStoreException(this IDocumentClient documentClient, Func action, IAsyncPolicy retryPolicy = null) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNull(action, nameof(action)); + + try + { + if (retryPolicy != null) + { + await retryPolicy.ExecuteAsync(() => action(documentClient)); + } + else + { + await action(documentClient); + } + } + catch (DocumentClientException e) + { + throw new IndexDataStoreException(e.StatusCode, e); + } + } + + public static async Task GetorCreateDocumentAsync( + this IDocumentClient documentClient, + string databaseId, + string collectionId, + string documentId, + RequestOptions requestOptions, + T defaultValue, + CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + + try + { + Uri documentUri = UriFactory.CreateDocumentUri(databaseId, collectionId, documentId); + DocumentResponse response = await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + return response.Document; + } + catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Attempt to create the document as it does not exist. + return await documentClient.CreateOrGetDocumentAsync(databaseId, collectionId, documentId, requestOptions, defaultValue, cancellationToken); + } + } + + public static async Task CreateOrGetDocumentAsync( + this IDocumentClient documentClient, + string databaseId, + string collectionId, + string documentId, + RequestOptions requestOptions, + T defaultValue, + CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + + try + { + Uri collectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, collectionId); + ResourceResponse result = await documentClient.CreateDocumentAsync(collectionUri, defaultValue, requestOptions, disableAutomaticIdGeneration: true, cancellationToken: cancellationToken); + + return JsonConvert.DeserializeObject(result.Resource.ToString(), requestOptions.JsonSerializerSettings); + } + catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.Conflict) + { + // Attempt to read the document as it already exists. + return await documentClient.GetorCreateDocumentAsync(databaseId, collectionId, documentId, requestOptions, defaultValue, cancellationToken); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs new file mode 100644 index 0000000000..b78031654c --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -0,0 +1,64 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Dicom; +using EnsureThat; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + internal class QueryInstance + { + [JsonConstructor] + public QueryInstance(string sopInstanceUID, IList<(DicomTag, object)> indexedAttributes) + : this(sopInstanceUID) + { + IndexedAttributes = indexedAttributes; + } + + private QueryInstance(string sopInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + SopInstanceUID = sopInstanceUID; + } + + public string SopInstanceUID { get; } + + public IList<(DicomTag, object)> IndexedAttributes { get; } = new List<(DicomTag, object)>(); + + public override int GetHashCode() + { + return SopInstanceUID.GetHashCode(StringComparison.Ordinal); + } + + public override bool Equals(object obj) + { + if (!(obj is QueryInstance instance)) + { + return false; + } + + return SopInstanceUID.Equals(instance.SopInstanceUID, StringComparison.Ordinal); + } + + public static QueryInstance Create(DicomDataset dicomItems, IEnumerable indexTags) + { + var sopInstanceUID = dicomItems.GetSingleValue(DicomTag.SOPInstanceUID); + var result = new QueryInstance(sopInstanceUID); + + foreach (DicomTag tag in indexTags) + { + if (dicomItems.TryGetString(tag, out string value)) + { + result.IndexedAttributes.Add((tag, value)); + } + } + + return result; + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs new file mode 100644 index 0000000000..8a2c4e0aff --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -0,0 +1,92 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Dicom; +using EnsureThat; +using Microsoft.Health.CosmosDb.Features.Storage; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + internal class QuerySeriesDocument + { + public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + EnsureArg.IsFalse(studyInstanceUID == seriesInstanceUID, nameof(seriesInstanceUID)); + + StudyInstanceUID = studyInstanceUID; + SeriesInstanceUID = seriesInstanceUID; + Id = GetDocumentId(studyInstanceUID, seriesInstanceUID); + PartitionKey = GetPartitionKey(studyInstanceUID); + } + + [JsonProperty(KnownDocumentProperties.Id)] + public string Id { get; } + + [JsonProperty(KnownDocumentProperties.PartitionKey)] + public string PartitionKey { get; } + + [JsonProperty(KnownDocumentProperties.ETag)] + public string ETag { get; set; } + + public string StudyInstanceUID { get; } + + public string SeriesInstanceUID { get; } + + public HashSet Instances { get; } = new HashSet(); + + public Dictionary> DistinctIndexedAttributes + { + get + { + var result = new Dictionary>(); + + foreach (QueryInstance instance in Instances) + { + foreach ((DicomTag key, object value) in instance.IndexedAttributes) + { + if (!result.ContainsKey(key)) + { + result[key] = new HashSet(); + } + + result[key].Add(value); + } + } + + return result; + } + } + + public static string GetDocumentId(string studyInstanceUID, string seriesInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + EnsureArg.IsFalse(studyInstanceUID == seriesInstanceUID); + return studyInstanceUID + seriesInstanceUID; + } + + public static string GetPartitionKey(string studyInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + return studyInstanceUID; + } + + public bool AddInstance(QueryInstance instance) + { + EnsureArg.IsNotNull(instance, nameof(instance)); + return Instances.Add(instance); + } + + public bool RemoveInstance(string sopInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + return Instances.RemoveWhere(x => x.SopInstanceUID == sopInstanceUID) > 0; + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs new file mode 100644 index 0000000000..6f481c889e --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning +{ + public class DicomCollectionUpgradeManager : CollectionUpgradeManager + { + public DicomCollectionUpgradeManager( + IEnumerable collectionUpdater, + CosmosDataStoreConfiguration configuration, + IOptionsMonitor namedCosmosCollectionConfigurationAccessor, + ICosmosDbDistributedLockFactory lockFactory, + ILogger logger) + : base( + collectionUpdater, + configuration, + GetCosmosCollectionConfiguration(namedCosmosCollectionConfigurationAccessor, Constants.CollectionConfigurationName), + lockFactory, + logger) + { + } + + public override int CollectionSettingsVersion => 1; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs new file mode 100644 index 0000000000..737fcaef92 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning +{ + public interface IDicomCollectionUpdater : ICollectionUpdater + { + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj b/src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj new file mode 100644 index 0000000000..e09fc62f3f --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj @@ -0,0 +1,18 @@ + + + + netcoreapp2.2 + + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs new file mode 100644 index 0000000000..af57ba2a12 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Dicom.CosmosDb; +using Microsoft.Health.Dicom.CosmosDb.Features.Health; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Extensions.DependencyInjection; + +namespace Microsoft.Extensions.DependencyInjection +{ + public static class DicomCosmosDbRegistrationExtensions + { + public static IServiceCollection AddDicomCosmosDb(this IServiceCollection services, IConfiguration configuration) + { + EnsureArg.IsNotNull(services, nameof(services)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + services.AddCosmosDb(); + + services + .Configure( + Constants.CollectionConfigurationName, + cosmosCollectionConfiguration => configuration.GetSection("DicomWeb:CosmosDb") + .Bind(cosmosCollectionConfiguration)); + + services.Add(sp => + { + CosmosDataStoreConfiguration config = sp.GetService(); + DicomCollectionUpgradeManager upgradeManager = sp.GetService(); + ILoggerFactory loggerFactory = sp.GetService(); + IOptionsMonitor namedCosmosCollectionConfiguration = sp.GetService>(); + CosmosCollectionConfiguration cosmosCollectionConfiguration = namedCosmosCollectionConfiguration.Get(Constants.CollectionConfigurationName); + + return new CollectionInitializer( + cosmosCollectionConfiguration.CollectionId, + config, + cosmosCollectionConfiguration.InitialCollectionThroughput, + upgradeManager, + loggerFactory.CreateLogger()); + }) + .Singleton() + .AsService(); + + services.Add() + .Singleton() + .AsService(); + + services.Add() + .Scoped() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf() + .AsService(); + + services.AddHealthChecks() + .AddCheck(name: nameof(DicomCosmosHealthCheck)); + + return services; + } + } +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj b/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj index 8a77d3bfe8..c1edce6264 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj @@ -18,6 +18,7 @@ + diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs index b7e97f0eb4..51a5579ecd 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs @@ -8,10 +8,10 @@ using System.Net; using System.Threading.Tasks; using Microsoft.Azure.Storage; -using Microsoft.Health.Dicom.Blob.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; using Xunit; -namespace Microsoft.Health.DicomTests.Integration.Persistence +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence { public class DicomBlobStorageTests : IClassFixture { diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs index 8292374df8..36d960f837 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs @@ -13,10 +13,11 @@ using Microsoft.Health.Blob.Features.Storage; using Microsoft.Health.Dicom.Blob; using Microsoft.Health.Dicom.Blob.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; using NSubstitute; using Xunit; -namespace Microsoft.Health.DicomTests.Integration.Persistence +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence { public class DicomBlobStorageTestsFixture : IAsyncLifetime { diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs new file mode 100644 index 0000000000..05b4e3b896 --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs @@ -0,0 +1,101 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Extensions.DependencyInjection; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence +{ + public class DicomCosmosDataStoreTestsFixture : IAsyncLifetime + { + private static readonly SemaphoreSlim CollectionInitializationSemaphore = new SemaphoreSlim(1, 1); + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; + private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration; + private readonly DicomCosmosConfiguration _dicomCosmosConfiguration; + private IDocumentClient _documentClient; + + public DicomCosmosDataStoreTestsFixture() + { + _cosmosDataStoreConfiguration = new CosmosDataStoreConfiguration + { + Host = Environment.GetEnvironmentVariable("CosmosDb:Host") ?? CosmosDbLocalEmulator.Host, + Key = Environment.GetEnvironmentVariable("CosmosDb:Key") ?? CosmosDbLocalEmulator.Key, + DatabaseId = Environment.GetEnvironmentVariable("CosmosDb:DatabaseId") ?? "DicomTests", + AllowDatabaseCreation = true, + PreferredLocations = Environment.GetEnvironmentVariable("CosmosDb:PreferredLocations")?.Split(';', StringSplitOptions.RemoveEmptyEntries), + }; + + _cosmosCollectionConfiguration = new CosmosCollectionConfiguration + { + CollectionId = Guid.NewGuid().ToString(), + }; + + _dicomCosmosConfiguration = new DicomCosmosConfiguration(); + } + + public IDicomIndexDataStore DicomIndexDataStore { get; private set; } + + public async Task InitializeAsync() + { + IOptionsMonitor optionsMonitor = Substitute.For>(); + + optionsMonitor.Get(CosmosDb.Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); + + var updaters = new IDicomCollectionUpdater[] { }; + + var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); + + var upgradeManager = new DicomCollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + IDocumentClientTestProvider testProvider = new DocumentClientReadWriteTestProvider(); + + var documentClientInitializer = new DicomDocumentClientInitializer(testProvider, NullLogger.Instance); + _documentClient = documentClientInitializer.CreateDocumentClient(_cosmosDataStoreConfiguration); + var collectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration.CollectionId, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration.InitialCollectionThroughput, upgradeManager, NullLogger.Instance); + + // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. + // Use the semaphore to only allow one initialization at a time. + await CollectionInitializationSemaphore.WaitAsync(); + + try + { + await documentClientInitializer.InitializeDataStore(_documentClient, _cosmosDataStoreConfiguration, new List { collectionInitializer }); + } + finally + { + CollectionInitializationSemaphore.Release(); + } + + var documentClient = new NonDisposingScope(_documentClient); + + DicomIndexDataStore = new DicomCosmosDataStore( + documentClient, + _cosmosDataStoreConfiguration, + optionsMonitor, + _dicomCosmosConfiguration); + } + + public async Task DisposeAsync() + { + using (_documentClient as IDisposable) + { + await _documentClient?.DeleteDocumentCollectionAsync(_cosmosDataStoreConfiguration.GetRelativeCollectionUri(_cosmosCollectionConfiguration.CollectionId)); + } + } + } +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs new file mode 100644 index 0000000000..57a670035a --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Dicom; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence +{ + public class DicomIndexDataStoreTests : IClassFixture + { + private readonly IDicomIndexDataStore _indexDataStore; + + public DicomIndexDataStoreTests(DicomCosmosDataStoreTestsFixture fixture) + { + _indexDataStore = fixture.DicomIndexDataStore; + } + + [Fact] + public async Task GivenAValidInstance_WhenIndexing_CanBeRetrieved() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + + await _indexDataStore.IndexInstanceAsync(testInstance); + + IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + + Assert.Single(instancesInStudy); + Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInStudy.First().SopInstanceUID); + + IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); + + Assert.Single(instancesInSeries); + Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInSeries.First().SopInstanceUID); + } + + private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceUID, string seriesInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID); + + var result = new DicomDataset(); + result.Add(DicomTag.StudyInstanceUID, studyInstanceUID); + result.Add(DicomTag.SeriesInstanceUID, seriesInstanceUID); + result.Add(DicomTag.SOPInstanceUID, Guid.NewGuid().ToString()); + return result; + } + } +} diff --git a/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj b/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj index 13b5dd04ae..19224d39f1 100644 --- a/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj +++ b/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj @@ -58,4 +58,6 @@ + + From 44835bd4fc35597721906c9c3c3ad8c40cdf0ce0 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Wed, 19 Jun 2019 16:30:35 +0100 Subject: [PATCH 02/22] Added settings updater and fixed all tests --- .../Features/Persistence/DicomIdentity.cs | 7 +- .../Validation/DicomIdentifierValidator.cs | 2 +- .../Storage/Documents/QueryInstanceTests.cs | 59 ++++++++++ .../Documents/QuerySeriesDocumentTests.cs | 4 + .../DicomCollectionUpgradeManagerTests.cs | 2 +- .../Features/Storage/DicomCosmosDataStore.cs | 40 +++---- .../Storage/DocumentClientExtensions.cs | 8 +- .../Storage/Documents/QueryInstance.cs | 34 ++++-- .../Storage/Documents/QuerySeriesDocument.cs | 21 +++- .../DicomCollectionSettingsUpdater.cs | 99 +++++++++++++++++ .../DicomCosmosDataStoreTestsFixture.cs | 17 ++- .../Persistence/DicomIndexDataStoreTests.cs | 104 ++++++++++++++++++ .../DocumentClientExtensionsTests.cs | 87 +++++++++++++++ 13 files changed, 438 insertions(+), 46 deletions(-) create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs create mode 100644 test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs index 219e18c4ec..8fcaab9e66 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs @@ -40,10 +40,11 @@ public static DicomIdentity Create(DicomDataset dicomDataset) { EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + // Note: Here we 'GetSingleValueOrDefault' and let the constructor validate the identifier. return new DicomIdentity( - dicomDataset.GetSingleValue(DicomTag.StudyInstanceUID), - dicomDataset.GetSingleValue(DicomTag.SeriesInstanceUID), - dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID)); + dicomDataset.GetSingleValueOrDefault(DicomTag.StudyInstanceUID, string.Empty), + dicomDataset.GetSingleValueOrDefault(DicomTag.SeriesInstanceUID, string.Empty), + dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty)); } } } diff --git a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs index 1635a99683..e93814b603 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs @@ -14,7 +14,7 @@ namespace Microsoft.Health.Dicom.Core.Features.Validation /// public class DicomIdentifierValidator : RegularExpressionValidator { - internal const string IdentifierRegex = "^[A-Za-z0-9\\-\\.]{1,64}$"; + public const string IdentifierRegex = "^[A-Za-z0-9\\-\\.]{1,64}$"; public DicomIdentifierValidator() : base(IdentifierRegex, RegexOptions.Singleline | RegexOptions.Compiled) diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs new file mode 100644 index 0000000000..cddd177409 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Dicom; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class QueryInstanceTests + { + [Fact] + public void GivenInvalidParameters_WhenCreatingQueryInstance_ExceptionsThrown() + { + Assert.Throws(() => new QueryInstance(null, new Dictionary())); + Assert.Throws(() => new QueryInstance(string.Empty, new Dictionary())); + Assert.Throws(() => new QueryInstance(new string('a', 65), new Dictionary())); + Assert.Throws(() => new QueryInstance("AA#AA", new Dictionary())); + Assert.Throws(() => new QueryInstance(Guid.NewGuid().ToString(), null)); + + Assert.Throws(() => QueryInstance.Create(null, null)); + Assert.Throws(() => QueryInstance.Create(new DicomDataset(), null)); + } + + [Fact] + public void GivenDicomTagsToIndex_WhenCreatingQueryInstance_TagsAreExtractedAndIndexed() + { + var sequenceDataset = new DicomDataset() + { + { DicomTag.ReferringPhysicianName, "TestPhysician" }, + }; + + var dicomDataset = new DicomDataset + { + { DicomTag.StudyInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.SeriesInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.SOPInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.PatientName, Guid.NewGuid().ToString() }, + { DicomTag.StudyDate, DateTime.UtcNow }, + }; + dicomDataset.Add(sequenceDataset); + + var instance = QueryInstance.Create( + dicomDataset, new[] { DicomTag.PatientName, DicomTag.StudyDate, DicomTag.ReferringPhysicianName, DicomTag.StudyTime }); + + Assert.NotNull(instance); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID), instance.SopInstanceUID); + + Assert.Equal(3, instance.IndexedAttributes.Count); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.IndexedAttributes[DicomTag.PatientName]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.IndexedAttributes[DicomTag.StudyDate]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.ReferringPhysicianName), instance.IndexedAttributes[DicomTag.ReferringPhysicianName]); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs index 46dd046685..2570c449ec 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -18,12 +18,16 @@ public void GivenInvalidInstanceIdentifers_WhenCreatingQuerySeriesDocument_Excep { Assert.Throws(() => new QuerySeriesDocument(null, Guid.NewGuid().ToString())); Assert.Throws(() => new QuerySeriesDocument(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(new string('a', 65), Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument("?...", Guid.NewGuid().ToString())); Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), null)); Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), string.Empty)); Assert.Throws(() => new QuerySeriesDocument("sameid", "sameid")); Assert.Throws(() => QuerySeriesDocument.GetDocumentId(null, Guid.NewGuid().ToString())); Assert.Throws(() => QuerySeriesDocument.GetDocumentId(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), new string('a', 65))); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), "?...")); Assert.Throws(() => QuerySeriesDocument.GetDocumentId(Guid.NewGuid().ToString(), null)); Assert.Throws(() => QuerySeriesDocument.GetDocumentId(Guid.NewGuid().ToString(), string.Empty)); Assert.Throws(() => QuerySeriesDocument.GetDocumentId("sameid", "sameid")); diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs index eed14282c0..394e5eaaa9 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs @@ -60,7 +60,7 @@ public DicomCollectionUpgradeManagerTests() collectionVersionWrappers.AsDocumentQuery().ExecuteNextAsync().Returns(new FeedResponse(new List())); - var updaters = new IDicomCollectionUpdater[] { }; + var updaters = new IDicomCollectionUpdater[] { new DicomCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance) }; _manager = new DicomCollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index 49d9d1117f..b1ae926032 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -27,7 +27,6 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage { internal class DicomCosmosDataStore : IDicomIndexDataStore { - private const string PreConditionFailedExceptionMessage = "one of the specified pre-condition is not met"; private readonly IDocumentClient _documentClient; private readonly Uri _collectionUri; private readonly DicomCosmosConfiguration _dicomConfiguration; @@ -64,12 +63,12 @@ public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToke RequestOptions requestOptions = CreateRequestOptions(defaultDocument.PartitionKey); - // Retry when the pre-condition fails on replace. + // Retry when the pre-condition fails on replace (ETag check). IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); await _documentClient.ThrowIndexDataStoreException( async (documentClient) => { - QuerySeriesDocument document = await _documentClient.GetorCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); + QuerySeriesDocument document = await documentClient.GetorCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); var instance = QueryInstance.Create(dicomDataset, _dicomConfiguration.QueryAttributes); @@ -78,10 +77,10 @@ await _documentClient.ThrowIndexDataStoreException( throw new IndexDataStoreException(HttpStatusCode.Conflict); } - // Note, we do a replace in-case the document was deleted. The entire read and replace should be retried on ETag check failed. + // Note, we do a replace (rather than upsert) in case the document is deleted. requestOptions.AccessCondition = new AccessCondition() { Condition = document.ETag, Type = AccessConditionType.IfMatch }; - var documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, document.Id); + Uri documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, document.Id); await documentClient.ReplaceDocumentAsync(documentUri, document, requestOptions, cancellationToken: cancellationToken); }, retryPolicy); @@ -103,18 +102,8 @@ public async Task> DeleteSeriesIndexAsync( return await _documentClient.ThrowIndexDataStoreException( async (documentClient) => { - DocumentResponse response; - - try - { - // Read document to work-out which instances are about to be removed. - response = await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); - } - catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { - // Do nothing when the series document does not exist; must be deleted. - return Array.Empty(); - } + DocumentResponse response = + await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); // Update the ETag check on the request options. requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; @@ -140,13 +129,12 @@ public async Task DeleteInstanceIndexAsync(string studyInstanceUID, string serie _databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); - DocumentResponse response; - IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); await _documentClient.ThrowIndexDataStoreException( async (documentClient) => { - response = await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + DocumentResponse response = + await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); // If the instance does not exist, throw not found exception. if (!response.Document.RemoveInstance(sopInstanceUID)) @@ -156,7 +144,7 @@ await _documentClient.ThrowIndexDataStoreException( requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; - // On delete or replace we should retry the entire read and delete. + // Delete the entire series if no more instances, otherwise replace. if (response.Document.Instances.Count == 0) { await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); @@ -190,7 +178,13 @@ public async Task> GetInstancesInStudyAsync(string st results.AddRange(nextResults); } - return results; + if (results.Any()) + { + return results; + } + + // If no results, this study does not exist. + throw new IndexDataStoreException(HttpStatusCode.NotFound); }); } @@ -224,7 +218,7 @@ private FeedOptions CreateFeedOptions(string partitionKey) private static IAsyncPolicy CreatePreConditionFailedRetryPolicy() => Policy - .Handle(ex => ex.StatusCode == HttpStatusCode.BadRequest && ex.Message.Contains(PreConditionFailedExceptionMessage, StringComparison.InvariantCultureIgnoreCase)) + .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed) .RetryForeverAsync(); } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs index 30b61aeff1..bb6d8d253e 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -70,6 +70,9 @@ public static async Task GetorCreateDocumentAsync( CancellationToken cancellationToken = default) { EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNullOrWhiteSpace(databaseId, nameof(databaseId)); + EnsureArg.IsNotNullOrWhiteSpace(collectionId, nameof(collectionId)); + EnsureArg.IsNotNullOrWhiteSpace(documentId, nameof(documentId)); try { @@ -94,13 +97,16 @@ public static async Task CreateOrGetDocumentAsync( CancellationToken cancellationToken = default) { EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNullOrWhiteSpace(databaseId, nameof(databaseId)); + EnsureArg.IsNotNullOrWhiteSpace(collectionId, nameof(collectionId)); + EnsureArg.IsNotNullOrWhiteSpace(documentId, nameof(documentId)); try { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, collectionId); ResourceResponse result = await documentClient.CreateDocumentAsync(collectionUri, defaultValue, requestOptions, disableAutomaticIdGeneration: true, cancellationToken: cancellationToken); - return JsonConvert.DeserializeObject(result.Resource.ToString(), requestOptions.JsonSerializerSettings); + return JsonConvert.DeserializeObject(result.Resource.ToString(), requestOptions?.JsonSerializerSettings); } catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.Conflict) { diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs index b78031654c..8b73d60dd2 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -5,8 +5,10 @@ using System; using System.Collections.Generic; +using System.Text.RegularExpressions; using Dicom; using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; using Newtonsoft.Json; namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents @@ -14,21 +16,25 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents internal class QueryInstance { [JsonConstructor] - public QueryInstance(string sopInstanceUID, IList<(DicomTag, object)> indexedAttributes) + public QueryInstance(string sopInstanceUID, IDictionary indexedAttributes) : this(sopInstanceUID) { + EnsureArg.IsNotNull(indexedAttributes, nameof(indexedAttributes)); + IndexedAttributes = indexedAttributes; } private QueryInstance(string sopInstanceUID) { EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + SopInstanceUID = sopInstanceUID; } public string SopInstanceUID { get; } - public IList<(DicomTag, object)> IndexedAttributes { get; } = new List<(DicomTag, object)>(); + public IDictionary IndexedAttributes { get; } = new Dictionary(); public override int GetHashCode() { @@ -37,24 +43,32 @@ public override int GetHashCode() public override bool Equals(object obj) { - if (!(obj is QueryInstance instance)) + if (obj is QueryInstance instance) { - return false; + return SopInstanceUID.Equals(instance.SopInstanceUID, StringComparison.Ordinal); } - return SopInstanceUID.Equals(instance.SopInstanceUID, StringComparison.Ordinal); + return false; } - public static QueryInstance Create(DicomDataset dicomItems, IEnumerable indexTags) + public static QueryInstance Create(DicomDataset dicomDataset, IEnumerable indexTags) { - var sopInstanceUID = dicomItems.GetSingleValue(DicomTag.SOPInstanceUID); + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + + var sopInstanceUID = dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty); var result = new QueryInstance(sopInstanceUID); - foreach (DicomTag tag in indexTags) + if (indexTags != null) { - if (dicomItems.TryGetString(tag, out string value)) + foreach (DicomTag tag in indexTags) { - result.IndexedAttributes.Add((tag, value)); + // All indexed tags must have a value multiplicty of 1. + EnsureArg.IsTrue(tag.DictionaryEntry.ValueMultiplicity.Multiplicity == 1); + + if (dicomDataset.TryGetSingleValue(tag, out object value)) + { + result.IndexedAttributes[tag] = value; + } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs index 8a2c4e0aff..b52795e532 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -4,15 +4,23 @@ // ------------------------------------------------------------------------------------------------- using System.Collections.Generic; +using System.Text.RegularExpressions; using Dicom; using EnsureThat; using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Validation; using Newtonsoft.Json; namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents { internal class QuerySeriesDocument { + /// + /// Lists the characters that are not valid for a Cosmos resource identifier. + /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.documents.resource.id?view=azure-dotnet + /// + private const string DocumentIdRegex = "[^?/\\#]"; + public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); @@ -68,12 +76,23 @@ public static string GetDocumentId(string studyInstanceUID, string seriesInstanc EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); EnsureArg.IsFalse(studyInstanceUID == seriesInstanceUID); - return studyInstanceUID + seriesInstanceUID; + + EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + + string documentId = studyInstanceUID + seriesInstanceUID; + + // Double safety for the document identifier. If the study and series are valid identifiers, the document ID 'should' conform. + EnsureArg.IsTrue(Regex.IsMatch(documentId, DocumentIdRegex)); + + return documentId; } public static string GetPartitionKey(string studyInstanceUID) { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + return studyInstanceUID; } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs new file mode 100644 index 0000000000..ba705cec72 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs @@ -0,0 +1,99 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning +{ + public class DicomCollectionSettingsUpdater : IDicomCollectionUpdater + { + private readonly ILogger _logger; + private readonly CosmosDataStoreConfiguration _configuration; + private readonly CosmosCollectionConfiguration _collectionConfiguration; + private const int CollectionSettingsVersion = 1; + + public DicomCollectionSettingsUpdater( + CosmosDataStoreConfiguration configuration, + IOptionsMonitor namedCosmosCollectionConfigurationAccessor, + ILogger logger) + { + EnsureArg.IsNotNull(configuration, nameof(configuration)); + EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); + EnsureArg.IsNotNull(logger, nameof(logger)); + + _configuration = configuration; + _collectionConfiguration = namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName); + _logger = logger; + } + + public async Task ExecuteAsync(IDocumentClient client, DocumentCollection collection, Uri relativeCollectionUri) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(collection, nameof(collection)); + + CollectionVersion thisVersion = await GetLatestCollectionVersion(client, collection); + + if (thisVersion.Version < CollectionSettingsVersion) + { + _logger.LogDebug("Ensuring indexes are up-to-date {CollectionUri}", _configuration.GetAbsoluteCollectionUri(_collectionConfiguration.CollectionId)); + + collection.IndexingPolicy = new IndexingPolicy + { + IndexingMode = IndexingMode.Consistent, + Automatic = true, + IncludedPaths = new Collection + { + new IncludedPath + { + Path = "/*", + Indexes = new Collection(), + }, + }, + ExcludedPaths = new Collection() + { + new ExcludedPath + { + Path = "/\"_etag\"/?", + }, + }, + }; + + // Setting the DefaultTTL to -1 means that by default all documents in the collection will live forever + // but the Cosmos DB service should monitor this collection for documents that have overridden this default. + // See: https://docs.microsoft.com/en-us/azure/cosmos-db/time-to-live + collection.DefaultTimeToLive = -1; + + await client.ReplaceDocumentCollectionAsync(collection); + + thisVersion.Version = CollectionSettingsVersion; + await client.UpsertDocumentAsync(relativeCollectionUri, thisVersion); + } + } + + private static async Task GetLatestCollectionVersion(IDocumentClient documentClient, DocumentCollection collection) + { + IDocumentQuery query = documentClient.CreateDocumentQuery( + collection.SelfLink, + new SqlQuerySpec("SELECT * FROM root r"), + new FeedOptions { PartitionKey = new PartitionKey(CollectionVersion.CollectionVersionPartition) }) + .AsDocumentQuery(); + + FeedResponse result = await query.ExecuteNextAsync(); + + return result.FirstOrDefault() ?? new CollectionVersion(); + } + } +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs index 05b4e3b896..756c7f2137 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs @@ -28,7 +28,6 @@ public class DicomCosmosDataStoreTestsFixture : IAsyncLifetime private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration; private readonly DicomCosmosConfiguration _dicomCosmosConfiguration; - private IDocumentClient _documentClient; public DicomCosmosDataStoreTestsFixture() { @@ -51,6 +50,12 @@ public DicomCosmosDataStoreTestsFixture() public IDicomIndexDataStore DicomIndexDataStore { get; private set; } + public IDocumentClient DocumentClient { get; private set; } + + public string DatabaseId => _cosmosDataStoreConfiguration.DatabaseId; + + public string CollectionId => _cosmosCollectionConfiguration.CollectionId; + public async Task InitializeAsync() { IOptionsMonitor optionsMonitor = Substitute.For>(); @@ -65,7 +70,7 @@ public async Task InitializeAsync() IDocumentClientTestProvider testProvider = new DocumentClientReadWriteTestProvider(); var documentClientInitializer = new DicomDocumentClientInitializer(testProvider, NullLogger.Instance); - _documentClient = documentClientInitializer.CreateDocumentClient(_cosmosDataStoreConfiguration); + DocumentClient = documentClientInitializer.CreateDocumentClient(_cosmosDataStoreConfiguration); var collectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration.CollectionId, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration.InitialCollectionThroughput, upgradeManager, NullLogger.Instance); // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. @@ -74,14 +79,14 @@ public async Task InitializeAsync() try { - await documentClientInitializer.InitializeDataStore(_documentClient, _cosmosDataStoreConfiguration, new List { collectionInitializer }); + await documentClientInitializer.InitializeDataStore(DocumentClient, _cosmosDataStoreConfiguration, new List { collectionInitializer }); } finally { CollectionInitializationSemaphore.Release(); } - var documentClient = new NonDisposingScope(_documentClient); + var documentClient = new NonDisposingScope(DocumentClient); DicomIndexDataStore = new DicomCosmosDataStore( documentClient, @@ -92,9 +97,9 @@ public async Task InitializeAsync() public async Task DisposeAsync() { - using (_documentClient as IDisposable) + using (DocumentClient as IDisposable) { - await _documentClient?.DeleteDocumentCollectionAsync(_cosmosDataStoreConfiguration.GetRelativeCollectionUri(_cosmosCollectionConfiguration.CollectionId)); + await DocumentClient?.DeleteDocumentCollectionAsync(_cosmosDataStoreConfiguration.GetRelativeCollectionUri(_cosmosCollectionConfiguration.CollectionId)); } } } diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs index 57a670035a..bda6a4418b 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -6,10 +6,12 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Threading.Tasks; using Dicom; using EnsureThat; using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; using Xunit; namespace Microsoft.Health.Dicom.Tests.Integration.Persistence @@ -42,6 +44,95 @@ public async Task GivenAValidInstance_WhenIndexing_CanBeRetrieved() Assert.Single(instancesInSeries); Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInSeries.First().SopInstanceUID); + + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + + Assert.Single(deletedInstances); + Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == deletedInstances.First().SopInstanceUID); + } + + [Fact] + public async Task GivenMultipleValidInstances_WhenIndexingInParallel_AreStored() + { + const int numberOfInstancesToIndex = 5; + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstancesToIndex); + IList sopInstanceUIDs = instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID)).ToList(); + + IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + Assert.Equal(numberOfInstancesToIndex, instancesInStudy.Count()); + instancesInStudy.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + + IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); + Assert.Equal(numberOfInstancesToIndex, instancesInSeries.Count()); + instancesInSeries.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + Assert.Equal(numberOfInstancesToIndex, instancesInSeries.Count()); + deletedInstances.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + } + + [Fact] + public async Task GivenAValidSeries_WhenDeletingAnInstance_InstanceIsDeletedButSeriesRemains() + { + const int numberOfInstancesToIndex = 5; + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstancesToIndex); + string firstSopInstanceUID = instances[0].GetSingleValue(DicomTag.SOPInstanceUID); + IList otherSopInstanceUIDs = instances.Skip(1).Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID)).ToList(); + + await _indexDataStore.DeleteInstanceIndexAsync(studyInstanceUID, seriesInstanceUID, firstSopInstanceUID); + + IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + Assert.Equal(numberOfInstancesToIndex - 1, instancesInStudy.Count()); + + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + deletedInstances.Each(x => Assert.True(otherSopInstanceUIDs.Contains(x.SopInstanceUID))); + } + + [Fact] + public async Task GivenAnInstanceWithInvalidInstanceId_WhenStoring_ExceptionIsThrown() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = "?/#"; + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + + testInstance.AddOrUpdate(DicomTag.SeriesInstanceUID, new string('a', 65)); + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + + testInstance.Remove(DicomTag.SeriesInstanceUID); + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + } + + [Fact] + public async Task GivenNonExistentInstancesOrSeries_WhenDeleting_IndexDataStoreExceptionIsThrownWithNotFoundStatusCode() + { + IndexDataStoreException deleteInstanceException = await Assert.ThrowsAsync( + () => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Equal(HttpStatusCode.NotFound, deleteInstanceException.StatusCode); + + IndexDataStoreException deleteSeriesException = await Assert.ThrowsAsync( + () => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Equal(HttpStatusCode.NotFound, deleteSeriesException.StatusCode); + } + + [Fact] + public async Task GivenNonExistentSeriesOrStudy_WhenFetchingInstances_IndexDataStoreExceptionIsThrownWithNotFoundStatusCode() + { + IndexDataStoreException getStudyInstancesException = await Assert.ThrowsAsync( + () => _indexDataStore.GetInstancesInStudyAsync(Guid.NewGuid().ToString())); + Assert.Equal(HttpStatusCode.NotFound, getStudyInstancesException.StatusCode); + + IndexDataStoreException getSeriesInstancesException = await Assert.ThrowsAsync( + () => _indexDataStore.GetInstancesInSeriesAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Equal(HttpStatusCode.NotFound, getSeriesInstancesException.StatusCode); } private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceUID, string seriesInstanceUID) @@ -55,5 +146,18 @@ private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceU result.Add(DicomTag.SOPInstanceUID, Guid.NewGuid().ToString()); return result; } + + private async Task> CreateSeriesInParallelAsync(string studyInstanceUID, string seriesInstanceUID, int numberOfItemsInSeries) + { + IList instances = new List(); + + for (var i = 0; i < numberOfItemsInSeries; i++) + { + instances.Add(CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID)); + } + + await Task.WhenAll(instances.Select(x => _indexDataStore.IndexInstanceAsync(x))); + return instances; + } } } diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs new file mode 100644 index 0000000000..8a5fd0208e --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence +{ + public class DocumentClientExtensionsTests : IClassFixture + { + private readonly IDocumentClient _documentClient; + private readonly DicomCosmosDataStoreTestsFixture _fixture; + + public DocumentClientExtensionsTests(DicomCosmosDataStoreTestsFixture fixture) + { + _fixture = fixture; + _documentClient = fixture.DocumentClient; + } + + [Fact] + public async Task GivenAnExistingDocument_OnCreateOrGet_DocumentIsFetched() + { + var expectedDocument = new TestDocument() { Id = Guid.NewGuid().ToString(), PartitionKey = Guid.NewGuid().ToString() }; + var requestOptions = new RequestOptions() { PartitionKey = new PartitionKey(expectedDocument.PartitionKey) }; + + await _documentClient.CreateDocumentAsync( + UriFactory.CreateDocumentCollectionUri(_fixture.DatabaseId, _fixture.CollectionId), + expectedDocument, + requestOptions); + + DocumentResponse readResponse = await _documentClient.ReadDocumentAsync( + UriFactory.CreateDocumentUri(_fixture.DatabaseId, _fixture.CollectionId, expectedDocument.Id), + requestOptions); + + expectedDocument = readResponse.Document; + + TestDocument actualDocument = await _documentClient.CreateOrGetDocumentAsync( + _fixture.DatabaseId, + _fixture.CollectionId, + expectedDocument.Id, + requestOptions, + new TestDocument() { Id = expectedDocument.Id, PartitionKey = expectedDocument.PartitionKey }); + + Assert.Equal(expectedDocument.Id, actualDocument.Id); + Assert.Equal(expectedDocument.PartitionKey, actualDocument.PartitionKey); + Assert.Equal(expectedDocument.ETag, actualDocument.ETag); + } + + [Fact] + public async Task GivenANonExistentDocument_OnGetOrCreate_DocumentIsCreated() + { + var expectedDocument = new TestDocument() { Id = Guid.NewGuid().ToString(), PartitionKey = Guid.NewGuid().ToString() }; + var requestOptions = new RequestOptions() { PartitionKey = new PartitionKey(expectedDocument.PartitionKey) }; + + TestDocument actualDocument = await _documentClient.GetorCreateDocumentAsync( + _fixture.DatabaseId, + _fixture.CollectionId, + expectedDocument.Id, + requestOptions, + expectedDocument); + + Assert.Equal(expectedDocument.Id, actualDocument.Id); + Assert.Equal(expectedDocument.PartitionKey, actualDocument.PartitionKey); + Assert.False(string.IsNullOrWhiteSpace(actualDocument.ETag)); + } + + private class TestDocument + { + [JsonProperty(KnownDocumentProperties.Id)] + public string Id { get; set; } + + [JsonProperty(KnownDocumentProperties.PartitionKey)] + public string PartitionKey { get; set; } + + [JsonProperty(KnownDocumentProperties.ETag)] + public string ETag { get; set; } + } + } +} From 453b2b8ead9253815d516edc366629dc6b4e367f Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Wed, 19 Jun 2019 16:43:17 +0100 Subject: [PATCH 03/22] Updated the build file to install the Cosmos emulator and updated the default query attributes --- build/build.yml | 4 ++++ .../Config/DicomCosmosConfiguration.cs | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/build/build.yml b/build/build.yml index 067b6b3ca8..611e0e52f0 100644 --- a/build/build.yml +++ b/build/build.yml @@ -29,6 +29,10 @@ steps: condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) displayName: 'Start Azurite Storage Emulator (Windows)' + # We also install the Cosmos database emulator to run tests against Cosmos. +- task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 + displayName: 'Run Azure Cosmos DB Emulator container' + - task: DotNetCoreCLI@2 displayName: 'dotnet test UnitTests' inputs: diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs index f9bb476cf3..6f00ae4f67 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs @@ -10,6 +10,18 @@ namespace Microsoft.Health.Dicom.CosmosDb.Config { public class DicomCosmosConfiguration { - public IEnumerable QueryAttributes { get; } = new[] { DicomTag.PatientName, DicomTag.ReferringPhysicianName, DicomTag.Modality }; + /// + /// Gets the DICOM tags that should be indexed and made queryable. + /// The StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID will be indexed automatically. + /// + public IEnumerable QueryAttributes { get; } = new[] + { + DicomTag.PatientName, + DicomTag.PatientID, + DicomTag.AccessionNumber, + DicomTag.Modality, + DicomTag.ReferringPhysicianName, + DicomTag.StudyDate, + }; } } From 720095cd7ea39ca27e8cd27551565b0de0fb6657 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 15:49:28 +0100 Subject: [PATCH 04/22] Updated build script and added more tests --- build/build.yml | 53 ++++++------- build/test.yml | 38 ++++++++++ .../Features/Persistence/DicomIdentity.cs | 18 +++++ .../Persistence/IDicomIndexDataStore.cs | 14 ++++ .../Storage/DicomCosmosDataStoreTests.cs | 61 +++++++++++++++ .../Documents/DicomTagSerializerTests.cs | 54 +++++++++++++ .../Storage/Documents/QueryInstanceTests.cs | 15 ++-- .../Documents/QuerySeriesDocumentTests.cs | 37 ++++++++- .../Features/Storage/CosmosQueryBuilder.cs | 69 +++++++++++++++++ .../Features/Storage/DicomCosmosDataStore.cs | 58 ++++++++++++-- .../Features/Storage/DicomTagSerializer.cs | 54 +++++++++++++ .../Storage/DocumentClientExtensions.cs | 15 ++++ .../Storage/Documents/QueryInstance.cs | 12 +-- .../Storage/Documents/QuerySeriesDocument.cs | 7 +- .../Persistence/DicomIndexDataStoreTests.cs | 75 ++++++++++++++++--- 15 files changed, 518 insertions(+), 62 deletions(-) create mode 100644 build/test.yml create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs diff --git a/build/build.yml b/build/build.yml index 611e0e52f0..483e14d53e 100644 --- a/build/build.yml +++ b/build/build.yml @@ -1,43 +1,46 @@ parameters: + # Default values + name: Windows + queue: 'Hosted VS2017' packageArtifacts: true -steps: -- task: DotNetCoreInstaller@0 - inputs: - version: '2.2.103' +phases: +- phase: ${{ parameters.name }} + queue: '${{ parameters.queue }}' -- script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror - displayName: 'dotnet build $(buildConfiguration)' + variables: + solution: '**/*.sln' + buildPlatform: 'Any CPU' + buildConfiguration: 'Release' + +steps: -# To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). -# This allows us to test the blob storage providers without an Azure instance. -- task: NodeTool@0 - displayName: 'Use Node 8.x' +- task: NuGetToolInstaller@0 + displayName: 'Use NuGet 4.7.1' inputs: - versionSpec: 8.x + versionSpec: 4.7.1 checkLatest: true -- script: npm install -g azurite@2.7.0 - displayName: 'Install Azurite v2.7.0' - -# We start the Azurite as a separate process as the start call is blocking. -- script: azurite -s & - condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) - displayName: 'Start Azurite Storage Emulator (Linux)' +- task: DotNetCoreInstaller@0 + inputs: + version: '2.2.103' -- script: start azurite -s - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - displayName: 'Start Azurite Storage Emulator (Windows)' +- task: DotNetCoreCLI@2 + displayName: 'donet restore' + inputs: + command: restore + restoreSolution: '**\*.sln' + feedsToUse: config + nugetConfigPath: nuget.config - # We also install the Cosmos database emulator to run tests against Cosmos. -- task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 - displayName: 'Run Azure Cosmos DB Emulator container' +- script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror + displayName: 'dotnet build $(buildConfiguration)' - task: DotNetCoreCLI@2 displayName: 'dotnet test UnitTests' inputs: command: test - projects: '**\*Tests*.csproj' + projects: '**/*UnitTests/*.csproj' arguments: '--configuration $(buildConfiguration)' - ${{ if eq(parameters.packageArtifacts, 'true') }}: diff --git a/build/test.yml b/build/test.yml new file mode 100644 index 0000000000..9c5c936839 --- /dev/null +++ b/build/test.yml @@ -0,0 +1,38 @@ + +- task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts - deploy' + inputs: + artifactName: deploy + downloadPath: '$(System.ArtifactsDirectory)' + +- powershell: | + $env:CosmosDb:Host = "$(DataStore--Host)" + $env:CosmosDb:Key = "$(DataStore--Key)" + displayName: 'Setup Environment Variables' + +# To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). +# This allows us to test the blob storage providers without an Azure instance. +- task: NodeTool@0 + displayName: 'Use Node 8.x' + inputs: + versionSpec: 8.x + checkLatest: true + +- script: npm install -g azurite@2.7.0 + displayName: 'Install Azurite v2.7.0' + +# We start the Azurite as a separate process as the start call is blocking. +- script: azurite -s & + condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) + displayName: 'Start Azurite Storage Emulator (Linux)' + +- script: start azurite -s + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + displayName: 'Start Azurite Storage Emulator (Windows)' + +- task: DotNetCoreCLI@2 + displayName: 'dotnet test UnitTests' + inputs: + command: test + projects: '**\*Tests*.csproj' + arguments: '--configuration $(buildConfiguration)' \ No newline at end of file diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs index 8fcaab9e66..cc325b4031 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Text.RegularExpressions; using Dicom; using EnsureThat; @@ -13,6 +14,8 @@ namespace Microsoft.Health.Dicom.Core.Features.Persistence { public class DicomIdentity { + private const StringComparison EqualsStringComparison = StringComparison.Ordinal; + [JsonConstructor] public DicomIdentity(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) { @@ -46,5 +49,20 @@ public static DicomIdentity Create(DicomDataset dicomDataset) dicomDataset.GetSingleValueOrDefault(DicomTag.SeriesInstanceUID, string.Empty), dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty)); } + + public override bool Equals(object obj) + { + if (obj is DicomIdentity identity) + { + return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison) && + SeriesInstanceUID.Equals(identity.SeriesInstanceUID, EqualsStringComparison) && + SopInstanceUID.Equals(identity.SopInstanceUID, EqualsStringComparison); + } + + return false; + } + + public override int GetHashCode() + => (StudyInstanceUID + SeriesInstanceUID + SopInstanceUID).GetHashCode(EqualsStringComparison); } } diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs index 4a2c4d8f1e..df4aec0435 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs @@ -14,10 +14,24 @@ public interface IDicomIndexDataStore { Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default); + Task> QueryInstancesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomTag Attribute, string Value)> query = null, + CancellationToken cancellationToken = default); + Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default); Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); + /// + /// Delete the provided series index. + /// + /// The study instance unique identifier. + /// The series instance unique identifier. + /// The cancellation token. + /// The collection of instances that were deleted from this series. Task> DeleteSeriesIndexAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); Task DeleteInstanceIndexAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default); diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs new file mode 100644 index 0000000000..d2ba1842f7 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Extensions.DependencyInjection; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage +{ + public class DicomCosmosDataStoreTests + { + private readonly DicomCosmosDataStore _indexDataStore; + + public DicomCosmosDataStoreTests() + { + IOptionsMonitor namedCosmosCollectionConfigurationAccessor = Substitute.For>(); + namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName).Returns(new CosmosCollectionConfiguration() { CollectionId = "testcollection" }); + + _indexDataStore = new DicomCosmosDataStore( + Substitute.For>(), + new CosmosDataStoreConfiguration(), + namedCosmosCollectionConfigurationAccessor, + new DicomCosmosConfiguration()); + } + + [Fact] + public async Task GivenInvalidParameters_WhenCallingAllMethods_ArgumentExceptionThrown() + { + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(null)); + + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(null, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(string.Empty, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), null)); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), string.Empty)); + + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(null, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(string.Empty, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), null, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), string.Empty, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), null)); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), string.Empty)); + + await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInStudyAsync(null)); + await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInStudyAsync(string.Empty)); + + await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(null, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(string.Empty, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(Guid.NewGuid().ToString(), null)); + await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(Guid.NewGuid().ToString(), string.Empty)); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs new file mode 100644 index 0000000000..00a2d35db9 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Dicom; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class DicomTagSerializerTests + { + [Fact] + public void GivenPrivateDicomTag_WhenSerialized_IsDeserializedCorrectly() + { + var dicomDictionary = new DicomDictionary + { + new DicomDictionaryEntry(new DicomTag(0x0011, 0x0010), "TEST CREATOR", "TEST", DicomVM.VM_1, false, DicomVR.LO), + }; + + SerializeAndDeserialize(new DicomTag(0007, 0010)); + SerializeAndDeserialize(new DicomTag(0011, 0010, dicomDictionary.GetPrivateCreator("TEST"))); + } + + [Fact] + public void GivenValidDicomTag_WhenSerialized_IsDeserializedCorrectly() + { + SerializeAndDeserialize(DicomTag.PatientName); + SerializeAndDeserialize(DicomTag.SOPInstanceStatus); + SerializeAndDeserialize(DicomTag.SeriesDate); + SerializeAndDeserialize(DicomTag.SeriesTime); + SerializeAndDeserialize(DicomTag.AbortReason); + SerializeAndDeserialize(DicomTag.WindowCenter); + } + + [Fact] + public void GivenInvalidDicomTagSerializerParameters_OnSerializeOrDeserialize_ArgumentExceptionIsThrown() + { + Assert.Throws(() => DicomTagSerializer.Serialize(null)); + Assert.Throws(() => DicomTagSerializer.Deserialize(null)); + Assert.Throws(() => DicomTagSerializer.Deserialize(string.Empty)); + } + + private static void SerializeAndDeserialize(DicomTag dicomTag) + { + var serialized = DicomTagSerializer.Serialize(dicomTag); + DicomTag deserialized = DicomTagSerializer.Deserialize(serialized); + + Assert.Equal(dicomTag, deserialized); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs index cddd177409..6d708db80d 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using Dicom; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; using Xunit; @@ -16,10 +17,10 @@ public class QueryInstanceTests [Fact] public void GivenInvalidParameters_WhenCreatingQueryInstance_ExceptionsThrown() { - Assert.Throws(() => new QueryInstance(null, new Dictionary())); - Assert.Throws(() => new QueryInstance(string.Empty, new Dictionary())); - Assert.Throws(() => new QueryInstance(new string('a', 65), new Dictionary())); - Assert.Throws(() => new QueryInstance("AA#AA", new Dictionary())); + Assert.Throws(() => new QueryInstance(null, new Dictionary())); + Assert.Throws(() => new QueryInstance(string.Empty, new Dictionary())); + Assert.Throws(() => new QueryInstance(new string('a', 65), new Dictionary())); + Assert.Throws(() => new QueryInstance("AA#AA", new Dictionary())); Assert.Throws(() => new QueryInstance(Guid.NewGuid().ToString(), null)); Assert.Throws(() => QueryInstance.Create(null, null)); @@ -51,9 +52,9 @@ public void GivenDicomTagsToIndex_WhenCreatingQueryInstance_TagsAreExtractedAndI Assert.Equal(dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID), instance.SopInstanceUID); Assert.Equal(3, instance.IndexedAttributes.Count); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.IndexedAttributes[DicomTag.PatientName]); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.IndexedAttributes[DicomTag.StudyDate]); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.ReferringPhysicianName), instance.IndexedAttributes[DicomTag.ReferringPhysicianName]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.StudyDate)]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.ReferringPhysicianName), instance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.ReferringPhysicianName)]); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs index 2570c449ec..ab15dfa17c 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -6,7 +6,9 @@ using System; using System.Linq; using Dicom; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Newtonsoft.Json; using Xunit; namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents @@ -54,12 +56,45 @@ public void GivenExistingInstance_WhenAddingToInstancesHashSet_IsNotAdded() Assert.True(document.AddInstance(instanceDocument1)); Assert.False(document.AddInstance(instanceDocument2)); - Assert.Equal(testPatientName, document.DistinctIndexedAttributes[DicomTag.PatientName].First()); + Assert.Equal(testPatientName, document.DistinctIndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)].First()); Assert.Throws(() => document.RemoveInstance(null)); Assert.Throws(() => document.RemoveInstance(string.Empty)); Assert.True(document.RemoveInstance(sopInstanceUID)); Assert.False(document.RemoveInstance(sopInstanceUID)); } + + [Fact] + public void GivenSeriesDocument_WhenSerialized_IsDeserializedCorrectly() + { + var document = new QuerySeriesDocument(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()) + { + ETag = Guid.NewGuid().ToString(), + }; + + var dataset = new DicomDataset(); + var sopInstanceUID = Guid.NewGuid().ToString(); + var testPatientName = Guid.NewGuid().ToString(); + dataset.Add(DicomTag.SOPInstanceUID, sopInstanceUID); + dataset.Add(DicomTag.PatientName, testPatientName); + + var instanceDocument = QueryInstance.Create(dataset, new[] { DicomTag.PatientName }); + document.AddInstance(instanceDocument); + + var serialized = JsonConvert.SerializeObject(document); + QuerySeriesDocument deserialized = JsonConvert.DeserializeObject(serialized); + + Assert.Equal(document.Id, deserialized.Id); + Assert.Equal(document.PartitionKey, deserialized.PartitionKey); + Assert.Equal(document.ETag, deserialized.ETag); + Assert.Equal(document.StudyInstanceUID, deserialized.StudyInstanceUID); + Assert.Equal(document.SeriesInstanceUID, deserialized.SeriesInstanceUID); + Assert.Equal(document.Instances.Count, deserialized.Instances.Count); + + QueryInstance deserializedFirstInstance = deserialized.Instances.First(); + Assert.Equal(deserializedFirstInstance.SopInstanceUID, deserializedFirstInstance.SopInstanceUID); + Assert.Equal(deserializedFirstInstance.IndexedAttributes.Count, deserializedFirstInstance.IndexedAttributes.Count); + Assert.Equal(deserializedFirstInstance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)], deserializedFirstInstance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)]); + } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs new file mode 100644 index 0000000000..b57d6f79fd --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Dicom; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + internal class CosmosQueryBuilder + { + private const string OffsetParameterName = "@offset"; + private const string LimitParameterName = "@limit"; + private const string ItemParameterNameFormat = "@item{0}"; + private const string InstanceSqlQuerySearchFormat = "SELECT value {{ \"StudyInstanceUID\": c.StudyInstanceUID, \"SeriesInstanceUID\": c.SeriesInstanceUID, \"SOPInstanceUID\": f.SopInstanceUID }} FROM c JOIN f in c.Instances {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private readonly DicomCosmosConfiguration _dicomConfiguration; + private readonly IFormatProvider _stringFormatProvider; + + public CosmosQueryBuilder(DicomCosmosConfiguration dicomConfiguration) + { + EnsureArg.IsNotNull(dicomConfiguration, nameof(dicomConfiguration)); + + _dicomConfiguration = dicomConfiguration; + _stringFormatProvider = CultureInfo.InvariantCulture; + } + + public SqlQuerySpec BuildInstanceQuerySpec( + int offset, + int limit, + IEnumerable<(DicomTag Attribute, string Value)> query = null) + { + // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. + var sqlParameterCollection = new SqlParameterCollection() + { + new SqlParameter { Name = OffsetParameterName, Value = offset }, + new SqlParameter { Name = LimitParameterName, Value = limit }, + }; + + // If a null or empty query collection we should provide an empty string for the WHERE clause. + var whereClause = string.Empty; + + if (query != null && query.Any()) + { + var parameterNameIndex = 1; + var whereItems = new List(); + + foreach ((DicomTag attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + { + var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); + sqlParameterCollection.Add(new SqlParameter { Name = parameterName, Value = value }); + whereItems.Add($"f.{nameof(QueryInstance.IndexedAttributes)}[\"{DicomTagSerializer.Serialize(attribute)}\"] = {parameterName}"); + } + + // Now construct the WHERE query joining each item with an AND. + whereClause = $"WHERE {string.Join(" AND ", whereItems)}"; + } + + return new SqlQuerySpec(string.Format(_stringFormatProvider, InstanceSqlQuerySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index b1ae926032..df0e357cbc 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Dicom; @@ -30,8 +31,10 @@ internal class DicomCosmosDataStore : IDicomIndexDataStore private readonly IDocumentClient _documentClient; private readonly Uri _collectionUri; private readonly DicomCosmosConfiguration _dicomConfiguration; + private readonly CosmosQueryBuilder _queryBuilder; private readonly string _databaseId; private readonly string _collectionId; + private readonly Random _random = new Random(); public DicomCosmosDataStore( IScoped documentClient, @@ -51,6 +54,7 @@ public DicomCosmosDataStore( _databaseId = configuration.DatabaseId; _collectionId = cosmosCollectionConfiguration.CollectionId; _dicomConfiguration = dicomConfiguration; + _queryBuilder = new CosmosQueryBuilder(dicomConfiguration); } /// @@ -86,6 +90,32 @@ await _documentClient.ThrowIndexDataStoreException( retryPolicy); } + /// + public async Task> QueryInstancesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomTag Attribute, string Value)> query = null, + CancellationToken cancellationToken = default) + { + EnsureArg.IsTrue(offset >= 0, nameof(offset)); + EnsureArg.IsTrue(limit > 0, nameof(limit)); + + // If the study instance UID is provided we can run the query against a specific partition. + FeedOptions feedOptions = string.IsNullOrWhiteSpace(studyInstanceUID) ? CreateCrossPartitionFeedOptions() : CreateFeedOptions(studyInstanceUID); + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildInstanceQuerySpec(offset, limit, query); + + IDocumentQuery instanceQuery = _documentClient + .CreateDocumentQuery(_collectionUri, sqlQuerySpec, feedOptions) + .AsDocumentQuery(); + + return await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + return await instanceQuery.ExecuteQueryUntilCompleteAsync(cancellationToken); + }); + } + /// public async Task> DeleteSeriesIndexAsync( string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) @@ -93,8 +123,7 @@ public async Task> DeleteSeriesIndexAsync( EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); - Uri documentUri = UriFactory.CreateDocumentUri( - _databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); + Uri documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); // Retry when the pre-condition fails on delete. @@ -125,8 +154,7 @@ public async Task DeleteInstanceIndexAsync(string studyInstanceUID, string serie EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); - Uri documentUri = UriFactory.CreateDocumentUri( - _databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); + Uri documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); @@ -214,11 +242,25 @@ private RequestOptions CreateRequestOptions(string partitionKey) => new RequestOptions() { PartitionKey = new PartitionKey(partitionKey) }; private FeedOptions CreateFeedOptions(string partitionKey) - => new FeedOptions() { PartitionKey = new PartitionKey(partitionKey) }; + { + EnsureArg.IsNotNull(partitionKey, nameof(partitionKey)); + return new FeedOptions() { PartitionKey = new PartitionKey(partitionKey), EnableCrossPartitionQuery = false }; + } + + private FeedOptions CreateCrossPartitionFeedOptions() + { + var feedOptions = new FeedOptions() { EnableCrossPartitionQuery = true }; + + // TODO: This is a reflection hack to enable cross partition skip and take. This will be in the latest SDK soon. + PropertyInfo propertyInfo = feedOptions.GetType().GetProperty("EnableCrossPartitionSkipTake", BindingFlags.NonPublic | BindingFlags.Instance); + propertyInfo.SetValue(feedOptions, Convert.ChangeType(true, propertyInfo.PropertyType)); + + return feedOptions; + } - private static IAsyncPolicy CreatePreConditionFailedRetryPolicy() + private IAsyncPolicy CreatePreConditionFailedRetryPolicy() => Policy - .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed) - .RetryForeverAsync(); + .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed || ex.StatusCode == HttpStatusCode.TooManyRequests) + .WaitAndRetryForeverAsync(retryIndex => TimeSpan.FromMilliseconds((retryIndex - 1) * _random.Next(200, 500))); } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs new file mode 100644 index 0000000000..f304f69396 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Globalization; +using Dicom; +using EnsureThat; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + internal static class DicomTagSerializer + { + private const char Seperator = ','; + private const string GroupElementStringFormat = "X4"; + private static readonly IFormatProvider FormatProvider = CultureInfo.InvariantCulture; + + public static string Serialize(DicomTag dicomTag) + { + EnsureArg.IsNotNull(dicomTag, nameof(dicomTag)); + + string groupString = dicomTag.Group.ToString(GroupElementStringFormat, FormatProvider); + string elementString = dicomTag.Element.ToString(GroupElementStringFormat, FormatProvider); + + if (dicomTag.PrivateCreator == null) + { + return $"{groupString}{Seperator}{elementString}"; + } + + return $"{groupString}{Seperator}{elementString}{Seperator}{dicomTag.PrivateCreator.Creator}"; + } + + public static DicomTag Deserialize(string dictionaryElement) + { + EnsureArg.IsNotNullOrWhiteSpace(dictionaryElement, nameof(dictionaryElement)); + var split = dictionaryElement.Split(Seperator); + + EnsureArg.IsTrue(split.Length == 2 || split.Length == 3, nameof(dictionaryElement)); + EnsureArg.IsTrue(split[0].Length == 4, nameof(dictionaryElement)); + EnsureArg.IsTrue(split[1].Length == 4, nameof(dictionaryElement)); + + EnsureArg.IsTrue(ushort.TryParse(split[0], NumberStyles.HexNumber, FormatProvider, out ushort group), nameof(dictionaryElement)); + EnsureArg.IsTrue(ushort.TryParse(split[1], NumberStyles.HexNumber, FormatProvider, out ushort element), nameof(dictionaryElement)); + + if (split.Length == 2) + { + return new DicomTag(group, element); + } + + return new DicomTag(group, element, split[2]); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs index bb6d8d253e..481742eeba 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -4,12 +4,14 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; using Newtonsoft.Json; using Polly; @@ -60,6 +62,19 @@ public static async Task ThrowIndexDataStoreException(this IDocumentClient docum } } + public static async Task> ExecuteQueryUntilCompleteAsync(this IDocumentQuery documentQuery, CancellationToken cancellationToken) + { + var results = new List(); + + while (documentQuery.HasMoreResults) + { + FeedResponse nextResults = await documentQuery.ExecuteNextAsync(cancellationToken); + results.AddRange(nextResults); + } + + return results; + } + public static async Task GetorCreateDocumentAsync( this IDocumentClient documentClient, string databaseId, diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs index 8b73d60dd2..3ad177cd4a 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -16,7 +16,7 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents internal class QueryInstance { [JsonConstructor] - public QueryInstance(string sopInstanceUID, IDictionary indexedAttributes) + public QueryInstance(string sopInstanceUID, IDictionary indexedAttributes) : this(sopInstanceUID) { EnsureArg.IsNotNull(indexedAttributes, nameof(indexedAttributes)); @@ -34,7 +34,7 @@ private QueryInstance(string sopInstanceUID) public string SopInstanceUID { get; } - public IDictionary IndexedAttributes { get; } = new Dictionary(); + public IDictionary IndexedAttributes { get; } = new Dictionary(); public override int GetHashCode() { @@ -60,14 +60,14 @@ public static QueryInstance Create(DicomDataset dicomDataset, IEnumerable Instances { get; } = new HashSet(); - public Dictionary> DistinctIndexedAttributes + public Dictionary> DistinctIndexedAttributes { get { - var result = new Dictionary>(); + var result = new Dictionary>(); foreach (QueryInstance instance in Instances) { - foreach ((DicomTag key, object value) in instance.IndexedAttributes) + foreach ((string key, object value) in instance.IndexedAttributes) { if (!result.ContainsKey(key)) { diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs index bda6a4418b..82b8b0dca7 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -135,27 +135,80 @@ public async Task GivenNonExistentSeriesOrStudy_WhenFetchingInstances_IndexDataS Assert.Equal(HttpStatusCode.NotFound, getSeriesInstancesException.StatusCode); } + [Fact] + public async Task GivenIndexedInstance_WhenQueryingByPatientName_InstancesIsRetrieved() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + string referringPhysicianName = Guid.NewGuid().ToString(); + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + testInstance.Add(DicomTag.ReferringPhysicianName, referringPhysicianName); + + await _indexDataStore.IndexInstanceAsync(testInstance); + + var sopInstanceUID = testInstance.GetSingleValue(DicomTag.SOPInstanceUID); + IEnumerable instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (DicomTag.ReferringPhysicianName, referringPhysicianName) }); + Assert.Single(instances); + Assert.Equal(new DicomIdentity(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.First()); + } + + [Fact] + public async Task GivenIndexedSeries_WhenQueryingWithPaging_PagesReturnedCorrectly() + { + const int numberOfInstances = 20; + Assert.Equal(0, numberOfInstances % 2); + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstances); + var instanceUIDs = new HashSet(instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID))); + + // End of page + IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(numberOfInstances, numberOfInstances, studyInstanceUID); + Assert.Empty(queryInstances1); + + // First 5 items + IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, numberOfInstances / 2, studyInstanceUID); + Assert.Equal(numberOfInstances / 2, queryInstances2.Count()); + + foreach (DicomIdentity item in queryInstances2) + { + Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); + } + + // Last 5 items + IEnumerable queryInstances3 = await _indexDataStore.QueryInstancesAsync(numberOfInstances / 2, numberOfInstances, studyInstanceUID); + Assert.Equal(numberOfInstances / 2, queryInstances3.Count()); + + foreach (DicomIdentity item in queryInstances3) + { + Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); + } + + Assert.Empty(instanceUIDs); + } + private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceUID, string seriesInstanceUID) { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID); EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID); - var result = new DicomDataset(); - result.Add(DicomTag.StudyInstanceUID, studyInstanceUID); - result.Add(DicomTag.SeriesInstanceUID, seriesInstanceUID); - result.Add(DicomTag.SOPInstanceUID, Guid.NewGuid().ToString()); + var result = new DicomDataset + { + { DicomTag.StudyInstanceUID, studyInstanceUID }, + { DicomTag.SeriesInstanceUID, seriesInstanceUID }, + { DicomTag.SOPInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.PatientName, Guid.NewGuid().ToString() }, + }; return result; } private async Task> CreateSeriesInParallelAsync(string studyInstanceUID, string seriesInstanceUID, int numberOfItemsInSeries) { - IList instances = new List(); - - for (var i = 0; i < numberOfItemsInSeries; i++) - { - instances.Add(CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID)); - } - + IList instances = Enumerable.Range(0, numberOfItemsInSeries) + .Select(_ => CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID)) + .ToList(); await Task.WhenAll(instances.Select(x => _indexDataStore.IndexInstanceAsync(x))); return instances; } From 70879af3729c9e982631c64ca588ca59851172de Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 15:56:01 +0100 Subject: [PATCH 05/22] Attempt to get tests running on windows --- build/build.yml | 15 +++------------ build/test.yml | 19 +++++++------------ 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/build/build.yml b/build/build.yml index 483e14d53e..df568eb891 100644 --- a/build/build.yml +++ b/build/build.yml @@ -1,18 +1,6 @@ parameters: - # Default values - name: Windows - queue: 'Hosted VS2017' packageArtifacts: true -phases: -- phase: ${{ parameters.name }} - queue: '${{ parameters.queue }}' - - variables: - solution: '**/*.sln' - buildPlatform: 'Any CPU' - buildConfiguration: 'Release' - steps: - task: NuGetToolInstaller@0 @@ -43,5 +31,8 @@ steps: projects: '**/*UnitTests/*.csproj' arguments: '--configuration $(buildConfiguration)' +- template: test.yml + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + - ${{ if eq(parameters.packageArtifacts, 'true') }}: - template: package.yml diff --git a/build/test.yml b/build/test.yml index 9c5c936839..cbeeabb0f6 100644 --- a/build/test.yml +++ b/build/test.yml @@ -1,15 +1,3 @@ - -- task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts - deploy' - inputs: - artifactName: deploy - downloadPath: '$(System.ArtifactsDirectory)' - -- powershell: | - $env:CosmosDb:Host = "$(DataStore--Host)" - $env:CosmosDb:Key = "$(DataStore--Key)" - displayName: 'Setup Environment Variables' - # To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). # This allows us to test the blob storage providers without an Azure instance. - task: NodeTool@0 @@ -29,9 +17,16 @@ - script: start azurite -s condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) displayName: 'Start Azurite Storage Emulator (Windows)' + +# Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing +# if performed on windows. +- task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 + displayName: 'Run Azure Cosmos DB Emulator container' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - task: DotNetCoreCLI@2 displayName: 'dotnet test UnitTests' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) inputs: command: test projects: '**\*Tests*.csproj' From 1be66b9b9fc11758172f8a4bc41b31bbc6520479 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 16:05:47 +0100 Subject: [PATCH 06/22] Updated build file --- build/build.yml | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/build/build.yml b/build/build.yml index df568eb891..0ad173ad8d 100644 --- a/build/build.yml +++ b/build/build.yml @@ -3,36 +3,36 @@ parameters: steps: -- task: NuGetToolInstaller@0 - displayName: 'Use NuGet 4.7.1' - inputs: - versionSpec: 4.7.1 - checkLatest: true - -- task: DotNetCoreInstaller@0 - inputs: - version: '2.2.103' - -- task: DotNetCoreCLI@2 - displayName: 'donet restore' - inputs: - command: restore - restoreSolution: '**\*.sln' - feedsToUse: config - nugetConfigPath: nuget.config - -- script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror - displayName: 'dotnet build $(buildConfiguration)' - -- task: DotNetCoreCLI@2 - displayName: 'dotnet test UnitTests' - inputs: - command: test + - task: NuGetToolInstaller@0 + displayName: 'Use NuGet 4.7.1' + inputs: + versionSpec: 4.7.1 + checkLatest: true + + - task: DotNetCoreInstaller@0 + inputs: + version: '2.2.103' + + - task: DotNetCoreCLI@2 + displayName: 'donet restore' + inputs: + command: restore + restoreSolution: '**\*.sln' + feedsToUse: config + nugetConfigPath: nuget.config + + - script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror + displayName: 'dotnet build $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: 'dotnet test UnitTests' + inputs: + command: test projects: '**/*UnitTests/*.csproj' - arguments: '--configuration $(buildConfiguration)' + arguments: '--configuration $(buildConfiguration)' -- template: test.yml - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + - include: test.yml + when: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) -- ${{ if eq(parameters.packageArtifacts, 'true') }}: - - template: package.yml + - ${{ if eq(parameters.packageArtifacts, 'true') }}: + - template: package.yml From 9e4373dd04bb42164673547da6e0cbf911cbff24 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 16:06:52 +0100 Subject: [PATCH 07/22] Updated build file --- build/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/build.yml b/build/build.yml index 0ad173ad8d..4e951bda42 100644 --- a/build/build.yml +++ b/build/build.yml @@ -31,8 +31,8 @@ steps: projects: '**/*UnitTests/*.csproj' arguments: '--configuration $(buildConfiguration)' - - include: test.yml - when: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + - template: test.yml + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - ${{ if eq(parameters.packageArtifacts, 'true') }}: - template: package.yml From 15685edcb07ef25bc88a0e1bda106aa822671935 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 16:07:43 +0100 Subject: [PATCH 08/22] Removed condition from build test --- build/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/build.yml b/build/build.yml index 4e951bda42..6c199da8ed 100644 --- a/build/build.yml +++ b/build/build.yml @@ -32,7 +32,6 @@ steps: arguments: '--configuration $(buildConfiguration)' - template: test.yml - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - ${{ if eq(parameters.packageArtifacts, 'true') }}: - template: package.yml From 5afc1ccdc9e9f9f21204910dd0036850f219e64c Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 16:12:45 +0100 Subject: [PATCH 09/22] Added steps to the test yml file --- build/test.yml | 63 ++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/build/test.yml b/build/test.yml index cbeeabb0f6..fe68bbf8b9 100644 --- a/build/test.yml +++ b/build/test.yml @@ -1,33 +1,36 @@ -# To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). -# This allows us to test the blob storage providers without an Azure instance. -- task: NodeTool@0 - displayName: 'Use Node 8.x' - inputs: - versionSpec: 8.x - checkLatest: true -- script: npm install -g azurite@2.7.0 - displayName: 'Install Azurite v2.7.0' +steps: -# We start the Azurite as a separate process as the start call is blocking. -- script: azurite -s & - condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) - displayName: 'Start Azurite Storage Emulator (Linux)' - -- script: start azurite -s - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - displayName: 'Start Azurite Storage Emulator (Windows)' + # To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). + # This allows us to test the blob storage providers without an Azure instance. + - task: NodeTool@0 + displayName: 'Use Node 8.x' + inputs: + versionSpec: 8.x + checkLatest: true -# Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing -# if performed on windows. -- task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 - displayName: 'Run Azure Cosmos DB Emulator container' - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - -- task: DotNetCoreCLI@2 - displayName: 'dotnet test UnitTests' - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - inputs: - command: test - projects: '**\*Tests*.csproj' - arguments: '--configuration $(buildConfiguration)' \ No newline at end of file + - script: npm install -g azurite@2.7.0 + displayName: 'Install Azurite v2.7.0' + + # We start the Azurite as a separate process as the start call is blocking. + - script: azurite -s & + condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) + displayName: 'Start Azurite Storage Emulator (Linux)' + + - script: start azurite -s + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + displayName: 'Start Azurite Storage Emulator (Windows)' + + # Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing + # if performed on windows. + - task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 + displayName: 'Run Azure Cosmos DB Emulator container' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + + - task: DotNetCoreCLI@2 + displayName: 'dotnet test UnitTests' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + inputs: + command: test + projects: '**\*Tests*.csproj' + arguments: '--configuration $(buildConfiguration)' \ No newline at end of file From d0f569de4ff14bc4f696d7115402a47b462e9c4a Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Thu, 20 Jun 2019 17:09:43 +0100 Subject: [PATCH 10/22] Updated the cosmos tests yml file to use the correct endpoint --- build/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/test.yml b/build/test.yml index fe68bbf8b9..9c38ade303 100644 --- a/build/test.yml +++ b/build/test.yml @@ -24,11 +24,14 @@ steps: # Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing # if performed on windows. - task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 - displayName: 'Run Azure Cosmos DB Emulator container' + displayName: 'Start Azure Cosmos DB Emulator (Windows)' condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + + - powershell: | + $env:CosmosDb:Host = "$(CosmosDbEmulator.Endpoint)" - task: DotNetCoreCLI@2 - displayName: 'dotnet test UnitTests' + displayName: 'Run all tests (Windows)' condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) inputs: command: test From 3ccb0f1aec0baa0b19cbca7c4e9ac18d96214894 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Fri, 21 Jun 2019 15:05:11 +0100 Subject: [PATCH 11/22] Updated the test build script, added support for series and study querying --- build/test.yml | 24 ++- .../Persistence/DicomInstanceTests.cs | 74 +++++++ .../Features/Persistence/DicomDataStore.cs | 66 ------ .../{DicomIdentity.cs => DicomInstance.cs} | 29 +-- .../Features/Persistence/DicomSeries.cs | 43 ++++ .../Features/Persistence/DicomStudy.cs | 43 ++++ .../Features/Persistence/IDicomDataStore.cs | 16 -- .../Persistence/IDicomIndexDataStore.cs | 22 +- .../Storage/Documents/AttributeValuesTests.cs | 93 +++++++++ .../Documents/QuerySeriesDocumentTests.cs | 2 +- .../Features/Storage/CosmosQueryBuilder.cs | 89 +++++--- .../Features/Storage/DicomCosmosDataStore.cs | 73 ++++--- .../Storage/Documents/AttributeValues.cs | 48 +++++ .../Storage/Documents/QuerySeriesDocument.cs | 6 +- .../Persistence/DicomIndexDataStoreTests.cs | 190 ++++++++++++++++-- 15 files changed, 633 insertions(+), 185 deletions(-) create mode 100644 src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs delete mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs rename src/Microsoft.Health.Dicom.Core/Features/Persistence/{DicomIdentity.cs => DicomInstance.cs} (69%) create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs delete mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs diff --git a/build/test.yml b/build/test.yml index 9c38ade303..3838f95868 100644 --- a/build/test.yml +++ b/build/test.yml @@ -23,12 +23,26 @@ steps: # Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing # if performed on windows. - - task: azure-cosmosdb.emulator-public-preview.run-cosmosdbemulatorcontainer.CosmosDbEmulator@2 - displayName: 'Start Azure Cosmos DB Emulator (Windows)' - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - - powershell: | - $env:CosmosDb:Host = "$(CosmosDbEmulator.Endpoint)" + $cosmosemulator=[Environment]::GetEnvironmentVariable("ProgramFiles") + "\Azure Cosmos DB Emulator\CosmosDB.Emulator.exe" + & $cosmosemulator /noui + $location = 'https://localhost:8081/_explorer/index.html' + $statusCode = [int]0 + while ($statusCode -ne 200) + { + Try + { + $req = [system.Net.WebRequest]::Create($location) + $response = $req.GetResponse() + $statusCode = [int]$response.StatusCode + $location + " status code: " + $statusCode + $response.Close() + } Catch { + $PSItem.ToString() + } + } + displayName: 'Start Installed Cosmos Emulator' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - task: DotNetCoreCLI@2 displayName: 'Run all tests (Windows)' diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs new file mode 100644 index 0000000000..affa82507b --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence +{ + public class DicomInstanceTests + { + [Fact] + public void GivenDicomInstanceSeriesStudy_WhenConstructedWithInvalidParameters_ArgumentExceptionThrown() + { + Assert.Throws(() => new DicomStudy(null)); + Assert.Throws(() => new DicomStudy(string.Empty)); + Assert.Throws(() => new DicomStudy("#")); + Assert.Throws(() => new DicomStudy(new string('a', 65))); + + Assert.Throws(() => new DicomSeries(null, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries("#", Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries(new string('a', 65), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), null)); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), "#")); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), new string('a', 65))); + Assert.Throws(() => new DicomSeries("aaa", "aaa")); + + Assert.Throws(() => new DicomInstance(null, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(string.Empty, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance("#", Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(new string('a', 65), Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), null, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), "#", Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), new string('a', 65), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), null)); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "#")); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), new string('a', 65))); + Assert.Throws(() => new DicomInstance("aaa", "aaa", "bbb")); + Assert.Throws(() => new DicomInstance("aaa", "bbb", "aaa")); + Assert.Throws(() => new DicomInstance("bbb", "aaa", "aaa")); + } + + [Fact] + public void GivenDicomInstanceSeriesStudy_WhenCompared_CorrectEqualsResponseReturned() + { + Assert.Equal("aaa", new DicomStudy("aaa").StudyInstanceUID); + Assert.Equal("aaa".GetHashCode(), new DicomStudy("aaa").GetHashCode()); + Assert.Equal(new DicomStudy("aaa"), new DicomStudy("aaa")); + Assert.NotEqual(new DicomStudy("aAa"), new DicomStudy("aaa")); + + Assert.Equal("aaa", new DicomSeries("aaa", "bbb").StudyInstanceUID); + Assert.Equal("bbb", new DicomSeries("aaa", "bbb").SeriesInstanceUID); + Assert.Equal("aaabbb".GetHashCode(), new DicomSeries("aaa", "bbb").GetHashCode()); + Assert.Equal(new DicomSeries("aaa", "bbb"), new DicomSeries("aaa", "bbb")); + Assert.NotEqual(new DicomSeries("aAa", "bbb"), new DicomSeries("aaa", "bbb")); + Assert.NotEqual(new DicomSeries("aaa", "bBb"), new DicomSeries("aaa", "bbb")); + + Assert.Equal("aaa", new DicomInstance("aaa", "bbb", "ccc").StudyInstanceUID); + Assert.Equal("bbb", new DicomInstance("aaa", "bbb", "ccc").SeriesInstanceUID); + Assert.Equal("ccc", new DicomInstance("aaa", "bbb", "ccc").SopInstanceUID); + Assert.Equal("aaabbbccc".GetHashCode(), new DicomInstance("aaa", "bbb", "ccc").GetHashCode()); + Assert.Equal(new DicomInstance("aaa", "bbb", "ccc"), new DicomInstance("aaa", "bbb", "ccc")); + Assert.NotEqual(new DicomInstance("aAa", "bbb", "ccc"), new DicomInstance("aaa", "bbb", "ccc")); + Assert.NotEqual(new DicomInstance("aaa", "bBb", "ccc"), new DicomInstance("aaa", "bbb", "ccc")); + Assert.NotEqual(new DicomInstance("aaa", "bbb", "cCc"), new DicomInstance("aaa", "bbb", "ccc")); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs deleted file mode 100644 index 49a4c0ca03..0000000000 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomDataStore.cs +++ /dev/null @@ -1,66 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Dicom; -using EnsureThat; - -namespace Microsoft.Health.Dicom.Core.Features.Persistence -{ - public class DicomDataStore : IDicomDataStore - { - // This value should not be changed without taking into consideration concurrency. - private const bool OverwriteBlobsIfExists = false; - private readonly IDicomBlobDataStore _dicomBlobDataStore; - private readonly IDicomIndexDataStore _dicomIndexDataStore; - - public DicomDataStore( - IDicomBlobDataStore dicomBlobDataStore, - IDicomIndexDataStore dicomIndexDataStore) - { - EnsureArg.IsNotNull(dicomBlobDataStore, nameof(dicomBlobDataStore)); - EnsureArg.IsNotNull(dicomIndexDataStore, nameof(dicomIndexDataStore)); - - _dicomBlobDataStore = dicomBlobDataStore; - _dicomIndexDataStore = dicomIndexDataStore; - } - - public async Task StoreAsync(DicomFile dicomFile, CancellationToken cancellationToken = default) - { - var dicomIdentity = DicomIdentity.Create(dicomFile.Dataset); - var instanceBlobName = GetInstanceBlobName(dicomIdentity); - - // Step 1. Store the blob file - using (var memoryStream = new MemoryStream()) - { - await dicomFile.SaveAsync(memoryStream); - await _dicomBlobDataStore.AddFileAsStreamAsync(instanceBlobName, memoryStream, overwriteIfExists: OverwriteBlobsIfExists, cancellationToken); - } - - // Step 2. Attempt to index the file; this will fail if it exists, which should never happen. - await _dicomIndexDataStore.IndexInstanceAsync(dicomFile.Dataset, cancellationToken); - } - - public async Task DeleteInstanceAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default) - { - var instanceBlobName = GetInstanceBlobName(studyInstanceUID, seriesInstanceUID, sopInstanceUID); - - // Reverse process of store. - // 1. Delete the instance index - await _dicomIndexDataStore.DeleteInstanceIndexAsync(studyInstanceUID, seriesInstanceUID, sopInstanceUID, cancellationToken); - - // 2. Delete the instance file. - await _dicomBlobDataStore.DeleteFileIfExistsAsync(instanceBlobName, cancellationToken); - } - - private static string GetInstanceBlobName(DicomIdentity dicomIdentity) - => GetInstanceBlobName(dicomIdentity.StudyInstanceUID, dicomIdentity.SeriesInstanceUID, dicomIdentity.SopInstanceUID); - - private static string GetInstanceBlobName(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) - => $"{studyInstanceUID}\\{seriesInstanceUID}\\{sopInstanceUID}"; - } -} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs similarity index 69% rename from src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs rename to src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs index cc325b4031..8b52b0a855 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomIdentity.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Text.RegularExpressions; using Dicom; using EnsureThat; @@ -12,39 +11,29 @@ namespace Microsoft.Health.Dicom.Core.Features.Persistence { - public class DicomIdentity + public class DicomInstance : DicomSeries { - private const StringComparison EqualsStringComparison = StringComparison.Ordinal; - [JsonConstructor] - public DicomIdentity(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) + public DicomInstance(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) + : base(studyInstanceUID, seriesInstanceUID) { - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); - EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); - // Run the instance identifiers through the regular expression check. - EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); - EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsNotEqualTo(sopInstanceUID, studyInstanceUID); + EnsureArg.IsNotEqualTo(sopInstanceUID, seriesInstanceUID); - StudyInstanceUID = studyInstanceUID; - SeriesInstanceUID = seriesInstanceUID; SopInstanceUID = sopInstanceUID; } - public string StudyInstanceUID { get; } - - public string SeriesInstanceUID { get; } - public string SopInstanceUID { get; } - public static DicomIdentity Create(DicomDataset dicomDataset) + public static DicomInstance Create(DicomDataset dicomDataset) { EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); // Note: Here we 'GetSingleValueOrDefault' and let the constructor validate the identifier. - return new DicomIdentity( + return new DicomInstance( dicomDataset.GetSingleValueOrDefault(DicomTag.StudyInstanceUID, string.Empty), dicomDataset.GetSingleValueOrDefault(DicomTag.SeriesInstanceUID, string.Empty), dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty)); @@ -52,7 +41,7 @@ public static DicomIdentity Create(DicomDataset dicomDataset) public override bool Equals(object obj) { - if (obj is DicomIdentity identity) + if (obj is DicomInstance identity) { return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison) && SeriesInstanceUID.Equals(identity.SeriesInstanceUID, EqualsStringComparison) && diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs new file mode 100644 index 0000000000..2ef3b56731 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Text.RegularExpressions; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomSeries : DicomStudy + { + [JsonConstructor] + public DicomSeries(string studyInstanceUID, string seriesInstanceUID) + : base(studyInstanceUID) + { + // Run the instance identifiers through the regular expression check. + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsNotEqualTo(studyInstanceUID, seriesInstanceUID); + + SeriesInstanceUID = seriesInstanceUID; + } + + public string SeriesInstanceUID { get; } + + public override bool Equals(object obj) + { + if (obj is DicomSeries identity) + { + return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison) && + SeriesInstanceUID.Equals(identity.SeriesInstanceUID, EqualsStringComparison); + } + + return false; + } + + public override int GetHashCode() + => (StudyInstanceUID + SeriesInstanceUID).GetHashCode(EqualsStringComparison); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs new file mode 100644 index 0000000000..c26007e4fb --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Text.RegularExpressions; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomStudy + { + protected const StringComparison EqualsStringComparison = StringComparison.Ordinal; + + [JsonConstructor] + public DicomStudy(string studyInstanceUID) + { + // Run the instance identifiers through the regular expression check. + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + + StudyInstanceUID = studyInstanceUID; + } + + public string StudyInstanceUID { get; } + + public override bool Equals(object obj) + { + if (obj is DicomStudy identity) + { + return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison); + } + + return false; + } + + public override int GetHashCode() + => StudyInstanceUID.GetHashCode(EqualsStringComparison); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs deleted file mode 100644 index 10af4300a5..0000000000 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomDataStore.cs +++ /dev/null @@ -1,16 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System.Threading; -using System.Threading.Tasks; -using Dicom; - -namespace Microsoft.Health.Dicom.Core.Features.Persistence -{ - public interface IDicomDataStore - { - Task StoreAsync(DicomFile dicomFile, CancellationToken cancellationToken = default); - } -} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs index df4aec0435..375b1df708 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs @@ -14,16 +14,30 @@ public interface IDicomIndexDataStore { Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default); - Task> QueryInstancesAsync( + Task> QueryStudiesAsync( int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default); - Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default); + Task> QuerySeriesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomTag Attribute, string Value)> query = null, + CancellationToken cancellationToken = default); + + Task> QueryInstancesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomTag Attribute, string Value)> query = null, + CancellationToken cancellationToken = default); + + Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default); - Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); + Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); /// /// Delete the provided series index. @@ -32,7 +46,7 @@ Task> QueryInstancesAsync( /// The series instance unique identifier. /// The cancellation token. /// The collection of instances that were deleted from this series. - Task> DeleteSeriesIndexAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); + Task> DeleteSeriesIndexAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); Task DeleteInstanceIndexAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default); } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs new file mode 100644 index 0000000000..9680cdee5a --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs @@ -0,0 +1,93 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class AttributeValuesTests + { + [Fact] + public void GivenACollectionOfDifferentTypes_WhenAddedToAttributeValues_AreStoredAndMinMaxDateTimeValuesCalculated() + { + const int numberOfItemsToAdd = 300; + var minDateTime = new DateTime(2019, 6, 21); + var maxDateTime = new DateTime(2020, 6, 21); + Assert.True((maxDateTime - minDateTime).Days > numberOfItemsToAdd); + + var attributeValues = new AttributeValues(); + + for (int i = 0; i < numberOfItemsToAdd; i++) + { + // Add DateTime, int, and string + Assert.True(attributeValues.Add(minDateTime.AddDays(i))); + Assert.True(attributeValues.Add(i)); + Assert.True(attributeValues.Add(new string('a', i))); + } + + Assert.True(attributeValues.Add(maxDateTime)); + + Assert.Equal(minDateTime, attributeValues.MinDateTimeValue); + Assert.Equal(maxDateTime, attributeValues.MaxDateTimeValue); + } + + [Fact] + public void GivenExistingItem_WhenAddedToAttributeValues_IsNotAdded() + { + var attributeValues = new AttributeValues(); + + Assert.True(attributeValues.Add(1)); + Assert.False(attributeValues.Add(1)); + + Assert.True(attributeValues.Add("HelloWorld")); + Assert.False(attributeValues.Add("HelloWorld")); + + Assert.Null(new AttributeValues().MinDateTimeValue); + Assert.Null(new AttributeValues().MaxDateTimeValue); + + Assert.True(attributeValues.Add(new DateTime(2019, 6, 22))); + Assert.False(attributeValues.Add(new DateTime(2019, 6, 22))); + + Assert.NotNull(attributeValues.MinDateTimeValue); + Assert.NotNull(attributeValues.MaxDateTimeValue); + + Assert.False(attributeValues.Add(null)); + } + + [Fact] + public void GivenAttributeValues_WhenCreatedWithInvalidParameters_ArgumentExceptionThrown() + { + Assert.Throws(() => new AttributeValues(null)); + Assert.NotNull(new AttributeValues().Values); + Assert.Null(new AttributeValues().MinDateTimeValue); + Assert.Null(new AttributeValues().MaxDateTimeValue); + } + + [Fact] + public void GivenAttributeValues_WhenSerialized_DeserializedCorrectly() + { + const int numberOfItemsToAdd = 300; + var dateTime = new DateTime(2019, 6, 20); + var attributeValues = new AttributeValues(); + + for (int i = 0; i < numberOfItemsToAdd; i++) + { + // Add DateTime, int, and string + Assert.True(attributeValues.Add(dateTime.AddDays(i))); + Assert.True(attributeValues.Add(i)); + Assert.True(attributeValues.Add(new string('a', i))); + } + + var json = JsonConvert.SerializeObject(attributeValues); + AttributeValues deserialized = JsonConvert.DeserializeObject(json); + + Assert.Equal(attributeValues.MinDateTimeValue, deserialized.MinDateTimeValue); + Assert.Equal(attributeValues.MaxDateTimeValue, deserialized.MaxDateTimeValue); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs index ab15dfa17c..e2582fa8fc 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -56,7 +56,7 @@ public void GivenExistingInstance_WhenAddingToInstancesHashSet_IsNotAdded() Assert.True(document.AddInstance(instanceDocument1)); Assert.False(document.AddInstance(instanceDocument2)); - Assert.Equal(testPatientName, document.DistinctIndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)].First()); + Assert.Equal(testPatientName, document.DistinctIndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)].Values.First()); Assert.Throws(() => document.RemoveInstance(null)); Assert.Throws(() => document.RemoveInstance(string.Empty)); diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs index b57d6f79fd..31cb462718 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -20,7 +20,9 @@ internal class CosmosQueryBuilder private const string OffsetParameterName = "@offset"; private const string LimitParameterName = "@limit"; private const string ItemParameterNameFormat = "@item{0}"; - private const string InstanceSqlQuerySearchFormat = "SELECT value {{ \"StudyInstanceUID\": c.StudyInstanceUID, \"SeriesInstanceUID\": c.SeriesInstanceUID, \"SOPInstanceUID\": f.SopInstanceUID }} FROM c JOIN f in c.Instances {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string StudySqlQuerySearchFormat = "SELECT DISTINCT c.StudyInstanceUID FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string SeriesSqlQuerySearchFormat = "SELECT VALUE {{ \"StudyInstanceUID\": c.StudyInstanceUID, \"SeriesInstanceUID\": c.SeriesInstanceUID }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string InstanceSqlQuerySearchFormat = "SELECT VALUE {{ \"StudyInstanceUID\": c.StudyInstanceUID, \"SeriesInstanceUID\": c.SeriesInstanceUID, \"SOPInstanceUID\": f.SopInstanceUID }} FROM c JOIN f in c.Instances {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; private readonly DicomCosmosConfiguration _dicomConfiguration; private readonly IFormatProvider _stringFormatProvider; @@ -32,38 +34,79 @@ public CosmosQueryBuilder(DicomCosmosConfiguration dicomConfiguration) _stringFormatProvider = CultureInfo.InvariantCulture; } + public SqlQuerySpec BuildStudyQuerySpec( + int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query = null) + => BuildSeriesLevelQuerySpec(StudySqlQuerySearchFormat, offset, limit, query); + + public SqlQuerySpec BuildSeriesQuerySpec( + int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query = null) + => BuildSeriesLevelQuerySpec(SeriesSqlQuerySearchFormat, offset, limit, query); + public SqlQuerySpec BuildInstanceQuerySpec( - int offset, - int limit, - IEnumerable<(DicomTag Attribute, string Value)> query = null) + int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query = null) { // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. - var sqlParameterCollection = new SqlParameterCollection() - { - new SqlParameter { Name = OffsetParameterName, Value = offset }, - new SqlParameter { Name = LimitParameterName, Value = limit }, - }; + SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); - // If a null or empty query collection we should provide an empty string for the WHERE clause. - var whereClause = string.Empty; + string whereClause = GenerateWhereClause( + query, + (tag, parameter) => + { + sqlParameterCollection.Add(parameter); + return $"f.{nameof(QueryInstance.IndexedAttributes)}[\"{DicomTagSerializer.Serialize(tag)}\"] = {parameter.Name}"; + }); - if (query != null && query.Any()) - { - var parameterNameIndex = 1; - var whereItems = new List(); + return new SqlQuerySpec(string.Format(_stringFormatProvider, InstanceSqlQuerySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + } + + private SqlQuerySpec BuildSeriesLevelQuerySpec( + string querySearchFormat, int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query) + { + // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. + SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); - foreach ((DicomTag attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + string whereClause = GenerateWhereClause( + query, + (tag, parameter) => { - var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); - sqlParameterCollection.Add(new SqlParameter { Name = parameterName, Value = value }); - whereItems.Add($"f.{nameof(QueryInstance.IndexedAttributes)}[\"{DicomTagSerializer.Serialize(attribute)}\"] = {parameterName}"); - } + sqlParameterCollection.Add(parameter); + return $"ARRAY_CONTAINS(c.{nameof(QuerySeriesDocument.DistinctIndexedAttributes)}[\"{DicomTagSerializer.Serialize(tag)}\"].{nameof(AttributeValues.Values)}, {parameter.Name})"; + }); + + return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + } - // Now construct the WHERE query joining each item with an AND. - whereClause = $"WHERE {string.Join(" AND ", whereItems)}"; + private string GenerateWhereClause(IEnumerable<(DicomTag Attribute, string Value)> query, Func createQueryItem) + { + // If a null or empty query collection we should provide an empty string for the WHERE clause. + if (query == null || !query.Any()) + { + return string.Empty; } - return new SqlQuerySpec(string.Format(_stringFormatProvider, InstanceSqlQuerySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + var parameterNameIndex = 1; + var queryItems = new List(); + + foreach ((DicomTag attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + { + var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); + queryItems.Add(createQueryItem(attribute, new SqlParameter { Name = parameterName, Value = value })); + } + + // Now construct the WHERE query joining each item with an AND. + return $"WHERE {string.Join(" AND ", queryItems)}"; + } + + private static SqlParameterCollection CreateQueryParameterCollection(int offset, int limit) + { + EnsureArg.IsTrue(offset >= 0, nameof(offset)); + EnsureArg.IsTrue(limit > 0, nameof(limit)); + + return new SqlParameterCollection() + { + new SqlParameter { Name = OffsetParameterName, Value = offset }, + new SqlParameter { Name = LimitParameterName, Value = limit }, + }; } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index df0e357cbc..c6c444d389 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -62,8 +62,8 @@ public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToke { EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); - var dicomIdentity = DicomIdentity.Create(dicomDataset); - var defaultDocument = new QuerySeriesDocument(dicomIdentity.StudyInstanceUID, dicomIdentity.SeriesInstanceUID); + var dicomInstance = DicomInstance.Create(dicomDataset); + var defaultDocument = new QuerySeriesDocument(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID); RequestOptions requestOptions = CreateRequestOptions(defaultDocument.PartitionKey); @@ -91,33 +91,31 @@ await _documentClient.ThrowIndexDataStoreException( } /// - public async Task> QueryInstancesAsync( - int offset, - int limit, - string studyInstanceUID = null, - IEnumerable<(DicomTag Attribute, string Value)> query = null, - CancellationToken cancellationToken = default) + public async Task> QueryStudiesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default) { - EnsureArg.IsTrue(offset >= 0, nameof(offset)); - EnsureArg.IsTrue(limit > 0, nameof(limit)); - - // If the study instance UID is provided we can run the query against a specific partition. - FeedOptions feedOptions = string.IsNullOrWhiteSpace(studyInstanceUID) ? CreateCrossPartitionFeedOptions() : CreateFeedOptions(studyInstanceUID); - SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildInstanceQuerySpec(offset, limit, query); + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildStudyQuerySpec(offset, limit, query); + return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); + } - IDocumentQuery instanceQuery = _documentClient - .CreateDocumentQuery(_collectionUri, sqlQuerySpec, feedOptions) - .AsDocumentQuery(); + /// + public async Task> QuerySeriesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + { + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildSeriesQuerySpec(offset, limit, query); + return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); + } - return await _documentClient.ThrowIndexDataStoreException( - async (documentClient) => - { - return await instanceQuery.ExecuteQueryUntilCompleteAsync(cancellationToken); - }); + /// + public async Task> QueryInstancesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + { + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildInstanceQuerySpec(offset, limit, query); + return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); } /// - public async Task> DeleteSeriesIndexAsync( + public async Task> DeleteSeriesIndexAsync( string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); @@ -141,7 +139,7 @@ public async Task> DeleteSeriesIndexAsync( await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); return response.Document.Instances - .Select(x => new DicomIdentity(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)) + .Select(x => new DicomInstance(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)) .ToArray(); }, retryPolicy); @@ -186,7 +184,7 @@ await _documentClient.ThrowIndexDataStoreException( } /// - public async Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default) + public async Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default) { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); @@ -198,15 +196,15 @@ public async Task> GetInstancesInStudyAsync(string st return await _documentClient.ThrowIndexDataStoreException( async (documentClient) => { - var results = new List(); + var results = new List(); while (identityQuery.HasMoreResults) { - FeedResponse nextResults = await identityQuery.ExecuteNextAsync(cancellationToken); + FeedResponse nextResults = await identityQuery.ExecuteNextAsync(cancellationToken); results.AddRange(nextResults); } - if (results.Any()) + if (results.Count > 0) { return results; } @@ -217,7 +215,7 @@ public async Task> GetInstancesInStudyAsync(string st } /// - public async Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) + public async Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); @@ -234,7 +232,7 @@ public async Task> GetInstancesInSeriesAsync(string s cancellationToken: cancellationToken); return response.Document.Instances.Select( - x => new DicomIdentity(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)); + x => new DicomInstance(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)); }); } @@ -258,6 +256,21 @@ private FeedOptions CreateCrossPartitionFeedOptions() return feedOptions; } + private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpec, string studyInstanceUID, CancellationToken cancellationToken) + { + // If the study instance UID is provided we can run the query against a specific partition. + FeedOptions feedOptions = string.IsNullOrWhiteSpace(studyInstanceUID) ? CreateCrossPartitionFeedOptions() : CreateFeedOptions(studyInstanceUID); + IDocumentQuery documentQuery = _documentClient + .CreateDocumentQuery(_collectionUri, sqlQuerySpec, feedOptions) + .AsDocumentQuery(); + + return await _documentClient.ThrowIndexDataStoreException( + async (documentClient) => + { + return await documentQuery.ExecuteQueryUntilCompleteAsync(cancellationToken); + }); + } + private IAsyncPolicy CreatePreConditionFailedRetryPolicy() => Policy .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed || ex.StatusCode == HttpStatusCode.TooManyRequests) diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs new file mode 100644 index 0000000000..e317071938 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + public class AttributeValues + { + public AttributeValues() + { + } + + [JsonConstructor] + public AttributeValues(HashSet values) + { + EnsureArg.IsNotNull(values, nameof(values)); + + Values = values; + } + + /// + /// Gets the minimum date time value from the array of values. + /// We pre-calculate this to support range queries on DateTime values. + /// This value does not need to be serialized if the array does not contain any DateTime values. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public DateTime? MinDateTimeValue => Values.Where(x => x is DateTime).Min(x => (DateTime?)x); + + /// + /// Gets the maximum date time value from the array of values. + /// We pre-calculate this to support range queries on DateTime values. + /// This value does not need to be serialized if the array does not contain any DateTime values. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public DateTime? MaxDateTimeValue => Values.Where(x => x is DateTime).Max(x => (DateTime?)x); + + public HashSet Values { get; } = new HashSet(); + + public bool Add(object value) => value == null ? false : Values.Add(value); + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs index f98a602609..d86db0cd49 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -47,11 +47,11 @@ public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) public HashSet Instances { get; } = new HashSet(); - public Dictionary> DistinctIndexedAttributes + public Dictionary DistinctIndexedAttributes { get { - var result = new Dictionary>(); + var result = new Dictionary(); foreach (QueryInstance instance in Instances) { @@ -59,7 +59,7 @@ public Dictionary> DistinctIndexedAttributes { if (!result.ContainsKey(key)) { - result[key] = new HashSet(); + result[key] = new AttributeValues(); } result[key].Add(value); diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs index 82b8b0dca7..6299158aa5 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -35,17 +35,17 @@ public async Task GivenAValidInstance_WhenIndexing_CanBeRetrieved() await _indexDataStore.IndexInstanceAsync(testInstance); - IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); Assert.Single(instancesInStudy); Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInStudy.First().SopInstanceUID); - IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); + IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); Assert.Single(instancesInSeries); Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInSeries.First().SopInstanceUID); - IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); Assert.Single(deletedInstances); Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == deletedInstances.First().SopInstanceUID); @@ -61,15 +61,15 @@ public async Task GivenMultipleValidInstances_WhenIndexingInParallel_AreStored() IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstancesToIndex); IList sopInstanceUIDs = instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID)).ToList(); - IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); Assert.Equal(numberOfInstancesToIndex, instancesInStudy.Count()); instancesInStudy.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); - IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); + IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); Assert.Equal(numberOfInstancesToIndex, instancesInSeries.Count()); instancesInSeries.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); - IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); Assert.Equal(numberOfInstancesToIndex, instancesInSeries.Count()); deletedInstances.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); } @@ -87,10 +87,10 @@ public async Task GivenAValidSeries_WhenDeletingAnInstance_InstanceIsDeletedButS await _indexDataStore.DeleteInstanceIndexAsync(studyInstanceUID, seriesInstanceUID, firstSopInstanceUID); - IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); Assert.Equal(numberOfInstancesToIndex - 1, instancesInStudy.Count()); - IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); deletedInstances.Each(x => Assert.True(otherSopInstanceUIDs.Contains(x.SopInstanceUID))); } @@ -135,6 +135,61 @@ public async Task GivenNonExistentSeriesOrStudy_WhenFetchingInstances_IndexDataS Assert.Equal(HttpStatusCode.NotFound, getSeriesInstancesException.StatusCode); } + [Fact] + public async Task GivenAStoredInstance_WhenStoringAgain_ConflictExceptionThrown() + { + DicomDataset testInstance = CreateTestInstanceDicomDataset(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + await _indexDataStore.IndexInstanceAsync(testInstance); + IndexDataStoreException dataStoreException = await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + Assert.Equal(HttpStatusCode.Conflict, dataStoreException.StatusCode); + + var dicomInstance = DicomInstance.Create(testInstance); + await _indexDataStore.DeleteInstanceIndexAsync(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID, dicomInstance.SopInstanceUID); + } + + [Fact] + public async Task GivenIndexDataStoreQuery_WhenQueryingWithInvalidParameters_ArgumentExceptionThrown() + { + await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(0, 0)); + } + + [Fact] + public async Task GivenIndexDataStoreQuery_WhenQueryingOnNonIndexAttribute_IsIgnoredAndQueryReturns() + { + DicomTag unsupportedDicomTag = DicomTag.AbortFlag; + DicomTag supportedDicomTag = DicomTag.PatientName; + + var studyInstanceUID = Guid.NewGuid().ToString(); + var testInstance1PatientName = "Test1"; + DicomDataset testInstance1 = CreateTestInstanceDicomDataset(studyInstanceUID, Guid.NewGuid().ToString(), testInstance1PatientName); + DicomDataset testInstance2 = CreateTestInstanceDicomDataset(studyInstanceUID, Guid.NewGuid().ToString(), "Test2"); + + testInstance1.AddOrUpdate(unsupportedDicomTag, "DifferentValue"); + + await _indexDataStore.IndexInstanceAsync(testInstance1); + await _indexDataStore.IndexInstanceAsync(testInstance2); + + var dicomInstance = DicomInstance.Create(testInstance1); + (DicomTag, string)[] queryTags = new[] { (supportedDicomTag, testInstance1PatientName), (unsupportedDicomTag, "test") }; + + IEnumerable studyResults = await _indexDataStore.QueryStudiesAsync(0, 10, studyInstanceUID, query: queryTags); + Assert.Single(studyResults); + Assert.Equal(new DicomStudy(dicomInstance.StudyInstanceUID), studyResults.First()); + + IEnumerable seriesResults = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID, query: queryTags); + Assert.Single(seriesResults); + Assert.Equal(new DicomSeries(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID), seriesResults.First()); + + IEnumerable instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, query: queryTags); + Assert.Single(instanceResults); + Assert.Equal(dicomInstance, instanceResults.First()); + } + [Fact] public async Task GivenIndexedInstance_WhenQueryingByPatientName_InstancesIsRetrieved() { @@ -148,9 +203,9 @@ public async Task GivenIndexedInstance_WhenQueryingByPatientName_InstancesIsRetr await _indexDataStore.IndexInstanceAsync(testInstance); var sopInstanceUID = testInstance.GetSingleValue(DicomTag.SOPInstanceUID); - IEnumerable instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (DicomTag.ReferringPhysicianName, referringPhysicianName) }); + IEnumerable instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (DicomTag.ReferringPhysicianName, referringPhysicianName) }); Assert.Single(instances); - Assert.Equal(new DicomIdentity(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.First()); + Assert.Equal(new DicomInstance(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.First()); } [Fact] @@ -165,23 +220,23 @@ public async Task GivenIndexedSeries_WhenQueryingWithPaging_PagesReturnedCorrect var instanceUIDs = new HashSet(instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID))); // End of page - IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(numberOfInstances, numberOfInstances, studyInstanceUID); + IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(numberOfInstances, numberOfInstances, studyInstanceUID); Assert.Empty(queryInstances1); // First 5 items - IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, numberOfInstances / 2, studyInstanceUID); + IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, numberOfInstances / 2, studyInstanceUID); Assert.Equal(numberOfInstances / 2, queryInstances2.Count()); - foreach (DicomIdentity item in queryInstances2) + foreach (DicomInstance item in queryInstances2) { Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); } // Last 5 items - IEnumerable queryInstances3 = await _indexDataStore.QueryInstancesAsync(numberOfInstances / 2, numberOfInstances, studyInstanceUID); + IEnumerable queryInstances3 = await _indexDataStore.QueryInstancesAsync(numberOfInstances / 2, numberOfInstances, studyInstanceUID); Assert.Equal(numberOfInstances / 2, queryInstances3.Count()); - foreach (DicomIdentity item in queryInstances3) + foreach (DicomInstance item in queryInstances3) { Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); } @@ -189,7 +244,107 @@ public async Task GivenIndexedSeries_WhenQueryingWithPaging_PagesReturnedCorrect Assert.Empty(instanceUIDs); } - private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceUID, string seriesInstanceUID) + [Fact] + public async Task GivenMultipleStudies_WhenQueryingWithOrWithoutStudyInstanceUID_QueryResultsReturnedCorrectly() + { + var seriesPerStudy = 2; + var instancesPerSeries = 2; + var patientName = Guid.NewGuid().ToString(); + var studyInstanceUIDs = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + var seriesInstanceUIDs = new List(); + var totalItems = studyInstanceUIDs.Length * seriesPerStudy * instancesPerSeries; + + foreach (string studyInstanceUID in studyInstanceUIDs) + { + for (var i = 0; i < seriesPerStudy; i++) + { + var seriesInstanceUID = Guid.NewGuid().ToString(); + seriesInstanceUIDs.Add(seriesInstanceUID); + + for (int ii = 0; ii < instancesPerSeries; ii++) + { + await _indexDataStore.IndexInstanceAsync(CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID, patientName)); + } + } + } + + // Validate Study Searching + IEnumerable queryStudiesResults1 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1); + Assert.Equal(studyInstanceUIDs.Length, queryStudiesResults1.Count()); + queryStudiesResults1.Each(x => Assert.Contains(x.StudyInstanceUID, studyInstanceUIDs)); + + IEnumerable queryStudiesResults2 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, studyInstanceUIDs[0]); + Assert.Single(queryStudiesResults2); + + IEnumerable queryStudiesResults3 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); + Assert.Empty(queryStudiesResults3); + + // Validate Series Searching + IEnumerable querySeriesResults1 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1); + Assert.Equal(seriesInstanceUIDs.Count, querySeriesResults1.Count()); + querySeriesResults1.Each(x => Assert.Contains(x.SeriesInstanceUID, seriesInstanceUIDs)); + + IEnumerable querySeriesResults2 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, studyInstanceUIDs[0]); + Assert.Equal(seriesPerStudy, querySeriesResults2.Count()); + + IEnumerable querySeriesResults3 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); + Assert.Empty(querySeriesResults3); + } + + [Fact] + public async Task GivenInstanceWithInjectedSql_WhenQuerying_IsReturnedCorrectly() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + testInstance.Add(DicomTag.ReferringPhysicianName, " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\""); + + await _indexDataStore.IndexInstanceAsync(testInstance); + + IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.Single(queryInstances1); + Assert.Equal(DicomInstance.Create(testInstance), queryInstances1.First()); + + IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, new[] { (DicomTag.PatientName, " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\"") }); + Assert.Empty(queryInstances2); + } + + [Fact] + public async Task GivenWorstCaseStoreScenario_WhenAllInstancesHaveDifferentTagValues_CanIndexAndQueryCorrectly() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + var startDateTime = new DateTime(2019, 6, 21); + const int numberOfItemsToInsert = 100; + var patientNames = Enumerable.Range(0, numberOfItemsToInsert).Select(_ => Guid.NewGuid().ToString()).ToArray(); + + for (var i = 0; i < numberOfItemsToInsert; i++) + { + DicomDataset dataset = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID, patientNames[i]); + dataset.AddOrUpdate(DicomTag.StudyDate, startDateTime.AddDays(i)); + await _indexDataStore.IndexInstanceAsync(dataset); + } + + IEnumerable queryStudies1 = await _indexDataStore.QueryStudiesAsync(0, 10); + Assert.Single(queryStudies1); + Assert.Equal(studyInstanceUID, queryStudies1.First().StudyInstanceUID); + + IEnumerable queryStudies2 = await _indexDataStore.QueryStudiesAsync(0, 10, query: new[] { (DicomTag.PatientName, patientNames[numberOfItemsToInsert / 2]) }); + Assert.Single(queryStudies2); + Assert.Equal(studyInstanceUID, queryStudies2.First().StudyInstanceUID); + + IEnumerable querySeries1 = await _indexDataStore.QuerySeriesAsync(0, 10); + Assert.Single(querySeries1); + Assert.Equal(seriesInstanceUID, querySeries1.First().SeriesInstanceUID); + + IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, numberOfItemsToInsert + 1); + Assert.Equal(numberOfItemsToInsert, queryInstances1.Count()); + + await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + } + + private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceUID, string seriesInstanceUID, string patientName = "Patient Test") { EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID); EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID); @@ -199,7 +354,8 @@ private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceU { DicomTag.StudyInstanceUID, studyInstanceUID }, { DicomTag.SeriesInstanceUID, seriesInstanceUID }, { DicomTag.SOPInstanceUID, Guid.NewGuid().ToString() }, - { DicomTag.PatientName, Guid.NewGuid().ToString() }, + { DicomTag.PatientName, patientName }, + { DicomTag.StudyDate, new DateTime(2019, 6, 21) }, }; return result; } From 5f333c1ca55ed8adedf33b42e386fd82779afaae Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Fri, 21 Jun 2019 15:23:30 +0100 Subject: [PATCH 12/22] Fixed tests when running in parallel --- .../Persistence/DicomIndexDataStoreTests.cs | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs index 6299158aa5..7d8061d282 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -143,8 +143,7 @@ public async Task GivenAStoredInstance_WhenStoringAgain_ConflictExceptionThrown( IndexDataStoreException dataStoreException = await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); Assert.Equal(HttpStatusCode.Conflict, dataStoreException.StatusCode); - var dicomInstance = DicomInstance.Create(testInstance); - await _indexDataStore.DeleteInstanceIndexAsync(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID, dicomInstance.SopInstanceUID); + await DeleteInstancesAsync(testInstance); } [Fact] @@ -188,6 +187,8 @@ public async Task GivenIndexDataStoreQuery_WhenQueryingOnNonIndexAttribute_IsIgn IEnumerable instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, query: queryTags); Assert.Single(instanceResults); Assert.Equal(dicomInstance, instanceResults.First()); + + await DeleteInstancesAsync(testInstance1, testInstance2); } [Fact] @@ -206,6 +207,8 @@ public async Task GivenIndexedInstance_WhenQueryingByPatientName_InstancesIsRetr IEnumerable instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (DicomTag.ReferringPhysicianName, referringPhysicianName) }); Assert.Single(instances); Assert.Equal(new DicomInstance(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.First()); + + await DeleteInstancesAsync(testInstance); } [Fact] @@ -242,6 +245,8 @@ public async Task GivenIndexedSeries_WhenQueryingWithPaging_PagesReturnedCorrect } Assert.Empty(instanceUIDs); + + await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); } [Fact] @@ -251,19 +256,19 @@ public async Task GivenMultipleStudies_WhenQueryingWithOrWithoutStudyInstanceUID var instancesPerSeries = 2; var patientName = Guid.NewGuid().ToString(); var studyInstanceUIDs = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - var seriesInstanceUIDs = new List(); + var dicomSeries = new List(); var totalItems = studyInstanceUIDs.Length * seriesPerStudy * instancesPerSeries; foreach (string studyInstanceUID in studyInstanceUIDs) { for (var i = 0; i < seriesPerStudy; i++) { - var seriesInstanceUID = Guid.NewGuid().ToString(); - seriesInstanceUIDs.Add(seriesInstanceUID); + var series = new DicomSeries(studyInstanceUID, Guid.NewGuid().ToString()); + dicomSeries.Add(series); for (int ii = 0; ii < instancesPerSeries; ii++) { - await _indexDataStore.IndexInstanceAsync(CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID, patientName)); + await _indexDataStore.IndexInstanceAsync(CreateTestInstanceDicomDataset(studyInstanceUID, series.SeriesInstanceUID, patientName)); } } } @@ -281,14 +286,19 @@ public async Task GivenMultipleStudies_WhenQueryingWithOrWithoutStudyInstanceUID // Validate Series Searching IEnumerable querySeriesResults1 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1); - Assert.Equal(seriesInstanceUIDs.Count, querySeriesResults1.Count()); - querySeriesResults1.Each(x => Assert.Contains(x.SeriesInstanceUID, seriesInstanceUIDs)); + Assert.Equal(dicomSeries.Count, querySeriesResults1.Count()); + querySeriesResults1.Each(x => Assert.Contains(x, dicomSeries)); IEnumerable querySeriesResults2 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, studyInstanceUIDs[0]); Assert.Equal(seriesPerStudy, querySeriesResults2.Count()); IEnumerable querySeriesResults3 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); Assert.Empty(querySeriesResults3); + + foreach (DicomSeries series in dicomSeries) + { + await _indexDataStore.DeleteSeriesIndexAsync(series.StudyInstanceUID, series.SeriesInstanceUID); + } } [Fact] @@ -308,6 +318,8 @@ public async Task GivenInstanceWithInjectedSql_WhenQuerying_IsReturnedCorrectly( IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, new[] { (DicomTag.PatientName, " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\"") }); Assert.Empty(queryInstances2); + + await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); } [Fact] @@ -368,5 +380,14 @@ private async Task> CreateSeriesInParallelAsync(string study await Task.WhenAll(instances.Select(x => _indexDataStore.IndexInstanceAsync(x))); return instances; } + + private async Task DeleteInstancesAsync(params DicomDataset[] datasets) + { + foreach (DicomDataset instance in datasets) + { + var dicomInstance = DicomInstance.Create(instance); + await _indexDataStore.DeleteInstanceIndexAsync(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID, dicomInstance.SopInstanceUID); + } + } } } From 3d9423e5ab2ccc656f0eb19442a82b96830686e7 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 25 Jun 2019 13:36:52 +0100 Subject: [PATCH 13/22] Refactored PR based on comments and added Dicom Attribute for being able to store sequence tags. Updated code to also index fields with a multiplicity greater than 1 --- .../Persistence/DicomAttributeIdExtensions.cs | 63 ++++++ .../Persistence/DicomAttributeIdTests.cs | 63 ++++++ .../Exceptions/DataStoreExceptionTests.cs | 46 +++++ .../Features/Persistence/DicomAttributeId.cs | 152 ++++++++++++++ .../Persistence/DicomAttributeIdExtensions.cs | 54 +++++ .../Exceptions/DataStoreException.cs | 67 ++++++ .../Exceptions/IndexDataStoreException.cs | 47 ----- .../Persistence/IDicomIndexDataStore.cs | 16 +- .../Features/Persistence/QueryResult.cs | 22 ++ .../Storage/DicomCosmosDataStoreTests.cs | 8 - .../Documents/DicomTagSerializerTests.cs | 54 ----- .../Storage/Documents/QueryInstanceTests.cs | 38 ++-- .../Documents/QuerySeriesDocumentTests.cs | 14 +- .../Config/DicomCosmosConfiguration.cs | 15 +- .../Features/Storage/CosmosQueryBuilder.cs | 22 +- .../Features/Storage/DicomCosmosDataStore.cs | 82 ++------ .../Features/Storage/DicomTagSerializer.cs | 54 ----- .../Storage/DocumentClientExtensions.cs | 8 +- .../Storage/Documents/QueryInstance.cs | 25 +-- .../Storage/Documents/QuerySeriesDocument.cs | 9 +- .../Persistence/DicomIndexDataStoreTests.cs | 194 ++++++++++-------- 21 files changed, 669 insertions(+), 384 deletions(-) create mode 100644 src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs create mode 100644 src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs create mode 100644 src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs delete mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs delete mode 100644 src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs delete mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs new file mode 100644 index 0000000000..30f539dccc --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs @@ -0,0 +1,63 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence +{ + public class DicomAttributeIdExtensions + { + [Fact] + public void GivenDatasetAndInvalidParameters_WhenFetchedAttributeValues_ArgumentExceptionIsThrown() + { + var dicomDataset = new DicomDataset(); + Assert.Throws(() => dicomDataset.TryGetValues((DicomAttributeId)null, out object[] values)); + Assert.Throws(() => dicomDataset.TryGetValues(new DicomAttributeId(DicomTag.StudyDate), out object[] values, -1)); + } + + [Fact] + public void GivenDatasetWithSequence_WhenFetchingAttributeValues_IsReturnedCorrectly() + { + var sequence1 = new DicomSequence( + DicomTag.ReferencedPatientSequence, + new DicomDataset() { { DicomTag.PatientName, "Patient1" } }, + new DicomDataset() { { DicomTag.PatientName, "Patient2" } }, + new DicomDataset() { { DicomTag.PatientName, "Patient3" } }); + + var fetchedValues1 = new DicomDataset() { sequence1 }.TryGetValues( + new DicomAttributeId(DicomTag.ReferencedPatientSequence, DicomTag.PatientName), + out object[] values1); + Assert.True(fetchedValues1); + Assert.Equal(3, values1.Length); + Assert.Contains("Patient1", values1); + Assert.Contains("Patient2", values1); + Assert.Contains("Patient3", values1); + + var sequence2 = new DicomSequence( + DicomTag.ReferencedStudySequence, + new DicomDataset() { { DicomTag.StudyDate, new DateTime(2019, 6, 25) } }, + new DicomDataset() { { new DicomSequence(DicomTag.ReferencedStudySequence, new DicomDataset() { { DicomTag.StudyDate, new DateTime(2019, 6, 24) } }) } }, + new DicomDataset() { { DicomTag.StudyDate, new DateTime(2019, 6, 23) } }); + + var fetchedValues2 = new DicomDataset() { sequence2 }.TryGetValues( + new DicomAttributeId(DicomTag.ReferencedStudySequence, DicomTag.StudyDate), + out object[] values2); + Assert.True(fetchedValues2); + Assert.Equal(2, values2.Length); + Assert.Contains(new DateTime(2019, 6, 25), values2); + Assert.Contains(new DateTime(2019, 6, 23), values2); + + var fetchedValues3 = new DicomDataset() { sequence2 }.TryGetValues( + new DicomAttributeId(DicomTag.ReferencedStudySequence, DicomTag.ReferencedStudySequence, DicomTag.StudyDate), + out object[] values3); + Assert.True(fetchedValues3); + Assert.Single(values3); + Assert.Contains(new DateTime(2019, 6, 24), values3); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs new file mode 100644 index 0000000000..6f0fc5f957 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs @@ -0,0 +1,63 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- +using System; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence +{ + public class DicomAttributeIdTests + { + [Fact] + public void GivenDicomAttributeId_WhenConstructingWithInvalidParameters_ArgumentExceptionThrown() + { + Assert.Throws(() => new DicomAttributeId((DicomTag[])null)); + Assert.Throws(() => new DicomAttributeId(Array.Empty())); + Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence, DicomTag.ROIContourSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate, DicomTag.RightLensSequence)); + + Assert.Throws(() => new DicomAttributeId((string)null)); + Assert.Throws(() => new DicomAttributeId(string.Empty)); + Assert.Throws(() => new DicomAttributeId("INVALID")); + Assert.Throws(() => new DicomAttributeId("INVALID.INVALID")); + } + + [Fact] + public void GivenValidDicomAttributeId_WhenSerialized_IsDeserializedCorrectly() + { + SerializeAndDeserialize(new DicomAttributeId(DicomTag.StudyDate)); + SerializeAndDeserialize(new DicomAttributeId(DicomTag.RelatedSeriesSequence, DicomTag.RegistrationSequence, DicomTag.RegisteredLocalizerUnits)); + SerializeAndDeserialize(new DicomAttributeId(DicomTag.RegionPixelShiftSequence, DicomTag.NumberOfVerticalPixels)); + SerializeAndDeserialize(new DicomAttributeId("0020000D")); + SerializeAndDeserialize(new DicomAttributeId("00101002.00100020")); + SerializeAndDeserialize(new DicomAttributeId("00101002.00100024.00400032")); + SerializeAndDeserialize(new DicomAttributeId("StudyInstanceUID")); + SerializeAndDeserialize(new DicomAttributeId("OtherPatientIDsSequence.PatientID")); + SerializeAndDeserialize(new DicomAttributeId("OtherPatientIDsSequence.IssuerOfPatientIDQualifiersSequence.UniversalEntityID")); + } + + [Fact] + public void GivenValidDicomAttributeId_WhenCompared_IsComparedCorrectly() + { + Assert.Equal(new DicomAttributeId(DicomTag.StudyInstanceUID), new DicomAttributeId("StudyInstanceUID")); + Assert.Equal(new DicomAttributeId(DicomTag.OtherPatientIDsSequence, DicomTag.PatientID), new DicomAttributeId("OtherPatientIDsSequence.PatientID")); + + Assert.Equal( + new DicomAttributeId(DicomTag.OtherPatientIDsSequence, DicomTag.IssuerOfPatientIDQualifiersSequence, DicomTag.UniversalEntityID), + new DicomAttributeId("OtherPatientIDsSequence.IssuerOfPatientIDQualifiersSequence.UniversalEntityID")); + } + + private void SerializeAndDeserialize(DicomAttributeId dicomAttributeId) + { + var json = JsonConvert.SerializeObject(dicomAttributeId); + var deserialized = JsonConvert.DeserializeObject(json); + + Assert.Equal(dicomAttributeId, deserialized); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs new file mode 100644 index 0000000000..6b7bee1038 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using System.Net; +using System.Runtime.Serialization.Formatters.Binary; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence.Exceptions +{ + public class DataStoreExceptionTests + { + [Fact] + public void GivenDataStoreException_WhenNoStatusCodeProvided_DefaultsToInternalServerError() + { + var innerException = new Exception("foo"); + Assert.Equal((int)HttpStatusCode.InternalServerError, new DataStoreException((HttpStatusCode?)null, innerException).StatusCode); + Assert.Equal((int)HttpStatusCode.InternalServerError, new DataStoreException((int?)null, innerException).StatusCode); + } + + [Fact] + public void GivenDataStoreException_WhenSerialized_IsDeserializedCorrectly() + { + var innerException = new Exception("foo"); + var exception = new DataStoreException(HttpStatusCode.NotFound, innerException); + + var buffer = new byte[4096]; + using (var memoryStream1 = new MemoryStream(buffer)) + using (var memoryStream2 = new MemoryStream(buffer)) + { + var formatter = new BinaryFormatter(); + + formatter.Serialize(memoryStream1, exception); + var deserializedException = (DataStoreException)formatter.Deserialize(memoryStream2); + + Assert.Equal(exception.StatusCode, deserializedException.StatusCode); + Assert.Equal(exception.InnerException.Message, deserializedException.InnerException.Message); + Assert.Equal(exception.Message, deserializedException.Message); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs new file mode 100644 index 0000000000..70f6b5ea56 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs @@ -0,0 +1,152 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; +using Dicom; +using EnsureThat; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + /// + /// Implements the DICOM attribute parsing and serialization as defined here: http://dicom.nema.org/Dicom/2013/output/chtml/part18/sect_6.7.html. + /// The order of tags are important in the class, all items but the last must be sequence elements. + /// This class supports serialization/ deserialization from keywords or Hex Group/ Element. + /// Examples of valid values for {attributeID}: + /// 0020000D + /// StudyInstanceUID + /// 00101002.00100020 + /// OtherPatientIDsSequence.PatientID + /// 00101002.00100024.00400032 + /// OtherPatientIDsSequence.IssuerOfPatientIDQualifiersSequence.UniversalEntityID + /// + public class DicomAttributeId + { + private const char Seperator = '.'; + private const string GroupElementStringFormat = "X4"; + private const NumberStyles GroupElementNumberStyles = NumberStyles.HexNumber; + private static readonly IFormatProvider FormatProvider = CultureInfo.InvariantCulture; + private readonly bool _writeTagsAsKeywords = false; + private readonly DicomTag[] _dicomTags; + private static readonly IDictionary KnownKeywordDicomTags = + typeof(DicomTag).GetFields(BindingFlags.Static | BindingFlags.Public) + .Select(x => x.GetValue(null) as DicomTag) + .Where(x => !string.IsNullOrWhiteSpace(x?.DictionaryEntry?.Keyword)) + .Distinct(new KeywordComparer()) + .ToDictionary(x => x.DictionaryEntry.Keyword, x => x); + + [JsonConstructor] + public DicomAttributeId(string attributeId) + : this(DeserializeAttributeId(attributeId)) + { + } + + public DicomAttributeId(params DicomTag[] dicomTags) + { + EnsureArg.IsNotNull(dicomTags, nameof(dicomTags)); + EnsureArg.IsTrue(dicomTags.Length > 0, nameof(dicomTags)); + + _dicomTags = dicomTags; + Validate(); + + AttributeId = string.Join(Seperator, dicomTags.Select(x => ConvertToString(x, _writeTagsAsKeywords))); + } + + public string AttributeId { get; } + + [JsonIgnore] + public int Length => _dicomTags.Length; + + [JsonIgnore] + public DicomTag LeafDicomTag => _dicomTags[Length - 1]; + + public DicomTag GetDicomTag(int index = -1) + { + EnsureArg.IsLt(index, _dicomTags.Length, nameof(index)); + return index < 0 ? _dicomTags[Length - 1] : _dicomTags[index]; + } + + public override int GetHashCode() + { + return AttributeId.GetHashCode(StringComparison.Ordinal); + } + + public override bool Equals(object obj) + { + if (obj is DicomAttributeId instance) + { + return AttributeId.Equals(instance.AttributeId, StringComparison.Ordinal); + } + + return false; + } + + public static DicomTag[] DeserializeAttributeId(string attributeId) + { + EnsureArg.IsNotNullOrWhiteSpace(attributeId, nameof(attributeId)); + + string[] split = attributeId.Split(Seperator); + var result = new DicomTag[split.Length]; + + for (int i = 0; i < split.Length; i++) + { + // First attempt Hex decimal parse of the string if correct length. + if (split[i].Length == 8 && + ushort.TryParse(split[i].Substring(0, 4), GroupElementNumberStyles, FormatProvider, out ushort group) && + ushort.TryParse(split[i].Substring(4, 4), GroupElementNumberStyles, FormatProvider, out ushort element)) + { + result[i] = new DicomTag(group, element); + continue; + } + + // Otherwise, attempt to look-up the DICOM tag keyword. + EnsureArg.IsTrue(KnownKeywordDicomTags.ContainsKey(split[i]), split[i]); + result[i] = KnownKeywordDicomTags[split[i]]; + } + + return result; + } + + private static string ConvertToString(DicomTag dicomTag, bool writeTagsAsKeywords) + { + if (writeTagsAsKeywords) + { + return dicomTag.DictionaryEntry.Keyword; + } + + string groupString = dicomTag.Group.ToString(GroupElementStringFormat, FormatProvider); + string elementString = dicomTag.Element.ToString(GroupElementStringFormat, FormatProvider); + return groupString + elementString; + } + + private void Validate() + { + // Validate all but the leaf tag are sequence elements + for (var i = 0; i < Length - 1; i++) + { + EnsureArg.IsTrue(_dicomTags[i].DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); + } + + EnsureArg.IsFalse(LeafDicomTag.DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); + } + + private class KeywordComparer : IEqualityComparer + { + public bool Equals(DicomTag dicomTag1, DicomTag dicomTag2) + { + return dicomTag1.DictionaryEntry.Keyword == dicomTag2.DictionaryEntry.Keyword; + } + + public int GetHashCode(DicomTag dicomTag) + { + return dicomTag.DictionaryEntry.Keyword.GetHashCode(StringComparison.InvariantCulture); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs new file mode 100644 index 0000000000..e085a12f0f --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Dicom; +using EnsureThat; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public static class DicomAttributeIdExtensions + { + public static bool TryGetValues(this DicomDataset dicomDataset, DicomAttributeId attributeId, out TItem[] values, int startAttributeIndex = 0) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + EnsureArg.IsNotNull(attributeId, nameof(attributeId)); + EnsureArg.IsGte(startAttributeIndex, 0, nameof(startAttributeIndex)); + + var nestedValues = new List(); + + // Handle for now sequence elements (last item in attribute ID). + if (startAttributeIndex == attributeId.Length - 1) + { + // Now first validate this DICOM tag exists in the dataset + if (dicomDataset.TryGetValue(attributeId.LeafDicomTag, 0, out TItem firstItem)) + { + nestedValues.Add(firstItem); + + for (int i = 1; i < dicomDataset.GetValueCount(attributeId.LeafDicomTag); i++) + { + if (dicomDataset.TryGetValue(attributeId.LeafDicomTag, i, out TItem value)) + { + nestedValues.Add(value); + } + } + } + } + else if (dicomDataset.TryGetSequence(attributeId.GetDicomTag(startAttributeIndex), out DicomSequence dicomSequence)) + { + foreach (DicomDataset sequenceDataset in dicomSequence.Items) + { + if (sequenceDataset.TryGetValues(attributeId, out TItem[] sequenceValues, startAttributeIndex + 1)) + { + nestedValues.AddRange(sequenceValues); + } + } + } + + values = nestedValues.Count > 0 ? nestedValues.ToArray() : null; + return nestedValues.Count > 0; + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs new file mode 100644 index 0000000000..cc37119ec0 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs @@ -0,0 +1,67 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Net; +using System.Runtime.Serialization; +using System.Security.Permissions; +using EnsureThat; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions +{ + [Serializable] + public class DataStoreException : Exception + { + public DataStoreException(HttpStatusCode httpStatusCode) + { + StatusCode = (int)httpStatusCode; + } + + public DataStoreException(HttpStatusCode? httpStatusCode, Exception innerException) + : base(innerException.Message, innerException) + { + StatusCode = httpStatusCode.HasValue ? (int)httpStatusCode.Value : (int)HttpStatusCode.InternalServerError; + } + + public DataStoreException(int? statusCode, Exception innerException) + : base(innerException.Message, innerException) + { + StatusCode = statusCode ?? (int)HttpStatusCode.InternalServerError; + } + + public DataStoreException() + { + } + + public DataStoreException(string message) + : base(message) + { + } + + public DataStoreException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected DataStoreException(SerializationInfo serializationInfo, StreamingContext streamingContext) + : base(serializationInfo, streamingContext) + { + EnsureArg.IsNotNull(serializationInfo, nameof(serializationInfo)); + + StatusCode = serializationInfo.GetInt32(nameof(StatusCode)); + } + + public int StatusCode { get; } + + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + EnsureArg.IsNotNull(serializationInfo, nameof(serializationInfo)); + + serializationInfo.AddValue(nameof(StatusCode), StatusCode); + base.GetObjectData(serializationInfo, context); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs deleted file mode 100644 index 89e8ed94ce..0000000000 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/IndexDataStoreException.cs +++ /dev/null @@ -1,47 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Net; -using System.Runtime.Serialization; - -namespace Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions -{ - [Serializable] - public class IndexDataStoreException : Exception - { - public IndexDataStoreException(HttpStatusCode httpStatusCode) - { - StatusCode = httpStatusCode; - } - - public IndexDataStoreException(HttpStatusCode? httpStatusCode, Exception innerException) - : base(innerException.Message, innerException) - { - StatusCode = httpStatusCode ?? HttpStatusCode.InternalServerError; - } - - public IndexDataStoreException() - { - } - - public IndexDataStoreException(string message) - : base(message) - { - } - - public IndexDataStoreException(string message, Exception innerException) - : base(message, innerException) - { - } - - protected IndexDataStoreException(SerializationInfo serializationInfo, StreamingContext streamingContext) - { - throw new NotImplementedException(); - } - - public HttpStatusCode StatusCode { get; } - } -} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs index 375b1df708..a7969eb49e 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs @@ -14,31 +14,27 @@ public interface IDicomIndexDataStore { Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default); - Task> QueryStudiesAsync( + Task> QueryStudiesAsync( int offset, int limit, string studyInstanceUID = null, - IEnumerable<(DicomTag Attribute, string Value)> query = null, + IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default); - Task> QuerySeriesAsync( + Task> QuerySeriesAsync( int offset, int limit, string studyInstanceUID = null, - IEnumerable<(DicomTag Attribute, string Value)> query = null, + IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default); - Task> QueryInstancesAsync( + Task> QueryInstancesAsync( int offset, int limit, string studyInstanceUID = null, - IEnumerable<(DicomTag Attribute, string Value)> query = null, + IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default); - Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default); - - Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); - /// /// Delete the provided series index. /// diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs new file mode 100644 index 0000000000..4218822cb0 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class QueryResult + { + public QueryResult(bool hasMoreResults, IEnumerable results) + { + HasMoreResults = hasMoreResults; + Results = results; + } + + public bool HasMoreResults { get; } + + public IEnumerable Results { get; } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs index d2ba1842f7..9bcba3f459 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs @@ -48,14 +48,6 @@ public async Task GivenInvalidParameters_WhenCallingAllMethods_ArgumentException await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), string.Empty, Guid.NewGuid().ToString())); await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), null)); await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), string.Empty)); - - await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInStudyAsync(null)); - await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInStudyAsync(string.Empty)); - - await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(null, Guid.NewGuid().ToString())); - await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(string.Empty, Guid.NewGuid().ToString())); - await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(Guid.NewGuid().ToString(), null)); - await Assert.ThrowsAsync(() => _indexDataStore.GetInstancesInSeriesAsync(Guid.NewGuid().ToString(), string.Empty)); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs deleted file mode 100644 index 00a2d35db9..0000000000 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/DicomTagSerializerTests.cs +++ /dev/null @@ -1,54 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using Dicom; -using Microsoft.Health.Dicom.CosmosDb.Features.Storage; -using Xunit; - -namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents -{ - public class DicomTagSerializerTests - { - [Fact] - public void GivenPrivateDicomTag_WhenSerialized_IsDeserializedCorrectly() - { - var dicomDictionary = new DicomDictionary - { - new DicomDictionaryEntry(new DicomTag(0x0011, 0x0010), "TEST CREATOR", "TEST", DicomVM.VM_1, false, DicomVR.LO), - }; - - SerializeAndDeserialize(new DicomTag(0007, 0010)); - SerializeAndDeserialize(new DicomTag(0011, 0010, dicomDictionary.GetPrivateCreator("TEST"))); - } - - [Fact] - public void GivenValidDicomTag_WhenSerialized_IsDeserializedCorrectly() - { - SerializeAndDeserialize(DicomTag.PatientName); - SerializeAndDeserialize(DicomTag.SOPInstanceStatus); - SerializeAndDeserialize(DicomTag.SeriesDate); - SerializeAndDeserialize(DicomTag.SeriesTime); - SerializeAndDeserialize(DicomTag.AbortReason); - SerializeAndDeserialize(DicomTag.WindowCenter); - } - - [Fact] - public void GivenInvalidDicomTagSerializerParameters_OnSerializeOrDeserialize_ArgumentExceptionIsThrown() - { - Assert.Throws(() => DicomTagSerializer.Serialize(null)); - Assert.Throws(() => DicomTagSerializer.Deserialize(null)); - Assert.Throws(() => DicomTagSerializer.Deserialize(string.Empty)); - } - - private static void SerializeAndDeserialize(DicomTag dicomTag) - { - var serialized = DicomTagSerializer.Serialize(dicomTag); - DicomTag deserialized = DicomTagSerializer.Deserialize(serialized); - - Assert.Equal(dicomTag, deserialized); - } - } -} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs index 6d708db80d..db280dfbc2 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs @@ -6,7 +6,7 @@ using System; using System.Collections.Generic; using Dicom; -using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; using Xunit; @@ -17,23 +17,27 @@ public class QueryInstanceTests [Fact] public void GivenInvalidParameters_WhenCreatingQueryInstance_ExceptionsThrown() { - Assert.Throws(() => new QueryInstance(null, new Dictionary())); - Assert.Throws(() => new QueryInstance(string.Empty, new Dictionary())); - Assert.Throws(() => new QueryInstance(new string('a', 65), new Dictionary())); - Assert.Throws(() => new QueryInstance("AA#AA", new Dictionary())); + Assert.Throws(() => new QueryInstance(null, new Dictionary())); + Assert.Throws(() => new QueryInstance(string.Empty, new Dictionary())); + Assert.Throws(() => new QueryInstance(new string('a', 65), new Dictionary())); + Assert.Throws(() => new QueryInstance("AA#AA", new Dictionary())); Assert.Throws(() => new QueryInstance(Guid.NewGuid().ToString(), null)); Assert.Throws(() => QueryInstance.Create(null, null)); - Assert.Throws(() => QueryInstance.Create(new DicomDataset(), null)); + Assert.Throws(() => QueryInstance.Create(new DicomDataset(), null)); + Assert.Throws(() => QueryInstance.Create(new DicomDataset(), Array.Empty())); } [Fact] public void GivenDicomTagsToIndex_WhenCreatingQueryInstance_TagsAreExtractedAndIndexed() { - var sequenceDataset = new DicomDataset() - { - { DicomTag.ReferringPhysicianName, "TestPhysician" }, - }; + var referringPhysicianName = "TestPhysician"; + var dicomSequence = new DicomSequence( + DicomTag.ReferringPhysicianIdentificationSequence, + new DicomDataset() + { + { DicomTag.ReferringPhysicianName, referringPhysicianName }, + }); var dicomDataset = new DicomDataset { @@ -43,18 +47,22 @@ public void GivenDicomTagsToIndex_WhenCreatingQueryInstance_TagsAreExtractedAndI { DicomTag.PatientName, Guid.NewGuid().ToString() }, { DicomTag.StudyDate, DateTime.UtcNow }, }; - dicomDataset.Add(sequenceDataset); + dicomDataset.Add(dicomSequence); + var patientNameAttribute = new DicomAttributeId(DicomTag.PatientName); + var studyDateAttribute = new DicomAttributeId(DicomTag.StudyDate); + var referringPhysicianNameAttribute = new DicomAttributeId(DicomTag.ReferringPhysicianIdentificationSequence, DicomTag.ReferringPhysicianName); + var studyTimeAttribute = new DicomAttributeId(DicomTag.StudyTime); var instance = QueryInstance.Create( - dicomDataset, new[] { DicomTag.PatientName, DicomTag.StudyDate, DicomTag.ReferringPhysicianName, DicomTag.StudyTime }); + dicomDataset, new[] { patientNameAttribute, studyDateAttribute, referringPhysicianNameAttribute, studyTimeAttribute }); Assert.NotNull(instance); Assert.Equal(dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID), instance.SopInstanceUID); Assert.Equal(3, instance.IndexedAttributes.Count); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)]); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.StudyDate)]); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.ReferringPhysicianName), instance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.ReferringPhysicianName)]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.IndexedAttributes[patientNameAttribute.AttributeId][0]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.IndexedAttributes[studyDateAttribute.AttributeId][0]); + Assert.Equal(referringPhysicianName, instance.IndexedAttributes[referringPhysicianNameAttribute.AttributeId][0]); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs index e2582fa8fc..62730c92fc 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -6,7 +6,7 @@ using System; using System.Linq; using Dicom; -using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; using Newtonsoft.Json; using Xunit; @@ -49,14 +49,15 @@ public void GivenExistingInstance_WhenAddingToInstancesHashSet_IsNotAdded() dataset.Add(DicomTag.SOPInstanceUID, sopInstanceUID); dataset.Add(DicomTag.PatientName, testPatientName); - var instanceDocument1 = QueryInstance.Create(dataset, new[] { DicomTag.PatientName }); - var instanceDocument2 = QueryInstance.Create(dataset, new[] { DicomTag.PatientName }); + var patientNameAttributeId = new DicomAttributeId(DicomTag.PatientName); + var instanceDocument1 = QueryInstance.Create(dataset, new[] { patientNameAttributeId }); + var instanceDocument2 = QueryInstance.Create(dataset, new[] { patientNameAttributeId }); Assert.Throws(() => document.AddInstance(null)); Assert.True(document.AddInstance(instanceDocument1)); Assert.False(document.AddInstance(instanceDocument2)); - Assert.Equal(testPatientName, document.DistinctIndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)].Values.First()); + Assert.Equal(testPatientName, document.DistinctIndexedAttributes[patientNameAttributeId.AttributeId].Values.First()); Assert.Throws(() => document.RemoveInstance(null)); Assert.Throws(() => document.RemoveInstance(string.Empty)); @@ -78,7 +79,8 @@ public void GivenSeriesDocument_WhenSerialized_IsDeserializedCorrectly() dataset.Add(DicomTag.SOPInstanceUID, sopInstanceUID); dataset.Add(DicomTag.PatientName, testPatientName); - var instanceDocument = QueryInstance.Create(dataset, new[] { DicomTag.PatientName }); + var patientNameAttributeId = new DicomAttributeId(DicomTag.PatientName); + var instanceDocument = QueryInstance.Create(dataset, new[] { patientNameAttributeId }); document.AddInstance(instanceDocument); var serialized = JsonConvert.SerializeObject(document); @@ -94,7 +96,7 @@ public void GivenSeriesDocument_WhenSerialized_IsDeserializedCorrectly() QueryInstance deserializedFirstInstance = deserialized.Instances.First(); Assert.Equal(deserializedFirstInstance.SopInstanceUID, deserializedFirstInstance.SopInstanceUID); Assert.Equal(deserializedFirstInstance.IndexedAttributes.Count, deserializedFirstInstance.IndexedAttributes.Count); - Assert.Equal(deserializedFirstInstance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)], deserializedFirstInstance.IndexedAttributes[DicomTagSerializer.Serialize(DicomTag.PatientName)]); + Assert.Equal(deserializedFirstInstance.IndexedAttributes[patientNameAttributeId.AttributeId], deserializedFirstInstance.IndexedAttributes[patientNameAttributeId.AttributeId]); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs index 6f00ae4f67..c7e840f2ba 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; namespace Microsoft.Health.Dicom.CosmosDb.Config { @@ -14,14 +15,14 @@ public class DicomCosmosConfiguration /// Gets the DICOM tags that should be indexed and made queryable. /// The StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID will be indexed automatically. /// - public IEnumerable QueryAttributes { get; } = new[] + public IEnumerable QueryAttributes { get; } = new[] { - DicomTag.PatientName, - DicomTag.PatientID, - DicomTag.AccessionNumber, - DicomTag.Modality, - DicomTag.ReferringPhysicianName, - DicomTag.StudyDate, + new DicomAttributeId(DicomTag.PatientName), + new DicomAttributeId(DicomTag.PatientID), + new DicomAttributeId(DicomTag.AccessionNumber), + new DicomAttributeId(DicomTag.Modality), + new DicomAttributeId(DicomTag.ReferringPhysicianName), + new DicomAttributeId(DicomTag.StudyDate), }; } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs index 31cb462718..abe0c0c06c 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -7,9 +7,9 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using Dicom; using EnsureThat; using Microsoft.Azure.Documents; +using Microsoft.Health.Dicom.Core.Features.Persistence; using Microsoft.Health.Dicom.CosmosDb.Config; using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; @@ -35,15 +35,15 @@ public CosmosQueryBuilder(DicomCosmosConfiguration dicomConfiguration) } public SqlQuerySpec BuildStudyQuerySpec( - int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query = null) + int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) => BuildSeriesLevelQuerySpec(StudySqlQuerySearchFormat, offset, limit, query); public SqlQuerySpec BuildSeriesQuerySpec( - int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query = null) + int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) => BuildSeriesLevelQuerySpec(SeriesSqlQuerySearchFormat, offset, limit, query); public SqlQuerySpec BuildInstanceQuerySpec( - int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query = null) + int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) { // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); @@ -53,14 +53,14 @@ public SqlQuerySpec BuildInstanceQuerySpec( (tag, parameter) => { sqlParameterCollection.Add(parameter); - return $"f.{nameof(QueryInstance.IndexedAttributes)}[\"{DicomTagSerializer.Serialize(tag)}\"] = {parameter.Name}"; + return $"ARRAY_CONTAINS(f.{nameof(QueryInstance.IndexedAttributes)}[\"{tag.AttributeId}\"], {parameter.Name})"; }); return new SqlQuerySpec(string.Format(_stringFormatProvider, InstanceSqlQuerySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); } private SqlQuerySpec BuildSeriesLevelQuerySpec( - string querySearchFormat, int offset, int limit, IEnumerable<(DicomTag Attribute, string Value)> query) + string querySearchFormat, int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query) { // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); @@ -70,13 +70,13 @@ private SqlQuerySpec BuildSeriesLevelQuerySpec( (tag, parameter) => { sqlParameterCollection.Add(parameter); - return $"ARRAY_CONTAINS(c.{nameof(QuerySeriesDocument.DistinctIndexedAttributes)}[\"{DicomTagSerializer.Serialize(tag)}\"].{nameof(AttributeValues.Values)}, {parameter.Name})"; + return $"ARRAY_CONTAINS(c.{nameof(QuerySeriesDocument.DistinctIndexedAttributes)}[\"{tag.AttributeId}\"].{nameof(AttributeValues.Values)}, {parameter.Name})"; }); return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); } - private string GenerateWhereClause(IEnumerable<(DicomTag Attribute, string Value)> query, Func createQueryItem) + private string GenerateWhereClause(IEnumerable<(DicomAttributeId Attribute, string Value)> query, Func createQueryItem) { // If a null or empty query collection we should provide an empty string for the WHERE clause. if (query == null || !query.Any()) @@ -87,7 +87,7 @@ private string GenerateWhereClause(IEnumerable<(DicomTag Attribute, string Value var parameterNameIndex = 1; var queryItems = new List(); - foreach ((DicomTag attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + foreach ((DicomAttributeId attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) { var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); queryItems.Add(createQueryItem(attribute, new SqlParameter { Name = parameterName, Value = value })); @@ -99,8 +99,8 @@ private string GenerateWhereClause(IEnumerable<(DicomTag Attribute, string Value private static SqlParameterCollection CreateQueryParameterCollection(int offset, int limit) { - EnsureArg.IsTrue(offset >= 0, nameof(offset)); - EnsureArg.IsTrue(limit > 0, nameof(limit)); + EnsureArg.IsGte(offset, 0, nameof(offset)); + EnsureArg.IsGt(limit, 0, nameof(limit)); return new SqlParameterCollection() { diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index c6c444d389..d179ccd470 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -69,7 +69,7 @@ public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToke // Retry when the pre-condition fails on replace (ETag check). IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); - await _documentClient.ThrowIndexDataStoreException( + await _documentClient.ThrowDataStoreException( async (documentClient) => { QuerySeriesDocument document = await documentClient.GetorCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); @@ -78,7 +78,7 @@ await _documentClient.ThrowIndexDataStoreException( if (!document.AddInstance(instance)) { - throw new IndexDataStoreException(HttpStatusCode.Conflict); + throw new DataStoreException(HttpStatusCode.Conflict); } // Note, we do a replace (rather than upsert) in case the document is deleted. @@ -91,24 +91,24 @@ await _documentClient.ThrowIndexDataStoreException( } /// - public async Task> QueryStudiesAsync( - int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + public async Task> QueryStudiesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default) { SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildStudyQuerySpec(offset, limit, query); return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); } /// - public async Task> QuerySeriesAsync( - int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + public async Task> QuerySeriesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default) { SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildSeriesQuerySpec(offset, limit, query); return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); } /// - public async Task> QueryInstancesAsync( - int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomTag Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + public async Task> QueryInstancesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default) { SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildInstanceQuerySpec(offset, limit, query); return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); @@ -126,7 +126,7 @@ public async Task> DeleteSeriesIndexAsync( // Retry when the pre-condition fails on delete. IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); - return await _documentClient.ThrowIndexDataStoreException( + return await _documentClient.ThrowDataStoreException( async (documentClient) => { DocumentResponse response = @@ -156,7 +156,7 @@ public async Task DeleteInstanceIndexAsync(string studyInstanceUID, string serie RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); - await _documentClient.ThrowIndexDataStoreException( + await _documentClient.ThrowDataStoreException( async (documentClient) => { DocumentResponse response = @@ -165,7 +165,7 @@ await _documentClient.ThrowIndexDataStoreException( // If the instance does not exist, throw not found exception. if (!response.Document.RemoveInstance(sopInstanceUID)) { - throw new IndexDataStoreException(HttpStatusCode.NotFound); + throw new DataStoreException(HttpStatusCode.NotFound); } requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; @@ -183,59 +183,6 @@ await _documentClient.ThrowIndexDataStoreException( retryPolicy); } - /// - public async Task> GetInstancesInStudyAsync(string studyInstanceUID, CancellationToken cancellationToken = default) - { - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - - FeedOptions feedOptions = CreateFeedOptions(studyInstanceUID); - var identityQuery = _documentClient.CreateDocumentQuery(_collectionUri, feedOptions) - .SelectMany(x => x.Instances.Select(y => new { x.StudyInstanceUID, x.SeriesInstanceUID, y.SopInstanceUID })) - .AsDocumentQuery(); - - return await _documentClient.ThrowIndexDataStoreException( - async (documentClient) => - { - var results = new List(); - - while (identityQuery.HasMoreResults) - { - FeedResponse nextResults = await identityQuery.ExecuteNextAsync(cancellationToken); - results.AddRange(nextResults); - } - - if (results.Count > 0) - { - return results; - } - - // If no results, this study does not exist. - throw new IndexDataStoreException(HttpStatusCode.NotFound); - }); - } - - /// - public async Task> GetInstancesInSeriesAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) - { - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); - - string documentId = QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID); - RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); - - return await _documentClient.ThrowIndexDataStoreException( - async (documentClient) => - { - DocumentResponse response = await documentClient.ReadDocumentAsync( - UriFactory.CreateDocumentUri(_databaseId, _collectionId, documentId), - requestOptions, - cancellationToken: cancellationToken); - - return response.Document.Instances.Select( - x => new DicomInstance(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)); - }); - } - private RequestOptions CreateRequestOptions(string partitionKey) => new RequestOptions() { PartitionKey = new PartitionKey(partitionKey) }; @@ -256,7 +203,7 @@ private FeedOptions CreateCrossPartitionFeedOptions() return feedOptions; } - private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpec, string studyInstanceUID, CancellationToken cancellationToken) + private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpec, string studyInstanceUID, CancellationToken cancellationToken) { // If the study instance UID is provided we can run the query against a specific partition. FeedOptions feedOptions = string.IsNullOrWhiteSpace(studyInstanceUID) ? CreateCrossPartitionFeedOptions() : CreateFeedOptions(studyInstanceUID); @@ -264,10 +211,11 @@ private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpe .CreateDocumentQuery(_collectionUri, sqlQuerySpec, feedOptions) .AsDocumentQuery(); - return await _documentClient.ThrowIndexDataStoreException( + return await _documentClient.ThrowDataStoreException( async (documentClient) => { - return await documentQuery.ExecuteQueryUntilCompleteAsync(cancellationToken); + IEnumerable results = await documentQuery.ExecuteQueryUntilCompleteAsync(cancellationToken); + return new QueryResult(documentQuery.HasMoreResults, results.ToArray()); }); } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs deleted file mode 100644 index f304f69396..0000000000 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomTagSerializer.cs +++ /dev/null @@ -1,54 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Globalization; -using Dicom; -using EnsureThat; - -namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage -{ - internal static class DicomTagSerializer - { - private const char Seperator = ','; - private const string GroupElementStringFormat = "X4"; - private static readonly IFormatProvider FormatProvider = CultureInfo.InvariantCulture; - - public static string Serialize(DicomTag dicomTag) - { - EnsureArg.IsNotNull(dicomTag, nameof(dicomTag)); - - string groupString = dicomTag.Group.ToString(GroupElementStringFormat, FormatProvider); - string elementString = dicomTag.Element.ToString(GroupElementStringFormat, FormatProvider); - - if (dicomTag.PrivateCreator == null) - { - return $"{groupString}{Seperator}{elementString}"; - } - - return $"{groupString}{Seperator}{elementString}{Seperator}{dicomTag.PrivateCreator.Creator}"; - } - - public static DicomTag Deserialize(string dictionaryElement) - { - EnsureArg.IsNotNullOrWhiteSpace(dictionaryElement, nameof(dictionaryElement)); - var split = dictionaryElement.Split(Seperator); - - EnsureArg.IsTrue(split.Length == 2 || split.Length == 3, nameof(dictionaryElement)); - EnsureArg.IsTrue(split[0].Length == 4, nameof(dictionaryElement)); - EnsureArg.IsTrue(split[1].Length == 4, nameof(dictionaryElement)); - - EnsureArg.IsTrue(ushort.TryParse(split[0], NumberStyles.HexNumber, FormatProvider, out ushort group), nameof(dictionaryElement)); - EnsureArg.IsTrue(ushort.TryParse(split[1], NumberStyles.HexNumber, FormatProvider, out ushort element), nameof(dictionaryElement)); - - if (split.Length == 2) - { - return new DicomTag(group, element); - } - - return new DicomTag(group, element, split[2]); - } - } -} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs index 481742eeba..4bb9106b2d 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -20,7 +20,7 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage { public static class DocumentClientExtensions { - public static async Task ThrowIndexDataStoreException(this IDocumentClient documentClient, Func> action, IAsyncPolicy retryPolicy = null) + public static async Task ThrowDataStoreException(this IDocumentClient documentClient, Func> action, IAsyncPolicy retryPolicy = null) { EnsureArg.IsNotNull(documentClient, nameof(documentClient)); EnsureArg.IsNotNull(action, nameof(action)); @@ -36,11 +36,11 @@ public static async Task ThrowIndexDataStoreException(this IDocumentClient } catch (DocumentClientException e) { - throw new IndexDataStoreException(e.StatusCode, e); + throw new DataStoreException(e.StatusCode, e); } } - public static async Task ThrowIndexDataStoreException(this IDocumentClient documentClient, Func action, IAsyncPolicy retryPolicy = null) + public static async Task ThrowDataStoreException(this IDocumentClient documentClient, Func action, IAsyncPolicy retryPolicy = null) { EnsureArg.IsNotNull(documentClient, nameof(documentClient)); EnsureArg.IsNotNull(action, nameof(action)); @@ -58,7 +58,7 @@ public static async Task ThrowIndexDataStoreException(this IDocumentClient docum } catch (DocumentClientException e) { - throw new IndexDataStoreException(e.StatusCode, e); + throw new DataStoreException(e.StatusCode, e); } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs index 3ad177cd4a..2d1f44c1b4 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -8,6 +8,7 @@ using System.Text.RegularExpressions; using Dicom; using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Persistence; using Microsoft.Health.Dicom.Core.Features.Validation; using Newtonsoft.Json; @@ -16,7 +17,7 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents internal class QueryInstance { [JsonConstructor] - public QueryInstance(string sopInstanceUID, IDictionary indexedAttributes) + public QueryInstance(string sopInstanceUID, IDictionary indexedAttributes) : this(sopInstanceUID) { EnsureArg.IsNotNull(indexedAttributes, nameof(indexedAttributes)); @@ -26,18 +27,19 @@ public QueryInstance(string sopInstanceUID, IDictionary indexedA private QueryInstance(string sopInstanceUID) { - EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); - EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(sopInstanceUID)); SopInstanceUID = sopInstanceUID; } public string SopInstanceUID { get; } - public IDictionary IndexedAttributes { get; } = new Dictionary(); + public IDictionary IndexedAttributes { get; } = new Dictionary(); public override int GetHashCode() { + // Note: We override Equals() and GetHashCode() so if this class is added to a HashSet + // we only compare on SopInstanceUID. return SopInstanceUID.GetHashCode(StringComparison.Ordinal); } @@ -51,24 +53,19 @@ public override bool Equals(object obj) return false; } - public static QueryInstance Create(DicomDataset dicomDataset, IEnumerable indexTags) + public static QueryInstance Create(DicomDataset dicomDataset, IEnumerable indexableAttributes) { EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + EnsureArg.IsNotNull(indexableAttributes, nameof(indexableAttributes)); var sopInstanceUID = dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty); var result = new QueryInstance(sopInstanceUID); - if (indexTags != null) + foreach (DicomAttributeId attribute in indexableAttributes) { - foreach (DicomTag dicomTag in indexTags) + if (dicomDataset.TryGetValues(attribute, out object[] values)) { - // All indexed tags must have a value multiplicty of 1. - EnsureArg.IsTrue(dicomTag.DictionaryEntry.ValueMultiplicity.Multiplicity == 1); - - if (dicomDataset.TryGetSingleValue(dicomTag, out object value)) - { - result.IndexedAttributes[DicomTagSerializer.Serialize(dicomTag)] = value; - } + result.IndexedAttributes[attribute.AttributeId] = values; } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs index d86db0cd49..91467f7676 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -47,7 +47,7 @@ public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) public HashSet Instances { get; } = new HashSet(); - public Dictionary DistinctIndexedAttributes + public IDictionary DistinctIndexedAttributes { get { @@ -55,14 +55,17 @@ public Dictionary DistinctIndexedAttributes foreach (QueryInstance instance in Instances) { - foreach ((string key, object value) in instance.IndexedAttributes) + foreach ((string key, object[] values) in instance.IndexedAttributes) { if (!result.ContainsKey(key)) { result[key] = new AttributeValues(); } - result[key].Add(value); + foreach (object value in values) + { + result[key].Add(value); + } } } diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs index 7d8061d282..958dd9f527 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -26,7 +26,7 @@ public DicomIndexDataStoreTests(DicomCosmosDataStoreTestsFixture fixture) } [Fact] - public async Task GivenAValidInstance_WhenIndexing_CanBeRetrieved() + public async Task GivenAValidInstance_WhenIndexing_CanBeQueried() { string studyInstanceUID = Guid.NewGuid().ToString(); string seriesInstanceUID = Guid.NewGuid().ToString(); @@ -35,15 +35,15 @@ public async Task GivenAValidInstance_WhenIndexing_CanBeRetrieved() await _indexDataStore.IndexInstanceAsync(testInstance); - IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); + QueryResult instancesInStudy = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.Single(instancesInStudy.Results); + Assert.False(instancesInStudy.HasMoreResults); + Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInStudy.Results.First().SopInstanceUID); - Assert.Single(instancesInStudy); - Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInStudy.First().SopInstanceUID); - - IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); - - Assert.Single(instancesInSeries); - Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInSeries.First().SopInstanceUID); + QueryResult seriesInStudy = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID); + Assert.False(seriesInStudy.HasMoreResults); + Assert.Single(seriesInStudy.Results); + Assert.True(seriesInstanceUID == seriesInStudy.Results.First().SeriesInstanceUID); IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); @@ -61,16 +61,18 @@ public async Task GivenMultipleValidInstances_WhenIndexingInParallel_AreStored() IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstancesToIndex); IList sopInstanceUIDs = instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID)).ToList(); - IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); - Assert.Equal(numberOfInstancesToIndex, instancesInStudy.Count()); - instancesInStudy.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + QueryResult instancesInStudy = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.False(instancesInStudy.HasMoreResults); + Assert.Equal(numberOfInstancesToIndex, instancesInStudy.Results.Count()); + instancesInStudy.Results.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); - IEnumerable instancesInSeries = await _indexDataStore.GetInstancesInSeriesAsync(studyInstanceUID, seriesInstanceUID); - Assert.Equal(numberOfInstancesToIndex, instancesInSeries.Count()); - instancesInSeries.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + QueryResult seriesInStudy = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID); + Assert.Single(seriesInStudy.Results); + Assert.False(seriesInStudy.HasMoreResults); + Assert.True(seriesInstanceUID == seriesInStudy.Results.First().SeriesInstanceUID); IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); - Assert.Equal(numberOfInstancesToIndex, instancesInSeries.Count()); + Assert.Equal(numberOfInstancesToIndex, deletedInstances.Count()); deletedInstances.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); } @@ -87,8 +89,9 @@ public async Task GivenAValidSeries_WhenDeletingAnInstance_InstanceIsDeletedButS await _indexDataStore.DeleteInstanceIndexAsync(studyInstanceUID, seriesInstanceUID, firstSopInstanceUID); - IEnumerable instancesInStudy = await _indexDataStore.GetInstancesInStudyAsync(studyInstanceUID); - Assert.Equal(numberOfInstancesToIndex - 1, instancesInStudy.Count()); + QueryResult instancesInStudy = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.False(instancesInStudy.HasMoreResults); + Assert.Equal(numberOfInstancesToIndex - 1, instancesInStudy.Results.Count()); IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); deletedInstances.Each(x => Assert.True(otherSopInstanceUIDs.Contains(x.SopInstanceUID))); @@ -114,25 +117,29 @@ public async Task GivenAnInstanceWithInvalidInstanceId_WhenStoring_ExceptionIsTh [Fact] public async Task GivenNonExistentInstancesOrSeries_WhenDeleting_IndexDataStoreExceptionIsThrownWithNotFoundStatusCode() { - IndexDataStoreException deleteInstanceException = await Assert.ThrowsAsync( + DataStoreException deleteInstanceException = await Assert.ThrowsAsync( () => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); - Assert.Equal(HttpStatusCode.NotFound, deleteInstanceException.StatusCode); + Assert.Equal((int)HttpStatusCode.NotFound, deleteInstanceException.StatusCode); - IndexDataStoreException deleteSeriesException = await Assert.ThrowsAsync( + DataStoreException deleteSeriesException = await Assert.ThrowsAsync( () => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); - Assert.Equal(HttpStatusCode.NotFound, deleteSeriesException.StatusCode); + Assert.Equal((int)HttpStatusCode.NotFound, deleteSeriesException.StatusCode); } [Fact] - public async Task GivenNonExistentSeriesOrStudy_WhenFetchingInstances_IndexDataStoreExceptionIsThrownWithNotFoundStatusCode() + public async Task GivenNonExistentStudy_WhenQuerying_NoExceptionIsThrown() { - IndexDataStoreException getStudyInstancesException = await Assert.ThrowsAsync( - () => _indexDataStore.GetInstancesInStudyAsync(Guid.NewGuid().ToString())); - Assert.Equal(HttpStatusCode.NotFound, getStudyInstancesException.StatusCode); + QueryResult studyResults = await _indexDataStore.QueryStudiesAsync(0, 10, Guid.NewGuid().ToString()); + Assert.Empty(studyResults.Results); + Assert.False(studyResults.HasMoreResults); + + QueryResult seriesResults = await _indexDataStore.QuerySeriesAsync(0, 10, Guid.NewGuid().ToString()); + Assert.Empty(seriesResults.Results); + Assert.False(studyResults.HasMoreResults); - IndexDataStoreException getSeriesInstancesException = await Assert.ThrowsAsync( - () => _indexDataStore.GetInstancesInSeriesAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); - Assert.Equal(HttpStatusCode.NotFound, getSeriesInstancesException.StatusCode); + QueryResult instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, Guid.NewGuid().ToString()); + Assert.Empty(instanceResults.Results); + Assert.False(studyResults.HasMoreResults); } [Fact] @@ -140,8 +147,8 @@ public async Task GivenAStoredInstance_WhenStoringAgain_ConflictExceptionThrown( { DicomDataset testInstance = CreateTestInstanceDicomDataset(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); await _indexDataStore.IndexInstanceAsync(testInstance); - IndexDataStoreException dataStoreException = await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); - Assert.Equal(HttpStatusCode.Conflict, dataStoreException.StatusCode); + DataStoreException dataStoreException = await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + Assert.Equal((int)HttpStatusCode.Conflict, dataStoreException.StatusCode); await DeleteInstancesAsync(testInstance); } @@ -174,19 +181,22 @@ public async Task GivenIndexDataStoreQuery_WhenQueryingOnNonIndexAttribute_IsIgn await _indexDataStore.IndexInstanceAsync(testInstance2); var dicomInstance = DicomInstance.Create(testInstance1); - (DicomTag, string)[] queryTags = new[] { (supportedDicomTag, testInstance1PatientName), (unsupportedDicomTag, "test") }; + (DicomAttributeId, string)[] queryAttributes = new[] { (new DicomAttributeId(supportedDicomTag), testInstance1PatientName), (new DicomAttributeId(unsupportedDicomTag), "test") }; - IEnumerable studyResults = await _indexDataStore.QueryStudiesAsync(0, 10, studyInstanceUID, query: queryTags); - Assert.Single(studyResults); - Assert.Equal(new DicomStudy(dicomInstance.StudyInstanceUID), studyResults.First()); + QueryResult studyResults = await _indexDataStore.QueryStudiesAsync(0, 10, studyInstanceUID, query: queryAttributes); + Assert.Single(studyResults.Results); + Assert.False(studyResults.HasMoreResults); + Assert.Equal(new DicomStudy(dicomInstance.StudyInstanceUID), studyResults.Results.First()); - IEnumerable seriesResults = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID, query: queryTags); - Assert.Single(seriesResults); - Assert.Equal(new DicomSeries(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID), seriesResults.First()); + QueryResult seriesResults = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID, query: queryAttributes); + Assert.Single(seriesResults.Results); + Assert.False(seriesResults.HasMoreResults); + Assert.Equal(new DicomSeries(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID), seriesResults.Results.First()); - IEnumerable instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, query: queryTags); - Assert.Single(instanceResults); - Assert.Equal(dicomInstance, instanceResults.First()); + QueryResult instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, query: queryAttributes); + Assert.Single(instanceResults.Results); + Assert.False(instanceResults.HasMoreResults); + Assert.Equal(dicomInstance, instanceResults.Results.First()); await DeleteInstancesAsync(testInstance1, testInstance2); } @@ -204,9 +214,10 @@ public async Task GivenIndexedInstance_WhenQueryingByPatientName_InstancesIsRetr await _indexDataStore.IndexInstanceAsync(testInstance); var sopInstanceUID = testInstance.GetSingleValue(DicomTag.SOPInstanceUID); - IEnumerable instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (DicomTag.ReferringPhysicianName, referringPhysicianName) }); - Assert.Single(instances); - Assert.Equal(new DicomInstance(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.First()); + QueryResult instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (new DicomAttributeId(DicomTag.ReferringPhysicianName), referringPhysicianName) }); + Assert.Single(instances.Results); + Assert.False(instances.HasMoreResults); + Assert.Equal(new DicomInstance(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.Results.First()); await DeleteInstancesAsync(testInstance); } @@ -223,23 +234,26 @@ public async Task GivenIndexedSeries_WhenQueryingWithPaging_PagesReturnedCorrect var instanceUIDs = new HashSet(instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID))); // End of page - IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(numberOfInstances, numberOfInstances, studyInstanceUID); - Assert.Empty(queryInstances1); + QueryResult queryInstances1 = await _indexDataStore.QueryInstancesAsync(numberOfInstances, numberOfInstances, studyInstanceUID); + Assert.Empty(queryInstances1.Results); + Assert.False(queryInstances1.HasMoreResults); - // First 5 items - IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, numberOfInstances / 2, studyInstanceUID); - Assert.Equal(numberOfInstances / 2, queryInstances2.Count()); + // First half of items items + QueryResult queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, numberOfInstances / 2, studyInstanceUID); + Assert.Equal(numberOfInstances / 2, queryInstances2.Results.Count()); + Assert.False(queryInstances2.HasMoreResults); - foreach (DicomInstance item in queryInstances2) + foreach (DicomInstance item in queryInstances2.Results) { Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); } - // Last 5 items - IEnumerable queryInstances3 = await _indexDataStore.QueryInstancesAsync(numberOfInstances / 2, numberOfInstances, studyInstanceUID); - Assert.Equal(numberOfInstances / 2, queryInstances3.Count()); + // Last half of items + QueryResult queryInstances3 = await _indexDataStore.QueryInstancesAsync(numberOfInstances / 2, numberOfInstances, studyInstanceUID); + Assert.Equal(numberOfInstances / 2, queryInstances3.Results.Count()); + Assert.False(queryInstances3.HasMoreResults); - foreach (DicomInstance item in queryInstances3) + foreach (DicomInstance item in queryInstances3.Results) { Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); } @@ -274,26 +288,32 @@ public async Task GivenMultipleStudies_WhenQueryingWithOrWithoutStudyInstanceUID } // Validate Study Searching - IEnumerable queryStudiesResults1 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1); - Assert.Equal(studyInstanceUIDs.Length, queryStudiesResults1.Count()); - queryStudiesResults1.Each(x => Assert.Contains(x.StudyInstanceUID, studyInstanceUIDs)); + QueryResult queryStudiesResults1 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1); + Assert.Equal(studyInstanceUIDs.Length, queryStudiesResults1.Results.Count()); + Assert.False(queryStudiesResults1.HasMoreResults); + queryStudiesResults1.Results.Each(x => Assert.Contains(x.StudyInstanceUID, studyInstanceUIDs)); - IEnumerable queryStudiesResults2 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, studyInstanceUIDs[0]); - Assert.Single(queryStudiesResults2); + QueryResult queryStudiesResults2 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, studyInstanceUIDs[0]); + Assert.Single(queryStudiesResults2.Results); + Assert.False(queryStudiesResults2.HasMoreResults); - IEnumerable queryStudiesResults3 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); - Assert.Empty(queryStudiesResults3); + QueryResult queryStudiesResults3 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); + Assert.Empty(queryStudiesResults3.Results); + Assert.False(queryStudiesResults3.HasMoreResults); // Validate Series Searching - IEnumerable querySeriesResults1 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1); - Assert.Equal(dicomSeries.Count, querySeriesResults1.Count()); - querySeriesResults1.Each(x => Assert.Contains(x, dicomSeries)); + QueryResult querySeriesResults1 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1); + Assert.Equal(dicomSeries.Count, querySeriesResults1.Results.Count()); + Assert.False(querySeriesResults1.HasMoreResults); + querySeriesResults1.Results.Each(x => Assert.Contains(x, dicomSeries)); - IEnumerable querySeriesResults2 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, studyInstanceUIDs[0]); - Assert.Equal(seriesPerStudy, querySeriesResults2.Count()); + QueryResult querySeriesResults2 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, studyInstanceUIDs[0]); + Assert.Equal(seriesPerStudy, querySeriesResults2.Results.Count()); + Assert.False(querySeriesResults2.HasMoreResults); - IEnumerable querySeriesResults3 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); - Assert.Empty(querySeriesResults3); + QueryResult querySeriesResults3 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); + Assert.Empty(querySeriesResults3.Results); + Assert.False(querySeriesResults3.HasMoreResults); foreach (DicomSeries series in dicomSeries) { @@ -312,12 +332,14 @@ public async Task GivenInstanceWithInjectedSql_WhenQuerying_IsReturnedCorrectly( await _indexDataStore.IndexInstanceAsync(testInstance); - IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); - Assert.Single(queryInstances1); - Assert.Equal(DicomInstance.Create(testInstance), queryInstances1.First()); + QueryResult queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.False(queryInstances1.HasMoreResults); + Assert.Single(queryInstances1.Results); + Assert.Equal(DicomInstance.Create(testInstance), queryInstances1.Results.First()); - IEnumerable queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, new[] { (DicomTag.PatientName, " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\"") }); - Assert.Empty(queryInstances2); + QueryResult queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, new[] { (new DicomAttributeId(DicomTag.PatientName), " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\"") }); + Assert.False(queryInstances2.HasMoreResults); + Assert.Empty(queryInstances2.Results); await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); } @@ -338,20 +360,24 @@ public async Task GivenWorstCaseStoreScenario_WhenAllInstancesHaveDifferentTagVa await _indexDataStore.IndexInstanceAsync(dataset); } - IEnumerable queryStudies1 = await _indexDataStore.QueryStudiesAsync(0, 10); - Assert.Single(queryStudies1); - Assert.Equal(studyInstanceUID, queryStudies1.First().StudyInstanceUID); + QueryResult queryStudies1 = await _indexDataStore.QueryStudiesAsync(0, 10); + Assert.False(queryStudies1.HasMoreResults); + Assert.Single(queryStudies1.Results); + Assert.Equal(studyInstanceUID, queryStudies1.Results.First().StudyInstanceUID); - IEnumerable queryStudies2 = await _indexDataStore.QueryStudiesAsync(0, 10, query: new[] { (DicomTag.PatientName, patientNames[numberOfItemsToInsert / 2]) }); - Assert.Single(queryStudies2); - Assert.Equal(studyInstanceUID, queryStudies2.First().StudyInstanceUID); + QueryResult queryStudies2 = await _indexDataStore.QueryStudiesAsync(0, 10, query: new[] { (new DicomAttributeId(DicomTag.PatientName), patientNames[numberOfItemsToInsert / 2]) }); + Assert.Single(queryStudies2.Results); + Assert.False(queryStudies2.HasMoreResults); + Assert.Equal(studyInstanceUID, queryStudies2.Results.First().StudyInstanceUID); - IEnumerable querySeries1 = await _indexDataStore.QuerySeriesAsync(0, 10); - Assert.Single(querySeries1); - Assert.Equal(seriesInstanceUID, querySeries1.First().SeriesInstanceUID); + QueryResult querySeries1 = await _indexDataStore.QuerySeriesAsync(0, 10); + Assert.False(querySeries1.HasMoreResults); + Assert.Single(querySeries1.Results); + Assert.Equal(seriesInstanceUID, querySeries1.Results.First().SeriesInstanceUID); - IEnumerable queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, numberOfItemsToInsert + 1); - Assert.Equal(numberOfItemsToInsert, queryInstances1.Count()); + QueryResult queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, numberOfItemsToInsert + 1); + Assert.False(queryInstances1.HasMoreResults); + Assert.Equal(numberOfItemsToInsert, queryInstances1.Results.Count()); await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); } From 01dd8de3429947c39e9dfafb7af2788ac271e94d Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 25 Jun 2019 13:52:44 +0100 Subject: [PATCH 14/22] Fixed broken test and resolved more PR comments --- .../Features/Persistence/DicomAttributeIdTests.cs | 3 +++ .../Features/Persistence/DicomAttributeId.cs | 14 +++++++++----- .../Persistence/DicomAttributeIdExtensions.cs | 6 +++--- .../Config/DicomCosmosConfiguration.cs | 1 + .../Persistence/DicomIndexDataStoreTests.cs | 12 ++++++------ 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs index 6f0fc5f957..3a78cfb103 100644 --- a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs @@ -25,6 +25,9 @@ public void GivenDicomAttributeId_WhenConstructingWithInvalidParameters_Argument Assert.Throws(() => new DicomAttributeId(string.Empty)); Assert.Throws(() => new DicomAttributeId("INVALID")); Assert.Throws(() => new DicomAttributeId("INVALID.INVALID")); + + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate).GetDicomTag(-1)); + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate).GetDicomTag(1)); } [Fact] diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs index 70f6b5ea56..900593f2dc 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs @@ -63,13 +63,17 @@ public DicomAttributeId(params DicomTag[] dicomTags) [JsonIgnore] public int Length => _dicomTags.Length; + /// + /// Gets the non-sequence DICOM tag. + /// [JsonIgnore] - public DicomTag LeafDicomTag => _dicomTags[Length - 1]; + public DicomTag InstanceDicomTag => _dicomTags[Length - 1]; - public DicomTag GetDicomTag(int index = -1) + public DicomTag GetDicomTag(int index) { + EnsureArg.IsGte(index, 0, nameof(index)); EnsureArg.IsLt(index, _dicomTags.Length, nameof(index)); - return index < 0 ? _dicomTags[Length - 1] : _dicomTags[index]; + return _dicomTags[index]; } public override int GetHashCode() @@ -87,7 +91,7 @@ public override bool Equals(object obj) return false; } - public static DicomTag[] DeserializeAttributeId(string attributeId) + private static DicomTag[] DeserializeAttributeId(string attributeId) { EnsureArg.IsNotNullOrWhiteSpace(attributeId, nameof(attributeId)); @@ -133,7 +137,7 @@ private void Validate() EnsureArg.IsTrue(_dicomTags[i].DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); } - EnsureArg.IsFalse(LeafDicomTag.DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); + EnsureArg.IsFalse(InstanceDicomTag.DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); } private class KeywordComparer : IEqualityComparer diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs index e085a12f0f..e63b4bd9c8 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs @@ -23,13 +23,13 @@ public static bool TryGetValues(this DicomDataset dicomDataset, DicomAttr if (startAttributeIndex == attributeId.Length - 1) { // Now first validate this DICOM tag exists in the dataset - if (dicomDataset.TryGetValue(attributeId.LeafDicomTag, 0, out TItem firstItem)) + if (dicomDataset.TryGetValue(attributeId.InstanceDicomTag, 0, out TItem firstItem)) { nestedValues.Add(firstItem); - for (int i = 1; i < dicomDataset.GetValueCount(attributeId.LeafDicomTag); i++) + for (int i = 1; i < dicomDataset.GetValueCount(attributeId.InstanceDicomTag); i++) { - if (dicomDataset.TryGetValue(attributeId.LeafDicomTag, i, out TItem value)) + if (dicomDataset.TryGetValue(attributeId.InstanceDicomTag, i, out TItem value)) { nestedValues.Add(value); } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs index c7e840f2ba..35f919d8e7 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs @@ -14,6 +14,7 @@ public class DicomCosmosConfiguration /// /// Gets the DICOM tags that should be indexed and made queryable. /// The StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID will be indexed automatically. + /// TODO: We should validate that the attributes defined here have a sensible value representation. /// public IEnumerable QueryAttributes { get; } = new[] { diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs index 958dd9f527..39917197e2 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -156,12 +156,12 @@ public async Task GivenAStoredInstance_WhenStoringAgain_ConflictExceptionThrown( [Fact] public async Task GivenIndexDataStoreQuery_WhenQueryingWithInvalidParameters_ArgumentExceptionThrown() { - await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(-1, 10)); - await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(0, 0)); - await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(-1, 10)); - await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(0, 0)); - await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(-1, 10)); - await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(0, 0)); } [Fact] From 1fb69cdd82e09ce9329ac04809f5657503518843 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 25 Jun 2019 15:03:09 +0100 Subject: [PATCH 15/22] Added missing code comment change for the DICOM identifier validator --- .../Features/Persistence/DicomInstance.cs | 7 +++---- .../Features/Persistence/DicomSeries.cs | 5 ++--- .../Features/Persistence/DicomStudy.cs | 3 +-- .../Features/Validation/DicomIdentifierValidator.cs | 3 ++- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs index 8b52b0a855..e45f5aab88 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs @@ -18,10 +18,9 @@ public DicomInstance(string studyInstanceUID, string seriesInstanceUID, string s : base(studyInstanceUID, seriesInstanceUID) { // Run the instance identifiers through the regular expression check. - EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); - EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex)); - EnsureArg.IsNotEqualTo(sopInstanceUID, studyInstanceUID); - EnsureArg.IsNotEqualTo(sopInstanceUID, seriesInstanceUID); + EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(sopInstanceUID)); + EnsureArg.IsNotEqualTo(sopInstanceUID, studyInstanceUID, nameof(sopInstanceUID)); + EnsureArg.IsNotEqualTo(sopInstanceUID, seriesInstanceUID, nameof(sopInstanceUID)); SopInstanceUID = sopInstanceUID; } diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs index 2ef3b56731..a6ab2bead3 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs @@ -17,9 +17,8 @@ public DicomSeries(string studyInstanceUID, string seriesInstanceUID) : base(studyInstanceUID) { // Run the instance identifiers through the regular expression check. - EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); - EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex)); - EnsureArg.IsNotEqualTo(studyInstanceUID, seriesInstanceUID); + EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(seriesInstanceUID)); + EnsureArg.IsNotEqualTo(studyInstanceUID, seriesInstanceUID, nameof(seriesInstanceUID)); SeriesInstanceUID = seriesInstanceUID; } diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs index c26007e4fb..34869fb30f 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs @@ -19,8 +19,7 @@ public class DicomStudy public DicomStudy(string studyInstanceUID) { // Run the instance identifiers through the regular expression check. - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(studyInstanceUID)); StudyInstanceUID = studyInstanceUID; } diff --git a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs index e93814b603..ea62f47179 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs @@ -9,7 +9,8 @@ namespace Microsoft.Health.Dicom.Core.Features.Validation { /// - /// Validates a unique identifer conforms to the rules from http://dicom.nema.org/dicom/2013/output/chtml/part05/chapter_9.html + /// Validates a unique identifer conforms to the character rules from http://dicom.nema.org/dicom/2013/output/chtml/part05/chapter_9.html. + /// This validator does not validate the format of the identifier. /// /// public class DicomIdentifierValidator : RegularExpressionValidator From 454ddebec9a2e7708f5560d7445eebf92dbe2246 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 25 Jun 2019 17:11:14 +0100 Subject: [PATCH 16/22] Updated the Regex so it is compiled --- Microsoft.Health.Dicom.sln | 11 +++++++++-- .../Features/Persistence/DicomInstance.cs | 3 +-- .../Features/Persistence/DicomSeries.cs | 3 +-- .../Features/Persistence/DicomStudy.cs | 3 +-- .../Validation/DicomIdentifierValidator.cs | 4 ++-- .../Features/Storage/Documents/QueryInstance.cs | 3 +-- .../Storage/Documents/QuerySeriesDocument.cs | 14 +++++--------- 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/Microsoft.Health.Dicom.sln b/Microsoft.Health.Dicom.sln index 2923d5fc31..0ed9169608 100644 --- a/Microsoft.Health.Dicom.sln +++ b/Microsoft.Health.Dicom.sln @@ -34,7 +34,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Core EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.CosmosDb", "src\Microsoft.Health.Dicom.CosmosDb\Microsoft.Health.Dicom.CosmosDb.csproj", "{E78AB378-6CF4-442A-BC34-81A1B7431ECD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.CosmosDb.UnitTests", "src\Microsoft.Health.Dicom.CosmosDb.UnitTests\Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj", "{FA0B32A4-E623-4B24-902D-BFDC6E42E612}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.CosmosDb.UnitTests", "src\Microsoft.Health.Dicom.CosmosDb.UnitTests\Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj", "{FA0B32A4-E623-4B24-902D-BFDC6E42E612}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Metadata", "src\Microsoft.Health.Dicom.Metadata\Microsoft.Health.Dicom.Metadata.csproj", "{40CEC679-D599-445D-90E9-0F357D947AEB}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -90,6 +92,10 @@ Global {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.Build.0 = Release|Any CPU + {40CEC679-D599-445D-90E9-0F357D947AEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {40CEC679-D599-445D-90E9-0F357D947AEB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {40CEC679-D599-445D-90E9-0F357D947AEB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {40CEC679-D599-445D-90E9-0F357D947AEB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -107,9 +113,10 @@ Global {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D} = {176641B3-297C-4E04-A83D-8F80F80485E8} {E78AB378-6CF4-442A-BC34-81A1B7431ECD} = {176641B3-297C-4E04-A83D-8F80F80485E8} {FA0B32A4-E623-4B24-902D-BFDC6E42E612} = {176641B3-297C-4E04-A83D-8F80F80485E8} + {40CEC679-D599-445D-90E9-0F357D947AEB} = {176641B3-297C-4E04-A83D-8F80F80485E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - RESX_SortFileContentOnSave = True SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} + RESX_SortFileContentOnSave = True EndGlobalSection EndGlobal diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs index e45f5aab88..13cf2dd6b6 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Text.RegularExpressions; using Dicom; using EnsureThat; using Microsoft.Health.Dicom.Core.Features.Validation; @@ -18,7 +17,7 @@ public DicomInstance(string studyInstanceUID, string seriesInstanceUID, string s : base(studyInstanceUID, seriesInstanceUID) { // Run the instance identifiers through the regular expression check. - EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(sopInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(sopInstanceUID), nameof(sopInstanceUID)); EnsureArg.IsNotEqualTo(sopInstanceUID, studyInstanceUID, nameof(sopInstanceUID)); EnsureArg.IsNotEqualTo(sopInstanceUID, seriesInstanceUID, nameof(sopInstanceUID)); diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs index a6ab2bead3..3df139f49d 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Text.RegularExpressions; using EnsureThat; using Microsoft.Health.Dicom.Core.Features.Validation; using Newtonsoft.Json; @@ -17,7 +16,7 @@ public DicomSeries(string studyInstanceUID, string seriesInstanceUID) : base(studyInstanceUID) { // Run the instance identifiers through the regular expression check. - EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(seriesInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(seriesInstanceUID), nameof(seriesInstanceUID)); EnsureArg.IsNotEqualTo(studyInstanceUID, seriesInstanceUID, nameof(seriesInstanceUID)); SeriesInstanceUID = seriesInstanceUID; diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs index 34869fb30f..505ea97e12 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs @@ -4,7 +4,6 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Text.RegularExpressions; using EnsureThat; using Microsoft.Health.Dicom.Core.Features.Validation; using Newtonsoft.Json; @@ -19,7 +18,7 @@ public class DicomStudy public DicomStudy(string studyInstanceUID) { // Run the instance identifiers through the regular expression check. - EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(studyInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyInstanceUID), nameof(studyInstanceUID)); StudyInstanceUID = studyInstanceUID; } diff --git a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs index ea62f47179..3d4cb500ef 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs @@ -15,10 +15,10 @@ namespace Microsoft.Health.Dicom.Core.Features.Validation /// public class DicomIdentifierValidator : RegularExpressionValidator { - public const string IdentifierRegex = "^[A-Za-z0-9\\-\\.]{1,64}$"; + public static readonly Regex IdentifierRegex = new Regex("^[A-Za-z0-9\\-\\.]{1,64}$", RegexOptions.Singleline | RegexOptions.Compiled); public DicomIdentifierValidator() - : base(IdentifierRegex, RegexOptions.Singleline | RegexOptions.Compiled) + : base(IdentifierRegex) { } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs index 2d1f44c1b4..a8b5f85a9f 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.Text.RegularExpressions; using Dicom; using EnsureThat; using Microsoft.Health.Dicom.Core.Features.Persistence; @@ -27,7 +26,7 @@ public QueryInstance(string sopInstanceUID, IDictionary indexe private QueryInstance(string sopInstanceUID) { - EnsureArg.IsTrue(Regex.IsMatch(sopInstanceUID, DicomIdentifierValidator.IdentifierRegex), nameof(sopInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(sopInstanceUID), nameof(sopInstanceUID)); SopInstanceUID = sopInstanceUID; } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs index 91467f7676..683ad8da0f 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -18,7 +18,7 @@ internal class QuerySeriesDocument /// Lists the characters that are not valid for a Cosmos resource identifier. /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.documents.resource.id?view=azure-dotnet /// - private const string DocumentIdRegex = "[^?/\\#]"; + private static readonly Regex DocumentIdRegex = new Regex("[^?/\\#]", RegexOptions.Singleline | RegexOptions.Compiled); public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) { @@ -75,25 +75,21 @@ public IDictionary DistinctIndexedAttributes public static string GetDocumentId(string studyInstanceUID, string seriesInstanceUID) { - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); EnsureArg.IsFalse(studyInstanceUID == seriesInstanceUID); - - EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); - EnsureArg.IsTrue(Regex.IsMatch(seriesInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(seriesInstanceUID)); string documentId = studyInstanceUID + seriesInstanceUID; // Double safety for the document identifier. If the study and series are valid identifiers, the document ID 'should' conform. - EnsureArg.IsTrue(Regex.IsMatch(documentId, DocumentIdRegex)); + EnsureArg.IsTrue(DocumentIdRegex.IsMatch(documentId)); return documentId; } public static string GetPartitionKey(string studyInstanceUID) { - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - EnsureArg.IsTrue(Regex.IsMatch(studyInstanceUID, DicomIdentifierValidator.IdentifierRegex)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyInstanceUID)); return studyInstanceUID; } From 9baa990d7c9d5d53a069467f47b01fc2c68c9334 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 25 Jun 2019 17:13:42 +0100 Subject: [PATCH 17/22] Updated solution file to remove missing project --- Microsoft.Health.Dicom.sln | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Microsoft.Health.Dicom.sln b/Microsoft.Health.Dicom.sln index 0ed9169608..1f275ea82a 100644 --- a/Microsoft.Health.Dicom.sln +++ b/Microsoft.Health.Dicom.sln @@ -36,8 +36,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Cosm EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.CosmosDb.UnitTests", "src\Microsoft.Health.Dicom.CosmosDb.UnitTests\Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj", "{FA0B32A4-E623-4B24-902D-BFDC6E42E612}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Metadata", "src\Microsoft.Health.Dicom.Metadata\Microsoft.Health.Dicom.Metadata.csproj", "{40CEC679-D599-445D-90E9-0F357D947AEB}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -92,10 +90,6 @@ Global {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.Build.0 = Release|Any CPU - {40CEC679-D599-445D-90E9-0F357D947AEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {40CEC679-D599-445D-90E9-0F357D947AEB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {40CEC679-D599-445D-90E9-0F357D947AEB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {40CEC679-D599-445D-90E9-0F357D947AEB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -113,10 +107,9 @@ Global {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D} = {176641B3-297C-4E04-A83D-8F80F80485E8} {E78AB378-6CF4-442A-BC34-81A1B7431ECD} = {176641B3-297C-4E04-A83D-8F80F80485E8} {FA0B32A4-E623-4B24-902D-BFDC6E42E612} = {176641B3-297C-4E04-A83D-8F80F80485E8} - {40CEC679-D599-445D-90E9-0F357D947AEB} = {176641B3-297C-4E04-A83D-8F80F80485E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} RESX_SortFileContentOnSave = True + SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} EndGlobalSection EndGlobal From 275fcc020a665909282e6d528c593c19000295a2 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Wed, 26 Jun 2019 13:23:43 +0100 Subject: [PATCH 18/22] Refactored based on PR comments --- .../Persistence/DicomAttributeIdTests.cs | 11 ++-- .../Features/Persistence/DicomAttributeId.cs | 52 ++++++++++++---- .../Config/DicomCosmosConfiguration.cs | 2 +- .../Features/Storage/CosmosQueryBuilder.cs | 61 +++++++++---------- .../Features/Storage/DicomCosmosDataStore.cs | 23 ++++--- .../Storage/DocumentClientExtensions.cs | 19 +----- .../DocumentClientExtensionsTests.cs | 2 +- 7 files changed, 93 insertions(+), 77 deletions(-) diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs index 3a78cfb103..1fc9fae990 100644 --- a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs @@ -17,14 +17,15 @@ public void GivenDicomAttributeId_WhenConstructingWithInvalidParameters_Argument { Assert.Throws(() => new DicomAttributeId((DicomTag[])null)); Assert.Throws(() => new DicomAttributeId(Array.Empty())); - Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence)); - Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence, DicomTag.ROIContourSequence)); - Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate, DicomTag.RightLensSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence, DicomTag.ROIContourSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate, DicomTag.RightLensSequence)); Assert.Throws(() => new DicomAttributeId((string)null)); Assert.Throws(() => new DicomAttributeId(string.Empty)); - Assert.Throws(() => new DicomAttributeId("INVALID")); - Assert.Throws(() => new DicomAttributeId("INVALID.INVALID")); + Assert.Throws(() => new DicomAttributeId("0020000D.0020000D")); + Assert.Throws(() => new DicomAttributeId("INVALID")); + Assert.Throws(() => new DicomAttributeId("INVALID.INVALID")); Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate).GetDicomTag(-1)); Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate).GetDicomTag(1)); diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs index 900593f2dc..52425d3381 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs @@ -31,6 +31,7 @@ public class DicomAttributeId private const char Seperator = '.'; private const string GroupElementStringFormat = "X4"; private const NumberStyles GroupElementNumberStyles = NumberStyles.HexNumber; + private const StringComparison _stringComparison = StringComparison.InvariantCulture; private static readonly IFormatProvider FormatProvider = CultureInfo.InvariantCulture; private readonly bool _writeTagsAsKeywords = false; private readonly DicomTag[] _dicomTags; @@ -38,7 +39,7 @@ public class DicomAttributeId typeof(DicomTag).GetFields(BindingFlags.Static | BindingFlags.Public) .Select(x => x.GetValue(null) as DicomTag) .Where(x => !string.IsNullOrWhiteSpace(x?.DictionaryEntry?.Keyword)) - .Distinct(new KeywordComparer()) + .Distinct(new KeywordComparer(_stringComparison)) .ToDictionary(x => x.DictionaryEntry.Keyword, x => x); [JsonConstructor] @@ -78,14 +79,23 @@ public DicomTag GetDicomTag(int index) public override int GetHashCode() { - return AttributeId.GetHashCode(StringComparison.Ordinal); + return AttributeId.GetHashCode(_stringComparison); } public override bool Equals(object obj) { - if (obj is DicomAttributeId instance) + if (obj is DicomAttributeId instance && instance.Length == Length) { - return AttributeId.Equals(instance.AttributeId, StringComparison.Ordinal); + // Validate all the DICOM tag keywords are the same + for (var i = 0; i < _dicomTags.Length; i++) + { + if (!string.Equals(_dicomTags[i].DictionaryEntry.Keyword, instance._dicomTags[i].DictionaryEntry.Keyword, _stringComparison)) + { + return false; + } + } + + return true; } return false; @@ -106,12 +116,15 @@ private static DicomTag[] DeserializeAttributeId(string attributeId) ushort.TryParse(split[i].Substring(4, 4), GroupElementNumberStyles, FormatProvider, out ushort element)) { result[i] = new DicomTag(group, element); - continue; } - - // Otherwise, attempt to look-up the DICOM tag keyword. - EnsureArg.IsTrue(KnownKeywordDicomTags.ContainsKey(split[i]), split[i]); - result[i] = KnownKeywordDicomTags[split[i]]; + else if (KnownKeywordDicomTags.TryGetValue(split[i], out DicomTag dicomTag)) + { + result[i] = dicomTag; + } + else + { + throw new FormatException($"Could not fromat the attribute '{split[i]}' from the provided attribute string '{attributeId}' to a known DICOM tag."); + } } return result; @@ -134,22 +147,35 @@ private void Validate() // Validate all but the leaf tag are sequence elements for (var i = 0; i < Length - 1; i++) { - EnsureArg.IsTrue(_dicomTags[i].DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); + if (!_dicomTags[i].DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)) + { + throw new FormatException($"All tags but the last must be sequence elements. The provided DICOM tag {_dicomTags[i].DictionaryEntry.Keyword} does not have a 'sequence' value type."); + } } - EnsureArg.IsFalse(InstanceDicomTag.DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)); + if (InstanceDicomTag.DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)) + { + throw new FormatException($"The last DICOM tag must not have the value representation 'sequence'. The provided DICOM tag {InstanceDicomTag.DictionaryEntry.Keyword} is known as having a 'sequence' value type."); + } } private class KeywordComparer : IEqualityComparer { + private readonly StringComparison _comparisonType; + + public KeywordComparer(StringComparison comparisonType) + { + _comparisonType = comparisonType; + } + public bool Equals(DicomTag dicomTag1, DicomTag dicomTag2) { - return dicomTag1.DictionaryEntry.Keyword == dicomTag2.DictionaryEntry.Keyword; + return string.Equals(dicomTag1.DictionaryEntry.Keyword, dicomTag2.DictionaryEntry.Keyword, _comparisonType); } public int GetHashCode(DicomTag dicomTag) { - return dicomTag.DictionaryEntry.Keyword.GetHashCode(StringComparison.InvariantCulture); + return dicomTag.DictionaryEntry.Keyword.GetHashCode(_comparisonType); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs index 35f919d8e7..60248f56d2 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs @@ -16,7 +16,7 @@ public class DicomCosmosConfiguration /// The StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID will be indexed automatically. /// TODO: We should validate that the attributes defined here have a sensible value representation. /// - public IEnumerable QueryAttributes { get; } = new[] + public HashSet QueryAttributes { get; } = new HashSet() { new DicomAttributeId(DicomTag.PatientName), new DicomAttributeId(DicomTag.PatientID), diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs index abe0c0c06c..c33e340752 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -45,56 +45,53 @@ public SqlQuerySpec BuildSeriesQuerySpec( public SqlQuerySpec BuildInstanceQuerySpec( int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) { - // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. - SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); - - string whereClause = GenerateWhereClause( + return GenerateQuerySpec( + InstanceSqlQuerySearchFormat, + offset, + limit, query, - (tag, parameter) => - { - sqlParameterCollection.Add(parameter); - return $"ARRAY_CONTAINS(f.{nameof(QueryInstance.IndexedAttributes)}[\"{tag.AttributeId}\"], {parameter.Name})"; - }); - - return new SqlQuerySpec(string.Format(_stringFormatProvider, InstanceSqlQuerySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + (tag, parameter) => $"ARRAY_CONTAINS(f.{nameof(QueryInstance.IndexedAttributes)}[\"{tag.AttributeId}\"], {parameter.Name})"); } private SqlQuerySpec BuildSeriesLevelQuerySpec( string querySearchFormat, int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query) { - // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. - SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); - - string whereClause = GenerateWhereClause( + return GenerateQuerySpec( + querySearchFormat, + offset, + limit, query, - (tag, parameter) => - { - sqlParameterCollection.Add(parameter); - return $"ARRAY_CONTAINS(c.{nameof(QuerySeriesDocument.DistinctIndexedAttributes)}[\"{tag.AttributeId}\"].{nameof(AttributeValues.Values)}, {parameter.Name})"; - }); - - return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + (tag, parameter) => $"ARRAY_CONTAINS(c.{nameof(QuerySeriesDocument.DistinctIndexedAttributes)}[\"{tag.AttributeId}\"].{nameof(AttributeValues.Values)}, {parameter.Name})"); } - private string GenerateWhereClause(IEnumerable<(DicomAttributeId Attribute, string Value)> query, Func createQueryItem) + private SqlQuerySpec GenerateQuerySpec( + string querySearchFormat, + int offset, + int limit, + IEnumerable<(DicomAttributeId Attribute, string Value)> query, + Func createQueryItem) { - // If a null or empty query collection we should provide an empty string for the WHERE clause. - if (query == null || !query.Any()) - { - return string.Empty; - } + // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. + SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); var parameterNameIndex = 1; var queryItems = new List(); - foreach ((DicomAttributeId attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + if (query != null && _dicomConfiguration.QueryAttributes != null) { - var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); - queryItems.Add(createQueryItem(attribute, new SqlParameter { Name = parameterName, Value = value })); + foreach ((DicomAttributeId attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + { + var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); + var sqlParameter = new SqlParameter { Name = parameterName, Value = value }; + + sqlParameterCollection.Add(sqlParameter); + queryItems.Add(createQueryItem(attribute, sqlParameter)); + } } // Now construct the WHERE query joining each item with an AND. - return $"WHERE {string.Join(" AND ", queryItems)}"; + var whereClause = queryItems.Count > 0 ? $"WHERE {string.Join(" AND ", queryItems)}" : string.Empty; + return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); } private static SqlParameterCollection CreateQueryParameterCollection(int offset, int limit) diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index d179ccd470..9a0268e842 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -72,7 +72,7 @@ public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToke await _documentClient.ThrowDataStoreException( async (documentClient) => { - QuerySeriesDocument document = await documentClient.GetorCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); + QuerySeriesDocument document = await documentClient.GetOrCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); var instance = QueryInstance.Create(dicomDataset, _dicomConfiguration.QueryAttributes); @@ -203,7 +203,7 @@ private FeedOptions CreateCrossPartitionFeedOptions() return feedOptions; } - private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpec, string studyInstanceUID, CancellationToken cancellationToken) + private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpec, string studyInstanceUID, CancellationToken cancellationToken) { // If the study instance UID is provided we can run the query against a specific partition. FeedOptions feedOptions = string.IsNullOrWhiteSpace(studyInstanceUID) ? CreateCrossPartitionFeedOptions() : CreateFeedOptions(studyInstanceUID); @@ -211,12 +211,19 @@ private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpe .CreateDocumentQuery(_collectionUri, sqlQuerySpec, feedOptions) .AsDocumentQuery(); - return await _documentClient.ThrowDataStoreException( - async (documentClient) => - { - IEnumerable results = await documentQuery.ExecuteQueryUntilCompleteAsync(cancellationToken); - return new QueryResult(documentQuery.HasMoreResults, results.ToArray()); - }); + var results = new List(); + while (documentQuery.HasMoreResults) + { + // Each loop has its own retry handler so we don't retry the entire query on failure. + await _documentClient.ThrowDataStoreException( + async (documentClient) => + { + FeedResponse nextResults = await documentQuery.ExecuteNextAsync(cancellationToken); + results.AddRange(nextResults); + }); + } + + return new QueryResult(documentQuery.HasMoreResults, results); } private IAsyncPolicy CreatePreConditionFailedRetryPolicy() diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs index 4bb9106b2d..83f29af90b 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -4,14 +4,12 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; -using Microsoft.Azure.Documents.Linq; using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; using Newtonsoft.Json; using Polly; @@ -62,20 +60,7 @@ public static async Task ThrowDataStoreException(this IDocumentClient documentCl } } - public static async Task> ExecuteQueryUntilCompleteAsync(this IDocumentQuery documentQuery, CancellationToken cancellationToken) - { - var results = new List(); - - while (documentQuery.HasMoreResults) - { - FeedResponse nextResults = await documentQuery.ExecuteNextAsync(cancellationToken); - results.AddRange(nextResults); - } - - return results; - } - - public static async Task GetorCreateDocumentAsync( + public static async Task GetOrCreateDocumentAsync( this IDocumentClient documentClient, string databaseId, string collectionId, @@ -126,7 +111,7 @@ public static async Task CreateOrGetDocumentAsync( catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.Conflict) { // Attempt to read the document as it already exists. - return await documentClient.GetorCreateDocumentAsync(databaseId, collectionId, documentId, requestOptions, defaultValue, cancellationToken); + return await documentClient.GetOrCreateDocumentAsync(databaseId, collectionId, documentId, requestOptions, defaultValue, cancellationToken); } } } diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs index 8a5fd0208e..89e09c0a53 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs @@ -60,7 +60,7 @@ public async Task GivenANonExistentDocument_OnGetOrCreate_DocumentIsCreated() var expectedDocument = new TestDocument() { Id = Guid.NewGuid().ToString(), PartitionKey = Guid.NewGuid().ToString() }; var requestOptions = new RequestOptions() { PartitionKey = new PartitionKey(expectedDocument.PartitionKey) }; - TestDocument actualDocument = await _documentClient.GetorCreateDocumentAsync( + TestDocument actualDocument = await _documentClient.GetOrCreateDocumentAsync( _fixture.DatabaseId, _fixture.CollectionId, expectedDocument.Id, From a6cb0ee2577ec9666d3ea9bbacf4b28349809d70 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Wed, 26 Jun 2019 13:35:11 +0100 Subject: [PATCH 19/22] Fixed typo in test build script file --- build/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/test.yml b/build/test.yml index 3838f95868..21e5096bd4 100644 --- a/build/test.yml +++ b/build/test.yml @@ -22,7 +22,7 @@ steps: displayName: 'Start Azurite Storage Emulator (Windows)' # Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing - # if performed on windows. + # is performed on windows. - powershell: | $cosmosemulator=[Environment]::GetEnvironmentVariable("ProgramFiles") + "\Azure Cosmos DB Emulator\CosmosDB.Emulator.exe" & $cosmosemulator /noui From a1fce50e446d9611af118edd969a048e11c98ef4 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 2 Jul 2019 11:42:45 +0100 Subject: [PATCH 20/22] Removed hard-coded paramter names from query strings and shortened unique identifier names --- ....cs => DicomAttributeIdExtensionsTests.cs} | 13 ++++- .../Features/Persistence/DicomAttributeId.cs | 5 +- .../Persistence/DicomAttributeIdExtensions.cs | 33 ++++++++----- .../Storage/Documents/QueryInstanceTests.cs | 10 ++-- .../Documents/QuerySeriesDocumentTests.cs | 10 ++-- .../Features/Storage/CosmosQueryBuilder.cs | 10 ++-- .../Features/Storage/DicomCosmosDataStore.cs | 10 ++-- .../Storage/DocumentClientExtensions.cs | 4 +- .../Storage/Documents/QueryInstance.cs | 24 +++++----- .../Storage/Documents/QuerySeriesDocument.cs | 47 ++++++++++--------- 10 files changed, 95 insertions(+), 71 deletions(-) rename src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/{DicomAttributeIdExtensions.cs => DicomAttributeIdExtensionsTests.cs} (88%) diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensionsTests.cs similarity index 88% rename from src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs rename to src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensionsTests.cs index 30f539dccc..108ce46e12 100644 --- a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensions.cs +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensionsTests.cs @@ -10,14 +10,23 @@ namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence { - public class DicomAttributeIdExtensions + public class DicomAttributeIdExtensionsTests { [Fact] public void GivenDatasetAndInvalidParameters_WhenFetchedAttributeValues_ArgumentExceptionIsThrown() { var dicomDataset = new DicomDataset(); Assert.Throws(() => dicomDataset.TryGetValues((DicomAttributeId)null, out object[] values)); - Assert.Throws(() => dicomDataset.TryGetValues(new DicomAttributeId(DicomTag.StudyDate), out object[] values, -1)); + } + + [Fact] + public void GivenDataset_WhenFetchingMissingAttribute_FalseIsReturned() + { + var dicomDataset = new DicomDataset(); + var result = dicomDataset.TryGetValues(new DicomAttributeId(DicomTag.PatientName), out object[] values); + + Assert.False(result); + Assert.Null(values); } [Fact] diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs index 52425d3381..5ac4cb3778 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs @@ -44,8 +44,11 @@ public class DicomAttributeId [JsonConstructor] public DicomAttributeId(string attributeId) - : this(DeserializeAttributeId(attributeId)) { + _dicomTags = DeserializeAttributeId(attributeId); + Validate(); + + AttributeId = attributeId; } public DicomAttributeId(params DicomTag[] dicomTags) diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs index e63b4bd9c8..64f11363e3 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs @@ -11,27 +11,40 @@ namespace Microsoft.Health.Dicom.Core.Features.Persistence { public static class DicomAttributeIdExtensions { - public static bool TryGetValues(this DicomDataset dicomDataset, DicomAttributeId attributeId, out TItem[] values, int startAttributeIndex = 0) + public static bool TryGetValues(this DicomDataset dicomDataset, DicomAttributeId attributeId, out TItem[] values) { EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); EnsureArg.IsNotNull(attributeId, nameof(attributeId)); - EnsureArg.IsGte(startAttributeIndex, 0, nameof(startAttributeIndex)); - var nestedValues = new List(); + var result = new List(); + dicomDataset.TryAppendValuesToList(result, attributeId); + + values = result.Count > 0 ? result.ToArray() : null; + return result.Count > 0; + } + + private static void TryAppendValuesToList( + this DicomDataset dicomDataset, + List list, + DicomAttributeId attributeId, + int startAttributeIndex = 0) + { + EnsureArg.IsGte(startAttributeIndex, 0, nameof(startAttributeIndex)); // Handle for now sequence elements (last item in attribute ID). if (startAttributeIndex == attributeId.Length - 1) { - // Now first validate this DICOM tag exists in the dataset + // Now first validate this DICOM tag exists in the dataset. + // Note: GetValueCount throws exception when the tag does not exist. if (dicomDataset.TryGetValue(attributeId.InstanceDicomTag, 0, out TItem firstItem)) { - nestedValues.Add(firstItem); + list.Add(firstItem); for (int i = 1; i < dicomDataset.GetValueCount(attributeId.InstanceDicomTag); i++) { if (dicomDataset.TryGetValue(attributeId.InstanceDicomTag, i, out TItem value)) { - nestedValues.Add(value); + list.Add(value); } } } @@ -40,15 +53,9 @@ public static bool TryGetValues(this DicomDataset dicomDataset, DicomAttr { foreach (DicomDataset sequenceDataset in dicomSequence.Items) { - if (sequenceDataset.TryGetValues(attributeId, out TItem[] sequenceValues, startAttributeIndex + 1)) - { - nestedValues.AddRange(sequenceValues); - } + sequenceDataset.TryAppendValuesToList(list, attributeId, startAttributeIndex + 1); } } - - values = nestedValues.Count > 0 ? nestedValues.ToArray() : null; - return nestedValues.Count > 0; } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs index db280dfbc2..fcc1987d05 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs @@ -57,12 +57,12 @@ public void GivenDicomTagsToIndex_WhenCreatingQueryInstance_TagsAreExtractedAndI dicomDataset, new[] { patientNameAttribute, studyDateAttribute, referringPhysicianNameAttribute, studyTimeAttribute }); Assert.NotNull(instance); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID), instance.SopInstanceUID); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID), instance.InstanceUID); - Assert.Equal(3, instance.IndexedAttributes.Count); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.IndexedAttributes[patientNameAttribute.AttributeId][0]); - Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.IndexedAttributes[studyDateAttribute.AttributeId][0]); - Assert.Equal(referringPhysicianName, instance.IndexedAttributes[referringPhysicianNameAttribute.AttributeId][0]); + Assert.Equal(3, instance.Attributes.Count); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.Attributes[patientNameAttribute.AttributeId][0]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.Attributes[studyDateAttribute.AttributeId][0]); + Assert.Equal(referringPhysicianName, instance.Attributes[referringPhysicianNameAttribute.AttributeId][0]); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs index 62730c92fc..398357c345 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -89,14 +89,14 @@ public void GivenSeriesDocument_WhenSerialized_IsDeserializedCorrectly() Assert.Equal(document.Id, deserialized.Id); Assert.Equal(document.PartitionKey, deserialized.PartitionKey); Assert.Equal(document.ETag, deserialized.ETag); - Assert.Equal(document.StudyInstanceUID, deserialized.StudyInstanceUID); - Assert.Equal(document.SeriesInstanceUID, deserialized.SeriesInstanceUID); + Assert.Equal(document.StudyUID, deserialized.StudyUID); + Assert.Equal(document.SeriesUID, deserialized.SeriesUID); Assert.Equal(document.Instances.Count, deserialized.Instances.Count); QueryInstance deserializedFirstInstance = deserialized.Instances.First(); - Assert.Equal(deserializedFirstInstance.SopInstanceUID, deserializedFirstInstance.SopInstanceUID); - Assert.Equal(deserializedFirstInstance.IndexedAttributes.Count, deserializedFirstInstance.IndexedAttributes.Count); - Assert.Equal(deserializedFirstInstance.IndexedAttributes[patientNameAttributeId.AttributeId], deserializedFirstInstance.IndexedAttributes[patientNameAttributeId.AttributeId]); + Assert.Equal(deserializedFirstInstance.InstanceUID, deserializedFirstInstance.InstanceUID); + Assert.Equal(deserializedFirstInstance.Attributes.Count, deserializedFirstInstance.Attributes.Count); + Assert.Equal(deserializedFirstInstance.Attributes[patientNameAttributeId.AttributeId], deserializedFirstInstance.Attributes[patientNameAttributeId.AttributeId]); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs index c33e340752..064b3b75da 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -20,9 +20,9 @@ internal class CosmosQueryBuilder private const string OffsetParameterName = "@offset"; private const string LimitParameterName = "@limit"; private const string ItemParameterNameFormat = "@item{0}"; - private const string StudySqlQuerySearchFormat = "SELECT DISTINCT c.StudyInstanceUID FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; - private const string SeriesSqlQuerySearchFormat = "SELECT VALUE {{ \"StudyInstanceUID\": c.StudyInstanceUID, \"SeriesInstanceUID\": c.SeriesInstanceUID }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; - private const string InstanceSqlQuerySearchFormat = "SELECT VALUE {{ \"StudyInstanceUID\": c.StudyInstanceUID, \"SeriesInstanceUID\": c.SeriesInstanceUID, \"SOPInstanceUID\": f.SopInstanceUID }} FROM c JOIN f in c.Instances {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string StudySqlQuerySearchFormat = "SELECT DISTINCT VALUE {{ \"" + nameof(DicomStudy.StudyInstanceUID) + "\": c." + nameof(QuerySeriesDocument.StudyUID) + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string SeriesSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomSeries.StudyInstanceUID) + "\": c." + nameof(QuerySeriesDocument.StudyUID) + ", \"" + nameof(DicomSeries.SeriesInstanceUID) + "\": c." + nameof(QuerySeriesDocument.SeriesUID) + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string InstanceSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomInstance.StudyInstanceUID) + "\": c." + nameof(QuerySeriesDocument.StudyUID) + ", \"" + nameof(DicomInstance.SeriesInstanceUID) + "\": c." + nameof(QuerySeriesDocument.SeriesUID) + ", \"" + nameof(DicomInstance.SopInstanceUID) + "\": f." + nameof(QueryInstance.InstanceUID) + " }} FROM c JOIN f in c." + nameof(QuerySeriesDocument.Instances) + " {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; private readonly DicomCosmosConfiguration _dicomConfiguration; private readonly IFormatProvider _stringFormatProvider; @@ -50,7 +50,7 @@ public SqlQuerySpec BuildInstanceQuerySpec( offset, limit, query, - (tag, parameter) => $"ARRAY_CONTAINS(f.{nameof(QueryInstance.IndexedAttributes)}[\"{tag.AttributeId}\"], {parameter.Name})"); + (tag, parameter) => $"ARRAY_CONTAINS(f.{nameof(QueryInstance.Attributes)}[\"{tag.AttributeId}\"], {parameter.Name})"); } private SqlQuerySpec BuildSeriesLevelQuerySpec( @@ -91,7 +91,7 @@ private SqlQuerySpec GenerateQuerySpec( // Now construct the WHERE query joining each item with an AND. var whereClause = queryItems.Count > 0 ? $"WHERE {string.Join(" AND ", queryItems)}" : string.Empty; - return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause.ToString(_stringFormatProvider)), sqlParameterCollection); + return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause), sqlParameterCollection); } private static SqlParameterCollection CreateQueryParameterCollection(int offset, int limit) diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index 9a0268e842..d2527a58e2 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -69,7 +69,7 @@ public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToke // Retry when the pre-condition fails on replace (ETag check). IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); - await _documentClient.ThrowDataStoreException( + await _documentClient.CatchClientExceptionAndThrowDataStoreException( async (documentClient) => { QuerySeriesDocument document = await documentClient.GetOrCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); @@ -126,7 +126,7 @@ public async Task> DeleteSeriesIndexAsync( // Retry when the pre-condition fails on delete. IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); - return await _documentClient.ThrowDataStoreException( + return await _documentClient.CatchClientExceptionAndThrowDataStoreException( async (documentClient) => { DocumentResponse response = @@ -139,7 +139,7 @@ public async Task> DeleteSeriesIndexAsync( await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); return response.Document.Instances - .Select(x => new DicomInstance(response.Document.StudyInstanceUID, response.Document.SeriesInstanceUID, x.SopInstanceUID)) + .Select(x => new DicomInstance(response.Document.StudyUID, response.Document.SeriesUID, x.InstanceUID)) .ToArray(); }, retryPolicy); @@ -156,7 +156,7 @@ public async Task DeleteInstanceIndexAsync(string studyInstanceUID, string serie RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); - await _documentClient.ThrowDataStoreException( + await _documentClient.CatchClientExceptionAndThrowDataStoreException( async (documentClient) => { DocumentResponse response = @@ -215,7 +215,7 @@ private async Task> ExecuteQueryAsync(SqlQuerySpec while (documentQuery.HasMoreResults) { // Each loop has its own retry handler so we don't retry the entire query on failure. - await _documentClient.ThrowDataStoreException( + await _documentClient.CatchClientExceptionAndThrowDataStoreException( async (documentClient) => { FeedResponse nextResults = await documentQuery.ExecuteNextAsync(cancellationToken); diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs index 83f29af90b..2eeb186096 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -18,7 +18,7 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage { public static class DocumentClientExtensions { - public static async Task ThrowDataStoreException(this IDocumentClient documentClient, Func> action, IAsyncPolicy retryPolicy = null) + public static async Task CatchClientExceptionAndThrowDataStoreException(this IDocumentClient documentClient, Func> action, IAsyncPolicy retryPolicy = null) { EnsureArg.IsNotNull(documentClient, nameof(documentClient)); EnsureArg.IsNotNull(action, nameof(action)); @@ -38,7 +38,7 @@ public static async Task ThrowDataStoreException(this IDocumentClient docu } } - public static async Task ThrowDataStoreException(this IDocumentClient documentClient, Func action, IAsyncPolicy retryPolicy = null) + public static async Task CatchClientExceptionAndThrowDataStoreException(this IDocumentClient documentClient, Func action, IAsyncPolicy retryPolicy = null) { EnsureArg.IsNotNull(documentClient, nameof(documentClient)); EnsureArg.IsNotNull(action, nameof(action)); diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs index a8b5f85a9f..964481866e 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -16,37 +16,37 @@ namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents internal class QueryInstance { [JsonConstructor] - public QueryInstance(string sopInstanceUID, IDictionary indexedAttributes) - : this(sopInstanceUID) + public QueryInstance(string instanceUID, IDictionary attributes) + : this(instanceUID) { - EnsureArg.IsNotNull(indexedAttributes, nameof(indexedAttributes)); + EnsureArg.IsNotNull(attributes, nameof(attributes)); - IndexedAttributes = indexedAttributes; + Attributes = attributes; } - private QueryInstance(string sopInstanceUID) + private QueryInstance(string instanceUID) { - EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(sopInstanceUID), nameof(sopInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(instanceUID), nameof(instanceUID)); - SopInstanceUID = sopInstanceUID; + InstanceUID = instanceUID; } - public string SopInstanceUID { get; } + public string InstanceUID { get; } - public IDictionary IndexedAttributes { get; } = new Dictionary(); + public IDictionary Attributes { get; } = new Dictionary(); public override int GetHashCode() { // Note: We override Equals() and GetHashCode() so if this class is added to a HashSet // we only compare on SopInstanceUID. - return SopInstanceUID.GetHashCode(StringComparison.Ordinal); + return InstanceUID.GetHashCode(StringComparison.Ordinal); } public override bool Equals(object obj) { if (obj is QueryInstance instance) { - return SopInstanceUID.Equals(instance.SopInstanceUID, StringComparison.Ordinal); + return InstanceUID.Equals(instance.InstanceUID, StringComparison.Ordinal); } return false; @@ -64,7 +64,7 @@ public static QueryInstance Create(DicomDataset dicomDataset, IEnumerable + /// The seperator between the Study and Series instance identifiers when creating the document identifier. + /// + private const char DocumentIdSeperator = '_'; + /// /// Lists the characters that are not valid for a Cosmos resource identifier. /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.documents.resource.id?view=azure-dotnet /// private static readonly Regex DocumentIdRegex = new Regex("[^?/\\#]", RegexOptions.Singleline | RegexOptions.Compiled); - public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) + public QuerySeriesDocument(string studyUID, string seriesUID) { - EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); - EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); - EnsureArg.IsFalse(studyInstanceUID == seriesInstanceUID, nameof(seriesInstanceUID)); - - StudyInstanceUID = studyInstanceUID; - SeriesInstanceUID = seriesInstanceUID; - Id = GetDocumentId(studyInstanceUID, seriesInstanceUID); - PartitionKey = GetPartitionKey(studyInstanceUID); + EnsureArg.IsNotNullOrWhiteSpace(studyUID, nameof(studyUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesUID, nameof(seriesUID)); + EnsureArg.IsFalse(studyUID == seriesUID, nameof(seriesUID)); + + StudyUID = studyUID; + SeriesUID = seriesUID; + Id = GetDocumentId(studyUID, seriesUID); + PartitionKey = GetPartitionKey(studyUID); } [JsonProperty(KnownDocumentProperties.Id)] @@ -41,9 +46,9 @@ public QuerySeriesDocument(string studyInstanceUID, string seriesInstanceUID) [JsonProperty(KnownDocumentProperties.ETag)] public string ETag { get; set; } - public string StudyInstanceUID { get; } + public string StudyUID { get; } - public string SeriesInstanceUID { get; } + public string SeriesUID { get; } public HashSet Instances { get; } = new HashSet(); @@ -55,7 +60,7 @@ public IDictionary DistinctIndexedAttributes foreach (QueryInstance instance in Instances) { - foreach ((string key, object[] values) in instance.IndexedAttributes) + foreach ((string key, object[] values) in instance.Attributes) { if (!result.ContainsKey(key)) { @@ -73,13 +78,13 @@ public IDictionary DistinctIndexedAttributes } } - public static string GetDocumentId(string studyInstanceUID, string seriesInstanceUID) + public static string GetDocumentId(string studyUID, string seriesUID) { - EnsureArg.IsFalse(studyInstanceUID == seriesInstanceUID); - EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyInstanceUID)); - EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(seriesInstanceUID)); + EnsureArg.IsFalse(studyUID == seriesUID); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(seriesUID)); - string documentId = studyInstanceUID + seriesInstanceUID; + string documentId = $"{studyUID}{DocumentIdSeperator}{seriesUID}"; // Double safety for the document identifier. If the study and series are valid identifiers, the document ID 'should' conform. EnsureArg.IsTrue(DocumentIdRegex.IsMatch(documentId)); @@ -87,11 +92,11 @@ public static string GetDocumentId(string studyInstanceUID, string seriesInstanc return documentId; } - public static string GetPartitionKey(string studyInstanceUID) + public static string GetPartitionKey(string studyUID) { - EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyInstanceUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyUID)); - return studyInstanceUID; + return studyUID; } public bool AddInstance(QueryInstance instance) @@ -103,7 +108,7 @@ public bool AddInstance(QueryInstance instance) public bool RemoveInstance(string sopInstanceUID) { EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); - return Instances.RemoveWhere(x => x.SopInstanceUID == sopInstanceUID) > 0; + return Instances.RemoveWhere(x => x.InstanceUID == sopInstanceUID) > 0; } } } From 9f62bbfaf5a5aee4cb4810547d263103b9bedc6d Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Tue, 2 Jul 2019 13:14:04 +0100 Subject: [PATCH 21/22] Normalized document casing for Cosmos documents --- .../Storage/Documents/AttributeValuesTests.cs | 20 +++++++++---------- .../Documents/QuerySeriesDocumentTests.cs | 2 +- .../Features/Storage/CosmosQueryBuilder.cs | 10 +++++----- .../Storage/Documents/AttributeValues.cs | 9 +++++---- .../Storage/Documents/DocumentProperties.cs | 20 +++++++++++++++++++ .../Storage/Documents/QueryInstance.cs | 2 ++ .../Storage/Documents/QuerySeriesDocument.cs | 6 +++++- 7 files changed, 48 insertions(+), 21 deletions(-) create mode 100644 src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs index 9680cdee5a..30c1054228 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs @@ -32,8 +32,8 @@ public void GivenACollectionOfDifferentTypes_WhenAddedToAttributeValues_AreStore Assert.True(attributeValues.Add(maxDateTime)); - Assert.Equal(minDateTime, attributeValues.MinDateTimeValue); - Assert.Equal(maxDateTime, attributeValues.MaxDateTimeValue); + Assert.Equal(minDateTime, attributeValues.MinDateTime); + Assert.Equal(maxDateTime, attributeValues.MaxDateTime); } [Fact] @@ -47,14 +47,14 @@ public void GivenExistingItem_WhenAddedToAttributeValues_IsNotAdded() Assert.True(attributeValues.Add("HelloWorld")); Assert.False(attributeValues.Add("HelloWorld")); - Assert.Null(new AttributeValues().MinDateTimeValue); - Assert.Null(new AttributeValues().MaxDateTimeValue); + Assert.Null(new AttributeValues().MinDateTime); + Assert.Null(new AttributeValues().MaxDateTime); Assert.True(attributeValues.Add(new DateTime(2019, 6, 22))); Assert.False(attributeValues.Add(new DateTime(2019, 6, 22))); - Assert.NotNull(attributeValues.MinDateTimeValue); - Assert.NotNull(attributeValues.MaxDateTimeValue); + Assert.NotNull(attributeValues.MinDateTime); + Assert.NotNull(attributeValues.MaxDateTime); Assert.False(attributeValues.Add(null)); } @@ -64,8 +64,8 @@ public void GivenAttributeValues_WhenCreatedWithInvalidParameters_ArgumentExcept { Assert.Throws(() => new AttributeValues(null)); Assert.NotNull(new AttributeValues().Values); - Assert.Null(new AttributeValues().MinDateTimeValue); - Assert.Null(new AttributeValues().MaxDateTimeValue); + Assert.Null(new AttributeValues().MinDateTime); + Assert.Null(new AttributeValues().MaxDateTime); } [Fact] @@ -86,8 +86,8 @@ public void GivenAttributeValues_WhenSerialized_DeserializedCorrectly() var json = JsonConvert.SerializeObject(attributeValues); AttributeValues deserialized = JsonConvert.DeserializeObject(json); - Assert.Equal(attributeValues.MinDateTimeValue, deserialized.MinDateTimeValue); - Assert.Equal(attributeValues.MaxDateTimeValue, deserialized.MaxDateTimeValue); + Assert.Equal(attributeValues.MinDateTime, deserialized.MinDateTime); + Assert.Equal(attributeValues.MaxDateTime, deserialized.MaxDateTime); } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs index 398357c345..dea032a602 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -57,7 +57,7 @@ public void GivenExistingInstance_WhenAddingToInstancesHashSet_IsNotAdded() Assert.True(document.AddInstance(instanceDocument1)); Assert.False(document.AddInstance(instanceDocument2)); - Assert.Equal(testPatientName, document.DistinctIndexedAttributes[patientNameAttributeId.AttributeId].Values.First()); + Assert.Equal(testPatientName, document.DistinctAttributes[patientNameAttributeId.AttributeId].Values.First()); Assert.Throws(() => document.RemoveInstance(null)); Assert.Throws(() => document.RemoveInstance(string.Empty)); diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs index 064b3b75da..d995b59d2d 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -20,9 +20,9 @@ internal class CosmosQueryBuilder private const string OffsetParameterName = "@offset"; private const string LimitParameterName = "@limit"; private const string ItemParameterNameFormat = "@item{0}"; - private const string StudySqlQuerySearchFormat = "SELECT DISTINCT VALUE {{ \"" + nameof(DicomStudy.StudyInstanceUID) + "\": c." + nameof(QuerySeriesDocument.StudyUID) + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; - private const string SeriesSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomSeries.StudyInstanceUID) + "\": c." + nameof(QuerySeriesDocument.StudyUID) + ", \"" + nameof(DicomSeries.SeriesInstanceUID) + "\": c." + nameof(QuerySeriesDocument.SeriesUID) + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; - private const string InstanceSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomInstance.StudyInstanceUID) + "\": c." + nameof(QuerySeriesDocument.StudyUID) + ", \"" + nameof(DicomInstance.SeriesInstanceUID) + "\": c." + nameof(QuerySeriesDocument.SeriesUID) + ", \"" + nameof(DicomInstance.SopInstanceUID) + "\": f." + nameof(QueryInstance.InstanceUID) + " }} FROM c JOIN f in c." + nameof(QuerySeriesDocument.Instances) + " {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string StudySqlQuerySearchFormat = "SELECT DISTINCT VALUE {{ \"" + nameof(DicomStudy.StudyInstanceUID) + "\": c." + DocumentProperties.StudyInstanceUID + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string SeriesSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomSeries.StudyInstanceUID) + "\": c." + DocumentProperties.StudyInstanceUID + ", \"" + nameof(DicomSeries.SeriesInstanceUID) + "\": c." + DocumentProperties.SeriesInstanceUID + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string InstanceSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomInstance.StudyInstanceUID) + "\": c." + DocumentProperties.StudyInstanceUID + ", \"" + nameof(DicomInstance.SeriesInstanceUID) + "\": c." + DocumentProperties.SeriesInstanceUID + ", \"" + nameof(DicomInstance.SopInstanceUID) + "\": f." + DocumentProperties.SopInstanceUID + " }} FROM c JOIN f in c." + DocumentProperties.Instances + " {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; private readonly DicomCosmosConfiguration _dicomConfiguration; private readonly IFormatProvider _stringFormatProvider; @@ -50,7 +50,7 @@ public SqlQuerySpec BuildInstanceQuerySpec( offset, limit, query, - (tag, parameter) => $"ARRAY_CONTAINS(f.{nameof(QueryInstance.Attributes)}[\"{tag.AttributeId}\"], {parameter.Name})"); + (tag, parameter) => $"ARRAY_CONTAINS(f.{DocumentProperties.Attributes}[\"{tag.AttributeId}\"], {parameter.Name})"); } private SqlQuerySpec BuildSeriesLevelQuerySpec( @@ -61,7 +61,7 @@ private SqlQuerySpec BuildSeriesLevelQuerySpec( offset, limit, query, - (tag, parameter) => $"ARRAY_CONTAINS(c.{nameof(QuerySeriesDocument.DistinctIndexedAttributes)}[\"{tag.AttributeId}\"].{nameof(AttributeValues.Values)}, {parameter.Name})"); + (tag, parameter) => $"ARRAY_CONTAINS(c.{DocumentProperties.DistinctAttributes}[\"{tag.AttributeId}\"][\"{DocumentProperties.Values}\"], {parameter.Name})"); } private SqlQuerySpec GenerateQuerySpec( diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs index e317071938..346b7f742e 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs @@ -30,17 +30,18 @@ public AttributeValues(HashSet values) /// We pre-calculate this to support range queries on DateTime values. /// This value does not need to be serialized if the array does not contain any DateTime values. /// - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public DateTime? MinDateTimeValue => Values.Where(x => x is DateTime).Min(x => (DateTime?)x); + [JsonProperty(DocumentProperties.MinimumDateTimeValue, NullValueHandling = NullValueHandling.Ignore)] + public DateTime? MinDateTime => Values.Where(x => x is DateTime).Min(x => (DateTime?)x); /// /// Gets the maximum date time value from the array of values. /// We pre-calculate this to support range queries on DateTime values. /// This value does not need to be serialized if the array does not contain any DateTime values. /// - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public DateTime? MaxDateTimeValue => Values.Where(x => x is DateTime).Max(x => (DateTime?)x); + [JsonProperty(DocumentProperties.MaximumDateTimeValue, NullValueHandling = NullValueHandling.Ignore)] + public DateTime? MaxDateTime => Values.Where(x => x is DateTime).Max(x => (DateTime?)x); + [JsonProperty(DocumentProperties.Values)] public HashSet Values { get; } = new HashSet(); public bool Add(object value) => value == null ? false : Values.Add(value); diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs new file mode 100644 index 0000000000..9efc970805 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs @@ -0,0 +1,20 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + internal static class DocumentProperties + { + public const string StudyInstanceUID = "studyUID"; + public const string SeriesInstanceUID = "seriesUID"; + public const string SopInstanceUID = "instanceUID"; + public const string Instances = "instances"; + public const string Attributes = "attributes"; + public const string DistinctAttributes = "distinctAttributes"; + public const string Values = "values"; + public const string MinimumDateTimeValue = "minDateTime"; + public const string MaximumDateTimeValue = "maxDateTime"; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs index 964481866e..96ad50c0f3 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -31,8 +31,10 @@ private QueryInstance(string instanceUID) InstanceUID = instanceUID; } + [JsonProperty(DocumentProperties.SopInstanceUID)] public string InstanceUID { get; } + [JsonProperty(DocumentProperties.Attributes)] public IDictionary Attributes { get; } = new Dictionary(); public override int GetHashCode() diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs index 88ddbed8b1..18f60e60c0 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -46,13 +46,17 @@ public QuerySeriesDocument(string studyUID, string seriesUID) [JsonProperty(KnownDocumentProperties.ETag)] public string ETag { get; set; } + [JsonProperty(DocumentProperties.StudyInstanceUID)] public string StudyUID { get; } + [JsonProperty(DocumentProperties.SeriesInstanceUID)] public string SeriesUID { get; } + [JsonProperty(DocumentProperties.Instances)] public HashSet Instances { get; } = new HashSet(); - public IDictionary DistinctIndexedAttributes + [JsonProperty(DocumentProperties.DistinctAttributes)] + public IDictionary DistinctAttributes { get { From 1e712bbcd3f9b465176e6e8541644f04cdcc8bd5 Mon Sep 17 00:00:00 2001 From: Richard Lowe Date: Wed, 3 Jul 2019 10:21:19 +0100 Subject: [PATCH 22/22] Updated PR based on final comments --- .../Features/Persistence/DicomAttributeId.cs | 28 +------------------ .../Storage/DicomCosmosDataStoreTests.cs | 15 ++++++++++ .../Features/Storage/DicomCosmosDataStore.cs | 14 +++++++++- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs index 5ac4cb3778..1b827f76c6 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Reflection; using Dicom; using EnsureThat; using Newtonsoft.Json; @@ -35,12 +34,7 @@ public class DicomAttributeId private static readonly IFormatProvider FormatProvider = CultureInfo.InvariantCulture; private readonly bool _writeTagsAsKeywords = false; private readonly DicomTag[] _dicomTags; - private static readonly IDictionary KnownKeywordDicomTags = - typeof(DicomTag).GetFields(BindingFlags.Static | BindingFlags.Public) - .Select(x => x.GetValue(null) as DicomTag) - .Where(x => !string.IsNullOrWhiteSpace(x?.DictionaryEntry?.Keyword)) - .Distinct(new KeywordComparer(_stringComparison)) - .ToDictionary(x => x.DictionaryEntry.Keyword, x => x); + private static readonly IDictionary KnownKeywordDicomTags = DicomDictionary.Default.ToDictionary(x => x.Keyword, x => x.Tag); [JsonConstructor] public DicomAttributeId(string attributeId) @@ -161,25 +155,5 @@ private void Validate() throw new FormatException($"The last DICOM tag must not have the value representation 'sequence'. The provided DICOM tag {InstanceDicomTag.DictionaryEntry.Keyword} is known as having a 'sequence' value type."); } } - - private class KeywordComparer : IEqualityComparer - { - private readonly StringComparison _comparisonType; - - public KeywordComparer(StringComparison comparisonType) - { - _comparisonType = comparisonType; - } - - public bool Equals(DicomTag dicomTag1, DicomTag dicomTag2) - { - return string.Equals(dicomTag1.DictionaryEntry.Keyword, dicomTag2.DictionaryEntry.Keyword, _comparisonType); - } - - public int GetHashCode(DicomTag dicomTag) - { - return dicomTag.DictionaryEntry.Keyword.GetHashCode(_comparisonType); - } - } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs index 9bcba3f459..bde5d57309 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs @@ -4,8 +4,10 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Reflection; using System.Threading.Tasks; using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; using Microsoft.Extensions.Options; using Microsoft.Health.CosmosDb.Configs; using Microsoft.Health.Dicom.CosmosDb.Config; @@ -49,5 +51,18 @@ public async Task GivenInvalidParameters_WhenCallingAllMethods_ArgumentException await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), null)); await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), string.Empty)); } + + [Fact] + public void GivenDocumentClient_WhenEnablingCrossPartitionQuery_RequiresReflection() + { + // When the feed options natively supports this, remove this test and the extra code in the DicomCosmosDataStore for this hack. + var feedOptions = new FeedOptions(); + + PropertyInfo propertyInfo = feedOptions.GetType().GetProperty("EnableCrossPartitionSkipTake", BindingFlags.NonPublic | BindingFlags.Instance); + propertyInfo.SetValue(feedOptions, Convert.ChangeType(true, propertyInfo.PropertyType)); + + Assert.NotNull(propertyInfo); + Assert.True((bool)propertyInfo.GetValue(feedOptions)); + } } } diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs index d2527a58e2..8c47e9aa48 100644 --- a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -229,6 +229,18 @@ await _documentClient.CatchClientExceptionAndThrowDataStoreException( private IAsyncPolicy CreatePreConditionFailedRetryPolicy() => Policy .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed || ex.StatusCode == HttpStatusCode.TooManyRequests) - .WaitAndRetryForeverAsync(retryIndex => TimeSpan.FromMilliseconds((retryIndex - 1) * _random.Next(200, 500))); + .WaitAndRetryForeverAsync( + (retryIndex, ex, context) => + { + // Use RetryAfter header when TooManyRequests response is returned. + if (ex is DocumentClientException doc && doc.StatusCode == HttpStatusCode.TooManyRequests) + { + return doc.RetryAfter; + } + + // Otherwise, delay retry with some randomness. + return TimeSpan.FromMilliseconds((retryIndex - 1) * _random.Next(200, 500)); + }, + (exception, retryIndex, timespan, context) => Task.CompletedTask); } }