diff --git a/Microsoft.Health.Dicom.sln b/Microsoft.Health.Dicom.sln index 93f8d8963e..1f275ea82a 100644 --- a/Microsoft.Health.Dicom.sln +++ b/Microsoft.Health.Dicom.sln @@ -28,9 +28,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Core EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Api", "src\Microsoft.Health.Dicom.Api\Microsoft.Health.Dicom.Api.csproj", "{B0570D75-E376-44AC-870B-87ECB54F0AE3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.Api.UnitTests", "src\Microsoft.Health.Dicom.Api.UnitTests\Microsoft.Health.Dicom.Api.UnitTests.csproj", "{D7B538E5-8B3B-487C-8F6A-475F80C50DFE}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Api.UnitTests", "src\Microsoft.Health.Dicom.Api.UnitTests\Microsoft.Health.Dicom.Api.UnitTests.csproj", "{D7B538E5-8B3B-487C-8F6A-475F80C50DFE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.Core.UnitTests", "src\Microsoft.Health.Dicom.Core.UnitTests\Microsoft.Health.Dicom.Core.UnitTests.csproj", "{FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Core.UnitTests", "src\Microsoft.Health.Dicom.Core.UnitTests\Microsoft.Health.Dicom.Core.UnitTests.csproj", "{FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.CosmosDb", "src\Microsoft.Health.Dicom.CosmosDb\Microsoft.Health.Dicom.CosmosDb.csproj", "{E78AB378-6CF4-442A-BC34-81A1B7431ECD}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.CosmosDb.UnitTests", "src\Microsoft.Health.Dicom.CosmosDb.UnitTests\Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj", "{FA0B32A4-E623-4B24-902D-BFDC6E42E612}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -78,6 +82,14 @@ Global {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D}.Release|Any CPU.Build.0 = Release|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E78AB378-6CF4-442A-BC34-81A1B7431ECD}.Release|Any CPU.Build.0 = Release|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FA0B32A4-E623-4B24-902D-BFDC6E42E612}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -93,6 +105,8 @@ Global {B0570D75-E376-44AC-870B-87ECB54F0AE3} = {176641B3-297C-4E04-A83D-8F80F80485E8} {D7B538E5-8B3B-487C-8F6A-475F80C50DFE} = {176641B3-297C-4E04-A83D-8F80F80485E8} {FA0484A7-AA0C-4CC6-A75F-1D6B23DD847D} = {176641B3-297C-4E04-A83D-8F80F80485E8} + {E78AB378-6CF4-442A-BC34-81A1B7431ECD} = {176641B3-297C-4E04-A83D-8F80F80485E8} + {FA0B32A4-E623-4B24-902D-BFDC6E42E612} = {176641B3-297C-4E04-A83D-8F80F80485E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution RESX_SortFileContentOnSave = True diff --git a/build/build.yml b/build/build.yml index 067b6b3ca8..6c199da8ed 100644 --- a/build/build.yml +++ b/build/build.yml @@ -2,39 +2,36 @@ parameters: packageArtifacts: true steps: -- task: DotNetCoreInstaller@0 - inputs: - version: '2.2.103' - -- script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror - displayName: 'dotnet build $(buildConfiguration)' - -# To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). -# This allows us to test the blob storage providers without an Azure instance. -- task: NodeTool@0 - displayName: 'Use Node 8.x' - inputs: - versionSpec: 8.x - checkLatest: true - -- script: npm install -g azurite@2.7.0 - displayName: 'Install Azurite v2.7.0' - -# We start the Azurite as a separate process as the start call is blocking. -- script: azurite -s & - condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) - displayName: 'Start Azurite Storage Emulator (Linux)' - -- script: start azurite -s - condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) - displayName: 'Start Azurite Storage Emulator (Windows)' - -- task: DotNetCoreCLI@2 - displayName: 'dotnet test UnitTests' - inputs: - command: test - projects: '**\*Tests*.csproj' - arguments: '--configuration $(buildConfiguration)' - -- ${{ if eq(parameters.packageArtifacts, 'true') }}: - - template: package.yml + + - task: NuGetToolInstaller@0 + displayName: 'Use NuGet 4.7.1' + inputs: + versionSpec: 4.7.1 + checkLatest: true + + - task: DotNetCoreInstaller@0 + inputs: + version: '2.2.103' + + - task: DotNetCoreCLI@2 + displayName: 'donet restore' + inputs: + command: restore + restoreSolution: '**\*.sln' + feedsToUse: config + nugetConfigPath: nuget.config + + - script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror + displayName: 'dotnet build $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: 'dotnet test UnitTests' + inputs: + command: test + projects: '**/*UnitTests/*.csproj' + arguments: '--configuration $(buildConfiguration)' + + - template: test.yml + + - ${{ if eq(parameters.packageArtifacts, 'true') }}: + - template: package.yml diff --git a/build/test.yml b/build/test.yml new file mode 100644 index 0000000000..21e5096bd4 --- /dev/null +++ b/build/test.yml @@ -0,0 +1,53 @@ + +steps: + + # To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite). + # This allows us to test the blob storage providers without an Azure instance. + - task: NodeTool@0 + displayName: 'Use Node 8.x' + inputs: + versionSpec: 8.x + checkLatest: true + + - script: npm install -g azurite@2.7.0 + displayName: 'Install Azurite v2.7.0' + + # We start the Azurite as a separate process as the start call is blocking. + - script: azurite -s & + condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) + displayName: 'Start Azurite Storage Emulator (Linux)' + + - script: start azurite -s + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + displayName: 'Start Azurite Storage Emulator (Windows)' + + # Currently the Cosmos DB Emulator only runs on Windows. Therefore all end to end testing + # is performed on windows. + - powershell: | + $cosmosemulator=[Environment]::GetEnvironmentVariable("ProgramFiles") + "\Azure Cosmos DB Emulator\CosmosDB.Emulator.exe" + & $cosmosemulator /noui + $location = 'https://localhost:8081/_explorer/index.html' + $statusCode = [int]0 + while ($statusCode -ne 200) + { + Try + { + $req = [system.Net.WebRequest]::Create($location) + $response = $req.GetResponse() + $statusCode = [int]$response.StatusCode + $location + " status code: " + $statusCode + $response.Close() + } Catch { + $PSItem.ToString() + } + } + displayName: 'Start Installed Cosmos Emulator' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + + - task: DotNetCoreCLI@2 + displayName: 'Run all tests (Windows)' + condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' )) + inputs: + command: test + projects: '**\*Tests*.csproj' + arguments: '--configuration $(buildConfiguration)' \ No newline at end of file diff --git a/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs b/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs index 8f3cdf1c51..d9b3c26cea 100644 --- a/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs +++ b/src/Microsoft.Health.Dicom.Blob/Features/Storage/DicomBlobDataStore.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Blob.Configs; +using Microsoft.Health.Dicom.Core.Features.Persistence; namespace Microsoft.Health.Dicom.Blob.Features.Storage { diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensionsTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensionsTests.cs new file mode 100644 index 0000000000..108ce46e12 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdExtensionsTests.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence +{ + public class DicomAttributeIdExtensionsTests + { + [Fact] + public void GivenDatasetAndInvalidParameters_WhenFetchedAttributeValues_ArgumentExceptionIsThrown() + { + var dicomDataset = new DicomDataset(); + Assert.Throws(() => dicomDataset.TryGetValues((DicomAttributeId)null, out object[] values)); + } + + [Fact] + public void GivenDataset_WhenFetchingMissingAttribute_FalseIsReturned() + { + var dicomDataset = new DicomDataset(); + var result = dicomDataset.TryGetValues(new DicomAttributeId(DicomTag.PatientName), out object[] values); + + Assert.False(result); + Assert.Null(values); + } + + [Fact] + public void GivenDatasetWithSequence_WhenFetchingAttributeValues_IsReturnedCorrectly() + { + var sequence1 = new DicomSequence( + DicomTag.ReferencedPatientSequence, + new DicomDataset() { { DicomTag.PatientName, "Patient1" } }, + new DicomDataset() { { DicomTag.PatientName, "Patient2" } }, + new DicomDataset() { { DicomTag.PatientName, "Patient3" } }); + + var fetchedValues1 = new DicomDataset() { sequence1 }.TryGetValues( + new DicomAttributeId(DicomTag.ReferencedPatientSequence, DicomTag.PatientName), + out object[] values1); + Assert.True(fetchedValues1); + Assert.Equal(3, values1.Length); + Assert.Contains("Patient1", values1); + Assert.Contains("Patient2", values1); + Assert.Contains("Patient3", values1); + + var sequence2 = new DicomSequence( + DicomTag.ReferencedStudySequence, + new DicomDataset() { { DicomTag.StudyDate, new DateTime(2019, 6, 25) } }, + new DicomDataset() { { new DicomSequence(DicomTag.ReferencedStudySequence, new DicomDataset() { { DicomTag.StudyDate, new DateTime(2019, 6, 24) } }) } }, + new DicomDataset() { { DicomTag.StudyDate, new DateTime(2019, 6, 23) } }); + + var fetchedValues2 = new DicomDataset() { sequence2 }.TryGetValues( + new DicomAttributeId(DicomTag.ReferencedStudySequence, DicomTag.StudyDate), + out object[] values2); + Assert.True(fetchedValues2); + Assert.Equal(2, values2.Length); + Assert.Contains(new DateTime(2019, 6, 25), values2); + Assert.Contains(new DateTime(2019, 6, 23), values2); + + var fetchedValues3 = new DicomDataset() { sequence2 }.TryGetValues( + new DicomAttributeId(DicomTag.ReferencedStudySequence, DicomTag.ReferencedStudySequence, DicomTag.StudyDate), + out object[] values3); + Assert.True(fetchedValues3); + Assert.Single(values3); + Assert.Contains(new DateTime(2019, 6, 24), values3); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs new file mode 100644 index 0000000000..1fc9fae990 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomAttributeIdTests.cs @@ -0,0 +1,67 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- +using System; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence +{ + public class DicomAttributeIdTests + { + [Fact] + public void GivenDicomAttributeId_WhenConstructingWithInvalidParameters_ArgumentExceptionThrown() + { + Assert.Throws(() => new DicomAttributeId((DicomTag[])null)); + Assert.Throws(() => new DicomAttributeId(Array.Empty())); + Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.RightImageSequence, DicomTag.ROIContourSequence)); + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate, DicomTag.RightLensSequence)); + + Assert.Throws(() => new DicomAttributeId((string)null)); + Assert.Throws(() => new DicomAttributeId(string.Empty)); + Assert.Throws(() => new DicomAttributeId("0020000D.0020000D")); + Assert.Throws(() => new DicomAttributeId("INVALID")); + Assert.Throws(() => new DicomAttributeId("INVALID.INVALID")); + + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate).GetDicomTag(-1)); + Assert.Throws(() => new DicomAttributeId(DicomTag.StudyDate).GetDicomTag(1)); + } + + [Fact] + public void GivenValidDicomAttributeId_WhenSerialized_IsDeserializedCorrectly() + { + SerializeAndDeserialize(new DicomAttributeId(DicomTag.StudyDate)); + SerializeAndDeserialize(new DicomAttributeId(DicomTag.RelatedSeriesSequence, DicomTag.RegistrationSequence, DicomTag.RegisteredLocalizerUnits)); + SerializeAndDeserialize(new DicomAttributeId(DicomTag.RegionPixelShiftSequence, DicomTag.NumberOfVerticalPixels)); + SerializeAndDeserialize(new DicomAttributeId("0020000D")); + SerializeAndDeserialize(new DicomAttributeId("00101002.00100020")); + SerializeAndDeserialize(new DicomAttributeId("00101002.00100024.00400032")); + SerializeAndDeserialize(new DicomAttributeId("StudyInstanceUID")); + SerializeAndDeserialize(new DicomAttributeId("OtherPatientIDsSequence.PatientID")); + SerializeAndDeserialize(new DicomAttributeId("OtherPatientIDsSequence.IssuerOfPatientIDQualifiersSequence.UniversalEntityID")); + } + + [Fact] + public void GivenValidDicomAttributeId_WhenCompared_IsComparedCorrectly() + { + Assert.Equal(new DicomAttributeId(DicomTag.StudyInstanceUID), new DicomAttributeId("StudyInstanceUID")); + Assert.Equal(new DicomAttributeId(DicomTag.OtherPatientIDsSequence, DicomTag.PatientID), new DicomAttributeId("OtherPatientIDsSequence.PatientID")); + + Assert.Equal( + new DicomAttributeId(DicomTag.OtherPatientIDsSequence, DicomTag.IssuerOfPatientIDQualifiersSequence, DicomTag.UniversalEntityID), + new DicomAttributeId("OtherPatientIDsSequence.IssuerOfPatientIDQualifiersSequence.UniversalEntityID")); + } + + private void SerializeAndDeserialize(DicomAttributeId dicomAttributeId) + { + var json = JsonConvert.SerializeObject(dicomAttributeId); + var deserialized = JsonConvert.DeserializeObject(json); + + Assert.Equal(dicomAttributeId, deserialized); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs new file mode 100644 index 0000000000..affa82507b --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/DicomInstanceTests.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence +{ + public class DicomInstanceTests + { + [Fact] + public void GivenDicomInstanceSeriesStudy_WhenConstructedWithInvalidParameters_ArgumentExceptionThrown() + { + Assert.Throws(() => new DicomStudy(null)); + Assert.Throws(() => new DicomStudy(string.Empty)); + Assert.Throws(() => new DicomStudy("#")); + Assert.Throws(() => new DicomStudy(new string('a', 65))); + + Assert.Throws(() => new DicomSeries(null, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries("#", Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries(new string('a', 65), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), null)); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), "#")); + Assert.Throws(() => new DicomSeries(Guid.NewGuid().ToString(), new string('a', 65))); + Assert.Throws(() => new DicomSeries("aaa", "aaa")); + + Assert.Throws(() => new DicomInstance(null, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(string.Empty, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance("#", Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(new string('a', 65), Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), null, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), "#", Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), new string('a', 65), Guid.NewGuid().ToString())); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), null)); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "#")); + Assert.Throws(() => new DicomInstance(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), new string('a', 65))); + Assert.Throws(() => new DicomInstance("aaa", "aaa", "bbb")); + Assert.Throws(() => new DicomInstance("aaa", "bbb", "aaa")); + Assert.Throws(() => new DicomInstance("bbb", "aaa", "aaa")); + } + + [Fact] + public void GivenDicomInstanceSeriesStudy_WhenCompared_CorrectEqualsResponseReturned() + { + Assert.Equal("aaa", new DicomStudy("aaa").StudyInstanceUID); + Assert.Equal("aaa".GetHashCode(), new DicomStudy("aaa").GetHashCode()); + Assert.Equal(new DicomStudy("aaa"), new DicomStudy("aaa")); + Assert.NotEqual(new DicomStudy("aAa"), new DicomStudy("aaa")); + + Assert.Equal("aaa", new DicomSeries("aaa", "bbb").StudyInstanceUID); + Assert.Equal("bbb", new DicomSeries("aaa", "bbb").SeriesInstanceUID); + Assert.Equal("aaabbb".GetHashCode(), new DicomSeries("aaa", "bbb").GetHashCode()); + Assert.Equal(new DicomSeries("aaa", "bbb"), new DicomSeries("aaa", "bbb")); + Assert.NotEqual(new DicomSeries("aAa", "bbb"), new DicomSeries("aaa", "bbb")); + Assert.NotEqual(new DicomSeries("aaa", "bBb"), new DicomSeries("aaa", "bbb")); + + Assert.Equal("aaa", new DicomInstance("aaa", "bbb", "ccc").StudyInstanceUID); + Assert.Equal("bbb", new DicomInstance("aaa", "bbb", "ccc").SeriesInstanceUID); + Assert.Equal("ccc", new DicomInstance("aaa", "bbb", "ccc").SopInstanceUID); + Assert.Equal("aaabbbccc".GetHashCode(), new DicomInstance("aaa", "bbb", "ccc").GetHashCode()); + Assert.Equal(new DicomInstance("aaa", "bbb", "ccc"), new DicomInstance("aaa", "bbb", "ccc")); + Assert.NotEqual(new DicomInstance("aAa", "bbb", "ccc"), new DicomInstance("aaa", "bbb", "ccc")); + Assert.NotEqual(new DicomInstance("aaa", "bBb", "ccc"), new DicomInstance("aaa", "bbb", "ccc")); + Assert.NotEqual(new DicomInstance("aaa", "bbb", "cCc"), new DicomInstance("aaa", "bbb", "ccc")); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs new file mode 100644 index 0000000000..6b7bee1038 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core.UnitTests/Features/Persistence/Exceptions/DataStoreExceptionTests.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using System.Net; +using System.Runtime.Serialization.Formatters.Binary; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Xunit; + +namespace Microsoft.Health.Dicom.Core.UnitTests.Features.Persistence.Exceptions +{ + public class DataStoreExceptionTests + { + [Fact] + public void GivenDataStoreException_WhenNoStatusCodeProvided_DefaultsToInternalServerError() + { + var innerException = new Exception("foo"); + Assert.Equal((int)HttpStatusCode.InternalServerError, new DataStoreException((HttpStatusCode?)null, innerException).StatusCode); + Assert.Equal((int)HttpStatusCode.InternalServerError, new DataStoreException((int?)null, innerException).StatusCode); + } + + [Fact] + public void GivenDataStoreException_WhenSerialized_IsDeserializedCorrectly() + { + var innerException = new Exception("foo"); + var exception = new DataStoreException(HttpStatusCode.NotFound, innerException); + + var buffer = new byte[4096]; + using (var memoryStream1 = new MemoryStream(buffer)) + using (var memoryStream2 = new MemoryStream(buffer)) + { + var formatter = new BinaryFormatter(); + + formatter.Serialize(memoryStream1, exception); + var deserializedException = (DataStoreException)formatter.Deserialize(memoryStream2); + + Assert.Equal(exception.StatusCode, deserializedException.StatusCode); + Assert.Equal(exception.InnerException.Message, deserializedException.InnerException.Message); + Assert.Equal(exception.Message, deserializedException.Message); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs new file mode 100644 index 0000000000..1b827f76c6 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeId.cs @@ -0,0 +1,159 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Dicom; +using EnsureThat; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + /// + /// Implements the DICOM attribute parsing and serialization as defined here: http://dicom.nema.org/Dicom/2013/output/chtml/part18/sect_6.7.html. + /// The order of tags are important in the class, all items but the last must be sequence elements. + /// This class supports serialization/ deserialization from keywords or Hex Group/ Element. + /// Examples of valid values for {attributeID}: + /// 0020000D + /// StudyInstanceUID + /// 00101002.00100020 + /// OtherPatientIDsSequence.PatientID + /// 00101002.00100024.00400032 + /// OtherPatientIDsSequence.IssuerOfPatientIDQualifiersSequence.UniversalEntityID + /// + public class DicomAttributeId + { + private const char Seperator = '.'; + private const string GroupElementStringFormat = "X4"; + private const NumberStyles GroupElementNumberStyles = NumberStyles.HexNumber; + private const StringComparison _stringComparison = StringComparison.InvariantCulture; + private static readonly IFormatProvider FormatProvider = CultureInfo.InvariantCulture; + private readonly bool _writeTagsAsKeywords = false; + private readonly DicomTag[] _dicomTags; + private static readonly IDictionary KnownKeywordDicomTags = DicomDictionary.Default.ToDictionary(x => x.Keyword, x => x.Tag); + + [JsonConstructor] + public DicomAttributeId(string attributeId) + { + _dicomTags = DeserializeAttributeId(attributeId); + Validate(); + + AttributeId = attributeId; + } + + public DicomAttributeId(params DicomTag[] dicomTags) + { + EnsureArg.IsNotNull(dicomTags, nameof(dicomTags)); + EnsureArg.IsTrue(dicomTags.Length > 0, nameof(dicomTags)); + + _dicomTags = dicomTags; + Validate(); + + AttributeId = string.Join(Seperator, dicomTags.Select(x => ConvertToString(x, _writeTagsAsKeywords))); + } + + public string AttributeId { get; } + + [JsonIgnore] + public int Length => _dicomTags.Length; + + /// + /// Gets the non-sequence DICOM tag. + /// + [JsonIgnore] + public DicomTag InstanceDicomTag => _dicomTags[Length - 1]; + + public DicomTag GetDicomTag(int index) + { + EnsureArg.IsGte(index, 0, nameof(index)); + EnsureArg.IsLt(index, _dicomTags.Length, nameof(index)); + return _dicomTags[index]; + } + + public override int GetHashCode() + { + return AttributeId.GetHashCode(_stringComparison); + } + + public override bool Equals(object obj) + { + if (obj is DicomAttributeId instance && instance.Length == Length) + { + // Validate all the DICOM tag keywords are the same + for (var i = 0; i < _dicomTags.Length; i++) + { + if (!string.Equals(_dicomTags[i].DictionaryEntry.Keyword, instance._dicomTags[i].DictionaryEntry.Keyword, _stringComparison)) + { + return false; + } + } + + return true; + } + + return false; + } + + private static DicomTag[] DeserializeAttributeId(string attributeId) + { + EnsureArg.IsNotNullOrWhiteSpace(attributeId, nameof(attributeId)); + + string[] split = attributeId.Split(Seperator); + var result = new DicomTag[split.Length]; + + for (int i = 0; i < split.Length; i++) + { + // First attempt Hex decimal parse of the string if correct length. + if (split[i].Length == 8 && + ushort.TryParse(split[i].Substring(0, 4), GroupElementNumberStyles, FormatProvider, out ushort group) && + ushort.TryParse(split[i].Substring(4, 4), GroupElementNumberStyles, FormatProvider, out ushort element)) + { + result[i] = new DicomTag(group, element); + } + else if (KnownKeywordDicomTags.TryGetValue(split[i], out DicomTag dicomTag)) + { + result[i] = dicomTag; + } + else + { + throw new FormatException($"Could not fromat the attribute '{split[i]}' from the provided attribute string '{attributeId}' to a known DICOM tag."); + } + } + + return result; + } + + private static string ConvertToString(DicomTag dicomTag, bool writeTagsAsKeywords) + { + if (writeTagsAsKeywords) + { + return dicomTag.DictionaryEntry.Keyword; + } + + string groupString = dicomTag.Group.ToString(GroupElementStringFormat, FormatProvider); + string elementString = dicomTag.Element.ToString(GroupElementStringFormat, FormatProvider); + return groupString + elementString; + } + + private void Validate() + { + // Validate all but the leaf tag are sequence elements + for (var i = 0; i < Length - 1; i++) + { + if (!_dicomTags[i].DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)) + { + throw new FormatException($"All tags but the last must be sequence elements. The provided DICOM tag {_dicomTags[i].DictionaryEntry.Keyword} does not have a 'sequence' value type."); + } + } + + if (InstanceDicomTag.DictionaryEntry.ValueRepresentations.Contains(DicomVR.SQ)) + { + throw new FormatException($"The last DICOM tag must not have the value representation 'sequence'. The provided DICOM tag {InstanceDicomTag.DictionaryEntry.Keyword} is known as having a 'sequence' value type."); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs new file mode 100644 index 0000000000..64f11363e3 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomAttributeIdExtensions.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Dicom; +using EnsureThat; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public static class DicomAttributeIdExtensions + { + public static bool TryGetValues(this DicomDataset dicomDataset, DicomAttributeId attributeId, out TItem[] values) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + EnsureArg.IsNotNull(attributeId, nameof(attributeId)); + + var result = new List(); + dicomDataset.TryAppendValuesToList(result, attributeId); + + values = result.Count > 0 ? result.ToArray() : null; + return result.Count > 0; + } + + private static void TryAppendValuesToList( + this DicomDataset dicomDataset, + List list, + DicomAttributeId attributeId, + int startAttributeIndex = 0) + { + EnsureArg.IsGte(startAttributeIndex, 0, nameof(startAttributeIndex)); + + // Handle for now sequence elements (last item in attribute ID). + if (startAttributeIndex == attributeId.Length - 1) + { + // Now first validate this DICOM tag exists in the dataset. + // Note: GetValueCount throws exception when the tag does not exist. + if (dicomDataset.TryGetValue(attributeId.InstanceDicomTag, 0, out TItem firstItem)) + { + list.Add(firstItem); + + for (int i = 1; i < dicomDataset.GetValueCount(attributeId.InstanceDicomTag); i++) + { + if (dicomDataset.TryGetValue(attributeId.InstanceDicomTag, i, out TItem value)) + { + list.Add(value); + } + } + } + } + else if (dicomDataset.TryGetSequence(attributeId.GetDicomTag(startAttributeIndex), out DicomSequence dicomSequence)) + { + foreach (DicomDataset sequenceDataset in dicomSequence.Items) + { + sequenceDataset.TryAppendValuesToList(list, attributeId, startAttributeIndex + 1); + } + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs new file mode 100644 index 0000000000..13cf2dd6b6 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomInstance.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Dicom; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomInstance : DicomSeries + { + [JsonConstructor] + public DicomInstance(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID) + : base(studyInstanceUID, seriesInstanceUID) + { + // Run the instance identifiers through the regular expression check. + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(sopInstanceUID), nameof(sopInstanceUID)); + EnsureArg.IsNotEqualTo(sopInstanceUID, studyInstanceUID, nameof(sopInstanceUID)); + EnsureArg.IsNotEqualTo(sopInstanceUID, seriesInstanceUID, nameof(sopInstanceUID)); + + SopInstanceUID = sopInstanceUID; + } + + public string SopInstanceUID { get; } + + public static DicomInstance Create(DicomDataset dicomDataset) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + + // Note: Here we 'GetSingleValueOrDefault' and let the constructor validate the identifier. + return new DicomInstance( + dicomDataset.GetSingleValueOrDefault(DicomTag.StudyInstanceUID, string.Empty), + dicomDataset.GetSingleValueOrDefault(DicomTag.SeriesInstanceUID, string.Empty), + dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty)); + } + + public override bool Equals(object obj) + { + if (obj is DicomInstance identity) + { + return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison) && + SeriesInstanceUID.Equals(identity.SeriesInstanceUID, EqualsStringComparison) && + SopInstanceUID.Equals(identity.SopInstanceUID, EqualsStringComparison); + } + + return false; + } + + public override int GetHashCode() + => (StudyInstanceUID + SeriesInstanceUID + SopInstanceUID).GetHashCode(EqualsStringComparison); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs new file mode 100644 index 0000000000..3df139f49d --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomSeries.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomSeries : DicomStudy + { + [JsonConstructor] + public DicomSeries(string studyInstanceUID, string seriesInstanceUID) + : base(studyInstanceUID) + { + // Run the instance identifiers through the regular expression check. + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(seriesInstanceUID), nameof(seriesInstanceUID)); + EnsureArg.IsNotEqualTo(studyInstanceUID, seriesInstanceUID, nameof(seriesInstanceUID)); + + SeriesInstanceUID = seriesInstanceUID; + } + + public string SeriesInstanceUID { get; } + + public override bool Equals(object obj) + { + if (obj is DicomSeries identity) + { + return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison) && + SeriesInstanceUID.Equals(identity.SeriesInstanceUID, EqualsStringComparison); + } + + return false; + } + + public override int GetHashCode() + => (StudyInstanceUID + SeriesInstanceUID).GetHashCode(EqualsStringComparison); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs new file mode 100644 index 0000000000..505ea97e12 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/DicomStudy.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class DicomStudy + { + protected const StringComparison EqualsStringComparison = StringComparison.Ordinal; + + [JsonConstructor] + public DicomStudy(string studyInstanceUID) + { + // Run the instance identifiers through the regular expression check. + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyInstanceUID), nameof(studyInstanceUID)); + + StudyInstanceUID = studyInstanceUID; + } + + public string StudyInstanceUID { get; } + + public override bool Equals(object obj) + { + if (obj is DicomStudy identity) + { + return StudyInstanceUID.Equals(identity.StudyInstanceUID, EqualsStringComparison); + } + + return false; + } + + public override int GetHashCode() + => StudyInstanceUID.GetHashCode(EqualsStringComparison); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs new file mode 100644 index 0000000000..cc37119ec0 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/Exceptions/DataStoreException.cs @@ -0,0 +1,67 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Net; +using System.Runtime.Serialization; +using System.Security.Permissions; +using EnsureThat; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions +{ + [Serializable] + public class DataStoreException : Exception + { + public DataStoreException(HttpStatusCode httpStatusCode) + { + StatusCode = (int)httpStatusCode; + } + + public DataStoreException(HttpStatusCode? httpStatusCode, Exception innerException) + : base(innerException.Message, innerException) + { + StatusCode = httpStatusCode.HasValue ? (int)httpStatusCode.Value : (int)HttpStatusCode.InternalServerError; + } + + public DataStoreException(int? statusCode, Exception innerException) + : base(innerException.Message, innerException) + { + StatusCode = statusCode ?? (int)HttpStatusCode.InternalServerError; + } + + public DataStoreException() + { + } + + public DataStoreException(string message) + : base(message) + { + } + + public DataStoreException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected DataStoreException(SerializationInfo serializationInfo, StreamingContext streamingContext) + : base(serializationInfo, streamingContext) + { + EnsureArg.IsNotNull(serializationInfo, nameof(serializationInfo)); + + StatusCode = serializationInfo.GetInt32(nameof(StatusCode)); + } + + public int StatusCode { get; } + + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + EnsureArg.IsNotNull(serializationInfo, nameof(serializationInfo)); + + serializationInfo.AddValue(nameof(StatusCode), StatusCode); + base.GetObjectData(serializationInfo, context); + } + } +} diff --git a/src/Microsoft.Health.Dicom.Blob/Features/Storage/IDicomBlobDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomBlobDataStore.cs similarity index 93% rename from src/Microsoft.Health.Dicom.Blob/Features/Storage/IDicomBlobDataStore.cs rename to src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomBlobDataStore.cs index 68408ba268..3ba2f2bf41 100644 --- a/src/Microsoft.Health.Dicom.Blob/Features/Storage/IDicomBlobDataStore.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomBlobDataStore.cs @@ -8,7 +8,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Health.Dicom.Blob.Features.Storage +namespace Microsoft.Health.Dicom.Core.Features.Persistence { public interface IDicomBlobDataStore { diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs new file mode 100644 index 0000000000..a7969eb49e --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/IDicomIndexDataStore.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Dicom; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public interface IDicomIndexDataStore + { + Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default); + + Task> QueryStudiesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, + CancellationToken cancellationToken = default); + + Task> QuerySeriesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, + CancellationToken cancellationToken = default); + + Task> QueryInstancesAsync( + int offset, + int limit, + string studyInstanceUID = null, + IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, + CancellationToken cancellationToken = default); + + /// + /// Delete the provided series index. + /// + /// The study instance unique identifier. + /// The series instance unique identifier. + /// The cancellation token. + /// The collection of instances that were deleted from this series. + Task> DeleteSeriesIndexAsync(string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default); + + Task DeleteInstanceIndexAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default); + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs b/src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs new file mode 100644 index 0000000000..4218822cb0 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Persistence/QueryResult.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.Health.Dicom.Core.Features.Persistence +{ + public class QueryResult + { + public QueryResult(bool hasMoreResults, IEnumerable results) + { + HasMoreResults = hasMoreResults; + Results = results; + } + + public bool HasMoreResults { get; } + + public IEnumerable Results { get; } + } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs index ee735095f6..3d4cb500ef 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Validation/DicomIdentifierValidator.cs @@ -9,13 +9,16 @@ namespace Microsoft.Health.Dicom.Core.Features.Validation { /// - /// Validates a unique identifer conforms to the rules from http://dicom.nema.org/dicom/2013/output/chtml/part05/chapter_9.html + /// Validates a unique identifer conforms to the character rules from http://dicom.nema.org/dicom/2013/output/chtml/part05/chapter_9.html. + /// This validator does not validate the format of the identifier. /// /// public class DicomIdentifierValidator : RegularExpressionValidator { + public static readonly Regex IdentifierRegex = new Regex("^[A-Za-z0-9\\-\\.]{1,64}$", RegexOptions.Singleline | RegexOptions.Compiled); + public DicomIdentifierValidator() - : base("^[A-Za-z0-9\\-\\.]{1,64}$", RegexOptions.Singleline | RegexOptions.Compiled) + : base(IdentifierRegex) { } } diff --git a/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj b/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj index f6637217d5..6a1e4d99b1 100644 --- a/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj +++ b/src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj @@ -7,6 +7,7 @@ + diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs new file mode 100644 index 0000000000..bde5d57309 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/DicomCosmosDataStoreTests.cs @@ -0,0 +1,68 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Extensions.DependencyInjection; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage +{ + public class DicomCosmosDataStoreTests + { + private readonly DicomCosmosDataStore _indexDataStore; + + public DicomCosmosDataStoreTests() + { + IOptionsMonitor namedCosmosCollectionConfigurationAccessor = Substitute.For>(); + namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName).Returns(new CosmosCollectionConfiguration() { CollectionId = "testcollection" }); + + _indexDataStore = new DicomCosmosDataStore( + Substitute.For>(), + new CosmosDataStoreConfiguration(), + namedCosmosCollectionConfigurationAccessor, + new DicomCosmosConfiguration()); + } + + [Fact] + public async Task GivenInvalidParameters_WhenCallingAllMethods_ArgumentExceptionThrown() + { + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(null)); + + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(null, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(string.Empty, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), null)); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), string.Empty)); + + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(null, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(string.Empty, Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), null, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), string.Empty, Guid.NewGuid().ToString())); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), null)); + await Assert.ThrowsAsync(() => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), string.Empty)); + } + + [Fact] + public void GivenDocumentClient_WhenEnablingCrossPartitionQuery_RequiresReflection() + { + // When the feed options natively supports this, remove this test and the extra code in the DicomCosmosDataStore for this hack. + var feedOptions = new FeedOptions(); + + PropertyInfo propertyInfo = feedOptions.GetType().GetProperty("EnableCrossPartitionSkipTake", BindingFlags.NonPublic | BindingFlags.Instance); + propertyInfo.SetValue(feedOptions, Convert.ChangeType(true, propertyInfo.PropertyType)); + + Assert.NotNull(propertyInfo); + Assert.True((bool)propertyInfo.GetValue(feedOptions)); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs new file mode 100644 index 0000000000..30c1054228 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/AttributeValuesTests.cs @@ -0,0 +1,93 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class AttributeValuesTests + { + [Fact] + public void GivenACollectionOfDifferentTypes_WhenAddedToAttributeValues_AreStoredAndMinMaxDateTimeValuesCalculated() + { + const int numberOfItemsToAdd = 300; + var minDateTime = new DateTime(2019, 6, 21); + var maxDateTime = new DateTime(2020, 6, 21); + Assert.True((maxDateTime - minDateTime).Days > numberOfItemsToAdd); + + var attributeValues = new AttributeValues(); + + for (int i = 0; i < numberOfItemsToAdd; i++) + { + // Add DateTime, int, and string + Assert.True(attributeValues.Add(minDateTime.AddDays(i))); + Assert.True(attributeValues.Add(i)); + Assert.True(attributeValues.Add(new string('a', i))); + } + + Assert.True(attributeValues.Add(maxDateTime)); + + Assert.Equal(minDateTime, attributeValues.MinDateTime); + Assert.Equal(maxDateTime, attributeValues.MaxDateTime); + } + + [Fact] + public void GivenExistingItem_WhenAddedToAttributeValues_IsNotAdded() + { + var attributeValues = new AttributeValues(); + + Assert.True(attributeValues.Add(1)); + Assert.False(attributeValues.Add(1)); + + Assert.True(attributeValues.Add("HelloWorld")); + Assert.False(attributeValues.Add("HelloWorld")); + + Assert.Null(new AttributeValues().MinDateTime); + Assert.Null(new AttributeValues().MaxDateTime); + + Assert.True(attributeValues.Add(new DateTime(2019, 6, 22))); + Assert.False(attributeValues.Add(new DateTime(2019, 6, 22))); + + Assert.NotNull(attributeValues.MinDateTime); + Assert.NotNull(attributeValues.MaxDateTime); + + Assert.False(attributeValues.Add(null)); + } + + [Fact] + public void GivenAttributeValues_WhenCreatedWithInvalidParameters_ArgumentExceptionThrown() + { + Assert.Throws(() => new AttributeValues(null)); + Assert.NotNull(new AttributeValues().Values); + Assert.Null(new AttributeValues().MinDateTime); + Assert.Null(new AttributeValues().MaxDateTime); + } + + [Fact] + public void GivenAttributeValues_WhenSerialized_DeserializedCorrectly() + { + const int numberOfItemsToAdd = 300; + var dateTime = new DateTime(2019, 6, 20); + var attributeValues = new AttributeValues(); + + for (int i = 0; i < numberOfItemsToAdd; i++) + { + // Add DateTime, int, and string + Assert.True(attributeValues.Add(dateTime.AddDays(i))); + Assert.True(attributeValues.Add(i)); + Assert.True(attributeValues.Add(new string('a', i))); + } + + var json = JsonConvert.SerializeObject(attributeValues); + AttributeValues deserialized = JsonConvert.DeserializeObject(json); + + Assert.Equal(attributeValues.MinDateTime, deserialized.MinDateTime); + Assert.Equal(attributeValues.MaxDateTime, deserialized.MaxDateTime); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs new file mode 100644 index 0000000000..fcc1987d05 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QueryInstanceTests.cs @@ -0,0 +1,68 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class QueryInstanceTests + { + [Fact] + public void GivenInvalidParameters_WhenCreatingQueryInstance_ExceptionsThrown() + { + Assert.Throws(() => new QueryInstance(null, new Dictionary())); + Assert.Throws(() => new QueryInstance(string.Empty, new Dictionary())); + Assert.Throws(() => new QueryInstance(new string('a', 65), new Dictionary())); + Assert.Throws(() => new QueryInstance("AA#AA", new Dictionary())); + Assert.Throws(() => new QueryInstance(Guid.NewGuid().ToString(), null)); + + Assert.Throws(() => QueryInstance.Create(null, null)); + Assert.Throws(() => QueryInstance.Create(new DicomDataset(), null)); + Assert.Throws(() => QueryInstance.Create(new DicomDataset(), Array.Empty())); + } + + [Fact] + public void GivenDicomTagsToIndex_WhenCreatingQueryInstance_TagsAreExtractedAndIndexed() + { + var referringPhysicianName = "TestPhysician"; + var dicomSequence = new DicomSequence( + DicomTag.ReferringPhysicianIdentificationSequence, + new DicomDataset() + { + { DicomTag.ReferringPhysicianName, referringPhysicianName }, + }); + + var dicomDataset = new DicomDataset + { + { DicomTag.StudyInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.SeriesInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.SOPInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.PatientName, Guid.NewGuid().ToString() }, + { DicomTag.StudyDate, DateTime.UtcNow }, + }; + dicomDataset.Add(dicomSequence); + + var patientNameAttribute = new DicomAttributeId(DicomTag.PatientName); + var studyDateAttribute = new DicomAttributeId(DicomTag.StudyDate); + var referringPhysicianNameAttribute = new DicomAttributeId(DicomTag.ReferringPhysicianIdentificationSequence, DicomTag.ReferringPhysicianName); + var studyTimeAttribute = new DicomAttributeId(DicomTag.StudyTime); + var instance = QueryInstance.Create( + dicomDataset, new[] { patientNameAttribute, studyDateAttribute, referringPhysicianNameAttribute, studyTimeAttribute }); + + Assert.NotNull(instance); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.SOPInstanceUID), instance.InstanceUID); + + Assert.Equal(3, instance.Attributes.Count); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.PatientName), instance.Attributes[patientNameAttribute.AttributeId][0]); + Assert.Equal(dicomDataset.GetSingleValue(DicomTag.StudyDate), instance.Attributes[studyDateAttribute.AttributeId][0]); + Assert.Equal(referringPhysicianName, instance.Attributes[referringPhysicianNameAttribute.AttributeId][0]); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs new file mode 100644 index 0000000000..dea032a602 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Documents/QuerySeriesDocumentTests.cs @@ -0,0 +1,102 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Documents +{ + public class QuerySeriesDocumentTests + { + [Fact] + public void GivenInvalidInstanceIdentifers_WhenCreatingQuerySeriesDocument_ExceptionsThrown() + { + Assert.Throws(() => new QuerySeriesDocument(null, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(new string('a', 65), Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument("?...", Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), null)); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => new QuerySeriesDocument("sameid", "sameid")); + + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(null, Guid.NewGuid().ToString())); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(string.Empty, Guid.NewGuid().ToString())); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), new string('a', 65))); + Assert.Throws(() => new QuerySeriesDocument(Guid.NewGuid().ToString(), "?...")); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(Guid.NewGuid().ToString(), null)); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId(Guid.NewGuid().ToString(), string.Empty)); + Assert.Throws(() => QuerySeriesDocument.GetDocumentId("sameid", "sameid")); + + Assert.Throws(() => QuerySeriesDocument.GetPartitionKey(null)); + Assert.Throws(() => QuerySeriesDocument.GetPartitionKey(string.Empty)); + } + + [Fact] + public void GivenExistingInstance_WhenAddingToInstancesHashSet_IsNotAdded() + { + var document = new QuerySeriesDocument(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + + var dataset = new DicomDataset(); + var sopInstanceUID = Guid.NewGuid().ToString(); + var testPatientName = Guid.NewGuid().ToString(); + dataset.Add(DicomTag.SOPInstanceUID, sopInstanceUID); + dataset.Add(DicomTag.PatientName, testPatientName); + + var patientNameAttributeId = new DicomAttributeId(DicomTag.PatientName); + var instanceDocument1 = QueryInstance.Create(dataset, new[] { patientNameAttributeId }); + var instanceDocument2 = QueryInstance.Create(dataset, new[] { patientNameAttributeId }); + + Assert.Throws(() => document.AddInstance(null)); + Assert.True(document.AddInstance(instanceDocument1)); + Assert.False(document.AddInstance(instanceDocument2)); + + Assert.Equal(testPatientName, document.DistinctAttributes[patientNameAttributeId.AttributeId].Values.First()); + + Assert.Throws(() => document.RemoveInstance(null)); + Assert.Throws(() => document.RemoveInstance(string.Empty)); + Assert.True(document.RemoveInstance(sopInstanceUID)); + Assert.False(document.RemoveInstance(sopInstanceUID)); + } + + [Fact] + public void GivenSeriesDocument_WhenSerialized_IsDeserializedCorrectly() + { + var document = new QuerySeriesDocument(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()) + { + ETag = Guid.NewGuid().ToString(), + }; + + var dataset = new DicomDataset(); + var sopInstanceUID = Guid.NewGuid().ToString(); + var testPatientName = Guid.NewGuid().ToString(); + dataset.Add(DicomTag.SOPInstanceUID, sopInstanceUID); + dataset.Add(DicomTag.PatientName, testPatientName); + + var patientNameAttributeId = new DicomAttributeId(DicomTag.PatientName); + var instanceDocument = QueryInstance.Create(dataset, new[] { patientNameAttributeId }); + document.AddInstance(instanceDocument); + + var serialized = JsonConvert.SerializeObject(document); + QuerySeriesDocument deserialized = JsonConvert.DeserializeObject(serialized); + + Assert.Equal(document.Id, deserialized.Id); + Assert.Equal(document.PartitionKey, deserialized.PartitionKey); + Assert.Equal(document.ETag, deserialized.ETag); + Assert.Equal(document.StudyUID, deserialized.StudyUID); + Assert.Equal(document.SeriesUID, deserialized.SeriesUID); + Assert.Equal(document.Instances.Count, deserialized.Instances.Count); + + QueryInstance deserializedFirstInstance = deserialized.Instances.First(); + Assert.Equal(deserializedFirstInstance.InstanceUID, deserializedFirstInstance.InstanceUID); + Assert.Equal(deserializedFirstInstance.Attributes.Count, deserializedFirstInstance.Attributes.Count); + Assert.Equal(deserializedFirstInstance.Attributes[patientNameAttributeId.AttributeId], deserializedFirstInstance.Attributes[patientNameAttributeId.AttributeId]); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs new file mode 100644 index 0000000000..394e5eaaa9 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Features/Storage/Versioning/DicomCollectionUpgradeManagerTests.cs @@ -0,0 +1,102 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Dicom.CosmosDb.UnitTests.Features.Storage.Versioning +{ + public class DicomCollectionUpgradeManagerTests + { + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration = new CosmosDataStoreConfiguration + { + AllowDatabaseCreation = false, + ConnectionMode = ConnectionMode.Direct, + ConnectionProtocol = Protocol.Https, + DatabaseId = "testdatabaseid", + Host = "https://fakehost", + Key = "ZmFrZWtleQ==", // "fakekey" + PreferredLocations = null, + }; + + private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration = new CosmosCollectionConfiguration + { + CollectionId = "testcollectionid", + }; + + private readonly DicomCollectionUpgradeManager _manager; + private readonly IDocumentClient _client; + + public DicomCollectionUpgradeManagerTests() + { + ICosmosDbDistributedLockFactory factory = Substitute.For(); + ICosmosDbDistributedLock cosmosDbDistributedLock = Substitute.For(); + IQueryable collectionVersionWrappers = Substitute.For, IDocumentQuery>(); + var optionsMonitor = Substitute.For>(); + + optionsMonitor.Get(Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); + + factory.Create(Arg.Any(), Arg.Any(), Arg.Any()).Returns(cosmosDbDistributedLock); + cosmosDbDistributedLock.TryAcquireLock().Returns(true); + + _client = Substitute.For(); + _client.CreateDocumentQuery(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(collectionVersionWrappers); + + collectionVersionWrappers.AsDocumentQuery().ExecuteNextAsync().Returns(new FeedResponse(new List())); + + var updaters = new IDicomCollectionUpdater[] { new DicomCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance) }; + _manager = new DicomCollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionIndexIsUpdated() + { + var documentCollection = new DocumentCollection(); + + await UpdateCollectionAsync(documentCollection); + + await _client.Received(1).ReplaceDocumentCollectionAsync(Arg.Is(documentCollection)); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionVersionWrapperIsSaved() + { + var documentCollection = new DocumentCollection(); + + await UpdateCollectionAsync(documentCollection); + + await _client.Received(1).UpsertDocumentAsync(Arg.Is(_cosmosDataStoreConfiguration.GetRelativeCollectionUri(_cosmosCollectionConfiguration.CollectionId)), Arg.Is(x => x.Version == _manager.CollectionSettingsVersion)); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionTTLIsSetToNeg1() + { + var documentCollection = new DocumentCollection(); + + await UpdateCollectionAsync(documentCollection); + + Assert.Equal(-1, documentCollection.DefaultTimeToLive); + } + + private async Task UpdateCollectionAsync(DocumentCollection documentCollection) + { + await _manager.SetupCollectionAsync(_client, documentCollection); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj new file mode 100644 index 0000000000..efa36dc435 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb.UnitTests/Microsoft.Health.Dicom.CosmosDb.UnitTests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp2.2 + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs b/src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs new file mode 100644 index 0000000000..6b736f9d32 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/AssemblyInfo.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Resources; +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Health.Dicom.CosmosDb.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Dicom.Tests.Integration")] +[assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs new file mode 100644 index 0000000000..60248f56d2 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Config/DicomCosmosConfiguration.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Dicom; +using Microsoft.Health.Dicom.Core.Features.Persistence; + +namespace Microsoft.Health.Dicom.CosmosDb.Config +{ + public class DicomCosmosConfiguration + { + /// + /// Gets the DICOM tags that should be indexed and made queryable. + /// The StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID will be indexed automatically. + /// TODO: We should validate that the attributes defined here have a sensible value representation. + /// + public HashSet QueryAttributes { get; } = new HashSet() + { + new DicomAttributeId(DicomTag.PatientName), + new DicomAttributeId(DicomTag.PatientID), + new DicomAttributeId(DicomTag.AccessionNumber), + new DicomAttributeId(DicomTag.Modality), + new DicomAttributeId(DicomTag.ReferringPhysicianName), + new DicomAttributeId(DicomTag.StudyDate), + }; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Constants.cs b/src/Microsoft.Health.Dicom.CosmosDb/Constants.cs new file mode 100644 index 0000000000..5342d23aee --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Constants.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Dicom.CosmosDb +{ + internal static class Constants + { + public const string CollectionConfigurationName = "dicomCosmosDb"; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs new file mode 100644 index 0000000000..c124779aae --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Health/DicomCosmosHealthCheck.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Azure.Documents; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Health; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Extensions.DependencyInjection; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Health +{ + public class DicomCosmosHealthCheck : CosmosHealthCheck + { + /// + /// Initializes a new instance of the class. + /// + /// The document client factory/ + /// The CosmosDB configuration. + /// The IOptions accessor to get a named version. + /// The test provider + /// The logger. + public DicomCosmosHealthCheck( + IScoped documentClient, + CosmosDataStoreConfiguration configuration, + IOptionsSnapshot namedCosmosCollectionConfigurationAccessor, + IDocumentClientTestProvider testProvider, + ILogger logger) + : base( + documentClient, + configuration, + namedCosmosCollectionConfigurationAccessor, + Constants.CollectionConfigurationName, + testProvider, + logger) + { + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs new file mode 100644 index 0000000000..d995b59d2d --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/CosmosQueryBuilder.cs @@ -0,0 +1,109 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + internal class CosmosQueryBuilder + { + private const string OffsetParameterName = "@offset"; + private const string LimitParameterName = "@limit"; + private const string ItemParameterNameFormat = "@item{0}"; + private const string StudySqlQuerySearchFormat = "SELECT DISTINCT VALUE {{ \"" + nameof(DicomStudy.StudyInstanceUID) + "\": c." + DocumentProperties.StudyInstanceUID + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string SeriesSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomSeries.StudyInstanceUID) + "\": c." + DocumentProperties.StudyInstanceUID + ", \"" + nameof(DicomSeries.SeriesInstanceUID) + "\": c." + DocumentProperties.SeriesInstanceUID + " }} FROM c {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private const string InstanceSqlQuerySearchFormat = "SELECT VALUE {{ \"" + nameof(DicomInstance.StudyInstanceUID) + "\": c." + DocumentProperties.StudyInstanceUID + ", \"" + nameof(DicomInstance.SeriesInstanceUID) + "\": c." + DocumentProperties.SeriesInstanceUID + ", \"" + nameof(DicomInstance.SopInstanceUID) + "\": f." + DocumentProperties.SopInstanceUID + " }} FROM c JOIN f in c." + DocumentProperties.Instances + " {0} OFFSET " + OffsetParameterName + " LIMIT " + LimitParameterName; + private readonly DicomCosmosConfiguration _dicomConfiguration; + private readonly IFormatProvider _stringFormatProvider; + + public CosmosQueryBuilder(DicomCosmosConfiguration dicomConfiguration) + { + EnsureArg.IsNotNull(dicomConfiguration, nameof(dicomConfiguration)); + + _dicomConfiguration = dicomConfiguration; + _stringFormatProvider = CultureInfo.InvariantCulture; + } + + public SqlQuerySpec BuildStudyQuerySpec( + int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) + => BuildSeriesLevelQuerySpec(StudySqlQuerySearchFormat, offset, limit, query); + + public SqlQuerySpec BuildSeriesQuerySpec( + int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) + => BuildSeriesLevelQuerySpec(SeriesSqlQuerySearchFormat, offset, limit, query); + + public SqlQuerySpec BuildInstanceQuerySpec( + int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null) + { + return GenerateQuerySpec( + InstanceSqlQuerySearchFormat, + offset, + limit, + query, + (tag, parameter) => $"ARRAY_CONTAINS(f.{DocumentProperties.Attributes}[\"{tag.AttributeId}\"], {parameter.Name})"); + } + + private SqlQuerySpec BuildSeriesLevelQuerySpec( + string querySearchFormat, int offset, int limit, IEnumerable<(DicomAttributeId Attribute, string Value)> query) + { + return GenerateQuerySpec( + querySearchFormat, + offset, + limit, + query, + (tag, parameter) => $"ARRAY_CONTAINS(c.{DocumentProperties.DistinctAttributes}[\"{tag.AttributeId}\"][\"{DocumentProperties.Values}\"], {parameter.Name})"); + } + + private SqlQuerySpec GenerateQuerySpec( + string querySearchFormat, + int offset, + int limit, + IEnumerable<(DicomAttributeId Attribute, string Value)> query, + Func createQueryItem) + { + // As 'OFFSET' and 'LIMIT' are not supported in Linq, all queries must be run using SQL syntax. + SqlParameterCollection sqlParameterCollection = CreateQueryParameterCollection(offset, limit); + + var parameterNameIndex = 1; + var queryItems = new List(); + + if (query != null && _dicomConfiguration.QueryAttributes != null) + { + foreach ((DicomAttributeId attribute, string value) in query.Where(x => _dicomConfiguration.QueryAttributes.Contains(x.Attribute))) + { + var parameterName = string.Format(_stringFormatProvider, ItemParameterNameFormat, parameterNameIndex++); + var sqlParameter = new SqlParameter { Name = parameterName, Value = value }; + + sqlParameterCollection.Add(sqlParameter); + queryItems.Add(createQueryItem(attribute, sqlParameter)); + } + } + + // Now construct the WHERE query joining each item with an AND. + var whereClause = queryItems.Count > 0 ? $"WHERE {string.Join(" AND ", queryItems)}" : string.Empty; + return new SqlQuerySpec(string.Format(_stringFormatProvider, querySearchFormat, whereClause), sqlParameterCollection); + } + + private static SqlParameterCollection CreateQueryParameterCollection(int offset, int limit) + { + EnsureArg.IsGte(offset, 0, nameof(offset)); + EnsureArg.IsGt(limit, 0, nameof(limit)); + + return new SqlParameterCollection() + { + new SqlParameter { Name = OffsetParameterName, Value = offset }, + new SqlParameter { Name = LimitParameterName, Value = limit }, + }; + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs new file mode 100644 index 0000000000..8c47e9aa48 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomCosmosDataStore.cs @@ -0,0 +1,246 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Dicom; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents; +using Microsoft.Health.Extensions.DependencyInjection; +using Polly; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + internal class DicomCosmosDataStore : IDicomIndexDataStore + { + private readonly IDocumentClient _documentClient; + private readonly Uri _collectionUri; + private readonly DicomCosmosConfiguration _dicomConfiguration; + private readonly CosmosQueryBuilder _queryBuilder; + private readonly string _databaseId; + private readonly string _collectionId; + private readonly Random _random = new Random(); + + public DicomCosmosDataStore( + IScoped documentClient, + CosmosDataStoreConfiguration configuration, + IOptionsMonitor namedCosmosCollectionConfigurationAccessor, + DicomCosmosConfiguration dicomConfiguration) + { + EnsureArg.IsNotNull(documentClient?.Value, nameof(documentClient)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); + EnsureArg.IsNotNull(dicomConfiguration, nameof(dicomConfiguration)); + + CosmosCollectionConfiguration cosmosCollectionConfiguration = namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName); + + _documentClient = documentClient.Value; + _collectionUri = configuration.GetRelativeCollectionUri(cosmosCollectionConfiguration.CollectionId); + _databaseId = configuration.DatabaseId; + _collectionId = cosmosCollectionConfiguration.CollectionId; + _dicomConfiguration = dicomConfiguration; + _queryBuilder = new CosmosQueryBuilder(dicomConfiguration); + } + + /// + public async Task IndexInstanceAsync(DicomDataset dicomDataset, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + + var dicomInstance = DicomInstance.Create(dicomDataset); + var defaultDocument = new QuerySeriesDocument(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID); + + RequestOptions requestOptions = CreateRequestOptions(defaultDocument.PartitionKey); + + // Retry when the pre-condition fails on replace (ETag check). + IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); + await _documentClient.CatchClientExceptionAndThrowDataStoreException( + async (documentClient) => + { + QuerySeriesDocument document = await documentClient.GetOrCreateDocumentAsync(_databaseId, _collectionId, defaultDocument.Id, requestOptions, defaultDocument, cancellationToken); + + var instance = QueryInstance.Create(dicomDataset, _dicomConfiguration.QueryAttributes); + + if (!document.AddInstance(instance)) + { + throw new DataStoreException(HttpStatusCode.Conflict); + } + + // Note, we do a replace (rather than upsert) in case the document is deleted. + requestOptions.AccessCondition = new AccessCondition() { Condition = document.ETag, Type = AccessConditionType.IfMatch }; + + Uri documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, document.Id); + await documentClient.ReplaceDocumentAsync(documentUri, document, requestOptions, cancellationToken: cancellationToken); + }, + retryPolicy); + } + + /// + public async Task> QueryStudiesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + { + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildStudyQuerySpec(offset, limit, query); + return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); + } + + /// + public async Task> QuerySeriesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + { + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildSeriesQuerySpec(offset, limit, query); + return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); + } + + /// + public async Task> QueryInstancesAsync( + int offset, int limit, string studyInstanceUID = null, IEnumerable<(DicomAttributeId Attribute, string Value)> query = null, CancellationToken cancellationToken = default) + { + SqlQuerySpec sqlQuerySpec = _queryBuilder.BuildInstanceQuerySpec(offset, limit, query); + return await ExecuteQueryAsync(sqlQuerySpec, studyInstanceUID, cancellationToken); + } + + /// + public async Task> DeleteSeriesIndexAsync( + string studyInstanceUID, string seriesInstanceUID, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + + Uri documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); + RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); + + // Retry when the pre-condition fails on delete. + IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); + return await _documentClient.CatchClientExceptionAndThrowDataStoreException( + async (documentClient) => + { + DocumentResponse response = + await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + + // Update the ETag check on the request options. + requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; + + // Delete the entire series document; if this fails the entire read and delete should be retried. + await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); + + return response.Document.Instances + .Select(x => new DicomInstance(response.Document.StudyUID, response.Document.SeriesUID, x.InstanceUID)) + .ToArray(); + }, + retryPolicy); + } + + /// + public async Task DeleteInstanceIndexAsync(string studyInstanceUID, string seriesInstanceUID, string sopInstanceUID, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID, nameof(studyInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID, nameof(seriesInstanceUID)); + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + + Uri documentUri = UriFactory.CreateDocumentUri(_databaseId, _collectionId, QuerySeriesDocument.GetDocumentId(studyInstanceUID, seriesInstanceUID)); + RequestOptions requestOptions = CreateRequestOptions(QuerySeriesDocument.GetPartitionKey(studyInstanceUID)); + + IAsyncPolicy retryPolicy = CreatePreConditionFailedRetryPolicy(); + await _documentClient.CatchClientExceptionAndThrowDataStoreException( + async (documentClient) => + { + DocumentResponse response = + await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + + // If the instance does not exist, throw not found exception. + if (!response.Document.RemoveInstance(sopInstanceUID)) + { + throw new DataStoreException(HttpStatusCode.NotFound); + } + + requestOptions.AccessCondition = new AccessCondition() { Condition = response.Document.ETag, Type = AccessConditionType.IfMatch }; + + // Delete the entire series if no more instances, otherwise replace. + if (response.Document.Instances.Count == 0) + { + await documentClient.DeleteDocumentAsync(documentUri, requestOptions, cancellationToken); + } + else + { + await documentClient.ReplaceDocumentAsync(documentUri, response.Document, requestOptions, cancellationToken: cancellationToken); + } + }, + retryPolicy); + } + + private RequestOptions CreateRequestOptions(string partitionKey) + => new RequestOptions() { PartitionKey = new PartitionKey(partitionKey) }; + + private FeedOptions CreateFeedOptions(string partitionKey) + { + EnsureArg.IsNotNull(partitionKey, nameof(partitionKey)); + return new FeedOptions() { PartitionKey = new PartitionKey(partitionKey), EnableCrossPartitionQuery = false }; + } + + private FeedOptions CreateCrossPartitionFeedOptions() + { + var feedOptions = new FeedOptions() { EnableCrossPartitionQuery = true }; + + // TODO: This is a reflection hack to enable cross partition skip and take. This will be in the latest SDK soon. + PropertyInfo propertyInfo = feedOptions.GetType().GetProperty("EnableCrossPartitionSkipTake", BindingFlags.NonPublic | BindingFlags.Instance); + propertyInfo.SetValue(feedOptions, Convert.ChangeType(true, propertyInfo.PropertyType)); + + return feedOptions; + } + + private async Task> ExecuteQueryAsync(SqlQuerySpec sqlQuerySpec, string studyInstanceUID, CancellationToken cancellationToken) + { + // If the study instance UID is provided we can run the query against a specific partition. + FeedOptions feedOptions = string.IsNullOrWhiteSpace(studyInstanceUID) ? CreateCrossPartitionFeedOptions() : CreateFeedOptions(studyInstanceUID); + IDocumentQuery documentQuery = _documentClient + .CreateDocumentQuery(_collectionUri, sqlQuerySpec, feedOptions) + .AsDocumentQuery(); + + var results = new List(); + while (documentQuery.HasMoreResults) + { + // Each loop has its own retry handler so we don't retry the entire query on failure. + await _documentClient.CatchClientExceptionAndThrowDataStoreException( + async (documentClient) => + { + FeedResponse nextResults = await documentQuery.ExecuteNextAsync(cancellationToken); + results.AddRange(nextResults); + }); + } + + return new QueryResult(documentQuery.HasMoreResults, results); + } + + private IAsyncPolicy CreatePreConditionFailedRetryPolicy() + => Policy + .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed || ex.StatusCode == HttpStatusCode.TooManyRequests) + .WaitAndRetryForeverAsync( + (retryIndex, ex, context) => + { + // Use RetryAfter header when TooManyRequests response is returned. + if (ex is DocumentClientException doc && doc.StatusCode == HttpStatusCode.TooManyRequests) + { + return doc.RetryAfter; + } + + // Otherwise, delay retry with some randomness. + return TimeSpan.FromMilliseconds((retryIndex - 1) * _random.Next(200, 500)); + }, + (exception, retryIndex, timespan, context) => Task.CompletedTask); + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs new file mode 100644 index 0000000000..d8c0fe155f --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DicomDocumentClientInitializer.cs @@ -0,0 +1,145 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Extensions.Logging; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + public class DicomDocumentClientInitializer : IDocumentClientInitializer + { + private readonly IDocumentClientTestProvider _testProvider; + private readonly ILogger _logger; + + public DicomDocumentClientInitializer(IDocumentClientTestProvider testProvider, ILogger logger) + { + EnsureArg.IsNotNull(logger, nameof(logger)); + EnsureArg.IsNotNull(testProvider, nameof(testProvider)); + + _testProvider = testProvider; + _logger = logger; + } + + /// + public IDocumentClient CreateDocumentClient(CosmosDataStoreConfiguration configuration) + { + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + _logger.LogInformation("Creating DocumentClient instance for {DatabaseUrl}", configuration.RelativeDatabaseUri); + + var connectionPolicy = new ConnectionPolicy + { + ConnectionMode = configuration.ConnectionMode, + ConnectionProtocol = configuration.ConnectionProtocol, + RetryOptions = new RetryOptions + { + MaxRetryAttemptsOnThrottledRequests = configuration.RetryOptions.MaxNumberOfRetries, + MaxRetryWaitTimeInSeconds = configuration.RetryOptions.MaxWaitTimeInSeconds, + }, + }; + + if (configuration.PreferredLocations != null && configuration.PreferredLocations.Any()) + { + _logger.LogInformation("Setting DocumentClient PreferredLocations to {PreferredLocations}", string.Join(";", configuration.PreferredLocations)); + + foreach (var preferredLocation in configuration.PreferredLocations) + { + connectionPolicy.PreferredLocations.Add(preferredLocation); + } + } + + // Setting TypeNameHandling to any value other than 'None' will be flagged + // as causing potential security issues + var serializerSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + DateFormatHandling = DateFormatHandling.IsoDateFormat, + DateParseHandling = DateParseHandling.DateTimeOffset, + DateTimeZoneHandling = DateTimeZoneHandling.RoundtripKind, + TypeNameHandling = TypeNameHandling.None, + }; + + serializerSettings.Converters.Add(new StringEnumConverter()); + + // By default, the Json.NET serializer uses 'F' instead of 'f' for fractions. + // 'F' will omit the trailing digits if they are 0. You might end up getting something like '2018-02-07T20:04:49.97114+00:00' + // where the fraction is actually 971140. Because the ordering is done as string, + // if the values don't always have complete 7 digits, the comparison might not work properly. + serializerSettings.Converters.Add(new IsoDateTimeConverter { DateTimeFormat = "o" }); + + return new DocumentClient(new Uri(configuration.Host), configuration.Key, serializerSettings, connectionPolicy, configuration.DefaultConsistencyLevel); + } + + /// + public async Task OpenDocumentClient(IDocumentClient client, CosmosDataStoreConfiguration configuration, CosmosCollectionConfiguration cosmosCollectionConfiguration) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + Uri absoluteCollectionUri = configuration.GetAbsoluteCollectionUri(cosmosCollectionConfiguration.CollectionId); + + _logger.LogInformation("Opening DocumentClient connection to {CollectionUri}", absoluteCollectionUri); + try + { + await _testProvider.PerformTest(client, configuration, cosmosCollectionConfiguration); + + _logger.LogInformation("Established DocumentClient connection to {CollectionUri}", absoluteCollectionUri); + } + catch (Exception e) + { + _logger.LogCritical(e, "Failed to connect to DocumentClient collection {CollectionUri}", absoluteCollectionUri); + throw; + } + } + + /// + public async Task InitializeDataStore(IDocumentClient documentClient, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable collectionInitializers) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers)); + + try + { + _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); + + if (cosmosDataStoreConfiguration.AllowDatabaseCreation) + { + _logger.LogDebug("CreateDatabaseIfNotExists {DatabaseId})", cosmosDataStoreConfiguration.DatabaseId); + + var options = new RequestOptions { OfferThroughput = cosmosDataStoreConfiguration.InitialDatabaseThroughput }; + await documentClient.CreateDatabaseIfNotExistsAsync( + new Database + { + Id = cosmosDataStoreConfiguration.DatabaseId, + }, + options); + } + + foreach (var collectionInitializer in collectionInitializers) + { + await collectionInitializer.InitializeCollection(documentClient); + } + + _logger.LogInformation("Cosmos DB Database {DatabaseId} and collections successfully initialized", cosmosDataStoreConfiguration.DatabaseId); + } + catch (Exception ex) + { + _logger.LogCritical(ex, "Cosmos DB Database {DatabaseId} and collections initialization failed", cosmosDataStoreConfiguration.DatabaseId); + throw; + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs new file mode 100644 index 0000000000..2eeb186096 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/DocumentClientExtensions.cs @@ -0,0 +1,118 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Newtonsoft.Json; +using Polly; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage +{ + public static class DocumentClientExtensions + { + public static async Task CatchClientExceptionAndThrowDataStoreException(this IDocumentClient documentClient, Func> action, IAsyncPolicy retryPolicy = null) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNull(action, nameof(action)); + + try + { + if (retryPolicy != null) + { + return await retryPolicy.ExecuteAsync(() => action(documentClient)); + } + + return await action(documentClient); + } + catch (DocumentClientException e) + { + throw new DataStoreException(e.StatusCode, e); + } + } + + public static async Task CatchClientExceptionAndThrowDataStoreException(this IDocumentClient documentClient, Func action, IAsyncPolicy retryPolicy = null) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNull(action, nameof(action)); + + try + { + if (retryPolicy != null) + { + await retryPolicy.ExecuteAsync(() => action(documentClient)); + } + else + { + await action(documentClient); + } + } + catch (DocumentClientException e) + { + throw new DataStoreException(e.StatusCode, e); + } + } + + public static async Task GetOrCreateDocumentAsync( + this IDocumentClient documentClient, + string databaseId, + string collectionId, + string documentId, + RequestOptions requestOptions, + T defaultValue, + CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNullOrWhiteSpace(databaseId, nameof(databaseId)); + EnsureArg.IsNotNullOrWhiteSpace(collectionId, nameof(collectionId)); + EnsureArg.IsNotNullOrWhiteSpace(documentId, nameof(documentId)); + + try + { + Uri documentUri = UriFactory.CreateDocumentUri(databaseId, collectionId, documentId); + DocumentResponse response = await documentClient.ReadDocumentAsync(documentUri, requestOptions, cancellationToken: cancellationToken); + return response.Document; + } + catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Attempt to create the document as it does not exist. + return await documentClient.CreateOrGetDocumentAsync(databaseId, collectionId, documentId, requestOptions, defaultValue, cancellationToken); + } + } + + public static async Task CreateOrGetDocumentAsync( + this IDocumentClient documentClient, + string databaseId, + string collectionId, + string documentId, + RequestOptions requestOptions, + T defaultValue, + CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(documentClient, nameof(documentClient)); + EnsureArg.IsNotNullOrWhiteSpace(databaseId, nameof(databaseId)); + EnsureArg.IsNotNullOrWhiteSpace(collectionId, nameof(collectionId)); + EnsureArg.IsNotNullOrWhiteSpace(documentId, nameof(documentId)); + + try + { + Uri collectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, collectionId); + ResourceResponse result = await documentClient.CreateDocumentAsync(collectionUri, defaultValue, requestOptions, disableAutomaticIdGeneration: true, cancellationToken: cancellationToken); + + return JsonConvert.DeserializeObject(result.Resource.ToString(), requestOptions?.JsonSerializerSettings); + } + catch (DocumentClientException ex) when (ex.StatusCode == HttpStatusCode.Conflict) + { + // Attempt to read the document as it already exists. + return await documentClient.GetOrCreateDocumentAsync(databaseId, collectionId, documentId, requestOptions, defaultValue, cancellationToken); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs new file mode 100644 index 0000000000..346b7f742e --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/AttributeValues.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + public class AttributeValues + { + public AttributeValues() + { + } + + [JsonConstructor] + public AttributeValues(HashSet values) + { + EnsureArg.IsNotNull(values, nameof(values)); + + Values = values; + } + + /// + /// Gets the minimum date time value from the array of values. + /// We pre-calculate this to support range queries on DateTime values. + /// This value does not need to be serialized if the array does not contain any DateTime values. + /// + [JsonProperty(DocumentProperties.MinimumDateTimeValue, NullValueHandling = NullValueHandling.Ignore)] + public DateTime? MinDateTime => Values.Where(x => x is DateTime).Min(x => (DateTime?)x); + + /// + /// Gets the maximum date time value from the array of values. + /// We pre-calculate this to support range queries on DateTime values. + /// This value does not need to be serialized if the array does not contain any DateTime values. + /// + [JsonProperty(DocumentProperties.MaximumDateTimeValue, NullValueHandling = NullValueHandling.Ignore)] + public DateTime? MaxDateTime => Values.Where(x => x is DateTime).Max(x => (DateTime?)x); + + [JsonProperty(DocumentProperties.Values)] + public HashSet Values { get; } = new HashSet(); + + public bool Add(object value) => value == null ? false : Values.Add(value); + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs new file mode 100644 index 0000000000..9efc970805 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/DocumentProperties.cs @@ -0,0 +1,20 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + internal static class DocumentProperties + { + public const string StudyInstanceUID = "studyUID"; + public const string SeriesInstanceUID = "seriesUID"; + public const string SopInstanceUID = "instanceUID"; + public const string Instances = "instances"; + public const string Attributes = "attributes"; + public const string DistinctAttributes = "distinctAttributes"; + public const string Values = "values"; + public const string MinimumDateTimeValue = "minDateTime"; + public const string MaximumDateTimeValue = "maxDateTime"; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs new file mode 100644 index 0000000000..96ad50c0f3 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QueryInstance.cs @@ -0,0 +1,76 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Dicom; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + internal class QueryInstance + { + [JsonConstructor] + public QueryInstance(string instanceUID, IDictionary attributes) + : this(instanceUID) + { + EnsureArg.IsNotNull(attributes, nameof(attributes)); + + Attributes = attributes; + } + + private QueryInstance(string instanceUID) + { + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(instanceUID), nameof(instanceUID)); + + InstanceUID = instanceUID; + } + + [JsonProperty(DocumentProperties.SopInstanceUID)] + public string InstanceUID { get; } + + [JsonProperty(DocumentProperties.Attributes)] + public IDictionary Attributes { get; } = new Dictionary(); + + public override int GetHashCode() + { + // Note: We override Equals() and GetHashCode() so if this class is added to a HashSet + // we only compare on SopInstanceUID. + return InstanceUID.GetHashCode(StringComparison.Ordinal); + } + + public override bool Equals(object obj) + { + if (obj is QueryInstance instance) + { + return InstanceUID.Equals(instance.InstanceUID, StringComparison.Ordinal); + } + + return false; + } + + public static QueryInstance Create(DicomDataset dicomDataset, IEnumerable indexableAttributes) + { + EnsureArg.IsNotNull(dicomDataset, nameof(dicomDataset)); + EnsureArg.IsNotNull(indexableAttributes, nameof(indexableAttributes)); + + var sopInstanceUID = dicomDataset.GetSingleValueOrDefault(DicomTag.SOPInstanceUID, string.Empty); + var result = new QueryInstance(sopInstanceUID); + + foreach (DicomAttributeId attribute in indexableAttributes) + { + if (dicomDataset.TryGetValues(attribute, out object[] values)) + { + result.Attributes[attribute.AttributeId] = values; + } + } + + return result; + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs new file mode 100644 index 0000000000..18f60e60c0 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Documents/QuerySeriesDocument.cs @@ -0,0 +1,118 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Text.RegularExpressions; +using EnsureThat; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Validation; +using Newtonsoft.Json; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Documents +{ + internal class QuerySeriesDocument + { + /// + /// The seperator between the Study and Series instance identifiers when creating the document identifier. + /// + private const char DocumentIdSeperator = '_'; + + /// + /// Lists the characters that are not valid for a Cosmos resource identifier. + /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.documents.resource.id?view=azure-dotnet + /// + private static readonly Regex DocumentIdRegex = new Regex("[^?/\\#]", RegexOptions.Singleline | RegexOptions.Compiled); + + public QuerySeriesDocument(string studyUID, string seriesUID) + { + EnsureArg.IsNotNullOrWhiteSpace(studyUID, nameof(studyUID)); + EnsureArg.IsNotNullOrWhiteSpace(seriesUID, nameof(seriesUID)); + EnsureArg.IsFalse(studyUID == seriesUID, nameof(seriesUID)); + + StudyUID = studyUID; + SeriesUID = seriesUID; + Id = GetDocumentId(studyUID, seriesUID); + PartitionKey = GetPartitionKey(studyUID); + } + + [JsonProperty(KnownDocumentProperties.Id)] + public string Id { get; } + + [JsonProperty(KnownDocumentProperties.PartitionKey)] + public string PartitionKey { get; } + + [JsonProperty(KnownDocumentProperties.ETag)] + public string ETag { get; set; } + + [JsonProperty(DocumentProperties.StudyInstanceUID)] + public string StudyUID { get; } + + [JsonProperty(DocumentProperties.SeriesInstanceUID)] + public string SeriesUID { get; } + + [JsonProperty(DocumentProperties.Instances)] + public HashSet Instances { get; } = new HashSet(); + + [JsonProperty(DocumentProperties.DistinctAttributes)] + public IDictionary DistinctAttributes + { + get + { + var result = new Dictionary(); + + foreach (QueryInstance instance in Instances) + { + foreach ((string key, object[] values) in instance.Attributes) + { + if (!result.ContainsKey(key)) + { + result[key] = new AttributeValues(); + } + + foreach (object value in values) + { + result[key].Add(value); + } + } + } + + return result; + } + } + + public static string GetDocumentId(string studyUID, string seriesUID) + { + EnsureArg.IsFalse(studyUID == seriesUID); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyUID)); + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(seriesUID)); + + string documentId = $"{studyUID}{DocumentIdSeperator}{seriesUID}"; + + // Double safety for the document identifier. If the study and series are valid identifiers, the document ID 'should' conform. + EnsureArg.IsTrue(DocumentIdRegex.IsMatch(documentId)); + + return documentId; + } + + public static string GetPartitionKey(string studyUID) + { + EnsureArg.IsTrue(DicomIdentifierValidator.IdentifierRegex.IsMatch(studyUID)); + + return studyUID; + } + + public bool AddInstance(QueryInstance instance) + { + EnsureArg.IsNotNull(instance, nameof(instance)); + return Instances.Add(instance); + } + + public bool RemoveInstance(string sopInstanceUID) + { + EnsureArg.IsNotNullOrWhiteSpace(sopInstanceUID, nameof(sopInstanceUID)); + return Instances.RemoveWhere(x => x.InstanceUID == sopInstanceUID) > 0; + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs new file mode 100644 index 0000000000..ba705cec72 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionSettingsUpdater.cs @@ -0,0 +1,99 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Azure.Documents.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning +{ + public class DicomCollectionSettingsUpdater : IDicomCollectionUpdater + { + private readonly ILogger _logger; + private readonly CosmosDataStoreConfiguration _configuration; + private readonly CosmosCollectionConfiguration _collectionConfiguration; + private const int CollectionSettingsVersion = 1; + + public DicomCollectionSettingsUpdater( + CosmosDataStoreConfiguration configuration, + IOptionsMonitor namedCosmosCollectionConfigurationAccessor, + ILogger logger) + { + EnsureArg.IsNotNull(configuration, nameof(configuration)); + EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); + EnsureArg.IsNotNull(logger, nameof(logger)); + + _configuration = configuration; + _collectionConfiguration = namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName); + _logger = logger; + } + + public async Task ExecuteAsync(IDocumentClient client, DocumentCollection collection, Uri relativeCollectionUri) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(collection, nameof(collection)); + + CollectionVersion thisVersion = await GetLatestCollectionVersion(client, collection); + + if (thisVersion.Version < CollectionSettingsVersion) + { + _logger.LogDebug("Ensuring indexes are up-to-date {CollectionUri}", _configuration.GetAbsoluteCollectionUri(_collectionConfiguration.CollectionId)); + + collection.IndexingPolicy = new IndexingPolicy + { + IndexingMode = IndexingMode.Consistent, + Automatic = true, + IncludedPaths = new Collection + { + new IncludedPath + { + Path = "/*", + Indexes = new Collection(), + }, + }, + ExcludedPaths = new Collection() + { + new ExcludedPath + { + Path = "/\"_etag\"/?", + }, + }, + }; + + // Setting the DefaultTTL to -1 means that by default all documents in the collection will live forever + // but the Cosmos DB service should monitor this collection for documents that have overridden this default. + // See: https://docs.microsoft.com/en-us/azure/cosmos-db/time-to-live + collection.DefaultTimeToLive = -1; + + await client.ReplaceDocumentCollectionAsync(collection); + + thisVersion.Version = CollectionSettingsVersion; + await client.UpsertDocumentAsync(relativeCollectionUri, thisVersion); + } + } + + private static async Task GetLatestCollectionVersion(IDocumentClient documentClient, DocumentCollection collection) + { + IDocumentQuery query = documentClient.CreateDocumentQuery( + collection.SelfLink, + new SqlQuerySpec("SELECT * FROM root r"), + new FeedOptions { PartitionKey = new PartitionKey(CollectionVersion.CollectionVersionPartition) }) + .AsDocumentQuery(); + + FeedResponse result = await query.ExecuteNextAsync(); + + return result.FirstOrDefault() ?? new CollectionVersion(); + } + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs new file mode 100644 index 0000000000..6f481c889e --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/DicomCollectionUpgradeManager.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning +{ + public class DicomCollectionUpgradeManager : CollectionUpgradeManager + { + public DicomCollectionUpgradeManager( + IEnumerable collectionUpdater, + CosmosDataStoreConfiguration configuration, + IOptionsMonitor namedCosmosCollectionConfigurationAccessor, + ICosmosDbDistributedLockFactory lockFactory, + ILogger logger) + : base( + collectionUpdater, + configuration, + GetCosmosCollectionConfiguration(namedCosmosCollectionConfigurationAccessor, Constants.CollectionConfigurationName), + lockFactory, + logger) + { + } + + public override int CollectionSettingsVersion => 1; + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs new file mode 100644 index 0000000000..737fcaef92 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Features/Storage/Versioning/IDicomCollectionUpdater.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; + +namespace Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning +{ + public interface IDicomCollectionUpdater : ICollectionUpdater + { + } +} diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj b/src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj new file mode 100644 index 0000000000..e09fc62f3f --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Microsoft.Health.Dicom.CosmosDb.csproj @@ -0,0 +1,18 @@ + + + + netcoreapp2.2 + + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs new file mode 100644 index 0000000000..af57ba2a12 --- /dev/null +++ b/src/Microsoft.Health.Dicom.CosmosDb/Registration/DicomCosmosDbRegistrationExtensions.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Dicom.CosmosDb; +using Microsoft.Health.Dicom.CosmosDb.Features.Health; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Extensions.DependencyInjection; + +namespace Microsoft.Extensions.DependencyInjection +{ + public static class DicomCosmosDbRegistrationExtensions + { + public static IServiceCollection AddDicomCosmosDb(this IServiceCollection services, IConfiguration configuration) + { + EnsureArg.IsNotNull(services, nameof(services)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + services.AddCosmosDb(); + + services + .Configure( + Constants.CollectionConfigurationName, + cosmosCollectionConfiguration => configuration.GetSection("DicomWeb:CosmosDb") + .Bind(cosmosCollectionConfiguration)); + + services.Add(sp => + { + CosmosDataStoreConfiguration config = sp.GetService(); + DicomCollectionUpgradeManager upgradeManager = sp.GetService(); + ILoggerFactory loggerFactory = sp.GetService(); + IOptionsMonitor namedCosmosCollectionConfiguration = sp.GetService>(); + CosmosCollectionConfiguration cosmosCollectionConfiguration = namedCosmosCollectionConfiguration.Get(Constants.CollectionConfigurationName); + + return new CollectionInitializer( + cosmosCollectionConfiguration.CollectionId, + config, + cosmosCollectionConfiguration.InitialCollectionThroughput, + upgradeManager, + loggerFactory.CreateLogger()); + }) + .Singleton() + .AsService(); + + services.Add() + .Singleton() + .AsService(); + + services.Add() + .Scoped() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf() + .AsService(); + + services.AddHealthChecks() + .AddCheck(name: nameof(DicomCosmosHealthCheck)); + + return services; + } + } +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj b/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj index 8a77d3bfe8..c1edce6264 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Microsoft.Health.Dicom.Tests.Integration.csproj @@ -18,6 +18,7 @@ + diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs index b7e97f0eb4..51a5579ecd 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTests.cs @@ -8,10 +8,10 @@ using System.Net; using System.Threading.Tasks; using Microsoft.Azure.Storage; -using Microsoft.Health.Dicom.Blob.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; using Xunit; -namespace Microsoft.Health.DicomTests.Integration.Persistence +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence { public class DicomBlobStorageTests : IClassFixture { diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs index 8292374df8..36d960f837 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomBlobStorageTestsFixture.cs @@ -13,10 +13,11 @@ using Microsoft.Health.Blob.Features.Storage; using Microsoft.Health.Dicom.Blob; using Microsoft.Health.Dicom.Blob.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; using NSubstitute; using Xunit; -namespace Microsoft.Health.DicomTests.Integration.Persistence +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence { public class DicomBlobStorageTestsFixture : IAsyncLifetime { diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs new file mode 100644 index 0000000000..756c7f2137 --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomCosmosDataStoreTestsFixture.cs @@ -0,0 +1,106 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.CosmosDb.Configs; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.CosmosDb.Config; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Extensions.DependencyInjection; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence +{ + public class DicomCosmosDataStoreTestsFixture : IAsyncLifetime + { + private static readonly SemaphoreSlim CollectionInitializationSemaphore = new SemaphoreSlim(1, 1); + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; + private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration; + private readonly DicomCosmosConfiguration _dicomCosmosConfiguration; + + public DicomCosmosDataStoreTestsFixture() + { + _cosmosDataStoreConfiguration = new CosmosDataStoreConfiguration + { + Host = Environment.GetEnvironmentVariable("CosmosDb:Host") ?? CosmosDbLocalEmulator.Host, + Key = Environment.GetEnvironmentVariable("CosmosDb:Key") ?? CosmosDbLocalEmulator.Key, + DatabaseId = Environment.GetEnvironmentVariable("CosmosDb:DatabaseId") ?? "DicomTests", + AllowDatabaseCreation = true, + PreferredLocations = Environment.GetEnvironmentVariable("CosmosDb:PreferredLocations")?.Split(';', StringSplitOptions.RemoveEmptyEntries), + }; + + _cosmosCollectionConfiguration = new CosmosCollectionConfiguration + { + CollectionId = Guid.NewGuid().ToString(), + }; + + _dicomCosmosConfiguration = new DicomCosmosConfiguration(); + } + + public IDicomIndexDataStore DicomIndexDataStore { get; private set; } + + public IDocumentClient DocumentClient { get; private set; } + + public string DatabaseId => _cosmosDataStoreConfiguration.DatabaseId; + + public string CollectionId => _cosmosCollectionConfiguration.CollectionId; + + public async Task InitializeAsync() + { + IOptionsMonitor optionsMonitor = Substitute.For>(); + + optionsMonitor.Get(CosmosDb.Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); + + var updaters = new IDicomCollectionUpdater[] { }; + + var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); + + var upgradeManager = new DicomCollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + IDocumentClientTestProvider testProvider = new DocumentClientReadWriteTestProvider(); + + var documentClientInitializer = new DicomDocumentClientInitializer(testProvider, NullLogger.Instance); + DocumentClient = documentClientInitializer.CreateDocumentClient(_cosmosDataStoreConfiguration); + var collectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration.CollectionId, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration.InitialCollectionThroughput, upgradeManager, NullLogger.Instance); + + // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. + // Use the semaphore to only allow one initialization at a time. + await CollectionInitializationSemaphore.WaitAsync(); + + try + { + await documentClientInitializer.InitializeDataStore(DocumentClient, _cosmosDataStoreConfiguration, new List { collectionInitializer }); + } + finally + { + CollectionInitializationSemaphore.Release(); + } + + var documentClient = new NonDisposingScope(DocumentClient); + + DicomIndexDataStore = new DicomCosmosDataStore( + documentClient, + _cosmosDataStoreConfiguration, + optionsMonitor, + _dicomCosmosConfiguration); + } + + public async Task DisposeAsync() + { + using (DocumentClient as IDisposable) + { + await DocumentClient?.DeleteDocumentCollectionAsync(_cosmosDataStoreConfiguration.GetRelativeCollectionUri(_cosmosCollectionConfiguration.CollectionId)); + } + } + } +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs new file mode 100644 index 0000000000..39917197e2 --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DicomIndexDataStoreTests.cs @@ -0,0 +1,419 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Dicom; +using EnsureThat; +using Microsoft.Health.Dicom.Core.Features.Persistence; +using Microsoft.Health.Dicom.Core.Features.Persistence.Exceptions; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence +{ + public class DicomIndexDataStoreTests : IClassFixture + { + private readonly IDicomIndexDataStore _indexDataStore; + + public DicomIndexDataStoreTests(DicomCosmosDataStoreTestsFixture fixture) + { + _indexDataStore = fixture.DicomIndexDataStore; + } + + [Fact] + public async Task GivenAValidInstance_WhenIndexing_CanBeQueried() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + + await _indexDataStore.IndexInstanceAsync(testInstance); + + QueryResult instancesInStudy = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.Single(instancesInStudy.Results); + Assert.False(instancesInStudy.HasMoreResults); + Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == instancesInStudy.Results.First().SopInstanceUID); + + QueryResult seriesInStudy = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID); + Assert.False(seriesInStudy.HasMoreResults); + Assert.Single(seriesInStudy.Results); + Assert.True(seriesInstanceUID == seriesInStudy.Results.First().SeriesInstanceUID); + + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + + Assert.Single(deletedInstances); + Assert.True(testInstance.GetSingleValue(DicomTag.SOPInstanceUID) == deletedInstances.First().SopInstanceUID); + } + + [Fact] + public async Task GivenMultipleValidInstances_WhenIndexingInParallel_AreStored() + { + const int numberOfInstancesToIndex = 5; + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstancesToIndex); + IList sopInstanceUIDs = instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID)).ToList(); + + QueryResult instancesInStudy = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.False(instancesInStudy.HasMoreResults); + Assert.Equal(numberOfInstancesToIndex, instancesInStudy.Results.Count()); + instancesInStudy.Results.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + + QueryResult seriesInStudy = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID); + Assert.Single(seriesInStudy.Results); + Assert.False(seriesInStudy.HasMoreResults); + Assert.True(seriesInstanceUID == seriesInStudy.Results.First().SeriesInstanceUID); + + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + Assert.Equal(numberOfInstancesToIndex, deletedInstances.Count()); + deletedInstances.Each(x => Assert.True(sopInstanceUIDs.Contains(x.SopInstanceUID))); + } + + [Fact] + public async Task GivenAValidSeries_WhenDeletingAnInstance_InstanceIsDeletedButSeriesRemains() + { + const int numberOfInstancesToIndex = 5; + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstancesToIndex); + string firstSopInstanceUID = instances[0].GetSingleValue(DicomTag.SOPInstanceUID); + IList otherSopInstanceUIDs = instances.Skip(1).Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID)).ToList(); + + await _indexDataStore.DeleteInstanceIndexAsync(studyInstanceUID, seriesInstanceUID, firstSopInstanceUID); + + QueryResult instancesInStudy = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.False(instancesInStudy.HasMoreResults); + Assert.Equal(numberOfInstancesToIndex - 1, instancesInStudy.Results.Count()); + + IEnumerable deletedInstances = await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + deletedInstances.Each(x => Assert.True(otherSopInstanceUIDs.Contains(x.SopInstanceUID))); + } + + [Fact] + public async Task GivenAnInstanceWithInvalidInstanceId_WhenStoring_ExceptionIsThrown() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = "?/#"; + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + + testInstance.AddOrUpdate(DicomTag.SeriesInstanceUID, new string('a', 65)); + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + + testInstance.Remove(DicomTag.SeriesInstanceUID); + await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + } + + [Fact] + public async Task GivenNonExistentInstancesOrSeries_WhenDeleting_IndexDataStoreExceptionIsThrownWithNotFoundStatusCode() + { + DataStoreException deleteInstanceException = await Assert.ThrowsAsync( + () => _indexDataStore.DeleteInstanceIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Equal((int)HttpStatusCode.NotFound, deleteInstanceException.StatusCode); + + DataStoreException deleteSeriesException = await Assert.ThrowsAsync( + () => _indexDataStore.DeleteSeriesIndexAsync(Guid.NewGuid().ToString(), Guid.NewGuid().ToString())); + Assert.Equal((int)HttpStatusCode.NotFound, deleteSeriesException.StatusCode); + } + + [Fact] + public async Task GivenNonExistentStudy_WhenQuerying_NoExceptionIsThrown() + { + QueryResult studyResults = await _indexDataStore.QueryStudiesAsync(0, 10, Guid.NewGuid().ToString()); + Assert.Empty(studyResults.Results); + Assert.False(studyResults.HasMoreResults); + + QueryResult seriesResults = await _indexDataStore.QuerySeriesAsync(0, 10, Guid.NewGuid().ToString()); + Assert.Empty(seriesResults.Results); + Assert.False(studyResults.HasMoreResults); + + QueryResult instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, Guid.NewGuid().ToString()); + Assert.Empty(instanceResults.Results); + Assert.False(studyResults.HasMoreResults); + } + + [Fact] + public async Task GivenAStoredInstance_WhenStoringAgain_ConflictExceptionThrown() + { + DicomDataset testInstance = CreateTestInstanceDicomDataset(Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + await _indexDataStore.IndexInstanceAsync(testInstance); + DataStoreException dataStoreException = await Assert.ThrowsAsync(() => _indexDataStore.IndexInstanceAsync(testInstance)); + Assert.Equal((int)HttpStatusCode.Conflict, dataStoreException.StatusCode); + + await DeleteInstancesAsync(testInstance); + } + + [Fact] + public async Task GivenIndexDataStoreQuery_WhenQueryingWithInvalidParameters_ArgumentExceptionThrown() + { + await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryStudiesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QuerySeriesAsync(0, 0)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(-1, 10)); + await Assert.ThrowsAsync(() => _indexDataStore.QueryInstancesAsync(0, 0)); + } + + [Fact] + public async Task GivenIndexDataStoreQuery_WhenQueryingOnNonIndexAttribute_IsIgnoredAndQueryReturns() + { + DicomTag unsupportedDicomTag = DicomTag.AbortFlag; + DicomTag supportedDicomTag = DicomTag.PatientName; + + var studyInstanceUID = Guid.NewGuid().ToString(); + var testInstance1PatientName = "Test1"; + DicomDataset testInstance1 = CreateTestInstanceDicomDataset(studyInstanceUID, Guid.NewGuid().ToString(), testInstance1PatientName); + DicomDataset testInstance2 = CreateTestInstanceDicomDataset(studyInstanceUID, Guid.NewGuid().ToString(), "Test2"); + + testInstance1.AddOrUpdate(unsupportedDicomTag, "DifferentValue"); + + await _indexDataStore.IndexInstanceAsync(testInstance1); + await _indexDataStore.IndexInstanceAsync(testInstance2); + + var dicomInstance = DicomInstance.Create(testInstance1); + (DicomAttributeId, string)[] queryAttributes = new[] { (new DicomAttributeId(supportedDicomTag), testInstance1PatientName), (new DicomAttributeId(unsupportedDicomTag), "test") }; + + QueryResult studyResults = await _indexDataStore.QueryStudiesAsync(0, 10, studyInstanceUID, query: queryAttributes); + Assert.Single(studyResults.Results); + Assert.False(studyResults.HasMoreResults); + Assert.Equal(new DicomStudy(dicomInstance.StudyInstanceUID), studyResults.Results.First()); + + QueryResult seriesResults = await _indexDataStore.QuerySeriesAsync(0, 10, studyInstanceUID, query: queryAttributes); + Assert.Single(seriesResults.Results); + Assert.False(seriesResults.HasMoreResults); + Assert.Equal(new DicomSeries(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID), seriesResults.Results.First()); + + QueryResult instanceResults = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, query: queryAttributes); + Assert.Single(instanceResults.Results); + Assert.False(instanceResults.HasMoreResults); + Assert.Equal(dicomInstance, instanceResults.Results.First()); + + await DeleteInstancesAsync(testInstance1, testInstance2); + } + + [Fact] + public async Task GivenIndexedInstance_WhenQueryingByPatientName_InstancesIsRetrieved() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + string referringPhysicianName = Guid.NewGuid().ToString(); + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + testInstance.Add(DicomTag.ReferringPhysicianName, referringPhysicianName); + + await _indexDataStore.IndexInstanceAsync(testInstance); + + var sopInstanceUID = testInstance.GetSingleValue(DicomTag.SOPInstanceUID); + QueryResult instances = await _indexDataStore.QueryInstancesAsync(0, 10, query: new[] { (new DicomAttributeId(DicomTag.ReferringPhysicianName), referringPhysicianName) }); + Assert.Single(instances.Results); + Assert.False(instances.HasMoreResults); + Assert.Equal(new DicomInstance(studyInstanceUID, seriesInstanceUID, sopInstanceUID), instances.Results.First()); + + await DeleteInstancesAsync(testInstance); + } + + [Fact] + public async Task GivenIndexedSeries_WhenQueryingWithPaging_PagesReturnedCorrectly() + { + const int numberOfInstances = 20; + Assert.Equal(0, numberOfInstances % 2); + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + IList instances = await CreateSeriesInParallelAsync(studyInstanceUID, seriesInstanceUID, numberOfInstances); + var instanceUIDs = new HashSet(instances.Select(x => x.GetSingleValue(DicomTag.SOPInstanceUID))); + + // End of page + QueryResult queryInstances1 = await _indexDataStore.QueryInstancesAsync(numberOfInstances, numberOfInstances, studyInstanceUID); + Assert.Empty(queryInstances1.Results); + Assert.False(queryInstances1.HasMoreResults); + + // First half of items items + QueryResult queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, numberOfInstances / 2, studyInstanceUID); + Assert.Equal(numberOfInstances / 2, queryInstances2.Results.Count()); + Assert.False(queryInstances2.HasMoreResults); + + foreach (DicomInstance item in queryInstances2.Results) + { + Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); + } + + // Last half of items + QueryResult queryInstances3 = await _indexDataStore.QueryInstancesAsync(numberOfInstances / 2, numberOfInstances, studyInstanceUID); + Assert.Equal(numberOfInstances / 2, queryInstances3.Results.Count()); + Assert.False(queryInstances3.HasMoreResults); + + foreach (DicomInstance item in queryInstances3.Results) + { + Assert.True(instanceUIDs.Remove(item.SopInstanceUID)); + } + + Assert.Empty(instanceUIDs); + + await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + } + + [Fact] + public async Task GivenMultipleStudies_WhenQueryingWithOrWithoutStudyInstanceUID_QueryResultsReturnedCorrectly() + { + var seriesPerStudy = 2; + var instancesPerSeries = 2; + var patientName = Guid.NewGuid().ToString(); + var studyInstanceUIDs = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + var dicomSeries = new List(); + var totalItems = studyInstanceUIDs.Length * seriesPerStudy * instancesPerSeries; + + foreach (string studyInstanceUID in studyInstanceUIDs) + { + for (var i = 0; i < seriesPerStudy; i++) + { + var series = new DicomSeries(studyInstanceUID, Guid.NewGuid().ToString()); + dicomSeries.Add(series); + + for (int ii = 0; ii < instancesPerSeries; ii++) + { + await _indexDataStore.IndexInstanceAsync(CreateTestInstanceDicomDataset(studyInstanceUID, series.SeriesInstanceUID, patientName)); + } + } + } + + // Validate Study Searching + QueryResult queryStudiesResults1 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1); + Assert.Equal(studyInstanceUIDs.Length, queryStudiesResults1.Results.Count()); + Assert.False(queryStudiesResults1.HasMoreResults); + queryStudiesResults1.Results.Each(x => Assert.Contains(x.StudyInstanceUID, studyInstanceUIDs)); + + QueryResult queryStudiesResults2 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, studyInstanceUIDs[0]); + Assert.Single(queryStudiesResults2.Results); + Assert.False(queryStudiesResults2.HasMoreResults); + + QueryResult queryStudiesResults3 = await _indexDataStore.QueryStudiesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); + Assert.Empty(queryStudiesResults3.Results); + Assert.False(queryStudiesResults3.HasMoreResults); + + // Validate Series Searching + QueryResult querySeriesResults1 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1); + Assert.Equal(dicomSeries.Count, querySeriesResults1.Results.Count()); + Assert.False(querySeriesResults1.HasMoreResults); + querySeriesResults1.Results.Each(x => Assert.Contains(x, dicomSeries)); + + QueryResult querySeriesResults2 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, studyInstanceUIDs[0]); + Assert.Equal(seriesPerStudy, querySeriesResults2.Results.Count()); + Assert.False(querySeriesResults2.HasMoreResults); + + QueryResult querySeriesResults3 = await _indexDataStore.QuerySeriesAsync(0, totalItems + 1, Guid.NewGuid().ToString()); + Assert.Empty(querySeriesResults3.Results); + Assert.False(querySeriesResults3.HasMoreResults); + + foreach (DicomSeries series in dicomSeries) + { + await _indexDataStore.DeleteSeriesIndexAsync(series.StudyInstanceUID, series.SeriesInstanceUID); + } + } + + [Fact] + public async Task GivenInstanceWithInjectedSql_WhenQuerying_IsReturnedCorrectly() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + + DicomDataset testInstance = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID); + testInstance.Add(DicomTag.ReferringPhysicianName, " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\""); + + await _indexDataStore.IndexInstanceAsync(testInstance); + + QueryResult queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID); + Assert.False(queryInstances1.HasMoreResults); + Assert.Single(queryInstances1.Results); + Assert.Equal(DicomInstance.Create(testInstance), queryInstances1.Results.First()); + + QueryResult queryInstances2 = await _indexDataStore.QueryInstancesAsync(0, 10, studyInstanceUID, new[] { (new DicomAttributeId(DicomTag.PatientName), " AND f.IndexedAttributes[\"0010,0010\"] = \"invalid\"") }); + Assert.False(queryInstances2.HasMoreResults); + Assert.Empty(queryInstances2.Results); + + await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + } + + [Fact] + public async Task GivenWorstCaseStoreScenario_WhenAllInstancesHaveDifferentTagValues_CanIndexAndQueryCorrectly() + { + string studyInstanceUID = Guid.NewGuid().ToString(); + string seriesInstanceUID = Guid.NewGuid().ToString(); + var startDateTime = new DateTime(2019, 6, 21); + const int numberOfItemsToInsert = 100; + var patientNames = Enumerable.Range(0, numberOfItemsToInsert).Select(_ => Guid.NewGuid().ToString()).ToArray(); + + for (var i = 0; i < numberOfItemsToInsert; i++) + { + DicomDataset dataset = CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID, patientNames[i]); + dataset.AddOrUpdate(DicomTag.StudyDate, startDateTime.AddDays(i)); + await _indexDataStore.IndexInstanceAsync(dataset); + } + + QueryResult queryStudies1 = await _indexDataStore.QueryStudiesAsync(0, 10); + Assert.False(queryStudies1.HasMoreResults); + Assert.Single(queryStudies1.Results); + Assert.Equal(studyInstanceUID, queryStudies1.Results.First().StudyInstanceUID); + + QueryResult queryStudies2 = await _indexDataStore.QueryStudiesAsync(0, 10, query: new[] { (new DicomAttributeId(DicomTag.PatientName), patientNames[numberOfItemsToInsert / 2]) }); + Assert.Single(queryStudies2.Results); + Assert.False(queryStudies2.HasMoreResults); + Assert.Equal(studyInstanceUID, queryStudies2.Results.First().StudyInstanceUID); + + QueryResult querySeries1 = await _indexDataStore.QuerySeriesAsync(0, 10); + Assert.False(querySeries1.HasMoreResults); + Assert.Single(querySeries1.Results); + Assert.Equal(seriesInstanceUID, querySeries1.Results.First().SeriesInstanceUID); + + QueryResult queryInstances1 = await _indexDataStore.QueryInstancesAsync(0, numberOfItemsToInsert + 1); + Assert.False(queryInstances1.HasMoreResults); + Assert.Equal(numberOfItemsToInsert, queryInstances1.Results.Count()); + + await _indexDataStore.DeleteSeriesIndexAsync(studyInstanceUID, seriesInstanceUID); + } + + private static DicomDataset CreateTestInstanceDicomDataset(string studyInstanceUID, string seriesInstanceUID, string patientName = "Patient Test") + { + EnsureArg.IsNotNullOrWhiteSpace(studyInstanceUID); + EnsureArg.IsNotNullOrWhiteSpace(seriesInstanceUID); + + var result = new DicomDataset + { + { DicomTag.StudyInstanceUID, studyInstanceUID }, + { DicomTag.SeriesInstanceUID, seriesInstanceUID }, + { DicomTag.SOPInstanceUID, Guid.NewGuid().ToString() }, + { DicomTag.PatientName, patientName }, + { DicomTag.StudyDate, new DateTime(2019, 6, 21) }, + }; + return result; + } + + private async Task> CreateSeriesInParallelAsync(string studyInstanceUID, string seriesInstanceUID, int numberOfItemsInSeries) + { + IList instances = Enumerable.Range(0, numberOfItemsInSeries) + .Select(_ => CreateTestInstanceDicomDataset(studyInstanceUID, seriesInstanceUID)) + .ToList(); + await Task.WhenAll(instances.Select(x => _indexDataStore.IndexInstanceAsync(x))); + return instances; + } + + private async Task DeleteInstancesAsync(params DicomDataset[] datasets) + { + foreach (DicomDataset instance in datasets) + { + var dicomInstance = DicomInstance.Create(instance); + await _indexDataStore.DeleteInstanceIndexAsync(dicomInstance.StudyInstanceUID, dicomInstance.SeriesInstanceUID, dicomInstance.SopInstanceUID); + } + } + } +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs new file mode 100644 index 0000000000..89e09c0a53 --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/DocumentClientExtensionsTests.cs @@ -0,0 +1,87 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading.Tasks; +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; +using Microsoft.Health.CosmosDb.Features.Storage; +using Microsoft.Health.Dicom.CosmosDb.Features.Storage; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence +{ + public class DocumentClientExtensionsTests : IClassFixture + { + private readonly IDocumentClient _documentClient; + private readonly DicomCosmosDataStoreTestsFixture _fixture; + + public DocumentClientExtensionsTests(DicomCosmosDataStoreTestsFixture fixture) + { + _fixture = fixture; + _documentClient = fixture.DocumentClient; + } + + [Fact] + public async Task GivenAnExistingDocument_OnCreateOrGet_DocumentIsFetched() + { + var expectedDocument = new TestDocument() { Id = Guid.NewGuid().ToString(), PartitionKey = Guid.NewGuid().ToString() }; + var requestOptions = new RequestOptions() { PartitionKey = new PartitionKey(expectedDocument.PartitionKey) }; + + await _documentClient.CreateDocumentAsync( + UriFactory.CreateDocumentCollectionUri(_fixture.DatabaseId, _fixture.CollectionId), + expectedDocument, + requestOptions); + + DocumentResponse readResponse = await _documentClient.ReadDocumentAsync( + UriFactory.CreateDocumentUri(_fixture.DatabaseId, _fixture.CollectionId, expectedDocument.Id), + requestOptions); + + expectedDocument = readResponse.Document; + + TestDocument actualDocument = await _documentClient.CreateOrGetDocumentAsync( + _fixture.DatabaseId, + _fixture.CollectionId, + expectedDocument.Id, + requestOptions, + new TestDocument() { Id = expectedDocument.Id, PartitionKey = expectedDocument.PartitionKey }); + + Assert.Equal(expectedDocument.Id, actualDocument.Id); + Assert.Equal(expectedDocument.PartitionKey, actualDocument.PartitionKey); + Assert.Equal(expectedDocument.ETag, actualDocument.ETag); + } + + [Fact] + public async Task GivenANonExistentDocument_OnGetOrCreate_DocumentIsCreated() + { + var expectedDocument = new TestDocument() { Id = Guid.NewGuid().ToString(), PartitionKey = Guid.NewGuid().ToString() }; + var requestOptions = new RequestOptions() { PartitionKey = new PartitionKey(expectedDocument.PartitionKey) }; + + TestDocument actualDocument = await _documentClient.GetOrCreateDocumentAsync( + _fixture.DatabaseId, + _fixture.CollectionId, + expectedDocument.Id, + requestOptions, + expectedDocument); + + Assert.Equal(expectedDocument.Id, actualDocument.Id); + Assert.Equal(expectedDocument.PartitionKey, actualDocument.PartitionKey); + Assert.False(string.IsNullOrWhiteSpace(actualDocument.ETag)); + } + + private class TestDocument + { + [JsonProperty(KnownDocumentProperties.Id)] + public string Id { get; set; } + + [JsonProperty(KnownDocumentProperties.PartitionKey)] + public string PartitionKey { get; set; } + + [JsonProperty(KnownDocumentProperties.ETag)] + public string ETag { get; set; } + } + } +} diff --git a/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj b/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj index 13b5dd04ae..19224d39f1 100644 --- a/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj +++ b/test/Microsoft.Health.Dicom.Web.Tests.E2E/Microsoft.Health.Dicom.Web.Tests.E2E.csproj @@ -58,4 +58,6 @@ + +