Merge branch 'release/2.0.7'

CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.0.7] 2020-02-05
+
+## Added
+
+- Support for zip file notation when loading dicoms from zip files (e.g. load only `c:\myzip!1.dcm` and `c:\myzip!3.dcm` )
+- Upgraded to latest RDMP release (4.0.2)
 ...
 
 ## [2.0.6] 2020-01-06
@@ -118,7 +124,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Initial commit from private repo
 
-[Unreleased]: https://github.com/HicServices/RdmpDicom/compare/v2.0.6...develop
+[Unreleased]: https://github.com/HicServices/RdmpDicom/compare/v2.0.7...develop
+[2.0.7]: https://github.com/HicServices/RdmpDicom/compare/v2.0.6...v2.0.7
 [2.0.6]: https://github.com/HicServices/RdmpDicom/compare/v2.0.5...v2.0.6
 [2.0.5]: https://github.com/HicServices/RdmpDicom/compare/v2.0.4...v2.0.5
 [2.0.4]: https://github.com/HicServices/RdmpDicom/compare/v2.0.3...v2.0.4

Packages.md

@@ -11,4 +11,4 @@
 | ------- | ------------| --------| ------- | ------- | -------------------------- |
 | fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1)|[MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | |
 | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.1.2](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.1.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | |
-| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.1](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.1-rc2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
+| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |

Rdmp.Dicom.Library.nuspec

@@ -14,7 +14,7 @@
         <copyright>Copyright 2018-2019</copyright>
         <dependencies>
           <dependency id="HIC.DicomTypeTranslation" version="2.1.2" />
-          <dependency id="HIC.RDMP.Plugin" version="4.0.1" />
+          <dependency id="HIC.RDMP.Plugin" version="4.0.2" />
           <dependency id="fo-dicom" version="4.0.1" />
         </dependencies>
     </metadata>

Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj

@@ -37,7 +37,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="HIC.BadMedicine.Dicom" Version="0.0.4" />
-    <PackageReference Include="HIC.RDMP.Plugin.Test" Version="4.0.1" />
+    <PackageReference Include="HIC.RDMP.Plugin.Test" Version="4.0.2" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
     <PackageReference Include="NUnit" Version="3.12.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />

Rdmp.Dicom.Tests/Unit/AmbiguousFilePathTests.cs

@@ -147,5 +147,45 @@ public void GetDatasetFromZipFile_WithPooling_Test()
 
 
         }
+
+        [Test]
+        public void TestZipEntry_Exists()
+        {
+            var zipFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "my.zip"));
+            var rootDir = Directory.CreateDirectory(Path.Combine(TestContext.CurrentContext.WorkDirectory,nameof(TestZipEntry_Exists)));
+            var subDirectory = rootDir.CreateSubdirectory("subdir");
+
+            var sourceFile = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory,@"TestData/IM-0001-0013.dcm"));
+
+            sourceFile.CopyTo(Path.Combine(rootDir.FullName, "file1.dcm"),true);
+            sourceFile.CopyTo(Path.Combine(subDirectory.FullName,"file2.dcm"),true);
+
+            if(zipFile.Exists)
+                zipFile.Delete();
+
+            ZipFile.CreateFromDirectory(rootDir.FullName,zipFile.FullName);
+
+            FileAssert.Exists(zipFile.FullName);
+
+            var exists = new AmbiguousFilePath(zipFile.FullName + "!file1.dcm");
+            Assert.IsNotNull(exists.GetDataset());
+
+            var notexists = new AmbiguousFilePath(zipFile.FullName + "!file2.dcm");
+            var ex = Assert.Throws<AmbiguousFilePathResolutionException>(()=>notexists.GetDataset());
+
+            StringAssert.Contains("Could not find path 'file2.dcm' within zip archive",ex.Message);
+
+            var existsRelative = new AmbiguousFilePath(zipFile.DirectoryName,"my.zip!file1.dcm"); 
+            Assert.IsNotNull(existsRelative.GetDataset());
+
+            var existsRelativeWithLeadingSlash = new AmbiguousFilePath(zipFile.DirectoryName,"my.zip!/file1.dcm"); 
+            Assert.IsNotNull(existsRelativeWithLeadingSlash.GetDataset());
+
+            var existsRelativeWithLeadingSlashInSubdir = new AmbiguousFilePath(zipFile.DirectoryName,"my.zip!/subdir/file2.dcm"); 
+            Assert.IsNotNull(existsRelativeWithLeadingSlashInSubdir.GetDataset());
+
+            var existsRelativeWithLeadingBackSlashInSubdir = new AmbiguousFilePath(zipFile.DirectoryName,"my.zip!\\subdir\\file2.dcm"); 
+            Assert.IsNotNull(existsRelativeWithLeadingBackSlashInSubdir.GetDataset());
+        }
     }
 }

Rdmp.Dicom.Tests/Unit/DICOMFileCollectionSourceTests.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Data;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
@@ -158,5 +159,125 @@ public void Test_ZipFile(bool expressRelative)
             Assert.IsTrue(finalTable.Exists());
             finalTable.Drop();
         }
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void Test_ZipFileNotation(bool expressRelative)
+        {
+            //get a clean database to upload to
+            var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
+
+            //create a folder in which to generate some dicoms
+            var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation)));
+
+            if(dirToLoad.Exists)
+                dirToLoad.Delete(true);
+
+            dirToLoad.Create();
+
+            //generate some random dicoms
+            var r = new Random(999);
+            DicomDataGenerator generator = new DicomDataGenerator(r,dirToLoad,"CT");
+            generator.MaximumImages = 5;
+            var people = new PersonCollection();
+            people.GeneratePeople(1,r);
+            generator.GenerateTestDataFile(people,new FileInfo("./inventory.csv"),1);
+
+            //This generates
+            // Test_ZipFile
+            //      2015
+            //          3
+            //              18          
+            //                  751140 2.25.166922918107154891877498685128076062226.dcm
+            //                  751140 2.25.179610809676265137473873365625829826423.dcm
+            //                  751140 2.25.201969634959506849065133495434871450465.dcm
+            //                  751140 2.25.237492679533001779093365416814254319890.dcm
+            //                  751140 2.25.316241631782653383510844072713132248731.dcm
+
+            var yearDir = dirToLoad.GetDirectories().Single();
+            StringAssert.IsMatch("\\d{4}",yearDir.Name);
+
+            //should be 5 images in the zip file
+            var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories);
+            Assert.AreEqual(5,dicomFiles.Length);
+
+            //e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm
+            //e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm
+            var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length);
+            var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length);
+
+            //zip them up
+            FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"));Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
+
+            if(zip.Exists)
+                zip.Delete();
+
+            ZipFile.CreateFromDirectory(dirToLoad.FullName,zip.FullName);
+
+            //e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm
+            string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1;
+            string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2;
+
+            var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt"));
+
+            //tell the source to load the zip
+            File.WriteAllText(loadMeTextFile.FullName,string.Join(Environment.NewLine, pathToLoad1, pathToLoad2));
+
+            var f = new FlatFileToLoad(loadMeTextFile);
+
+            //Setup source
+            var source = new DicomFileCollectionSource();
+            source.FilenameField = "RelativeFileArchiveURI";
+
+            if (expressRelative)
+                source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
+
+            var worklist = new FlatFileToLoadDicomFileWorklist(f);
+
+            //Setup destination
+            var destination = new DataTableUploadDestination();
+            destination.AllowResizingColumnsAtUploadTime = true;
+
+            //setup pipeline
+            var contextFactory = new DataFlowPipelineContextFactory<DataTable>();
+            var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination);
+
+            //run pipeline
+            var pipe = new DataFlowPipelineEngine<DataTable>(context,source,destination,new ThrowImmediatelyDataLoadEventListener());
+            pipe.Initialize(db,worklist);
+            pipe.ExecutePipeline(new GracefulCancellationToken());
+
+            var finalTable = db.ExpectTable(destination.TargetTableName);
+
+            using (var dt = finalTable.GetDataTable())
+            {
+                //should be 2 rows (since we told it to only load 2 files out of the zip)
+                Assert.AreEqual(2,dt.Rows.Count);
+
+                string pathInDbToDicomFile = (string) dt.Rows[0]["RelativeFileArchiveURI"];
+
+                //We expect either something like:
+                // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
+                // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
+
+                //the path referenced should be the file read in relative/absolute format
+                StringAssert.IsMatch(
+                    expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
+                        $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
+                    pathInDbToDicomFile);
+
+                StringAssert.Contains(yearDir.Name,pathInDbToDicomFile,"Expected zip file to have subdirectories and for them to be loaded correctly");
+
+                //confirm we can read that out again
+                using (var pool = new ZipPool())
+                {
+                    var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
+                    Assert.IsNotNull(path.GetDataset(pool));
+                }
+            }
+
+            Assert.IsTrue(finalTable.Exists());
+            finalTable.Drop();
+        }
     }
 }

Rdmp.Dicom.Tests/Unit/DicomSourceUnitTests.cs

@@ -36,6 +36,8 @@ public void Test_Linux_Root()
         [TestCase(@"C:/bob/",@"C:\bob\fish/1.dcm","./fish/1.dcm")] //mixed slash directions!
         [TestCase("/bob/","/bob/fish/1.dcm","./fish/1.dcm")] //linux style paths
         [TestCase("/bob/",@"\bob\fish\1.dcm","./fish/1.dcm")]
+        [TestCase(@"\\myserver\bob",@"\\myserver\bob\fish\1.dcm","./fish/1.dcm")] // UNC server paths
+        [TestCase(@"\\myserver\bob",@"\\myOtherServer\bob\fish\1.dcm",@"\\myOtherServer/bob/fish/1.dcm")]
         [TestCase("/","/bob/fish/1.dcm","./bob/fish/1.dcm")]
         [TestCase(@"C:\bob\",@"D:\fish\1.dcm",@"D:/fish/1.dcm")] //not relative so just return verbatim string (with slash fixes)
         [TestCase(@"C:\bob\",@"D:/fish/1.dcm",@"D:/fish/1.dcm")]

Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj

@@ -12,7 +12,7 @@
     <ProjectReference Include="..\Rdmp.Dicom\Rdmp.Dicom.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="HIC.RDMP.Plugin.UI" Version="4.0.1" />
+    <PackageReference Include="HIC.RDMP.Plugin.UI" Version="4.0.2" />
     <PackageReference Include="System.Drawing.Common" Version="4.5.1" />
   </ItemGroup>
   <ItemGroup>

Rdmp.Dicom/Extraction/FoDicomBased/AmbiguousFilePath.cs

@@ -74,6 +74,27 @@ public DicomFile GetDataset(ZipPool pool = null)
                 {
                     var entry = zip.GetEntry(bits[1]);
 
+                    if (entry == null)
+                    {
+                        //Maybe user has formatted it dodgy
+                        //e.g. \2015\3\18\2.25.177481563701402448825228719253578992342.dcm
+                        string adjusted = bits[1].TrimStart('\\','/');
+
+                        //if that doesn't work
+                        if ((entry = zip.GetEntry(adjusted)) == null)
+                        {
+                            //try normalizing the slashes
+                            adjusted = adjusted.Replace('\\','/');
+
+                            //nope we just cannot get a legit path in this zip
+                            if ((entry = zip.GetEntry(adjusted)) == null)
+                                throw new AmbiguousFilePathResolutionException($"Could not find path '{bits[1]}' within zip archive '{bits[0]}'");
+                        }
+
+                        //we fixed it to something that actually exists so update our state that we don't make the same mistake again
+                        FullPath = bits[0] + '!' + adjusted;
+                    }
+
                     if (!IsDicomReference(bits[1]))
                         throw new AmbiguousFilePathResolutionException("Path provided '" + FullPath + "' was to a zip file but not to a dicom file entry");
 
@@ -106,7 +127,7 @@ private bool IsDicomReference(string fullPath)
             return fullPath.EndsWith(".dcm", StringComparison.CurrentCultureIgnoreCase);
         }
 
-        private bool IsZipReference(string path)
+        public static bool IsZipReference(string path)
         {
             switch (path.Count(c=>c=='!'))
             {

Rdmp.Dicom/PipelineComponents/DicomSources/DicomFileCollectionSource.cs

@@ -15,6 +15,7 @@
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.DataFlowPipeline.Requirements;
 using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Dicom.Extraction.FoDicomBased;
 
 namespace Rdmp.Dicom.PipelineComponents.DicomSources
 {
@@ -39,6 +40,8 @@ public class DicomFileCollectionSource : DicomSource, IPipelineRequirement<IDico
 
         private IDataLoadEventListener _listener;
 
+        private readonly ZipPool _zipPool = new ZipPool();
+
         public override DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
         {
             _listener = listener;
@@ -55,14 +58,14 @@ public override DataTable GetChunk(IDataLoadEventListener listener, GracefulCanc
 
             try
             {
-                FileInfo file;
+                AmbiguousFilePath file;
                 DirectoryInfo directory;
 
                 if (!_fileWorklist.GetNextFileOrDirectoryToProcess(out directory, out file))
                     return null;
 
                 if(file != null && directory == null)
-                    dt.TableName = QuerySyntaxHelper.MakeHeaderNameSensible(Path.GetFileNameWithoutExtension(file.Name));
+                    dt.TableName = QuerySyntaxHelper.MakeHeaderNameSensible(Path.GetFileNameWithoutExtension(file.FullPath));
                 else if (directory != null)
                     dt.TableName = QuerySyntaxHelper.MakeHeaderNameSensible(Path.GetFileNameWithoutExtension(directory.Name));
                 else
@@ -75,17 +78,16 @@ public override DataTable GetChunk(IDataLoadEventListener listener, GracefulCanc
                 }
                 else
                 //Input is a single zip file
-                if (file.Extension == ".zip")
+                if (file.FullPath.EndsWith(".zip"))
                 {
-                    ProcessZipArchive(dt, listener, file.FullName);
+                    ProcessZipArchive(dt, listener, file.FullPath);
                 }
-                else if (file.Extension == ".dcm")
+                else
                 {
-                    using (var fs = file.Open(FileMode.Open))
-                        ProcessFile(fs, dt, file.FullName, listener);
+                    var df = file.GetDataset(_zipPool);
+                    ProcessDataset(file.FullPath, df.Dataset, dt, listener);
                 }
-                else
-                    throw new Exception("Expected file to be either .zip or .dcm ");
+
             }
             finally
             {
@@ -299,5 +301,13 @@ public override DataTable TryGetPreview()
                 _stopwatch = new Stopwatch();
             }
         }
+
+        public override void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
+        {
+            _zipPool?.Dispose();
+
+            base.Dispose(listener, pipelineFailureExceptionIfAny);
+
+        }
     }
 }