hot fix (#155)

* Fix command-line options (#132) * [DotLiquid] Refine template loading performance (#138) * Refine template loading * Copy list of dictionary in template provider * Add TemplateLocalFileSystem * Fix typo * fix bug for pull templates CLI (#145) * update readme (#135) * update readme * update readme * update readme * update doc * refine CLI options and fix typo (#144) * fix typo and refine commandline description * fix typo * [DotLiquid] Add description for "-t" option (#147) * Fixed FHIR server name (#143) * Add description for -t * Remove extra period * Remove unused sentence * Refine template zip name Co-authored-by: Ranvijay Kumar <[email protected]> * Fixed FHIR server name (#143) (#150) Co-authored-by: Ranvijay Kumar <[email protected]> Co-authored-by: Boya Wu <[email protected]> Co-authored-by: sowu880 <[email protected]> Co-authored-by: Ranvijay Kumar <[email protected]>
microsoft · Dec 18, 2020 · c6a87f8 · c6a87f8
1 parent fd19cc2
commit c6a87f8
Show file tree

Hide file tree

Showing 17 changed files with 307 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ FHIR Converter with DotLiquid engine is integrated into the [FHIR Server for Azu
 
 This project consists of the following components:
 
-1. A command-line tool for converting data.
+1. A command-line tool for converting data and managing templates.
 2. [Templates](data/Templates) for HL7 v2 to FHIR conversion.
 3. [Sample data](data/SampleData) for testing purpose.
 
@@ -41,24 +41,46 @@ FHIR Converter is integrated into the FHIR Server for Azure to run as part of th
 
 ### Command-line tool
 
+**Convert Data**
+
 The command-line tool can be used to convert a folder containing HL7 v2 messages to FHIR resources.
 Here are the parameters that the tool accepts:
 
 | Option | Name | Optionality | Default | Description |
 | ----- | ----- | ----- |----- |----- |
 | -d | TemplateDirectory | Required | | Root directory of templates. |
-| -r | RootTemplate | Required | | Name of root template. |
+| -r | RootTemplate | Required | | Name of root template. Valid values are ADT_A01, OML_O21, ORU_R01, VXU_V04. |
 | -c | InputDataContent | Optional| | Input data content. Specify OutputDataFile to get the results. |
 | -f | OutputDataFile | Optional | | Output data file. |
-| -i | InputDataFolder | Optional | | Input data folder. Specify OutputDataFolder to get the results.. |
+| -i | InputDataFolder | Optional | | Input data folder. Specify OutputDataFolder to get the results. |
 | -o | OutputDataFolder | Optional | | Output data folder. |
+| -t | IsTraceInfo | Optional | | Provide trace information in the output if "-t" is set. |
 | --version | Version | Optional | | Display version information. |
 | --help | Help | Optional | | Display usage information of this tool. |
 
 Example usage to convert HL7 v2 messages to FHIR resources in a folder:
 ```
->.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe -d myTemplateDirectory -r ADT_A01 -i myInputDataFolder -o myOutputDataFolder
+>.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe convert -d myTemplateDirectory -r ADT_A01 -i myInputDataFolder -o myOutputDataFolder
+```
+
+**Manage Templates**
+
+The command-line tool also supports managing different versions of templates from Azure Container Registry (ACR). Users can customize templates and store them on ACR if default templates can not meet requirements. After [ACR authentication](docs/TemplateManagementCLI.md), users can pull and push templates from/to a remote ACR through our tool.
+
+Example command to push a collection of templates to ACR image from a folder:
+```
+>.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe push testacr.azurecr.io/templatetest:default myInputFolder
 ```
+Example usage of pulling an image of templates in a folder:
+
+```
+>.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe pull testacr.azurecr.io/templatetest@sha256:412ea84f1bb1a9d98345efb7b427ba89616ec29ac332d543eff9a2161ca12a58 myOutputFolder
+
+```
+More details of usage are given in [Template Management CLI tool](docs/TemplateManagementCLI.md).
+
+Besides current version of [templates](data/Templates) given in our project, other versions that released by Microsoft are stored in a public ACR: healthplatformregistry.azurecr.io, users can directly pull templates from ``` healthplatformregistry.azurecr.io/hl7v2defaulttemplates:<version> ``` without authentication.
+>Note!: Template version is aligned with the version of FHIR Converter. 
 
 ### A note on Resource ID generation 
 

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PullTemplateOptions.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PullTemplateOptions.cs
@@ -7,7 +7,7 @@
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models
 {
-    [Verb("pull", HelpText = "Pull template image to registry")]
+    [Verb("pull", HelpText = "Pull a template image from a registry")]
     public class PullTemplateOptions
     {
         [Value(0, Required = true, HelpText = "Image reference: <registry>/<imageName>:<imageTag> or <registry>/<imageName>@<imageDigest>")]
@@ -18,8 +18,5 @@ public class PullTemplateOptions
 
         [Option('f', "ForceOverride", Required = false, Default = false, HelpText = "Force to override existed files")]
         public bool ForceOverride { get; set; }
-
-        [Option('e', "ErrorJsonFile", Required = false, Default = null, HelpText = "Output error message File.")]
-        public string ErrorJsonFile { get; set; }
     }
 }
diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PushTemplateOptions.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PushTemplateOptions.cs
@@ -7,7 +7,7 @@
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models
 {
-    [Verb("push", HelpText = "Push template image to registry")]
+    [Verb("push", HelpText = "Push a template image to a registry")]
     public class PushTemplateOptions
     {
         [Value(0, Required = true, HelpText = "Image reference: <registry>/<imageName>:<imageTag>")]
@@ -18,8 +18,5 @@ public class PushTemplateOptions
 
         [Option('n', "NewBaseLayer", Required = false, Default = false, HelpText = "Build new base layer")]
         public bool BuildNewBaseLayer { get; set; }
-
-        [Option('e', "ErrorJsonFile", Required = false, Default = null, HelpText = "Output error message File.")]
-        public string ErrorJsonFile { get; set; }
     }
 }
diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs
@@ -4,9 +4,9 @@
 // -------------------------------------------------------------------------------------------------
 
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using CommandLine;
-using CommandLine.Text;
 using Microsoft.Health.Fhir.Liquid.Converter.Tool.Models;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.Tool
@@ -21,7 +21,7 @@ public static async Task<int> Main(string[] args)
                 parseResult.WithParsed<ConverterOptions>(options => ConverterLogicHandler.Convert(options));
                 await parseResult.WithParsedAsync<PullTemplateOptions>(options => TemplateManagementLogicHandler.PullAsync(options));
                 await parseResult.WithParsedAsync<PushTemplateOptions>(options => TemplateManagementLogicHandler.PushAsync(options));
-                parseResult.WithNotParsed((errors) => HandleOptionsParseError(parseResult));
+                parseResult.WithNotParsed((errors) => HandleOptionsParseError(errors));
                 return 0;
             }
             catch (Exception ex)
@@ -31,10 +31,12 @@ public static async Task<int> Main(string[] args)
             }
         }
 
-        private static void HandleOptionsParseError(ParserResult<object> parseResult)
+        private static void HandleOptionsParseError(IEnumerable<Error> errors)
         {
-            var usageText = HelpText.RenderUsageText(parseResult);
-            throw new InputParameterException(usageText);
+            if (!errors.IsHelp() && !errors.IsVersion())
+            {
+                throw new InputParameterException(@"The input option is invalid.");
+            }
         }
     }
 }
diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/TemplateManagementLogicHandler.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/TemplateManagementLogicHandler.cs
@@ -25,6 +25,7 @@ internal static async Task PullAsync(PullTemplateOptions options)
 
             OCIFileManager fileManager = new OCIFileManager(options.ImageReference, options.OutputTemplateFolder);
             await fileManager.PullOCIImageAsync();
+            fileManager.UnpackOCIImage();
             Console.WriteLine($"Successfully pulled templates to {options.OutputTemplateFolder} folder");
         }
 

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs
@@ -7,13 +7,17 @@
 using System.Globalization;
 using DotLiquid;
 using DotLiquid.Exceptions;
+using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
 using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
+using Microsoft.Health.Fhir.Liquid.Converter.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids
 {
     public class EvaluateTests
     {
+        public const string TemplateName = "TemplateName";
+
         public static IEnumerable<object[]> GetValidEvaluateTemplateContents()
         {
             yield return new object[] { @"{% evaluate bundleId using 'ID/Bundle' -%}" };
@@ -41,15 +45,15 @@ public static IEnumerable<object[]> GetInvalidEvaluateTemplateContents()
         public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultShouldBeReturned(string templateContent)
         {
             // Template should be parsed correctly
-            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
-            var template = Template.Parse(templateContent);
+            var template = TemplateUtility.ParseTemplate(TemplateName, templateContent);
             Assert.True(template.Root.NodeList.Count > 0);
 
             // Template should be rendered correctly
+            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
             var context = new Context(
                 environments: new List<Hash>(),
                 outerScope: new Hash(),
-                registers: Hash.FromAnonymousObject(new { file_system = templateProvider }),
+                registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }),
                 errorsOutputMode: ErrorsOutputMode.Rethrow,
                 maxIterations: 0,
                 timeout: 0,
@@ -62,17 +66,14 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh
         [MemberData(nameof(GetInvalidEvaluateTemplateContents))]
         public void GivenInvalidEvaluateTemplateContent_WhenParse_ExceptionsShouldBeThrown(string templateContent)
         {
-            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
-            Assert.Throws<SyntaxException>(() => Template.Parse(templateContent));
+            Assert.Throws<ConverterInitializeException>(() => TemplateUtility.ParseTemplate(TemplateName, templateContent));
         }
 
         [Fact]
         public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
         {
-            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
-
             // No template file system
-            var template = Template.Parse(@"{% evaluate bundleId using 'ID/Bundle' Data: hl7v2Data -%}");
+            var template = TemplateUtility.ParseTemplate(TemplateName, @"{% evaluate bundleId using 'ID/Bundle' Data: hl7v2Data -%}");
             var context = new Context(
                 environments: new List<Hash>(),
                 outerScope: new Hash(),
@@ -84,11 +85,12 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
             Assert.Throws<FileSystemException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
 
             // Valid template file system but no such template
-            template = Template.Parse(@"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}");
+            template = TemplateUtility.ParseTemplate(TemplateName, @"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}");
+            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
             context = new Context(
                 environments: new List<Hash>(),
                 outerScope: new Hash(),
-                registers: Hash.FromAnonymousObject(new { file_system = templateProvider }),
+                registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }),
                 errorsOutputMode: ErrorsOutputMode.Rethrow,
                 maxIterations: 0,
                 timeout: 0,

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/MemoryFileSystemTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/MemoryFileSystemTests.cs
@@ -7,8 +7,8 @@
 using System.Collections.Generic;
 using System.Globalization;
 using DotLiquid;
+using Microsoft.Health.Fhir.Liquid.Converter.DotLiquids;
 using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
-using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids
@@ -26,9 +26,11 @@ public void GivenAValidTemplateCollection_WhenGetTemplate_CorrectResultShouldBeR
                 },
             };
 
-            var templateProvider = new Hl7v2TemplateProvider(templateCollection);
-            Assert.Equal("hello world", templateProvider.GetTemplate("template1").Render());
-            Assert.Null(templateProvider.GetTemplate("template2"));
+            var memoryFileSystem = new MemoryFileSystem(templateCollection);
+            Assert.Equal("hello world", memoryFileSystem.GetTemplate("template1").Render());
+            Assert.Null(memoryFileSystem.GetTemplate("template2"));
+            Assert.Null(memoryFileSystem.GetTemplate(null));
+            Assert.Null(memoryFileSystem.GetTemplate(string.Empty));
         }
 
         [Fact]
@@ -55,11 +57,11 @@ public void GivenTwoValidTemplateCollection_WhenGetTemplate_CorrectResultShouldB
                 },
             };
 
-            var templateProvider = new Hl7v2TemplateProvider(templateCollection);
-            Assert.Null(templateProvider.GetTemplate("template1"));
-            Assert.Equal("template2 updated in customized layer", templateProvider.GetTemplate("template2").Render());
-            Assert.Equal("template3 added in base layer", templateProvider.GetTemplate("template3").Render());
-            Assert.Equal("template4 added in customized layer", templateProvider.GetTemplate("template4").Render());
+            var memoryFileSystem = new MemoryFileSystem(templateCollection);
+            Assert.Null(memoryFileSystem.GetTemplate("template1"));
+            Assert.Equal("template2 updated in customized layer", memoryFileSystem.GetTemplate("template2").Render());
+            Assert.Equal("template3 added in base layer", memoryFileSystem.GetTemplate("template3").Render());
+            Assert.Equal("template4 added in customized layer", memoryFileSystem.GetTemplate("template4").Render());
         }
 
         [Fact]
@@ -73,13 +75,15 @@ public void GivenAValidTemplateCollection_WhenGetTemplateWithContext_CorrectResu
                 },
             };
 
-            var templateProvider = new Hl7v2TemplateProvider(templateCollection);
+            var memoryFileSystem = new MemoryFileSystem(templateCollection);
             var context = new Context(CultureInfo.InvariantCulture);
             context["template1"] = "template1";
             context["template2"] = "template2";
-            Assert.Equal("hello world", templateProvider.GetTemplate(context, "template1").Render());
-            Assert.Throws<RenderException>(() => templateProvider.GetTemplate(context, "template2"));
-            Assert.Throws<RenderException>(() => templateProvider.GetTemplate(context, "template3"));
+            Assert.Equal("hello world", memoryFileSystem.GetTemplate(context, "template1").Render());
+            Assert.Throws<RenderException>(() => memoryFileSystem.GetTemplate(context, "template2"));
+            Assert.Throws<RenderException>(() => memoryFileSystem.GetTemplate(context, "template3"));
+            Assert.Throws<RenderException>(() => memoryFileSystem.GetTemplate(context, null));
+            Assert.Throws<RenderException>(() => memoryFileSystem.GetTemplate(context, string.Empty));
         }
 
         [Fact]
@@ -106,17 +110,17 @@ public void GivenTwoValidtTemplateCollection_WhenGetTemplateWithContext_CorrectR
                 },
             };
 
-            var templateProvider = new Hl7v2TemplateProvider(templateCollection);
+            var memoryFileSystem = new MemoryFileSystem(templateCollection);
             var context = new Context(CultureInfo.InvariantCulture);
             context["'folder/template1'"] = "folder/template1";
             context["template2"] = "template2";
             context["template3"] = "template3";
             context["template4"] = "template4";
 
-            Assert.Throws<Exceptions.RenderException>(() => templateProvider.GetTemplate(context, "'folder/template1'"));
-            Assert.Equal("template2 updated in customized layer", templateProvider.GetTemplate(context, "template2").Render());
-            Assert.Equal("template3 added in base layer", templateProvider.GetTemplate(context, "template3").Render());
-            Assert.Equal("template4 added in customized layer", templateProvider.GetTemplate(context, "template4").Render());
+            Assert.Throws<RenderException>(() => memoryFileSystem.GetTemplate(context, "'folder/template1'"));
+            Assert.Equal("template2 updated in customized layer", memoryFileSystem.GetTemplate(context, "template2").Render());
+            Assert.Equal("template3 added in base layer", memoryFileSystem.GetTemplate(context, "template3").Render());
+            Assert.Equal("template4 added in customized layer", memoryFileSystem.GetTemplate(context, "template4").Render());
         }
 
         [Fact]
@@ -130,10 +134,10 @@ public void GivenAValidTemplateCollection_WhenReadTemplateWithContext_ExceptionS
                 },
             };
 
-            var templateProvider = new Hl7v2TemplateProvider(templateCollection);
+            var memoryFileSystem = new MemoryFileSystem(templateCollection);
             var context = new Context(CultureInfo.InvariantCulture);
             context["hello"] = "hello";
-            Assert.Throws<NotImplementedException>(() => templateProvider.ReadTemplateFile(context, "hello"));
+            Assert.Throws<NotImplementedException>(() => memoryFileSystem.ReadTemplateFile(context, "hello"));
         }
     }
 }
diff --git a/...crosoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs b/...crosoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs
@@ -0,0 +1,56 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Globalization;
+using DotLiquid;
+using Microsoft.Health.Fhir.Liquid.Converter.DotLiquids;
+using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
+using Microsoft.Health.Fhir.Liquid.Converter.Models;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids
+{
+    public class TemplateLocalFileSystemTests
+    {
+        [Fact]
+        public void GivenAValidTemplateDirectory_WhenGetTemplate_CorrectResultsShouldBeReturned()
+        {
+            var templateLocalFileSystem = new TemplateLocalFileSystem(Constants.Hl7v2TemplateDirectory, DataType.Hl7v2);
+            var context = new Context(CultureInfo.InvariantCulture);
+
+            // Template exists
+            Assert.NotNull(templateLocalFileSystem.GetTemplate("ADT_A01"));
+
+            // Template does not exist
+            Assert.Null(templateLocalFileSystem.GetTemplate("Foo"));
+        }
+
+        [Fact]
+        public void GivenAValidTemplateDirectory_WhenGetTemplateWithContext_CorrectResultsShouldBeReturned()
+        {
+            var templateLocalFileSystem = new TemplateLocalFileSystem(Constants.Hl7v2TemplateDirectory, DataType.Hl7v2);
+            var context = new Context(CultureInfo.InvariantCulture);
+
+            // Template exists
+            context["ADT_A01"] = "ADT_A01";
+            Assert.NotNull(templateLocalFileSystem.GetTemplate(context, "ADT_A01"));
+
+            // Template does not exist
+            context["Foo"] = "Foo";
+            Assert.Throws<RenderException>(() => templateLocalFileSystem.GetTemplate(context, "Foo"));
+            Assert.Throws<RenderException>(() => templateLocalFileSystem.GetTemplate(context, "Bar"));
+        }
+
+        [Fact]
+        public void GivenAValidTemplateDirectory_WhenReadTemplateWithContext_ExceptionShouldBeThrown()
+        {
+            var templateLocalFileSystem = new TemplateLocalFileSystem(Constants.Hl7v2TemplateDirectory, DataType.Hl7v2);
+            var context = new Context(CultureInfo.InvariantCulture);
+            context["ADT_A01"] = "ADT_A01";
+            Assert.Throws<NotImplementedException>(() => templateLocalFileSystem.ReadTemplateFile(context, "hello"));
+        }
+    }
+}