From 07fd0c143bb0e33a7f32b4af04f417cd43deebc2 Mon Sep 17 00:00:00 2001 From: Valeriy Sedov Date: Thu, 1 Aug 2024 17:14:08 +0200 Subject: [PATCH 1/2] Rename etlLibraryPath to etlLibraryFolderPath --- .../Common/Utility/EtlLibrary.cs | 34 +++++++++---------- .../Desktop/Settings/BuildingSettings.cs | 2 +- .../Program.cs | 4 +-- .../appsettings.json | 2 +- .../Function.cs | 4 +-- .../Settings/BuildingSettings.cs | 4 +-- .../Settings/Settings.cs | 4 +-- sources/Tests/RunETL/App.config | 2 +- sources/Tests/RunETL/Program.cs | 2 +- sources/Tests/RunLocal/Program.cs | 8 ++--- 10 files changed, 33 insertions(+), 33 deletions(-) diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Utility/EtlLibrary.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Utility/EtlLibrary.cs index 92ce260e..da6dc53e 100644 --- a/sources/Framework/org.ohdsi.cdm.framework/Common/Utility/EtlLibrary.cs +++ b/sources/Framework/org.ohdsi.cdm.framework/Common/Utility/EtlLibrary.cs @@ -23,9 +23,9 @@ private static IEnumerable GetETLAssemblies(string path) } } - private static IEnumerable> FindAssemblyAndResource(string etlLibraryPath, string name) + private static IEnumerable> FindAssemblyAndResource(string etlLibraryFolderPath, string name) { - foreach (var assembly in GetETLAssemblies(etlLibraryPath)) + foreach (var assembly in GetETLAssemblies(etlLibraryFolderPath)) { foreach (var resourceName in assembly.GetManifestResourceNames()) { @@ -46,11 +46,11 @@ private static Resource ReadResource(Assembly assembly, string resourceName) return new Resource(resourceName.Split('.').TakeLast(2).First(), value); } - private static Resource GetResource(string etlLibraryPath, string name) + private static Resource GetResource(string etlLibraryFolderPath, string name) { Console.WriteLine("GetResource: " + name); - var result = FindAssemblyAndResource(etlLibraryPath, name).FirstOrDefault(); + var result = FindAssemblyAndResource(etlLibraryFolderPath, name).FirstOrDefault(); if (result != null) { var resource = ReadResource(result.Item1, result.Item2); @@ -58,14 +58,14 @@ private static Resource GetResource(string etlLibraryPath, string name) return resource; } - throw new KeyNotFoundException($"GetResource | Resource: {name}; LibraryPath: {etlLibraryPath} - not exists"); + throw new KeyNotFoundException($"GetResource | Resource: {name}; LibraryPath: {etlLibraryFolderPath} - not exists"); } - private static IEnumerable GetResources(string etlLibraryPath, string name) + private static IEnumerable GetResources(string etlLibraryFolderPath, string name) { Console.WriteLine("GetResources: " + name); - foreach (var ar in FindAssemblyAndResource(etlLibraryPath, name)) + foreach (var ar in FindAssemblyAndResource(etlLibraryFolderPath, name)) { var resource = ReadResource(ar.Item1, ar.Item2); if (resource != null) @@ -82,7 +82,7 @@ private static string NormalizeVendorName(string vendorName) return result; } - public static void LoadVendorSettings(string etlLibraryPath, IVendorSettings settings) + public static void LoadVendorSettings(string etlLibraryFolderPath, IVendorSettings settings) { Console.WriteLine("LoadVendorSettings | Vendor: " + settings.Vendor); @@ -93,16 +93,16 @@ public static void LoadVendorSettings(string etlLibraryPath, IVendorSettings set settings.SourceQueryDefinitions = []; settings.CombinedLookupDefinitions = []; - settings.BatchScript ??= EtlLibrary.GetResource(etlLibraryPath, vendorFolder + "." + batch).Value; + settings.BatchScript ??= EtlLibrary.GetResource(etlLibraryFolderPath, vendorFolder + "." + batch).Value; - foreach (var definition in GetResources(etlLibraryPath, vendorFolder + ".Definitions.")) + foreach (var definition in GetResources(etlLibraryFolderPath, vendorFolder + ".Definitions.")) { var qd = new QueryDefinition().DeserializeFromXml(definition.Value); qd.FileName = definition.Name; settings.SourceQueryDefinitions.Add(qd); } - foreach (var lookup in GetResources(etlLibraryPath, vendorFolder + ".CombinedLookups.")) + foreach (var lookup in GetResources(etlLibraryFolderPath, vendorFolder + ".CombinedLookups.")) { var ld = new LookupDefinition().DeserializeFromXml(lookup.Value); ld.FileName = lookup.Name; @@ -110,9 +110,9 @@ public static void LoadVendorSettings(string etlLibraryPath, IVendorSettings set } } - public static Vendor CreateVendorInstance(string etlLibraryPath, string name) + public static Vendor CreateVendorInstance(string etlLibraryFolderPath, string name) { - foreach (var assembly in GetETLAssemblies(etlLibraryPath)) + foreach (var assembly in GetETLAssemblies(etlLibraryFolderPath)) { var vendorTypes = assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(Vendor)) && !t.IsAbstract); var vendorType = vendorTypes.FirstOrDefault(a => a.Name.Contains(name, StringComparison.CurrentCultureIgnoreCase)); @@ -133,12 +133,12 @@ public static Vendor CreateVendorInstance(string etlLibraryPath, string name) } } - throw new KeyNotFoundException($"CreateVendorInstance | Vendor: {name}; LibraryPath: {etlLibraryPath} - not exists"); + throw new KeyNotFoundException($"CreateVendorInstance | Vendor: {name}; LibraryPath: {etlLibraryFolderPath} - not exists"); } - public static ConstructorInfo GetBuilderConstructor(string etlLibraryPath, Vendor vendor) + public static ConstructorInfo GetBuilderConstructor(string etlLibraryFolderPath, Vendor vendor) { - foreach (var assembly in GetETLAssemblies(etlLibraryPath)) + foreach (var assembly in GetETLAssemblies(etlLibraryFolderPath)) { var builderTypes = assembly.GetTypes(). Where(t => t.IsSubclassOf(typeof(PersonBuilder)) && !t.IsAbstract); @@ -151,7 +151,7 @@ public static ConstructorInfo GetBuilderConstructor(string etlLibraryPath, Vendo } } - throw new KeyNotFoundException($"GetBuilderConstructor | Vendor: {vendor}; LibraryPath: {etlLibraryPath} - not exists"); + throw new KeyNotFoundException($"GetBuilderConstructor | Vendor: {vendor}; LibraryPath: {etlLibraryFolderPath} - not exists"); } } } \ No newline at end of file diff --git a/sources/Framework/org.ohdsi.cdm.framework/Desktop/Settings/BuildingSettings.cs b/sources/Framework/org.ohdsi.cdm.framework/Desktop/Settings/BuildingSettings.cs index f0a6fe29..c750bbf5 100644 --- a/sources/Framework/org.ohdsi.cdm.framework/Desktop/Settings/BuildingSettings.cs +++ b/sources/Framework/org.ohdsi.cdm.framework/Desktop/Settings/BuildingSettings.cs @@ -234,7 +234,7 @@ private void SetSourceReleaseDate() private void SetVendorSettings() { - Console.WriteLine("etlLibraryPath: " + EtlLibraryPath); + Console.WriteLine("etlLibraryFolderPath: " + EtlLibraryPath); EtlLibrary.LoadVendorSettings(EtlLibraryPath, this); } diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs index 0129b206..0d6a53a0 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs @@ -86,7 +86,7 @@ static int Main(string[] arguments) IConfigurationRoot configuration = builder.Build(); - vendor = EtlLibrary.CreateVendorInstance(configuration.GetSection("AppSettings")["etlLibraryPath"], vendorName); + vendor = EtlLibrary.CreateVendorInstance(configuration.GetSection("AppSettings")["etlLibraryFolderPath"], vendorName); var builderConnectionString = configuration.GetConnectionString("Builder"); @@ -178,7 +178,7 @@ static int Main(string[] arguments) Settings.Current.Building.RawVocabularyConnectionString = vocabularyConnectionString; Settings.Current.Building.RawDestinationConnectionString = destinationConnectionString; Settings.Current.Building.Vendor = vendor; - Settings.Current.Building.EtlLibraryPath = configuration.GetSection("AppSettings")["etlLibraryPath"]; + Settings.Current.Building.EtlLibraryPath = configuration.GetSection("AppSettings")["etlLibraryFolderPath"]; if (!createNewBuildingId) { diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/appsettings.json b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/appsettings.json index 68d19c52..2f7b5421 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/appsettings.json +++ b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/appsettings.json @@ -13,7 +13,7 @@ "cdm_folder_csv": "", "cdm_folder": "", "vendor_settings": "", - "etlLibraryPath": "", + "etlLibraryFolderPath": "", "spectrum_cluster": "", "spectrum_db": "", diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Function.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Function.cs index cefc8a8a..e0e89b1f 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Function.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Function.cs @@ -107,7 +107,7 @@ public void Initialize() } public async Task FunctionHandler2(string s3AwsAccessKeyId, string s3AwsSecretAccessKey, string bucket, string folder, Vendor vendor, int buildingId, int chunkId, string prefix, - int attempt, string etlLibraryPath) + int attempt, string etlLibraryFolderPath) { Dictionary restorePoint = null; @@ -122,7 +122,7 @@ public async Task FunctionHandler2(string s3AwsAccessKeyId, string s3Aws return null; } - EtlLibraryPath = etlLibraryPath; + EtlLibraryPath = etlLibraryFolderPath; Settings.Current.S3AwsAccessKeyId = s3AwsAccessKeyId; Settings.Current.S3AwsSecretAccessKey = s3AwsSecretAccessKey; diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/BuildingSettings.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/BuildingSettings.cs index ac71ba93..53715c86 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/BuildingSettings.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/BuildingSettings.cs @@ -18,9 +18,9 @@ public class BuildingSettings : IVendorSettings public string BatchScript { get; set; } - public void SetVendorSettings(string etlLibraryPath) + public void SetVendorSettings(string etlLibraryFolderPath) { - EtlLibrary.LoadVendorSettings(etlLibraryPath, this); + EtlLibrary.LoadVendorSettings(etlLibraryFolderPath, this); } } } \ No newline at end of file diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/Settings.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/Settings.cs index 0b9358e7..f319b467 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/Settings.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Settings/Settings.cs @@ -49,10 +49,10 @@ static Settings() #region Methods - public static void Initialize(int buildingId, Vendor vendor, string etlLibraryPath) + public static void Initialize(int buildingId, Vendor vendor, string etlLibraryFolderPath) { Current.Building = new BuildingSettings { Id = buildingId, Vendor = vendor }; - Current.Building.SetVendorSettings(etlLibraryPath); + Current.Building.SetVendorSettings(etlLibraryFolderPath); } #endregion diff --git a/sources/Tests/RunETL/App.config b/sources/Tests/RunETL/App.config index 72b0c2ec..418c2ab9 100644 --- a/sources/Tests/RunETL/App.config +++ b/sources/Tests/RunETL/App.config @@ -4,6 +4,6 @@ - + \ No newline at end of file diff --git a/sources/Tests/RunETL/Program.cs b/sources/Tests/RunETL/Program.cs index c26b8c10..f752034a 100644 --- a/sources/Tests/RunETL/Program.cs +++ b/sources/Tests/RunETL/Program.cs @@ -30,7 +30,7 @@ static void Main(string[] args) { chunkscnt = o.ChunksCnt.Value; slicescnt = o.SlicesCnt.Value; - vendor = EtlLibrary.CreateVendorInstance(ConfigurationManager.AppSettings["etlLibraryPath"], o.Vendor); + vendor = EtlLibrary.CreateVendorInstance(ConfigurationManager.AppSettings["etlLibraryFolderPath"], o.Vendor); buildingid = o.Buildingid.Value; }); diff --git a/sources/Tests/RunLocal/Program.cs b/sources/Tests/RunLocal/Program.cs index ade1b43f..632dac7b 100644 --- a/sources/Tests/RunLocal/Program.cs +++ b/sources/Tests/RunLocal/Program.cs @@ -27,8 +27,8 @@ static void Main(string[] args) Console.WriteLine($"{Directory.GetCurrentDirectory()}"); - var etlLibraryPath = ""; - Process(EtlLibrary.CreateVendorInstance(etlLibraryPath, args[0]), int.Parse(args[1]), int.Parse(args[2]), args[3], bool.Parse(args[4]), etlLibraryPath); + var etlLibraryFolderPath = ""; + Process(EtlLibrary.CreateVendorInstance(etlLibraryFolderPath, args[0]), int.Parse(args[1]), int.Parse(args[2]), args[3], bool.Parse(args[4]), etlLibraryFolderPath); //int[] slicesNum = [24, 40, 48, 96, 192]; @@ -40,7 +40,7 @@ static void Main(string[] args) Console.ReadLine(); } - private static string Process(Vendor vendor, int buildingId, int chunkId, string prefix, bool clean, string etlLibraryPath) + private static string Process(Vendor vendor, int buildingId, int chunkId, string prefix, bool clean, string etlLibraryFolderPath) { try { @@ -86,7 +86,7 @@ private static string Process(Vendor vendor, int buildingId, int chunkId, string var client = new AmazonS3Client(_awsAccessKeyId, _awsSecretAccessKey, config); var func = new Function(client); - var t = func.FunctionHandler2(_awsAccessKeyId, _awsSecretAccessKey, _bucket, _cdmFolder, vendor, buildingId, chunkId, prefix, 0, etlLibraryPath); + var t = func.FunctionHandler2(_awsAccessKeyId, _awsSecretAccessKey, _bucket, _cdmFolder, vendor, buildingId, chunkId, prefix, 0, etlLibraryFolderPath); t.Wait(); Console.WriteLine($"chunkId={chunkId};prefix={prefix} - DONE"); From 17e1ea095f571d3480b0c337a65443fdc29c0e69 Mon Sep 17 00:00:00 2001 From: Valeriy Sedov Date: Thu, 1 Aug 2024 17:17:29 +0200 Subject: [PATCH 2/2] Add etlLibraryFolderPath to args --- sources/Tests/RunLocal/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sources/Tests/RunLocal/Program.cs b/sources/Tests/RunLocal/Program.cs index 632dac7b..edb2cecb 100644 --- a/sources/Tests/RunLocal/Program.cs +++ b/sources/Tests/RunLocal/Program.cs @@ -27,8 +27,8 @@ static void Main(string[] args) Console.WriteLine($"{Directory.GetCurrentDirectory()}"); - var etlLibraryFolderPath = ""; - Process(EtlLibrary.CreateVendorInstance(etlLibraryFolderPath, args[0]), int.Parse(args[1]), int.Parse(args[2]), args[3], bool.Parse(args[4]), etlLibraryFolderPath); + var vendor = EtlLibrary.CreateVendorInstance(args[5], args[0]); + Process(vendor, int.Parse(args[1]), int.Parse(args[2]), args[3], bool.Parse(args[4]), args[5]); //int[] slicesNum = [24, 40, 48, 96, 192];