Merge branch 'main' into feature/pythoncheck

.github/workflows/dotnet-core.yml

@@ -10,25 +10,54 @@ env:
 
 jobs:
   package:
-    runs-on: self-hosted
+    runs-on: windows-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/checkout@v4
         with:
           repository: HicServices/RDMP
           ref: develop
           path: RDMP
-      - uses: actions/setup-dotnet@v1
+      - uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: 6.0.x
+          dotnet-version: 7.0.x
       - name: Get version
         id: version
         shell: cmd
         run: |
           getversions >> %GITHUB_OUTPUT%
+      - name: Install MS SQL 2019 Express LocalDB
+        uses: crazy-max/ghaction-chocolatey@v3
+        with:
+          args: install -r sqllocaldb --no-progress
+      - name: Install python2
+        uses: crazy-max/ghaction-chocolatey@v3
+        with:
+          args: install -r python2
+      - name: Populate Databases.yaml 
+        shell: bash
+        run: |
+            find ./RDMP/Tools/rdmp/Databases.yaml -type f -exec sed -i 's/RDMP_/TEST_/g' {} \;
+      # - name: Initialise LocalDB
+      #   shell: bash
+      #   run: |
+      #     SqlLocalDB.exe create MSSQLLocalDB -s
+      #     sqlcmd -l 180 -S '(localdb)\MSSQLLocalDB' -Q "SELECT @@VERSION;"
+      #     sed -i'' -e 's/localhost/\(localdb\)\\MSSQLLocalDB/' RDMP/Tests.Common/TestDatabases.txt
+      - name: Install a SQL Server suite of tools
+        uses: potatoqualitee/[email protected]
+        with:
+          install: sqlengine, sqlclient, sqlpackage, localdb
+      - name: Run sqlclient
+        run: sqlcmd -S localhost -U sa -P dbatools.I0 -d tempdb -Q "SELECT @@version;"
+      - uses: shogo82148/actions-setup-mysql@v1
+        with:
+          mysql-version: '8.0'
+          root-password: 'YourStrong!Passw0rd'
+          auto-start: true
       - name: Set up database
         run: |
-          dotnet run --project RDMP/Tools/rdmp/rdmp.csproj -c Release -- install localhost TEST_ -d
+          dotnet run --project RDMP/Tools/rdmp/rdmp.csproj -c Release -- install --createdatabasetimeout 1800 localhost TEST_ -e     
       - name: Test
         run: |
           dotnet test

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/AutomationUserInterface.cs

@@ -60,7 +60,7 @@ public override object[] GetChildren(object model)
             case IProject p:
             {
                 var schedule = GetScheduleIfAny(p);
-                
+
                 if(schedule != null)
                     return new[] { schedule };
                 break;

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/BasicAutomationCommandExecution.cs

@@ -25,7 +25,7 @@ public BasicAutomationCommandExecution(IBasicActivateItems activator):base(activ
         {
             SetImpossible("There is no Automation Repository configured");
         }
-                
+
     }
 
 }

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Data/AutomateExtraction.cs

@@ -61,7 +61,7 @@ public bool Release
     #endregion
 
     #region Relationships
-        
+
     [NoMappingToDatabase]
     public IExtractionConfiguration ExtractionConfiguration => _repository.DataExportRepository.GetObjectByID<ExtractionConfiguration>(ExtractionConfigurationId);
 
@@ -99,7 +99,7 @@ public AutomateExtraction(PluginRepository repository, DbDataReader r)
     }
 
     private ExtractionConfiguration _cachedExtractionConfiguration;
-        
+
 
     public override string ToString()
     {
@@ -146,7 +146,7 @@ public void ClearBaselines()
 
         foreach (var r in _repository.GetAllObjectsWithParent<SuccessfullyExtractedResults>(this))
             r.DeleteInDatabase();
-            
+
         BaselineDate = null;
         SaveToDatabase();
     }

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Data/AutomateExtractionSchedule.cs

@@ -160,7 +160,7 @@ public void CheckTicketing(ICheckNotifier notifier)
             notifier.OnCheckPerformed(new CheckEventArgs("No Ticket specified, governance will not be checked",CheckResult.Warning));
             return;
         }
-            
+
         var ticketingSystem = _repository.CatalogueRepository.GetTicketingSystem();
 
 

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Data/QueuedExtraction.cs

@@ -19,7 +19,7 @@ public class QueuedExtraction : DatabaseEntity
     private DateTime _dueDate;
     private string _requester;
     private DateTime _requestDate;
-        
+
 
     public int ExtractionConfiguration_ID
     {

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Data/Repository/AutomateExtractionRepository.cs

@@ -15,7 +15,7 @@ public AutomateExtractionRepository(IRDMPPlatformRepositoryServiceLocator reposi
         CatalogueRepository = repositoryLocator.CatalogueRepository;
         DataExportRepository = repositoryLocator.DataExportRepository;
     }
-        
+
     protected override bool IsCompatibleType(Type type)
     {
         return typeof(DatabaseEntity).IsAssignableFrom(type);

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Data/Repository/AutomateExtractionRepositoryFinder.cs

@@ -9,17 +9,17 @@ namespace LoadModules.Extensions.AutomationPlugins.Data.Repository;
 public class AutomateExtractionRepositoryFinder : PluginRepositoryFinder
 {
     public static int Timeout = 5;
-        
+
     public AutomateExtractionRepositoryFinder(IRDMPPlatformRepositoryServiceLocator repositoryLocator) : base(repositoryLocator)
     {
-            
+
     }
 
     public override PluginRepository GetRepositoryIfAny()
     {
         if (RepositoryLocator.CatalogueRepository == null || RepositoryLocator.DataExportRepository == null)
             return null;
-            
+
         var patcher = new AutomateExtractionPluginPatcher();
 
         var compatibleServers = RepositoryLocator.CatalogueRepository.GetAllObjects<ExternalDatabaseServer>()

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Data/SuccessfullyExtractedResults.cs

@@ -14,7 +14,7 @@ public class SuccessfullyExtractedResults : DatabaseEntity
     private string _sQL;
     private int _extractableDataSet_ID;
     private int _automateExtraction_ID;
-        
+
     public string SQL
     {
         get => _sQL;

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/ExecuteCommandCreateNewAutomateExtraction.cs

@@ -25,14 +25,14 @@ public ExecuteCommandCreateNewAutomateExtraction(IBasicActivateItems activator,I
 
         // the project doesnt have a schedule
         Schedule = existingSchedules.FirstOrDefault(s => s.Project_ID == extractionConfiguration.Project_ID);
-            
+
         if (Schedule == null)
         {
             SetImpossible($"Project {extractionConfiguration.Project} does not have an {nameof(AutomateExtractionSchedule)}");
             return;
         }
 
-            
+
         ExtractionConfiguration = extractionConfiguration;
         var existingAutomateExtractions = AutomationRepository.GetAllObjects<AutomateExtraction>();
 

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/ExecuteCommandCreateNewAutomateExtractionSchedule.cs

@@ -21,7 +21,7 @@ public ExecuteCommandCreateNewAutomateExtractionSchedule(IBasicActivateItems act
         }
 
         var existing = AutomationRepository.GetAllObjects<AutomateExtractionSchedule>();
-            
+
         if(existing.Any(s=>s.Project_ID == project.ID))
         {
             SetImpossible($"Project already has an {nameof(AutomateExtractionSchedule)}");

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Execution/ExtractionPipeline/AutomatedExtractionPipelineChecker.cs

@@ -29,7 +29,7 @@ public void Check(ICheckNotifier notifier)
             else
                 notifier.OnCheckPerformed(new CheckEventArgs(
                     $"Automated Extraction can only take place through Pipelines that include a {nameof(SuccessfullyExtractedResultsDocumenter)} plugin component", CheckResult.Fail));
-            
+
             var source = _automateExtractionPipeline.Source;
 
             if (source == null)

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Execution/ExtractionPipeline/DeltaHacker.cs

@@ -32,8 +32,8 @@ public BaselineHackEvaluation ExecuteHackIfAllowed(IDataLoadEventListener listen
 
         if(validFromField == null)
             return BaselineHackEvaluation.DatasetNotCompatibleWithBaselining;
-            
-        //there must be an existing baseline 
+
+        //there must be an existing baseline
 
         //get the ids that the request is for
         var configId = _extractDatasetCommand.Configuration.ID;
@@ -52,7 +52,7 @@ public BaselineHackEvaluation ExecuteHackIfAllowed(IDataLoadEventListener listen
 
         //see if there is an audit of the previous execution success for this dataset
         var previousSuccess = automateExtraction[0].GetSuccessIfAnyFor(_extractDatasetCommand.DatasetBundle.DataSet);
-                
+
         //no there is no record of a succesful extraction so a full baseline is needed
         if(previousSuccess == null)
             return BaselineHackEvaluation.NoCompatibleBaselineAvailable;
@@ -82,12 +82,12 @@ public BaselineHackEvaluation ExecuteHackIfAllowed(IDataLoadEventListener listen
 
         //
         var tblForJoin = _repository.DiscoveredServer.GetCurrentDatabase().ExpectTable("ReleaseIdentifiersSeen");
-            
+
         if(!tblForJoin.Exists())
             throw new Exception($"Table '{tblForJoin} did not exist!");
 
         var tableForJoinName = tblForJoin.GetFullyQualifiedName();
-            
+
         hackSql = $@"
 AND
 (
@@ -120,7 +120,7 @@ public ColumnInfo GetValidFromField(IDataLoadEventListener listener)
         {
             var col = tableInfo.ColumnInfos.SingleOrDefault(c => c.GetRuntimeName() == validFromFieldName);
 
-            //table doesn't have a 
+            //table doesn't have a
             if(col == null)
             {
                 listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
@@ -146,7 +146,7 @@ public ColumnInfo GetValidFromField(IDataLoadEventListener listener)
                         ));
                         return null;
                     }
-                        
+
                     //we are not a primary and we have found a primary already so ignore this col
                 }
                 else //previous one was not a primary
@@ -175,7 +175,7 @@ public ColumnInfo GetValidFromField(IDataLoadEventListener listener)
                 $"No ColumnInfos were found called '{validFromFieldName}'"));
             return null;
         }
-            
+
         listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
             $"Found valid from field '{validFromField}'"));
         return validFromField;

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Execution/ExtractionPipeline/IdentifierAccumulator.cs

@@ -83,7 +83,7 @@ WHERE NOT EXISTS (SELECT 1 FROM ReleaseIdentifiersSeen A2 WHERE
                 var cmd = new SqlCommand(sql, con);
                 cmd.ExecuteNonQuery();
             }
-            
+
 
             tempTable.Drop();
         }

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Execution/ExtractionPipeline/SuccessfullyExtractedResultsDocumenter.cs

@@ -44,7 +44,7 @@ private DataTable ProcessPipelineData(ExtractDatasetCommand ds, DataTable toProc
         {
             _sql = ds.QueryBuilder.SQL;
             _dataset = ds.DatasetBundle.DataSet;
-                
+
             var finder = new AutomateExtractionRepositoryFinder(new RepositoryProvider(ds.DataExportRepository));
             _repo = finder.GetRepositoryIfAny() as AutomateExtractionRepository;
 
@@ -60,15 +60,15 @@ private DataTable ProcessPipelineData(ExtractDatasetCommand ds, DataTable toProc
 
             //index ensure you can't have multiple so this shouldn't blow up
             _automateExtraction = matches.Single();
-               
-            //delete any old baseline records 
+
+            //delete any old baseline records
             var success = _automateExtraction.GetSuccessIfAnyFor(ds.DatasetBundle.DataSet);
             success?.DeleteInDatabase();
 
             _accumulator = IdentifierAccumulator.GetInstance(_dataLoadInfo);
 
         }
-            
+
         foreach (var value in ds.ReleaseIdentifierSubstitutions
                      .SelectMany(substitution => toProcess.Rows.Cast<DataRow>(),
                          (substitution, dr) => dr[substitution.GetRuntimeName()])
@@ -79,7 +79,7 @@ private DataTable ProcessPipelineData(ExtractDatasetCommand ds, DataTable toProc
 
         return toProcess;
     }
-        
+
     private DataTable ProcessPipelineData(ExtractGlobalsCommand globalData, DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
     {
         listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Warning,
@@ -102,12 +102,12 @@ public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureEx
 
     public void Abort(IDataLoadEventListener listener)
     {
-            
+
     }
 
     public void Check(ICheckNotifier notifier)
     {
-            
+
     }
 
     public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener)

AutomationPlugins/LoadModules.Extensions.AutomationPlugins/Properties/AssemblyInfo.cs

@@ -1,7 +1,7 @@
 using System.Runtime.InteropServices;
 
-// Setting ComVisible to false makes the types in this assembly not visible 
-// to COM components.  If you need to access a type in this assembly from 
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
 // COM, set the ComVisible attribute to true on that type.
 [assembly: ComVisible(false)]
 

Interactive/LoadModules.Extensions.Interactive/DeAnonymise/DeAnonymiseAgainstCohort.cs

@@ -13,7 +13,7 @@ namespace LoadModules.Extensions.Interactive.DeAnonymise;
 public class DeAnonymiseAgainstCohort: IPluginDataFlowComponent<DataTable>, IPipelineRequirement<IBasicActivateItems>
 {
     /// <summary>
-    /// When null (default) we launch a new DeAnonymiseAgainstCohortUI in order that the user selects which cohort he wants to deanonymise against. If you set this then you can 
+    /// When null (default) we launch a new DeAnonymiseAgainstCohortUI in order that the user selects which cohort he wants to deanonymise against. If you set this then you can
     /// (for example in unit tests) specify an explicit implementation and dodge the gui.
     /// </summary>
     public IDeAnonymiseAgainstCohortConfigurationFulfiller ConfigurationGetter;
@@ -68,7 +68,7 @@ public void Abort(IDataLoadEventListener listener)
     }
     public void Check(ICheckNotifier notifier)
     {
-           
+
     }
 
     public void PreInitialize(IBasicActivateItems value, IDataLoadEventListener listener)

Interactive/LoadModules.Extensions.Interactive/DeAnonymise/DeAnonymiseAgainstCohortUI.cs

@@ -20,7 +20,7 @@ public partial class DeAnonymiseAgainstCohortUI : Form, IDeAnonymiseAgainstCohor
     private readonly IDataExportRepository _dataExportRepository;
     public IExtractableCohort ChosenCohort { get; set; }
     public string OverrideReleaseIdentifier { get; set; }
-        
+
     public DeAnonymiseAgainstCohortUI(DataTable toProcess, IBasicActivateItems activator)
     {
         _toProcess = toProcess;
@@ -39,7 +39,7 @@ public DeAnonymiseAgainstCohortUI(DataTable toProcess, IBasicActivateItems activ
 
         if (_dataExportRepository == null)
             throw new Exception("DataExportRepository was not set so cannot fetch list of cohorts to advertise to user at runtime");
-         
+
         foreach (DataColumn column in toProcess.Columns)
         {
             var b = new Button
@@ -50,14 +50,14 @@ public DeAnonymiseAgainstCohortUI(DataTable toProcess, IBasicActivateItems activ
             b.Click += b_Click;
         }
     }
-        
+
     void b_Click(object sender, EventArgs e)
     {
         OverrideReleaseIdentifier = ((Button) sender).Text;
         CheckCohortHasCorrectColumns();
     }
 
-       
+
     private void btnChooseCohort_Click(object sender, EventArgs e)
     {
         var dialog = new SelectDialog<IMapsDirectlyToDatabaseTable>(
@@ -81,7 +81,7 @@ private void CheckCohortHasCorrectColumns()
         else
             checksUI1.OnCheckPerformed(new CheckEventArgs($"Found column {release} in your DataTable",
                 CheckResult.Success));
-            
+
         lblExpectedReleaseIdentifierColumn.Text = release;
     }