VIH-10198 Add ref data migration pipeline (#606)

* Update ref data project to clean up statements before creating idempotnent scripts * add new pipeline * add stg env * add pr step * formatting * add missing fgenerate step * update database name * Change database name --------- Co-authored-by: Benjamin Garside (v1) <[email protected]> Co-authored-by: Benjamin Garside (v1) <[email protected]> Co-authored-by: Oliver Scott <[email protected]>
hmcts · Sep 29, 2023 · 1d7c1ce · 1d7c1ce
1 parent fd3f94b
commit 1d7c1ce
Show file tree

Hide file tree

Showing 11 changed files with 220 additions and 52 deletions.
diff --git a/RefData/DesignTimeHearingsContextFactory.cs b/RefData/DesignTimeHearingsContextFactory.cs
diff --git a/RefData/DesignTimeHearingsRefDataContextFactory.cs b/RefData/DesignTimeHearingsRefDataContextFactory.cs
@@ -0,0 +1,58 @@
+namespace RefData
+{
+    [ExcludeFromCodeCoverage]
+    public class DesignTimeHearingsRefDataContextFactory : IDesignTimeDbContextFactory<RefDataContext>
+    {
+        public RefDataContext CreateDbContext(string[] args)
+        {
+            var config = new ConfigurationBuilder()
+                .AddJsonFile("appsettings.json", true)
+                .AddUserSecrets("9AECE566-336D-4D16-88FA-7A76C27321CD")
+                .AddEnvironmentVariables()
+                .Build();
+            var builder = new DbContextOptionsBuilder<RefDataContext>();
+            builder.UseSqlServer(config.GetConnectionString("VideoApi"));
+            builder.ReplaceService<IRelationalCommandBuilderFactory, DynamicSqlRelationalCommandBuilderFactory>();
+            var context = new RefDataContext(builder.Options);
+            return context;
+        }
+    }
+
+    [ExcludeFromCodeCoverage]
+    public class DynamicSqlRelationalCommandBuilder : RelationalCommandBuilder
+    {
+        public DynamicSqlRelationalCommandBuilder(RelationalCommandBuilderDependencies dependencies) : base(dependencies)
+        {
+        }
+
+        public override IRelationalCommand Build()
+        {
+            var commandText = base.Build().CommandText;
+            commandText = commandText.Replace("SET XACT_ABORT ON;", string.Empty);
+            commandText = commandText.Replace("SET XACT_ABORT ON", string.Empty);
+            commandText = commandText.Replace("SET XACT_ABORT OFF;", string.Empty);
+            commandText = commandText.Replace("SET XACT_ABORT OFF", string.Empty);
+            commandText = commandText.Replace("BEGIN TRANSACTION;", string.Empty);
+            commandText = commandText.Replace("BEGIN TRANSACTION", string.Empty);
+
+            commandText = commandText.Replace("COMMIT TRANSACTION;", string.Empty);
+            commandText = commandText.Replace("COMMIT;", string.Empty);
+            commandText = commandText.Replace("COMMIT", string.Empty);
+            commandText = "EXECUTE ('" + commandText.Replace("'", "''") + "')";
+            return new RelationalCommand(Dependencies, commandText, Parameters);
+        }
+    }
+
+    [ExcludeFromCodeCoverage]
+    public class DynamicSqlRelationalCommandBuilderFactory : RelationalCommandBuilderFactory
+    {
+        public DynamicSqlRelationalCommandBuilderFactory(RelationalCommandBuilderDependencies dependencies) : base(dependencies)
+        {
+        }
+
+        public override IRelationalCommandBuilder Create()
+        {
+            return new DynamicSqlRelationalCommandBuilder(Dependencies);
+        }
+    }
+}
diff --git a/RefData/GlobalUsings.cs b/RefData/GlobalUsings.cs
@@ -0,0 +1,10 @@
+// Global using directives
+global using System;
+global using System.Diagnostics.CodeAnalysis;
+global using System.IO;
+global using Microsoft.EntityFrameworkCore;
+global using Microsoft.EntityFrameworkCore.Infrastructure;
+global using Microsoft.EntityFrameworkCore.Design;
+global using Microsoft.EntityFrameworkCore.Migrations;
+global using Microsoft.EntityFrameworkCore.Storage;
+global using Microsoft.Extensions.Configuration;
diff --git a/RefData/Migrations/20230718235233_UpdateVenueNames.cs b/RefData/Migrations/20230718235233_UpdateVenueNames.cs
@@ -1,34 +1,17 @@
-using Microsoft.EntityFrameworkCore.Migrations;
-
-#nullable disable
+#nullable disable
 
 namespace RefData.Migrations
 {
     public partial class UpdateVenueNames : Migration
     {
         protected override void Up(MigrationBuilder migrationBuilder)
         {
-            var sqlFile1 = Path.Combine("data/9777_update_venue_names.sql");
-            RunSqlFile(File.ReadAllText(sqlFile1), migrationBuilder);
+            SqlFileHelper.RunSqlFile("data/9777_update_venue_names.sql", migrationBuilder);
         }
 
         protected override void Down(MigrationBuilder migrationBuilder)
         {
 
         }
-
-        /// <summary>
-        /// EF Core does not like GO statements in the migration files, so we need to split the file into batches
-        /// </summary>
-        /// <param name="sql"></param>
-        /// <param name="migrationBuilder"></param>
-        private void RunSqlFile(string sql, MigrationBuilder migrationBuilder)
-        {
-            string[] batches = sql.Split(new [] {"GO;"}, StringSplitOptions.None);
-            foreach (string batch in batches)
-            {
-                migrationBuilder.Sql(batch);
-            }
-        }
     }
 }
diff --git a/RefData/Migrations/SqlFileHelper.cs b/RefData/Migrations/SqlFileHelper.cs
@@ -0,0 +1,19 @@
+namespace RefData.Migrations;
+
+public static class SqlFileHelper
+{
+    public static void RunSqlFile(string filePath, MigrationBuilder migrationBuilder)
+    {
+        Console.WriteLine("Path is " + filePath);
+        var sql = File.ReadAllText(filePath);
+        var batches = sql.Split(new [] {"GO;"}, StringSplitOptions.None);
+        foreach (var batch in batches)
+        {
+            if(string.IsNullOrWhiteSpace(batch)) continue;
+            Console.WriteLine("---Start--------------------");
+            Console.WriteLine(batch);
+            Console.WriteLine("---End--------------------");
+            migrationBuilder.Sql(batch);
+        }
+    }
+}
diff --git a/RefData/RefData.csproj b/RefData/RefData.csproj
@@ -25,6 +25,9 @@
     <None Update="data\*.sql">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>
diff --git a/RefData/appsettings.json b/RefData/appsettings.json
@@ -0,0 +1,5 @@
+{
+  "ConnectionStrings": {
+    "VideoApi": "Server=(LocalDb)\\MSSQLLocalDB;Database=VhVideoApi;Trusted_Connection=True;"
+  }
+}
diff --git a/RefData/data/9777_update_venue_names.sql b/RefData/data/9777_update_venue_names.sql
@@ -2,16 +2,13 @@ SET XACT_ABORT ON;
 GO;
 BEGIN TRANSACTION;
 
-SELECT DISTINCT HearingVenueName from Conference
-GO;
-
 CREATE OR ALTER PROC #Conference_UpdateHearingVenueName @oldVenueName nvarchar(max), @newVenueName nvarchar(max)
 As
 BEGIN
-    IF EXISTS (SELECT * FROM dbo.Conference WHERE HearingVenueName = TRIM(@oldVenueName))
+    IF EXISTS (SELECT * FROM VhVideo.dbo.Conference WHERE HearingVenueName = TRIM(@oldVenueName))
         BEGIN
             Print ('FOUND venue with the name: ' + @oldVenueName);
-            Update Conference Set HearingVenueName = @newVenueName Where HearingVenueName = @oldVenueName;
+            Update VhVideo.dbo.Conference Set HearingVenueName = @newVenueName Where HearingVenueName = @oldVenueName;
         END
     ELSE
         BEGIN
@@ -79,9 +76,6 @@ EXEC #Conference_UpdateHearingVenueName @oldVenueName='King''s Lynn Crown Court'
 EXEC #Conference_UpdateHearingVenueName @oldVenueName='Hereford Magistrates Court', @newVenueName='Hereford Justice Centre';
 GO;
 
-SELECT DISTINCT HearingVenueName from Conference
-GO;
-
 COMMIT;
 SET XACT_ABORT OFF
 GO;
diff --git a/azure-pipelines.sds.data.release.yml b/azure-pipelines.sds.data.release.yml
@@ -0,0 +1,85 @@
+name: $(Date:yyyyMMddHHmm)-$(Rev:r)
+
+trigger:
+  - master
+
+pr: none
+
+parameters:
+  - name: environments
+    type: object
+    default:
+      - dev
+      - stg
+      - test
+      - ithc
+      - demo
+      - prod
+
+resources:
+  repositories:
+    - repository: azTemplates
+      type: github
+      name: hmcts/azure-devops-templates
+      ref: master
+      endpoint: hmcts
+
+pool:
+  vmImage: windows-latest
+
+stages:
+  - stage: Generate_EF
+    displayName: "Generate Entity Framework"
+    jobs:
+      - job: Generate_EF
+        variables:
+          - template: variables/shared.yaml
+        displayName: "Generate Entity Framework Script"
+        steps:
+          - template: templates/Database/EntityFramework/generate-script.yaml@azTemplates
+            parameters:
+              outputPath: $(Build.StagingDirectory)
+              contextName: ${{ variables.efContextNameRefData }}
+              workingPath: $(System.DefaultWorkingDirectory)/RefData
+              migrationsPath: RefData/Migrations
+              projectName: ${{ variables.efProjectNameRefData }}
+
+  - ${{each env in parameters.environments}}:
+      - stage: Run_EF_${{ env }}
+        dependsOn: Generate_EF
+        jobs:
+          - ${{ if ne(env, 'dev') }}:
+              - job: Approve_EF_${{ env }}
+                displayName: "Approve EF to ${{ env }}"
+                pool: server
+                timeoutInMinutes: 10080 # 7 Days
+                steps:
+                  - task: ManualValidation@0
+                    timeoutInMinutes: 10080 # 7 Days
+                    inputs:
+                      instructions: "Please Approve EF run to ${{ env }}"
+                      onTimeout: "reject"
+          - job: Run_EF_RefData_${{ env }}
+            variables:
+              - template: variables/shared.yaml
+                parameters:
+                  env: ${{ env }}
+            pool:
+              vmImage: "windows-latest" # This job must be run on Windows
+            displayName: "Run Entity Framework Ref Data ${{ env }}"
+            ${{ if ne(env, 'dev') }}:
+              dependsOn: Approve_EF_${{ env }}
+            steps:
+              - download: current
+                displayName: Download Sql Artifact
+              - template: templates/Database/EntityFramework/run-entity-framework.yaml@azTemplates
+                parameters:
+                  sqlServerResourceGroup: ${{ variables.vhResourceGroup }}
+                  sqlServerName: ${{ variables.vhSQLServerName }}
+                  databaseName: ${{ variables.BookingsApiDbName }}
+                  azureSubscription: ${{ variables.subscriptionName }}
+                  sqlScriptLocation: "$(Pipeline.Workspace)/${{ variables.efContextNameRefData }}-$(Build.BuildId)/${{ variables.efContextNameRefData }}.sql"
+                  kvfirewallRequired: false
+                  kvName: ${{ variables.vhKeyVault }}
+                  kvSqlPasswordSecret: ${{ variables.vhSqlPasswordSecret }}
+                  kvSqlUsernameSecret: ${{ variables.vhSqlUsernameSecret }}
diff --git a/azure-pipelines.sds.pr-release.yml b/azure-pipelines.sds.pr-release.yml
@@ -66,6 +66,17 @@ stages:
               workingPath: $(System.DefaultWorkingDirectory)/${{ variables.appName }}/${{ variables.appName }}
               migrationsPath: ${{ variables.appName }}/${{ variables.appName }}.DAL/Migrations
               projectName: ${{ variables.efProjectName }}
+
+      - job: Generate_RefData_Entity_Framework_Script
+        displayName: "Generate RefData Entity Framework Script"
+        steps:
+          - template: templates/Database/EntityFramework/generate-script.yaml@azTemplates
+            parameters:
+              outputPath: $(Build.StagingDirectory)
+              contextName: ${{ variables.efContextNameRefData }}
+              workingPath: $(System.DefaultWorkingDirectory)/RefData
+              migrationsPath: RefData/Migrations
+              projectName: ${{ variables.efProjectNameRefData }}
 
       - job: package_nuget
         displayName: "Package NuGet Packages"
@@ -167,6 +178,27 @@ stages:
               kvName: ${{ variables.vhKeyVault }}
               kvSqlPasswordSecret: ${{ variables.vhSqlPasswordSecret }}
               kvSqlUsernameSecret: ${{ variables.vhSqlUsernameSecret }}
+
+      - job: Run_Entity_Framework_RefData_Dev
+        pool:
+          vmImage: "windows-latest" # This Job Must be Run on Windows
+        displayName: Run Entity Framework Ref Data Dev
+        dependsOn: Run_Entity_Framework_Dev
+        steps:
+          - download: current
+            displayName: Download Sql Artifact
+
+          - template: templates/Database/EntityFramework/run-entity-framework.yaml@azTemplates
+            parameters:
+              sqlServerResourceGroup: ${{ variables.vhResourceGroup }}
+              sqlServerName: ${{ variables.vhSQLServerName }}
+              databaseName: ${{ variables.videoApiDbName }}
+              azureSubscription: ${{ variables.subscriptionName }}
+              sqlScriptLocation: "$(Pipeline.Workspace)/${{ variables.efContextNameRefData }}-$(Build.BuildId)/${{ variables.efContextNameRefData }}.sql"
+              kvfirewallRequired: false
+              kvName: ${{ variables.vhKeyVault }}
+              kvSqlPasswordSecret: ${{ variables.vhSqlPasswordSecret }}
+              kvSqlUsernameSecret: ${{ variables.vhSqlUsernameSecret }}
 
   #####################################################
   # Deploy Helm Chart to Dev. #########################

diff --git a/variables/shared.yaml b/variables/shared.yaml
@@ -74,6 +74,10 @@ variables:
   - name: vhResourceGroup
     value: vh-infra-core-${{ parameters.env }}
 
+  - name: efContextNameRefData
+    value: RefDataContext
+  - name: efProjectNameRefData
+    value: RefData.csproj
   - name: efContextName
     value: VideoApiDbContext
   - name: efProjectName