From 1d7c1cefff596cb74a1b3154e70c234a1d869ddf Mon Sep 17 00:00:00 2001 From: shaed-parkar <41630528+shaed-parkar@users.noreply.github.com> Date: Fri, 29 Sep 2023 17:40:51 +0100 Subject: [PATCH] VIH-10198 Add ref data migration pipeline (#606) * Update ref data project to clean up statements before creating idempotnent scripts * add new pipeline * add stg env * add pr step * formatting * add missing fgenerate step * update database name * Change database name --------- Co-authored-by: Benjamin Garside (v1) Co-authored-by: Benjamin Garside (v1) <114483383+benjamin-v1@users.noreply.github.com> Co-authored-by: Oliver Scott --- RefData/DesignTimeHearingsContextFactory.cs | 25 ------ ...DesignTimeHearingsRefDataContextFactory.cs | 58 +++++++++++++ RefData/GlobalUsings.cs | 10 +++ .../20230718235233_UpdateVenueNames.cs | 21 +---- RefData/Migrations/SqlFileHelper.cs | 19 +++++ RefData/RefData.csproj | 3 + RefData/appsettings.json | 5 ++ RefData/data/9777_update_venue_names.sql | 10 +-- azure-pipelines.sds.data.release.yml | 85 +++++++++++++++++++ azure-pipelines.sds.pr-release.yml | 32 +++++++ variables/shared.yaml | 4 + 11 files changed, 220 insertions(+), 52 deletions(-) delete mode 100644 RefData/DesignTimeHearingsContextFactory.cs create mode 100644 RefData/DesignTimeHearingsRefDataContextFactory.cs create mode 100644 RefData/GlobalUsings.cs create mode 100644 RefData/Migrations/SqlFileHelper.cs create mode 100644 RefData/appsettings.json create mode 100644 azure-pipelines.sds.data.release.yml diff --git a/RefData/DesignTimeHearingsContextFactory.cs b/RefData/DesignTimeHearingsContextFactory.cs deleted file mode 100644 index 8e06d60cb..000000000 --- a/RefData/DesignTimeHearingsContextFactory.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Design; -using Microsoft.Extensions.Configuration; - -namespace RefData -{ - [ExcludeFromCodeCoverage] - public class DesignTimeHearingsContextFactory : IDesignTimeDbContextFactory - { - public RefDataContext CreateDbContext(string[] args) - { - var config = new ConfigurationBuilder() - .AddJsonFile("appsettings.json", true) - .AddUserSecrets("9AECE566-336D-4D16-88FA-7A76C27321CD") - .AddEnvironmentVariables() - .Build(); - var builder = new DbContextOptionsBuilder(); - builder.UseSqlServer(config.GetConnectionString("VideoApi")); - var context = new RefDataContext(builder.Options); - return context; - } - } - -} \ No newline at end of file diff --git a/RefData/DesignTimeHearingsRefDataContextFactory.cs b/RefData/DesignTimeHearingsRefDataContextFactory.cs new file mode 100644 index 000000000..860e9c3eb --- /dev/null +++ b/RefData/DesignTimeHearingsRefDataContextFactory.cs @@ -0,0 +1,58 @@ +namespace RefData +{ + [ExcludeFromCodeCoverage] + public class DesignTimeHearingsRefDataContextFactory : IDesignTimeDbContextFactory + { + public RefDataContext CreateDbContext(string[] args) + { + var config = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", true) + .AddUserSecrets("9AECE566-336D-4D16-88FA-7A76C27321CD") + .AddEnvironmentVariables() + .Build(); + var builder = new DbContextOptionsBuilder(); + builder.UseSqlServer(config.GetConnectionString("VideoApi")); + builder.ReplaceService(); + var context = new RefDataContext(builder.Options); + return context; + } + } + + [ExcludeFromCodeCoverage] + public class DynamicSqlRelationalCommandBuilder : RelationalCommandBuilder + { + public DynamicSqlRelationalCommandBuilder(RelationalCommandBuilderDependencies dependencies) : base(dependencies) + { + } + + public override IRelationalCommand Build() + { + var commandText = base.Build().CommandText; + commandText = commandText.Replace("SET XACT_ABORT ON;", string.Empty); + commandText = commandText.Replace("SET XACT_ABORT ON", string.Empty); + commandText = commandText.Replace("SET XACT_ABORT OFF;", string.Empty); + commandText = commandText.Replace("SET XACT_ABORT OFF", string.Empty); + commandText = commandText.Replace("BEGIN TRANSACTION;", string.Empty); + commandText = commandText.Replace("BEGIN TRANSACTION", string.Empty); + + commandText = commandText.Replace("COMMIT TRANSACTION;", string.Empty); + commandText = commandText.Replace("COMMIT;", string.Empty); + commandText = commandText.Replace("COMMIT", string.Empty); + commandText = "EXECUTE ('" + commandText.Replace("'", "''") + "')"; + return new RelationalCommand(Dependencies, commandText, Parameters); + } + } + + [ExcludeFromCodeCoverage] + public class DynamicSqlRelationalCommandBuilderFactory : RelationalCommandBuilderFactory + { + public DynamicSqlRelationalCommandBuilderFactory(RelationalCommandBuilderDependencies dependencies) : base(dependencies) + { + } + + public override IRelationalCommandBuilder Create() + { + return new DynamicSqlRelationalCommandBuilder(Dependencies); + } + } +} \ No newline at end of file diff --git a/RefData/GlobalUsings.cs b/RefData/GlobalUsings.cs new file mode 100644 index 000000000..b8825affd --- /dev/null +++ b/RefData/GlobalUsings.cs @@ -0,0 +1,10 @@ +// Global using directives +global using System; +global using System.Diagnostics.CodeAnalysis; +global using System.IO; +global using Microsoft.EntityFrameworkCore; +global using Microsoft.EntityFrameworkCore.Infrastructure; +global using Microsoft.EntityFrameworkCore.Design; +global using Microsoft.EntityFrameworkCore.Migrations; +global using Microsoft.EntityFrameworkCore.Storage; +global using Microsoft.Extensions.Configuration; \ No newline at end of file diff --git a/RefData/Migrations/20230718235233_UpdateVenueNames.cs b/RefData/Migrations/20230718235233_UpdateVenueNames.cs index ceca77ab4..c84cd8cdf 100644 --- a/RefData/Migrations/20230718235233_UpdateVenueNames.cs +++ b/RefData/Migrations/20230718235233_UpdateVenueNames.cs @@ -1,6 +1,4 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable +#nullable disable namespace RefData.Migrations { @@ -8,27 +6,12 @@ public partial class UpdateVenueNames : Migration { protected override void Up(MigrationBuilder migrationBuilder) { - var sqlFile1 = Path.Combine("data/9777_update_venue_names.sql"); - RunSqlFile(File.ReadAllText(sqlFile1), migrationBuilder); + SqlFileHelper.RunSqlFile("data/9777_update_venue_names.sql", migrationBuilder); } protected override void Down(MigrationBuilder migrationBuilder) { } - - /// - /// EF Core does not like GO statements in the migration files, so we need to split the file into batches - /// - /// - /// - private void RunSqlFile(string sql, MigrationBuilder migrationBuilder) - { - string[] batches = sql.Split(new [] {"GO;"}, StringSplitOptions.None); - foreach (string batch in batches) - { - migrationBuilder.Sql(batch); - } - } } } diff --git a/RefData/Migrations/SqlFileHelper.cs b/RefData/Migrations/SqlFileHelper.cs new file mode 100644 index 000000000..e3720fe2e --- /dev/null +++ b/RefData/Migrations/SqlFileHelper.cs @@ -0,0 +1,19 @@ +namespace RefData.Migrations; + +public static class SqlFileHelper +{ + public static void RunSqlFile(string filePath, MigrationBuilder migrationBuilder) + { + Console.WriteLine("Path is " + filePath); + var sql = File.ReadAllText(filePath); + var batches = sql.Split(new [] {"GO;"}, StringSplitOptions.None); + foreach (var batch in batches) + { + if(string.IsNullOrWhiteSpace(batch)) continue; + Console.WriteLine("---Start--------------------"); + Console.WriteLine(batch); + Console.WriteLine("---End--------------------"); + migrationBuilder.Sql(batch); + } + } +} \ No newline at end of file diff --git a/RefData/RefData.csproj b/RefData/RefData.csproj index b881fa09b..0c39c77bb 100644 --- a/RefData/RefData.csproj +++ b/RefData/RefData.csproj @@ -25,6 +25,9 @@ Always + + Always + diff --git a/RefData/appsettings.json b/RefData/appsettings.json new file mode 100644 index 000000000..219989055 --- /dev/null +++ b/RefData/appsettings.json @@ -0,0 +1,5 @@ +{ + "ConnectionStrings": { + "VideoApi": "Server=(LocalDb)\\MSSQLLocalDB;Database=VhVideoApi;Trusted_Connection=True;" + } +} \ No newline at end of file diff --git a/RefData/data/9777_update_venue_names.sql b/RefData/data/9777_update_venue_names.sql index 47ede5d6e..d857ea2eb 100644 --- a/RefData/data/9777_update_venue_names.sql +++ b/RefData/data/9777_update_venue_names.sql @@ -2,16 +2,13 @@ SET XACT_ABORT ON; GO; BEGIN TRANSACTION; -SELECT DISTINCT HearingVenueName from Conference -GO; - CREATE OR ALTER PROC #Conference_UpdateHearingVenueName @oldVenueName nvarchar(max), @newVenueName nvarchar(max) As BEGIN - IF EXISTS (SELECT * FROM dbo.Conference WHERE HearingVenueName = TRIM(@oldVenueName)) + IF EXISTS (SELECT * FROM VhVideo.dbo.Conference WHERE HearingVenueName = TRIM(@oldVenueName)) BEGIN Print ('FOUND venue with the name: ' + @oldVenueName); - Update Conference Set HearingVenueName = @newVenueName Where HearingVenueName = @oldVenueName; + Update VhVideo.dbo.Conference Set HearingVenueName = @newVenueName Where HearingVenueName = @oldVenueName; END ELSE BEGIN @@ -79,9 +76,6 @@ EXEC #Conference_UpdateHearingVenueName @oldVenueName='King''s Lynn Crown Court' EXEC #Conference_UpdateHearingVenueName @oldVenueName='Hereford Magistrates Court', @newVenueName='Hereford Justice Centre'; GO; -SELECT DISTINCT HearingVenueName from Conference -GO; - COMMIT; SET XACT_ABORT OFF GO; \ No newline at end of file diff --git a/azure-pipelines.sds.data.release.yml b/azure-pipelines.sds.data.release.yml new file mode 100644 index 000000000..942e8e601 --- /dev/null +++ b/azure-pipelines.sds.data.release.yml @@ -0,0 +1,85 @@ +name: $(Date:yyyyMMddHHmm)-$(Rev:r) + +trigger: + - master + +pr: none + +parameters: + - name: environments + type: object + default: + - dev + - stg + - test + - ithc + - demo + - prod + +resources: + repositories: + - repository: azTemplates + type: github + name: hmcts/azure-devops-templates + ref: master + endpoint: hmcts + +pool: + vmImage: windows-latest + +stages: + - stage: Generate_EF + displayName: "Generate Entity Framework" + jobs: + - job: Generate_EF + variables: + - template: variables/shared.yaml + displayName: "Generate Entity Framework Script" + steps: + - template: templates/Database/EntityFramework/generate-script.yaml@azTemplates + parameters: + outputPath: $(Build.StagingDirectory) + contextName: ${{ variables.efContextNameRefData }} + workingPath: $(System.DefaultWorkingDirectory)/RefData + migrationsPath: RefData/Migrations + projectName: ${{ variables.efProjectNameRefData }} + + - ${{each env in parameters.environments}}: + - stage: Run_EF_${{ env }} + dependsOn: Generate_EF + jobs: + - ${{ if ne(env, 'dev') }}: + - job: Approve_EF_${{ env }} + displayName: "Approve EF to ${{ env }}" + pool: server + timeoutInMinutes: 10080 # 7 Days + steps: + - task: ManualValidation@0 + timeoutInMinutes: 10080 # 7 Days + inputs: + instructions: "Please Approve EF run to ${{ env }}" + onTimeout: "reject" + - job: Run_EF_RefData_${{ env }} + variables: + - template: variables/shared.yaml + parameters: + env: ${{ env }} + pool: + vmImage: "windows-latest" # This job must be run on Windows + displayName: "Run Entity Framework Ref Data ${{ env }}" + ${{ if ne(env, 'dev') }}: + dependsOn: Approve_EF_${{ env }} + steps: + - download: current + displayName: Download Sql Artifact + - template: templates/Database/EntityFramework/run-entity-framework.yaml@azTemplates + parameters: + sqlServerResourceGroup: ${{ variables.vhResourceGroup }} + sqlServerName: ${{ variables.vhSQLServerName }} + databaseName: ${{ variables.BookingsApiDbName }} + azureSubscription: ${{ variables.subscriptionName }} + sqlScriptLocation: "$(Pipeline.Workspace)/${{ variables.efContextNameRefData }}-$(Build.BuildId)/${{ variables.efContextNameRefData }}.sql" + kvfirewallRequired: false + kvName: ${{ variables.vhKeyVault }} + kvSqlPasswordSecret: ${{ variables.vhSqlPasswordSecret }} + kvSqlUsernameSecret: ${{ variables.vhSqlUsernameSecret }} diff --git a/azure-pipelines.sds.pr-release.yml b/azure-pipelines.sds.pr-release.yml index 9cbed8256..2a22e3648 100644 --- a/azure-pipelines.sds.pr-release.yml +++ b/azure-pipelines.sds.pr-release.yml @@ -66,6 +66,17 @@ stages: workingPath: $(System.DefaultWorkingDirectory)/${{ variables.appName }}/${{ variables.appName }} migrationsPath: ${{ variables.appName }}/${{ variables.appName }}.DAL/Migrations projectName: ${{ variables.efProjectName }} + + - job: Generate_RefData_Entity_Framework_Script + displayName: "Generate RefData Entity Framework Script" + steps: + - template: templates/Database/EntityFramework/generate-script.yaml@azTemplates + parameters: + outputPath: $(Build.StagingDirectory) + contextName: ${{ variables.efContextNameRefData }} + workingPath: $(System.DefaultWorkingDirectory)/RefData + migrationsPath: RefData/Migrations + projectName: ${{ variables.efProjectNameRefData }} - job: package_nuget displayName: "Package NuGet Packages" @@ -167,6 +178,27 @@ stages: kvName: ${{ variables.vhKeyVault }} kvSqlPasswordSecret: ${{ variables.vhSqlPasswordSecret }} kvSqlUsernameSecret: ${{ variables.vhSqlUsernameSecret }} + + - job: Run_Entity_Framework_RefData_Dev + pool: + vmImage: "windows-latest" # This Job Must be Run on Windows + displayName: Run Entity Framework Ref Data Dev + dependsOn: Run_Entity_Framework_Dev + steps: + - download: current + displayName: Download Sql Artifact + + - template: templates/Database/EntityFramework/run-entity-framework.yaml@azTemplates + parameters: + sqlServerResourceGroup: ${{ variables.vhResourceGroup }} + sqlServerName: ${{ variables.vhSQLServerName }} + databaseName: ${{ variables.videoApiDbName }} + azureSubscription: ${{ variables.subscriptionName }} + sqlScriptLocation: "$(Pipeline.Workspace)/${{ variables.efContextNameRefData }}-$(Build.BuildId)/${{ variables.efContextNameRefData }}.sql" + kvfirewallRequired: false + kvName: ${{ variables.vhKeyVault }} + kvSqlPasswordSecret: ${{ variables.vhSqlPasswordSecret }} + kvSqlUsernameSecret: ${{ variables.vhSqlUsernameSecret }} ##################################################### # Deploy Helm Chart to Dev. ######################### diff --git a/variables/shared.yaml b/variables/shared.yaml index 17a827056..7eb1dbbf5 100644 --- a/variables/shared.yaml +++ b/variables/shared.yaml @@ -74,6 +74,10 @@ variables: - name: vhResourceGroup value: vh-infra-core-${{ parameters.env }} + - name: efContextNameRefData + value: RefDataContext + - name: efProjectNameRefData + value: RefData.csproj - name: efContextName value: VideoApiDbContext - name: efProjectName