From 6474ae169fa41312eaeb67ce17b4f77d253a5a6a Mon Sep 17 00:00:00 2001 From: Guillaume Escarieux Date: Thu, 16 Feb 2023 01:34:49 +0100 Subject: [PATCH] feat: Account for custom S3 compatible service V4 Signature for AWS Custom ServiceURL and Region for S3 compatible storage other than AWS Added and used some parameters Update AWSSDK.S3 to latest version possible (see https://github.com/aws/aws-sdk-net/issues/2540#issuecomment-1432217487) --- Estranged.Lfs.sln | 18 ++-- README.md | 69 ++++++++++-- .../Estranged.Lfs.Hosting.AspNet.csproj | 2 +- .../Estranged.Lfs.Hosting.AspNet/Startup.cs | 28 +++-- .../Estranged.Lfs.Hosting.Lambda/Startup.cs | 18 +++- .../Estranged.Lfs.Hosting.Lambda/modele.yaml | 102 ++++++++++++++++++ .../Estranged.Lfs.Adapter.S3.csproj | 2 +- src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs | 10 +- .../Estranged.Lfs.Tests.csproj | 2 +- 9 files changed, 217 insertions(+), 34 deletions(-) create mode 100644 hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml diff --git a/Estranged.Lfs.sln b/Estranged.Lfs.sln index dd233d8..f8c67bb 100644 --- a/Estranged.Lfs.sln +++ b/Estranged.Lfs.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 -VisualStudioVersion = 16.0.29806.167 +VisualStudioVersion = 16.0.30320.27 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{03ED5016-F8BF-4C18-A997-2FFE0E6EF896}" EndProject @@ -14,6 +14,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Adapters", "Adapters", "{299762B3-8879-42F1-B002-4A9AF7401A13}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Authenticators", "Authenticators", "{FD975DFA-BEA7-451F-B85F-262B6C658DAC}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{68EBAB1E-0B07-4701-A39D-096A8226A3A2}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Hosting.AspNet", "hosting\Estranged.Lfs.Hosting.AspNet\Estranged.Lfs.Hosting.AspNet.csproj", "{948329E9-DE90-4245-8C2E-BCD23746056A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Api", "src\Estranged.Lfs.Api\Estranged.Lfs.Api.csproj", "{EAFEAA0F-678A-4888-9F32-45DA467A4CD5}" @@ -28,12 +34,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Authenticator EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Authenticator.BitBucket", "src\Estranged.Lfs.Authenticator.BitBucket\Estranged.Lfs.Authenticator.BitBucket.csproj", "{711786E0-5F18-440C-A2FA-CB1E26FD2DEF}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Adapters", "Adapters", "{299762B3-8879-42F1-B002-4A9AF7401A13}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Authenticators", "Authenticators", "{FD975DFA-BEA7-451F-B85F-262B6C658DAC}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{68EBAB1E-0B07-4701-A39D-096A8226A3A2}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Tests", "tests\Estranged.Lfs.Tests\Estranged.Lfs.Tests.csproj", "{3EBBF34F-B0EF-4C60-A9F8-432A5E9C1DD2}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Adapter.Azure.Blob", "src\Estranged.Lfs.Adapter.Azure.Blob\Estranged.Lfs.Adapter.Azure.Blob.csproj", "{E4A09D4E-FF7F-4899-9384-5E8A90D0E9DA}" @@ -85,6 +85,8 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution + {299762B3-8879-42F1-B002-4A9AF7401A13} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} + {FD975DFA-BEA7-451F-B85F-262B6C658DAC} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} {948329E9-DE90-4245-8C2E-BCD23746056A} = {89E5DA04-5E1C-410C-9175-27C0713594AD} {EAFEAA0F-678A-4888-9F32-45DA467A4CD5} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} {CE95E105-4980-4F65-915C-879692BC0F23} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} @@ -92,8 +94,6 @@ Global {D4D0ABA0-A829-43C0-B3E8-C57AE9170ECF} = {89E5DA04-5E1C-410C-9175-27C0713594AD} {6467DCDA-D062-4DD4-A4A2-D8D551411B2A} = {FD975DFA-BEA7-451F-B85F-262B6C658DAC} {711786E0-5F18-440C-A2FA-CB1E26FD2DEF} = {FD975DFA-BEA7-451F-B85F-262B6C658DAC} - {299762B3-8879-42F1-B002-4A9AF7401A13} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} - {FD975DFA-BEA7-451F-B85F-262B6C658DAC} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} {3EBBF34F-B0EF-4C60-A9F8-432A5E9C1DD2} = {68EBAB1E-0B07-4701-A39D-096A8226A3A2} {E4A09D4E-FF7F-4899-9384-5E8A90D0E9DA} = {299762B3-8879-42F1-B002-4A9AF7401A13} EndGlobalSection diff --git a/README.md b/README.md index 067e797..4c45366 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A Git LFS backend which provides pluggable authentication and blob store adapter services.AddLfs(); ``` -2. Register an implementation for IBlobAdapter and IAuthenticator. Amazon AWS S3 and Azure Blob Storage are provided out of the box: +2. Register an implementation for IBlobAdapter and IAuthenticator. Amazon AWS S3, S3-compatible and Azure Blob Storage are provided out of the box: ```csharp var s3BlobConfig = new S3BlobAdapterConfig @@ -109,6 +109,40 @@ There are currently two hosting examples: The former is a simple example using only Asp.NET components, and the latter is an Asp.NET Lambda function which can be deployed directly to AWS Lambda, behind API Gateway. +### Asp.NET version + +1. Edit the variables values to suit to your environment + +``` + LfsBucket // Mandatory: Name of S3 bucket + S3AccessKeyId // Optional: _aws_access_key_id_ of the .aws/credential file for your custom s3 profile + S3AccessKeySecret // Optional: _aws_secret_access_key_ of the .aws/credential file for your custom s3 profile + S3Region // Optional: region in custom S3 + S3ServiceURL // Optional: endpoint of custom S3 +``` + +2. It can be launched in VS by choosing _Estranged.Lfs.Hosting.AspNet_ (not the default _IIS Express_ option that doesnt work). + +![image](https://user-images.githubusercontent.com/2952456/89800274-d82c9380-db2e-11ea-85bb-3fc8652e3e9d.png) + +3. Or it can be published in folder, then launched with _Estranged.Lfs.Hosting.AspNet.exe_ + +4. This is a console application that is listening for HTTP LFS requests on https://localhost:5001 + +![image](https://user-images.githubusercontent.com/2952456/89800695-6739ab80-db2f-11ea-8641-0eab8c501381.png) + +5. Change the .lfconfig to send request to the console app + +``` +[lfs] +url = https://localhost:5001/ +``` +6. From git repo Commit lfs file and Push, and enter when asked the user and password set by this line +` services.AddSingleton(x => new DictionaryAuthenticator(new Dictionary { { "usernametest", "passwordtest" } }));` + +7. The pushed file is now present in custom S3 Storage +![image](https://user-images.githubusercontent.com/2952456/89806464-5e4cd800-db37-11ea-85bd-9ce724e7ee0e.png) + #### Deploying to Lambda 1. Head over to the `Estranged.Lfs.Hosting.Lambda` project in the `hosting` folder. @@ -117,20 +151,41 @@ The former is a simple example using only Asp.NET components, and the latter is ```javascript { - "profile": "default", + "profile": "default", // AWS connexion profile "configuration": "Release", "framework": "net6.0", "function-handler": "Estranged.Lfs.Hosting.Lambda::Estranged.Lfs.Hosting.Lambda.LambdaEntryPoint::FunctionHandlerAsync", "function-memory-size": 256, "function-timeout": 30, "function-runtime": "dotnet6", - "region": "", - "s3-bucket": "", + "region": "", // AWS public region + "s3-bucket": "", // S3 bucket needed to upload the modele/output of the stack, must be outside of the stack (shared between all stacks) "s3-prefix": "", - "function-name": "", + "function-name": "", // lambda name must be same as stack name // Set other variables required by the Lambda function - "environment-variables": "LFS_BUCKET=;=" + "environment-variables": "LFS_BUCKET=;LFS_USERNAME=;LFS_PASSWORD=;S3_ACCESS_KEY=;S3_ACCESS_SECRET=;S3_REGION=;S3_SERVICE_URL=;=", // can be found and changed in Lambda configuration UI" } ``` +4. Run `dotnet lambda deploy-serverless` to deploy the stack +5. Run `dotnet lambda deploy-function` to deploy the code of the lambda function +6. Change the .lfconfig of the GIT project to send requests to the lambda function (the URL was in 4. output) +``` +[lfs] +url = https://xxxxxxxxx.execute-api.eu-west-1.amazonaws.com/lfs +``` + +8. Commit and push LFS files, when prompt enter AWS_STACK_ParameterUsername and AWS_STACK_ParameterPassword, the files can be seen in your S3 storage! + +**Instead of using user/password authentication, it is possible to use Github or Bitbucket authentication.** + +Edit the `aws-lambda-tools-defaults.json` file and redeploy the lambda (or edit directly in the lambda UI) + +(example with github): + + ``` "environment-variables": "GITHUB_ORGANISATION=REPO_ORGANISATION,GITHUB_REPOSITORY=REPO_NAME,..."``` + + (example with bitbucket): + + ``` "environment-variables": "BITBUCKET_WORKSPACE=REPO_WORKSPACE,BITBUCKET_REPOSITORY=REPO_NAME,..."``` -4. Run `dotnet-lambda deploy-serverless` to deploy the Lambda function + In this case, use your platform username and dedicated auth token to authenticate. \ No newline at end of file diff --git a/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj b/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj index f63b77b..af1fcd0 100644 --- a/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj +++ b/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj @@ -10,4 +10,4 @@ - + \ No newline at end of file diff --git a/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs b/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs index c4d946d..7fcc073 100644 --- a/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs +++ b/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs @@ -1,11 +1,11 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Estranged.Lfs.Api; -using Amazon.S3; -using Microsoft.Extensions.Configuration; +using Amazon.S3; using Estranged.Lfs.Adapter.S3; +using Estranged.Lfs.Api; using Estranged.Lfs.Data; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using System.Collections.Generic; namespace Estranged.Lfs.Hosting.AspNet @@ -23,7 +23,19 @@ public void ConfigureServices(IServiceCollection services) }); services.AddSingleton(); - services.AddLfsS3Adapter(new S3BlobAdapterConfig{Bucket = "estranged-lfs-test"}, new AmazonS3Client()); + const string LfsBucket = "estranged-lfs-test"; + const string S3AccessKeyId = ""; + const string S3AccessKeySecret = ""; + const string S3Region = ""; + const string S3ServiceURL = ""; + if (!string.IsNullOrWhiteSpace(S3ServiceURL) && !string.IsNullOrWhiteSpace(S3Region) && !string.IsNullOrWhiteSpace(S3AccessKeyId) && !string.IsNullOrWhiteSpace(S3AccessKeySecret)) + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = LfsBucket }, new AmazonS3Client(S3AccessKeyId, S3AccessKeySecret, new AmazonS3Config { ServiceURL = S3ServiceURL, AuthenticationRegion = S3Region, SignatureVersion = "V4" })); + } + else + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = LfsBucket }, new AmazonS3Client()); + } services.AddSingleton(x => new DictionaryAuthenticator(new Dictionary { { "usernametest", "passwordtest" } })); services.AddLfsApi(); } @@ -34,4 +46,4 @@ public void Configure(IApplicationBuilder app) app.UseEndpoints(endpoints => endpoints.MapControllers()); } } -} +} \ No newline at end of file diff --git a/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs b/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs index 660c04b..322e451 100644 --- a/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs +++ b/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs @@ -29,6 +29,10 @@ public void ConfigureServices(IServiceCollection services) const string S3AccelerationVariable = "S3_ACCELERATION"; const string LfsAzureStorageConnectionStringVariable = "LFS_AZUREBLOB_CONNECTIONSTRING"; const string LfsAzureStorageContainerNameVariable = "LFS_AZUREBLOB_CONTAINERNAME"; + const string S3ServiceURL = "S3_SERVICE_URL"; + const string S3Region = "S3_REGION"; + const string S3AccessKey = "S3_ACCESS_KEY"; + const string S3AccessSecret = "S3_ACCESS_SECRET"; var config = new ConfigurationBuilder() .AddEnvironmentVariables() @@ -43,6 +47,10 @@ public void ConfigureServices(IServiceCollection services) string bitBucketWorkspace = config[BitBucketWorkspaceVariable]; string bitBucketRepository = config[BitBucketRepositoryVariable]; bool s3Acceleration = bool.Parse(config[S3AccelerationVariable] ?? "false"); + string s3ServiceURL = config[S3ServiceURL]; + string s3Region = config[S3Region]; + string s3AccessKey = config[S3AccessKey]; + string s3AccessSecret = config[S3AccessSecret]; bool isS3Storage = !string.IsNullOrWhiteSpace(lfsBucket); bool isAzureStorage = !string.IsNullOrWhiteSpace(lfsAzureStorageConnectionString); @@ -74,7 +82,15 @@ public void ConfigureServices(IServiceCollection services) if (isS3Storage) { - services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration })); + if (!string.IsNullOrWhiteSpace(s3ServiceURL) && !string.IsNullOrWhiteSpace(s3Region) && !string.IsNullOrWhiteSpace(s3AccessKey) && !string.IsNullOrWhiteSpace(s3AccessSecret)) + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(s3AccessKey, s3AccessSecret, new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration, ServiceURL = s3ServiceURL, AuthenticationRegion = s3Region, SignatureVersion = "V4" })); + + } + else + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration })); + } } else if (isAzureStorage) { diff --git a/hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml b/hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml new file mode 100644 index 0000000..75ba5fd --- /dev/null +++ b/hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml @@ -0,0 +1,102 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + GitLfsUsername: + Type: String + Description: Username for authenticating against Git LFS endpoint + GitLfsPassword: + Type: String + Description: Password for authenticating against Git LFS endpoint +Outputs: + LfsEndpoint: + Description: The Git LFS endpoint to use + Value: !Sub 'https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/lfs' +Resources: + StorageBucket: + Type: AWS::S3::Bucket + DeletionPolicy: Retain + RestApi: + Type: AWS::ApiGateway::RestApi + Properties: + Body: + swagger: '2.0' + info: + description: 'Describes a proxy to a Lambda function to sign S3 requests.' + title: 'Git LFS REST API' + version: '1.0.0' + paths: + /{proxy+}: + x-amazon-apigateway-any-method: + produces: + - application/json + parameters: + - name: proxy + in: path + required: true + type: string + responses: {} + x-amazon-apigateway-integration: + responses: + default: + statusCode: 200 + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${SigningLambda}/invocations' + passthroughBehavior: when_no_match + httpMethod: POST + contentHandling: CONVERT_TO_TEXT + type: aws_proxy + Description: Git LFS endpoint + FailOnWarnings: true + Name: !Ref AWS::StackName + RestDeployment: + Type: AWS::ApiGateway::Deployment + Properties: + RestApiId: !Ref RestApi + StageName: lfs + SigningLambda: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Sub 'ae-infrastructure-${AWS::Region}' + S3Key: git-lfs/3.0.0/Estranged.Lfs.Hosting.Lambda.zip + Description: Generates S3 signed URLs for Git LFS + FunctionName: !Ref AWS::StackName + Handler: Estranged.Lfs.Hosting.Lambda::Estranged.Lfs.Hosting.Lambda.LambdaEntryPoint::FunctionHandlerAsync + MemorySize: 512 + Role: !GetAtt SigningLambdaRole.Arn + Runtime: dotnetcore3.1 + Timeout: 30 + Environment: + Variables: + LFS_BUCKET: !Ref StorageBucket + LFS_USERNAME: !Ref GitLfsUsername + LFS_PASSWORD: !Ref GitLfsPassword + GITHUB_ORGANISATION: !Ref GitOrganisation + GITHUB_REPOSITORY: !Ref GitHubRepositoryVariable + BITBUCKET_WORKSPACE: !Ref BitBucketWorkspaceVariable + BITBUCKET_REPOSITORY: !Ref BitBucketRepositoryVariable + S3Region: !Ref S3_REGION + S3ServiceURL: !Ref S3_SERVICE_URL + S3AccessKey: !Ref S3_ACCESS_KEY + S3AccessSecret: !Ref S3_ACCESS_SECRET + SigningLambdaGatewayPermission: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !GetAtt SigningLambda.Arn + Principal: apigateway.amazonaws.com + SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${RestApi}/* + SigningLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - + Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonS3FullAccess \ No newline at end of file diff --git a/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj b/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj index 24520a3..c6eb517 100644 --- a/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj +++ b/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs b/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs index c60cdfd..aacc597 100644 --- a/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs +++ b/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs @@ -43,6 +43,8 @@ public async Task UriForDownload(string oid, CancellationToken token } catch (AmazonS3Exception ex) { + Console.WriteLine($"[ERROR] - {ex.StatusCode} - {ex.Message}"); + Console.WriteLine(ex.StackTrace); return new SignedBlob { ErrorCode = (int)ex.StatusCode, @@ -62,12 +64,8 @@ public Task UriForUpload(string oid, long size, CancellationToken to { return Task.FromResult(new SignedBlob { - Uri = MakePreSignedUrl(oid, HttpVerb.PUT, BlobConstants.UploadMimeType), - Expiry = config.Expiry, - Headers = new Dictionary - { - {"Content-Type", BlobConstants.UploadMimeType} - } + Uri = MakePreSignedUrl(oid, HttpVerb.PUT, null), + Expiry = config.Expiry }); } } diff --git a/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj b/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj index db175ed..ebae046 100644 --- a/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj +++ b/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj @@ -25,4 +25,4 @@ - + \ No newline at end of file