diff --git a/Estranged.Lfs.sln b/Estranged.Lfs.sln index dd233d8..f8c67bb 100644 --- a/Estranged.Lfs.sln +++ b/Estranged.Lfs.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 -VisualStudioVersion = 16.0.29806.167 +VisualStudioVersion = 16.0.30320.27 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{03ED5016-F8BF-4C18-A997-2FFE0E6EF896}" EndProject @@ -14,6 +14,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Adapters", "Adapters", "{299762B3-8879-42F1-B002-4A9AF7401A13}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Authenticators", "Authenticators", "{FD975DFA-BEA7-451F-B85F-262B6C658DAC}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{68EBAB1E-0B07-4701-A39D-096A8226A3A2}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Hosting.AspNet", "hosting\Estranged.Lfs.Hosting.AspNet\Estranged.Lfs.Hosting.AspNet.csproj", "{948329E9-DE90-4245-8C2E-BCD23746056A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Api", "src\Estranged.Lfs.Api\Estranged.Lfs.Api.csproj", "{EAFEAA0F-678A-4888-9F32-45DA467A4CD5}" @@ -28,12 +34,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Authenticator EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Authenticator.BitBucket", "src\Estranged.Lfs.Authenticator.BitBucket\Estranged.Lfs.Authenticator.BitBucket.csproj", "{711786E0-5F18-440C-A2FA-CB1E26FD2DEF}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Adapters", "Adapters", "{299762B3-8879-42F1-B002-4A9AF7401A13}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Authenticators", "Authenticators", "{FD975DFA-BEA7-451F-B85F-262B6C658DAC}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{68EBAB1E-0B07-4701-A39D-096A8226A3A2}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Tests", "tests\Estranged.Lfs.Tests\Estranged.Lfs.Tests.csproj", "{3EBBF34F-B0EF-4C60-A9F8-432A5E9C1DD2}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Adapter.Azure.Blob", "src\Estranged.Lfs.Adapter.Azure.Blob\Estranged.Lfs.Adapter.Azure.Blob.csproj", "{E4A09D4E-FF7F-4899-9384-5E8A90D0E9DA}" @@ -85,6 +85,8 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution + {299762B3-8879-42F1-B002-4A9AF7401A13} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} + {FD975DFA-BEA7-451F-B85F-262B6C658DAC} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} {948329E9-DE90-4245-8C2E-BCD23746056A} = {89E5DA04-5E1C-410C-9175-27C0713594AD} {EAFEAA0F-678A-4888-9F32-45DA467A4CD5} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} {CE95E105-4980-4F65-915C-879692BC0F23} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} @@ -92,8 +94,6 @@ Global {D4D0ABA0-A829-43C0-B3E8-C57AE9170ECF} = {89E5DA04-5E1C-410C-9175-27C0713594AD} {6467DCDA-D062-4DD4-A4A2-D8D551411B2A} = {FD975DFA-BEA7-451F-B85F-262B6C658DAC} {711786E0-5F18-440C-A2FA-CB1E26FD2DEF} = {FD975DFA-BEA7-451F-B85F-262B6C658DAC} - {299762B3-8879-42F1-B002-4A9AF7401A13} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} - {FD975DFA-BEA7-451F-B85F-262B6C658DAC} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896} {3EBBF34F-B0EF-4C60-A9F8-432A5E9C1DD2} = {68EBAB1E-0B07-4701-A39D-096A8226A3A2} {E4A09D4E-FF7F-4899-9384-5E8A90D0E9DA} = {299762B3-8879-42F1-B002-4A9AF7401A13} EndGlobalSection diff --git a/README.md b/README.md index 067e797..4c45366 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A Git LFS backend which provides pluggable authentication and blob store adapter services.AddLfs(); ``` -2. Register an implementation for IBlobAdapter and IAuthenticator. Amazon AWS S3 and Azure Blob Storage are provided out of the box: +2. Register an implementation for IBlobAdapter and IAuthenticator. Amazon AWS S3, S3-compatible and Azure Blob Storage are provided out of the box: ```csharp var s3BlobConfig = new S3BlobAdapterConfig @@ -109,6 +109,40 @@ There are currently two hosting examples: The former is a simple example using only Asp.NET components, and the latter is an Asp.NET Lambda function which can be deployed directly to AWS Lambda, behind API Gateway. +### Asp.NET version + +1. Edit the variables values to suit to your environment + +``` + LfsBucket // Mandatory: Name of S3 bucket + S3AccessKeyId // Optional: _aws_access_key_id_ of the .aws/credential file for your custom s3 profile + S3AccessKeySecret // Optional: _aws_secret_access_key_ of the .aws/credential file for your custom s3 profile + S3Region // Optional: region in custom S3 + S3ServiceURL // Optional: endpoint of custom S3 +``` + +2. It can be launched in VS by choosing _Estranged.Lfs.Hosting.AspNet_ (not the default _IIS Express_ option that doesnt work). + +![image](https://user-images.githubusercontent.com/2952456/89800274-d82c9380-db2e-11ea-85bb-3fc8652e3e9d.png) + +3. Or it can be published in folder, then launched with _Estranged.Lfs.Hosting.AspNet.exe_ + +4. This is a console application that is listening for HTTP LFS requests on https://localhost:5001 + +![image](https://user-images.githubusercontent.com/2952456/89800695-6739ab80-db2f-11ea-8641-0eab8c501381.png) + +5. Change the .lfconfig to send request to the console app + +``` +[lfs] +url = https://localhost:5001/ +``` +6. From git repo Commit lfs file and Push, and enter when asked the user and password set by this line +` services.AddSingleton(x => new DictionaryAuthenticator(new Dictionary { { "usernametest", "passwordtest" } }));` + +7. The pushed file is now present in custom S3 Storage +![image](https://user-images.githubusercontent.com/2952456/89806464-5e4cd800-db37-11ea-85bd-9ce724e7ee0e.png) + #### Deploying to Lambda 1. Head over to the `Estranged.Lfs.Hosting.Lambda` project in the `hosting` folder. @@ -117,20 +151,41 @@ The former is a simple example using only Asp.NET components, and the latter is ```javascript { - "profile": "default", + "profile": "default", // AWS connexion profile "configuration": "Release", "framework": "net6.0", "function-handler": "Estranged.Lfs.Hosting.Lambda::Estranged.Lfs.Hosting.Lambda.LambdaEntryPoint::FunctionHandlerAsync", "function-memory-size": 256, "function-timeout": 30, "function-runtime": "dotnet6", - "region": "", - "s3-bucket": "", + "region": "", // AWS public region + "s3-bucket": "", // S3 bucket needed to upload the modele/output of the stack, must be outside of the stack (shared between all stacks) "s3-prefix": "", - "function-name": "", + "function-name": "", // lambda name must be same as stack name // Set other variables required by the Lambda function - "environment-variables": "LFS_BUCKET=;=" + "environment-variables": "LFS_BUCKET=;LFS_USERNAME=;LFS_PASSWORD=;S3_ACCESS_KEY=;S3_ACCESS_SECRET=;S3_REGION=;S3_SERVICE_URL=;=", // can be found and changed in Lambda configuration UI" } ``` +4. Run `dotnet lambda deploy-serverless` to deploy the stack +5. Run `dotnet lambda deploy-function` to deploy the code of the lambda function +6. Change the .lfconfig of the GIT project to send requests to the lambda function (the URL was in 4. output) +``` +[lfs] +url = https://xxxxxxxxx.execute-api.eu-west-1.amazonaws.com/lfs +``` + +8. Commit and push LFS files, when prompt enter AWS_STACK_ParameterUsername and AWS_STACK_ParameterPassword, the files can be seen in your S3 storage! + +**Instead of using user/password authentication, it is possible to use Github or Bitbucket authentication.** + +Edit the `aws-lambda-tools-defaults.json` file and redeploy the lambda (or edit directly in the lambda UI) + +(example with github): + + ``` "environment-variables": "GITHUB_ORGANISATION=REPO_ORGANISATION,GITHUB_REPOSITORY=REPO_NAME,..."``` + + (example with bitbucket): + + ``` "environment-variables": "BITBUCKET_WORKSPACE=REPO_WORKSPACE,BITBUCKET_REPOSITORY=REPO_NAME,..."``` -4. Run `dotnet-lambda deploy-serverless` to deploy the Lambda function + In this case, use your platform username and dedicated auth token to authenticate. \ No newline at end of file diff --git a/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj b/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj index f63b77b..af1fcd0 100644 --- a/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj +++ b/hosting/Estranged.Lfs.Hosting.AspNet/Estranged.Lfs.Hosting.AspNet.csproj @@ -10,4 +10,4 @@ - + \ No newline at end of file diff --git a/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs b/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs index c4d946d..7fcc073 100644 --- a/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs +++ b/hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs @@ -1,11 +1,11 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Estranged.Lfs.Api; -using Amazon.S3; -using Microsoft.Extensions.Configuration; +using Amazon.S3; using Estranged.Lfs.Adapter.S3; +using Estranged.Lfs.Api; using Estranged.Lfs.Data; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using System.Collections.Generic; namespace Estranged.Lfs.Hosting.AspNet @@ -23,7 +23,19 @@ public void ConfigureServices(IServiceCollection services) }); services.AddSingleton(); - services.AddLfsS3Adapter(new S3BlobAdapterConfig{Bucket = "estranged-lfs-test"}, new AmazonS3Client()); + const string LfsBucket = "estranged-lfs-test"; + const string S3AccessKeyId = ""; + const string S3AccessKeySecret = ""; + const string S3Region = ""; + const string S3ServiceURL = ""; + if (!string.IsNullOrWhiteSpace(S3ServiceURL) && !string.IsNullOrWhiteSpace(S3Region) && !string.IsNullOrWhiteSpace(S3AccessKeyId) && !string.IsNullOrWhiteSpace(S3AccessKeySecret)) + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = LfsBucket }, new AmazonS3Client(S3AccessKeyId, S3AccessKeySecret, new AmazonS3Config { ServiceURL = S3ServiceURL, AuthenticationRegion = S3Region, SignatureVersion = "V4" })); + } + else + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = LfsBucket }, new AmazonS3Client()); + } services.AddSingleton(x => new DictionaryAuthenticator(new Dictionary { { "usernametest", "passwordtest" } })); services.AddLfsApi(); } @@ -34,4 +46,4 @@ public void Configure(IApplicationBuilder app) app.UseEndpoints(endpoints => endpoints.MapControllers()); } } -} +} \ No newline at end of file diff --git a/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs b/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs index 660c04b..322e451 100644 --- a/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs +++ b/hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs @@ -29,6 +29,10 @@ public void ConfigureServices(IServiceCollection services) const string S3AccelerationVariable = "S3_ACCELERATION"; const string LfsAzureStorageConnectionStringVariable = "LFS_AZUREBLOB_CONNECTIONSTRING"; const string LfsAzureStorageContainerNameVariable = "LFS_AZUREBLOB_CONTAINERNAME"; + const string S3ServiceURL = "S3_SERVICE_URL"; + const string S3Region = "S3_REGION"; + const string S3AccessKey = "S3_ACCESS_KEY"; + const string S3AccessSecret = "S3_ACCESS_SECRET"; var config = new ConfigurationBuilder() .AddEnvironmentVariables() @@ -43,6 +47,10 @@ public void ConfigureServices(IServiceCollection services) string bitBucketWorkspace = config[BitBucketWorkspaceVariable]; string bitBucketRepository = config[BitBucketRepositoryVariable]; bool s3Acceleration = bool.Parse(config[S3AccelerationVariable] ?? "false"); + string s3ServiceURL = config[S3ServiceURL]; + string s3Region = config[S3Region]; + string s3AccessKey = config[S3AccessKey]; + string s3AccessSecret = config[S3AccessSecret]; bool isS3Storage = !string.IsNullOrWhiteSpace(lfsBucket); bool isAzureStorage = !string.IsNullOrWhiteSpace(lfsAzureStorageConnectionString); @@ -74,7 +82,15 @@ public void ConfigureServices(IServiceCollection services) if (isS3Storage) { - services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration })); + if (!string.IsNullOrWhiteSpace(s3ServiceURL) && !string.IsNullOrWhiteSpace(s3Region) && !string.IsNullOrWhiteSpace(s3AccessKey) && !string.IsNullOrWhiteSpace(s3AccessSecret)) + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(s3AccessKey, s3AccessSecret, new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration, ServiceURL = s3ServiceURL, AuthenticationRegion = s3Region, SignatureVersion = "V4" })); + + } + else + { + services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration })); + } } else if (isAzureStorage) { diff --git a/hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml b/hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml new file mode 100644 index 0000000..75ba5fd --- /dev/null +++ b/hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml @@ -0,0 +1,102 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + GitLfsUsername: + Type: String + Description: Username for authenticating against Git LFS endpoint + GitLfsPassword: + Type: String + Description: Password for authenticating against Git LFS endpoint +Outputs: + LfsEndpoint: + Description: The Git LFS endpoint to use + Value: !Sub 'https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/lfs' +Resources: + StorageBucket: + Type: AWS::S3::Bucket + DeletionPolicy: Retain + RestApi: + Type: AWS::ApiGateway::RestApi + Properties: + Body: + swagger: '2.0' + info: + description: 'Describes a proxy to a Lambda function to sign S3 requests.' + title: 'Git LFS REST API' + version: '1.0.0' + paths: + /{proxy+}: + x-amazon-apigateway-any-method: + produces: + - application/json + parameters: + - name: proxy + in: path + required: true + type: string + responses: {} + x-amazon-apigateway-integration: + responses: + default: + statusCode: 200 + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${SigningLambda}/invocations' + passthroughBehavior: when_no_match + httpMethod: POST + contentHandling: CONVERT_TO_TEXT + type: aws_proxy + Description: Git LFS endpoint + FailOnWarnings: true + Name: !Ref AWS::StackName + RestDeployment: + Type: AWS::ApiGateway::Deployment + Properties: + RestApiId: !Ref RestApi + StageName: lfs + SigningLambda: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Sub 'ae-infrastructure-${AWS::Region}' + S3Key: git-lfs/3.0.0/Estranged.Lfs.Hosting.Lambda.zip + Description: Generates S3 signed URLs for Git LFS + FunctionName: !Ref AWS::StackName + Handler: Estranged.Lfs.Hosting.Lambda::Estranged.Lfs.Hosting.Lambda.LambdaEntryPoint::FunctionHandlerAsync + MemorySize: 512 + Role: !GetAtt SigningLambdaRole.Arn + Runtime: dotnetcore3.1 + Timeout: 30 + Environment: + Variables: + LFS_BUCKET: !Ref StorageBucket + LFS_USERNAME: !Ref GitLfsUsername + LFS_PASSWORD: !Ref GitLfsPassword + GITHUB_ORGANISATION: !Ref GitOrganisation + GITHUB_REPOSITORY: !Ref GitHubRepositoryVariable + BITBUCKET_WORKSPACE: !Ref BitBucketWorkspaceVariable + BITBUCKET_REPOSITORY: !Ref BitBucketRepositoryVariable + S3Region: !Ref S3_REGION + S3ServiceURL: !Ref S3_SERVICE_URL + S3AccessKey: !Ref S3_ACCESS_KEY + S3AccessSecret: !Ref S3_ACCESS_SECRET + SigningLambdaGatewayPermission: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !GetAtt SigningLambda.Arn + Principal: apigateway.amazonaws.com + SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${RestApi}/* + SigningLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - + Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonS3FullAccess \ No newline at end of file diff --git a/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj b/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj index 24520a3..c6eb517 100644 --- a/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj +++ b/src/Estranged.Lfs.Adapter.S3/Estranged.Lfs.Adapter.S3.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs b/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs index c60cdfd..aacc597 100644 --- a/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs +++ b/src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs @@ -43,6 +43,8 @@ public async Task UriForDownload(string oid, CancellationToken token } catch (AmazonS3Exception ex) { + Console.WriteLine($"[ERROR] - {ex.StatusCode} - {ex.Message}"); + Console.WriteLine(ex.StackTrace); return new SignedBlob { ErrorCode = (int)ex.StatusCode, @@ -62,12 +64,8 @@ public Task UriForUpload(string oid, long size, CancellationToken to { return Task.FromResult(new SignedBlob { - Uri = MakePreSignedUrl(oid, HttpVerb.PUT, BlobConstants.UploadMimeType), - Expiry = config.Expiry, - Headers = new Dictionary - { - {"Content-Type", BlobConstants.UploadMimeType} - } + Uri = MakePreSignedUrl(oid, HttpVerb.PUT, null), + Expiry = config.Expiry }); } } diff --git a/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj b/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj index db175ed..ebae046 100644 --- a/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj +++ b/tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj @@ -25,4 +25,4 @@ - + \ No newline at end of file