Skip to content

Commit

Permalink
feat: Account for custom S3 compatible service
Browse files Browse the repository at this point in the history
V4 Signature for AWS
Custom ServiceURL and Region for S3 compatible storage other than AWS
Added and used some parameters
Update AWSSDK.S3 to latest version possible (see aws/aws-sdk-net#2540 (comment))
  • Loading branch information
Guillaume Escarieux committed Feb 16, 2023
1 parent 0b7a20c commit 6474ae1
Show file tree
Hide file tree
Showing 9 changed files with 217 additions and 34 deletions.
18 changes: 9 additions & 9 deletions Estranged.Lfs.sln
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29806.167
VisualStudioVersion = 16.0.30320.27
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{03ED5016-F8BF-4C18-A997-2FFE0E6EF896}"
EndProject
Expand All @@ -14,6 +14,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
README.md = README.md
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Adapters", "Adapters", "{299762B3-8879-42F1-B002-4A9AF7401A13}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Authenticators", "Authenticators", "{FD975DFA-BEA7-451F-B85F-262B6C658DAC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{68EBAB1E-0B07-4701-A39D-096A8226A3A2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Hosting.AspNet", "hosting\Estranged.Lfs.Hosting.AspNet\Estranged.Lfs.Hosting.AspNet.csproj", "{948329E9-DE90-4245-8C2E-BCD23746056A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Api", "src\Estranged.Lfs.Api\Estranged.Lfs.Api.csproj", "{EAFEAA0F-678A-4888-9F32-45DA467A4CD5}"
Expand All @@ -28,12 +34,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Authenticator
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Authenticator.BitBucket", "src\Estranged.Lfs.Authenticator.BitBucket\Estranged.Lfs.Authenticator.BitBucket.csproj", "{711786E0-5F18-440C-A2FA-CB1E26FD2DEF}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Adapters", "Adapters", "{299762B3-8879-42F1-B002-4A9AF7401A13}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Authenticators", "Authenticators", "{FD975DFA-BEA7-451F-B85F-262B6C658DAC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{68EBAB1E-0B07-4701-A39D-096A8226A3A2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Tests", "tests\Estranged.Lfs.Tests\Estranged.Lfs.Tests.csproj", "{3EBBF34F-B0EF-4C60-A9F8-432A5E9C1DD2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Estranged.Lfs.Adapter.Azure.Blob", "src\Estranged.Lfs.Adapter.Azure.Blob\Estranged.Lfs.Adapter.Azure.Blob.csproj", "{E4A09D4E-FF7F-4899-9384-5E8A90D0E9DA}"
Expand Down Expand Up @@ -85,15 +85,15 @@ Global
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{299762B3-8879-42F1-B002-4A9AF7401A13} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896}
{FD975DFA-BEA7-451F-B85F-262B6C658DAC} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896}
{948329E9-DE90-4245-8C2E-BCD23746056A} = {89E5DA04-5E1C-410C-9175-27C0713594AD}
{EAFEAA0F-678A-4888-9F32-45DA467A4CD5} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896}
{CE95E105-4980-4F65-915C-879692BC0F23} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896}
{C38DD1A3-E930-4BF2-8BB2-ADFD66225B25} = {299762B3-8879-42F1-B002-4A9AF7401A13}
{D4D0ABA0-A829-43C0-B3E8-C57AE9170ECF} = {89E5DA04-5E1C-410C-9175-27C0713594AD}
{6467DCDA-D062-4DD4-A4A2-D8D551411B2A} = {FD975DFA-BEA7-451F-B85F-262B6C658DAC}
{711786E0-5F18-440C-A2FA-CB1E26FD2DEF} = {FD975DFA-BEA7-451F-B85F-262B6C658DAC}
{299762B3-8879-42F1-B002-4A9AF7401A13} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896}
{FD975DFA-BEA7-451F-B85F-262B6C658DAC} = {03ED5016-F8BF-4C18-A997-2FFE0E6EF896}
{3EBBF34F-B0EF-4C60-A9F8-432A5E9C1DD2} = {68EBAB1E-0B07-4701-A39D-096A8226A3A2}
{E4A09D4E-FF7F-4899-9384-5E8A90D0E9DA} = {299762B3-8879-42F1-B002-4A9AF7401A13}
EndGlobalSection
Expand Down
69 changes: 62 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ A Git LFS backend which provides pluggable authentication and blob store adapter
services.AddLfs();
```

2. Register an implementation for IBlobAdapter and IAuthenticator. Amazon AWS S3 and Azure Blob Storage are provided out of the box:
2. Register an implementation for IBlobAdapter and IAuthenticator. Amazon AWS S3, S3-compatible and Azure Blob Storage are provided out of the box:

```csharp
var s3BlobConfig = new S3BlobAdapterConfig
Expand Down Expand Up @@ -109,6 +109,40 @@ There are currently two hosting examples:

The former is a simple example using only Asp.NET components, and the latter is an Asp.NET Lambda function which can be deployed directly to AWS Lambda, behind API Gateway.

### Asp.NET version

1. Edit the variables values to suit to your environment

```
LfsBucket // Mandatory: Name of S3 bucket
S3AccessKeyId // Optional: _aws_access_key_id_ of the .aws/credential file for your custom s3 profile
S3AccessKeySecret // Optional: _aws_secret_access_key_ of the .aws/credential file for your custom s3 profile
S3Region // Optional: region in custom S3
S3ServiceURL // Optional: endpoint of custom S3
```

2. It can be launched in VS by choosing _Estranged.Lfs.Hosting.AspNet_ (not the default _IIS Express_ option that doesnt work).

![image](https://user-images.githubusercontent.com/2952456/89800274-d82c9380-db2e-11ea-85bb-3fc8652e3e9d.png)

3. Or it can be published in folder, then launched with _Estranged.Lfs.Hosting.AspNet.exe_

4. This is a console application that is listening for HTTP LFS requests on https://localhost:5001

![image](https://user-images.githubusercontent.com/2952456/89800695-6739ab80-db2f-11ea-8641-0eab8c501381.png)

5. Change the .lfconfig to send request to the console app

```
[lfs]
url = https://localhost:5001/
```
6. From git repo Commit lfs file and Push, and enter when asked the user and password set by this line
` services.AddSingleton<IAuthenticator>(x => new DictionaryAuthenticator(new Dictionary<string, string> { { "usernametest", "passwordtest" } }));`

7. The pushed file is now present in custom S3 Storage
![image](https://user-images.githubusercontent.com/2952456/89806464-5e4cd800-db37-11ea-85bd-9ce724e7ee0e.png)

#### Deploying to Lambda

1. Head over to the `Estranged.Lfs.Hosting.Lambda` project in the `hosting` folder.
Expand All @@ -117,20 +151,41 @@ The former is a simple example using only Asp.NET components, and the latter is

```javascript
{
"profile": "default",
"profile": "default", // AWS connexion profile
"configuration": "Release",
"framework": "net6.0",
"function-handler": "Estranged.Lfs.Hosting.Lambda::Estranged.Lfs.Hosting.Lambda.LambdaEntryPoint::FunctionHandlerAsync",
"function-memory-size": 256,
"function-timeout": 30,
"function-runtime": "dotnet6",
"region": "<aws region>",
"s3-bucket": "<s3 bucket to upload the lambda to>",
"region": "<aws region>", // AWS public region
"s3-bucket": "<s3 bucket to upload the lambda to>", // S3 bucket needed to upload the modele/output of the stack, must be outside of the stack (shared between all stacks)
"s3-prefix": "<path in s3 to upload the lambda to>",
"function-name": "<lambda name to deploy or update>",
"function-name": "<lambda name to deploy or update>", // lambda name must be same as stack name
// Set other variables required by the Lambda function
"environment-variables": "LFS_BUCKET=<lfs s3 bucket>;<key>=<value>"
"environment-variables": "LFS_BUCKET=<lfs s3 bucket>;LFS_USERNAME=<AWS_STACK_ParameterUsername>;LFS_PASSWORD=<AWS_STACK_ParameterPassword>;S3_ACCESS_KEY=<S3 AccessKey>;S3_ACCESS_SECRET=<S3 AccessSecret>;S3_REGION=<Custom S3 Region>;S3_SERVICE_URL=<Custom S3 EndPoint>;<key>=<value>", // can be found and changed in Lambda configuration UI"
}
```
4. Run `dotnet lambda deploy-serverless` to deploy the stack
5. Run `dotnet lambda deploy-function` to deploy the code of the lambda function
6. Change the .lfconfig of the GIT project to send requests to the lambda function (the URL was in 4. output)
```
[lfs]
url = https://xxxxxxxxx.execute-api.eu-west-1.amazonaws.com/lfs
```

8. Commit and push LFS files, when prompt enter AWS_STACK_ParameterUsername and AWS_STACK_ParameterPassword, the files can be seen in your S3 storage!

**Instead of using user/password authentication, it is possible to use Github or Bitbucket authentication.**

Edit the `aws-lambda-tools-defaults.json` file and redeploy the lambda (or edit directly in the lambda UI)

(example with github):

``` "environment-variables": "GITHUB_ORGANISATION=REPO_ORGANISATION,GITHUB_REPOSITORY=REPO_NAME,..."```

(example with bitbucket):

``` "environment-variables": "BITBUCKET_WORKSPACE=REPO_WORKSPACE,BITBUCKET_REPOSITORY=REPO_NAME,..."```

4. Run `dotnet-lambda deploy-serverless` to deploy the Lambda function
In this case, use your platform username and dedicated auth token to authenticate.
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
<ProjectReference Include="..\..\src\Estranged.Lfs.Api\Estranged.Lfs.Api.csproj" />
</ItemGroup>

</Project>
</Project>
28 changes: 20 additions & 8 deletions hosting/Estranged.Lfs.Hosting.AspNet/Startup.cs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Estranged.Lfs.Api;
using Amazon.S3;
using Microsoft.Extensions.Configuration;
using Amazon.S3;
using Estranged.Lfs.Adapter.S3;
using Estranged.Lfs.Api;
using Estranged.Lfs.Data;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System.Collections.Generic;

namespace Estranged.Lfs.Hosting.AspNet
Expand All @@ -23,7 +23,19 @@ public void ConfigureServices(IServiceCollection services)
});

services.AddSingleton<IAmazonS3, AmazonS3Client>();
services.AddLfsS3Adapter(new S3BlobAdapterConfig{Bucket = "estranged-lfs-test"}, new AmazonS3Client());
const string LfsBucket = "estranged-lfs-test";
const string S3AccessKeyId = "";
const string S3AccessKeySecret = "";
const string S3Region = "";
const string S3ServiceURL = "";
if (!string.IsNullOrWhiteSpace(S3ServiceURL) && !string.IsNullOrWhiteSpace(S3Region) && !string.IsNullOrWhiteSpace(S3AccessKeyId) && !string.IsNullOrWhiteSpace(S3AccessKeySecret))
{
services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = LfsBucket }, new AmazonS3Client(S3AccessKeyId, S3AccessKeySecret, new AmazonS3Config { ServiceURL = S3ServiceURL, AuthenticationRegion = S3Region, SignatureVersion = "V4" }));
}
else
{
services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = LfsBucket }, new AmazonS3Client());
}
services.AddSingleton<IAuthenticator>(x => new DictionaryAuthenticator(new Dictionary<string, string> { { "usernametest", "passwordtest" } }));
services.AddLfsApi();
}
Expand All @@ -34,4 +46,4 @@ public void Configure(IApplicationBuilder app)
app.UseEndpoints(endpoints => endpoints.MapControllers());
}
}
}
}
18 changes: 17 additions & 1 deletion hosting/Estranged.Lfs.Hosting.Lambda/Startup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ public void ConfigureServices(IServiceCollection services)
const string S3AccelerationVariable = "S3_ACCELERATION";
const string LfsAzureStorageConnectionStringVariable = "LFS_AZUREBLOB_CONNECTIONSTRING";
const string LfsAzureStorageContainerNameVariable = "LFS_AZUREBLOB_CONTAINERNAME";
const string S3ServiceURL = "S3_SERVICE_URL";
const string S3Region = "S3_REGION";
const string S3AccessKey = "S3_ACCESS_KEY";
const string S3AccessSecret = "S3_ACCESS_SECRET";

var config = new ConfigurationBuilder()
.AddEnvironmentVariables()
Expand All @@ -43,6 +47,10 @@ public void ConfigureServices(IServiceCollection services)
string bitBucketWorkspace = config[BitBucketWorkspaceVariable];
string bitBucketRepository = config[BitBucketRepositoryVariable];
bool s3Acceleration = bool.Parse(config[S3AccelerationVariable] ?? "false");
string s3ServiceURL = config[S3ServiceURL];
string s3Region = config[S3Region];
string s3AccessKey = config[S3AccessKey];
string s3AccessSecret = config[S3AccessSecret];

bool isS3Storage = !string.IsNullOrWhiteSpace(lfsBucket);
bool isAzureStorage = !string.IsNullOrWhiteSpace(lfsAzureStorageConnectionString);
Expand Down Expand Up @@ -74,7 +82,15 @@ public void ConfigureServices(IServiceCollection services)

if (isS3Storage)
{
services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration }));
if (!string.IsNullOrWhiteSpace(s3ServiceURL) && !string.IsNullOrWhiteSpace(s3Region) && !string.IsNullOrWhiteSpace(s3AccessKey) && !string.IsNullOrWhiteSpace(s3AccessSecret))
{
services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(s3AccessKey, s3AccessSecret, new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration, ServiceURL = s3ServiceURL, AuthenticationRegion = s3Region, SignatureVersion = "V4" }));

}
else
{
services.AddLfsS3Adapter(new S3BlobAdapterConfig { Bucket = lfsBucket }, new AmazonS3Client(new AmazonS3Config { UseAccelerateEndpoint = s3Acceleration }));
}
}
else if (isAzureStorage)
{
Expand Down
102 changes: 102 additions & 0 deletions hosting/Estranged.Lfs.Hosting.Lambda/modele.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
AWSTemplateFormatVersion: "2010-09-09"
Parameters:
GitLfsUsername:
Type: String
Description: Username for authenticating against Git LFS endpoint
GitLfsPassword:
Type: String
Description: Password for authenticating against Git LFS endpoint
Outputs:
LfsEndpoint:
Description: The Git LFS endpoint to use
Value: !Sub 'https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/lfs'
Resources:
StorageBucket:
Type: AWS::S3::Bucket
DeletionPolicy: Retain
RestApi:
Type: AWS::ApiGateway::RestApi
Properties:
Body:
swagger: '2.0'
info:
description: 'Describes a proxy to a Lambda function to sign S3 requests.'
title: 'Git LFS REST API'
version: '1.0.0'
paths:
/{proxy+}:
x-amazon-apigateway-any-method:
produces:
- application/json
parameters:
- name: proxy
in: path
required: true
type: string
responses: {}
x-amazon-apigateway-integration:
responses:
default:
statusCode: 200
uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${SigningLambda}/invocations'
passthroughBehavior: when_no_match
httpMethod: POST
contentHandling: CONVERT_TO_TEXT
type: aws_proxy
Description: Git LFS endpoint
FailOnWarnings: true
Name: !Ref AWS::StackName
RestDeployment:
Type: AWS::ApiGateway::Deployment
Properties:
RestApiId: !Ref RestApi
StageName: lfs
SigningLambda:
Type: AWS::Lambda::Function
Properties:
Code:
S3Bucket: !Sub 'ae-infrastructure-${AWS::Region}'
S3Key: git-lfs/3.0.0/Estranged.Lfs.Hosting.Lambda.zip
Description: Generates S3 signed URLs for Git LFS
FunctionName: !Ref AWS::StackName
Handler: Estranged.Lfs.Hosting.Lambda::Estranged.Lfs.Hosting.Lambda.LambdaEntryPoint::FunctionHandlerAsync
MemorySize: 512
Role: !GetAtt SigningLambdaRole.Arn
Runtime: dotnetcore3.1
Timeout: 30
Environment:
Variables:
LFS_BUCKET: !Ref StorageBucket
LFS_USERNAME: !Ref GitLfsUsername
LFS_PASSWORD: !Ref GitLfsPassword
GITHUB_ORGANISATION: !Ref GitOrganisation
GITHUB_REPOSITORY: !Ref GitHubRepositoryVariable
BITBUCKET_WORKSPACE: !Ref BitBucketWorkspaceVariable
BITBUCKET_REPOSITORY: !Ref BitBucketRepositoryVariable
S3Region: !Ref S3_REGION
S3ServiceURL: !Ref S3_SERVICE_URL
S3AccessKey: !Ref S3_ACCESS_KEY
S3AccessSecret: !Ref S3_ACCESS_SECRET
SigningLambdaGatewayPermission:
Type: AWS::Lambda::Permission
Properties:
Action: lambda:InvokeFunction
FunctionName: !GetAtt SigningLambda.Arn
Principal: apigateway.amazonaws.com
SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${RestApi}/*
SigningLambdaRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
Action:
- sts:AssumeRole
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
- arn:aws:iam::aws:policy/AmazonS3FullAccess
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.9.30" />
<PackageReference Include="AWSSDK.S3" Version="3.7.10.1" />
</ItemGroup>

<ItemGroup>
Expand Down
10 changes: 4 additions & 6 deletions src/Estranged.Lfs.Adapter.S3/S3BlobAdapter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ public async Task<SignedBlob> UriForDownload(string oid, CancellationToken token
}
catch (AmazonS3Exception ex)
{
Console.WriteLine($"[ERROR] - {ex.StatusCode} - {ex.Message}");
Console.WriteLine(ex.StackTrace);
return new SignedBlob
{
ErrorCode = (int)ex.StatusCode,
Expand All @@ -62,12 +64,8 @@ public Task<SignedBlob> UriForUpload(string oid, long size, CancellationToken to
{
return Task.FromResult(new SignedBlob
{
Uri = MakePreSignedUrl(oid, HttpVerb.PUT, BlobConstants.UploadMimeType),
Expiry = config.Expiry,
Headers = new Dictionary<string, string>
{
{"Content-Type", BlobConstants.UploadMimeType}
}
Uri = MakePreSignedUrl(oid, HttpVerb.PUT, null),
Expiry = config.Expiry
});
}
}
Expand Down
2 changes: 1 addition & 1 deletion tests/Estranged.Lfs.Tests/Estranged.Lfs.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@
<ProjectReference Include="..\..\src\Estranged.Lfs.Authenticator.GitHub\Estranged.Lfs.Authenticator.GitHub.csproj" />
</ItemGroup>

</Project>
</Project>

0 comments on commit 6474ae1

Please sign in to comment.