feature: add support for S3 Express #225

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -22,7 +22,7 @@ set -e
 
 failed=0
 
-required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO"
+required=("S3_SERVICE" "S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO"
 "S3_REGION" "S3_STYLE" "ALLOW_DIRECTORY_LIST" "AWS_SIGS_VERSION"
 "CORS_ENABLED")
 
@@ -123,6 +123,7 @@ fi
 
 echo "S3 Backend Environment"
 echo "Access Key ID: ${AWS_ACCESS_KEY_ID}"
+echo "Service: ${S3_SERVICE}"
 echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}"
 echo "Region: ${S3_REGION}"
 echo "Addressing Style: ${S3_STYLE}"

common/etc/nginx/include/s3gateway.js

@@ -32,6 +32,7 @@ import awssig2 from "./awssig2.js";
 import awssig4 from "./awssig4.js";
 import utils from "./utils.js";
 
+_requireEnvVars('S3_SERVICE');
 _requireEnvVars('S3_BUCKET_NAME');
 _requireEnvVars('S3_SERVER');
 _requireEnvVars('S3_SERVER_PROTO');
@@ -84,9 +85,10 @@ const INDEX_PAGE = "index.html";
 
 /**
  * Constant defining the service requests are being signed for.
+ * can be 's3' or 's3express'.
  * @type {string}
  */
-const SERVICE = 's3';
+const SERVICE = process.env['S3_SERVICE'];
 
 /**
  * Transform the headers returned from S3 such that there isn't information

common/etc/nginx/nginx.conf

@@ -12,6 +12,7 @@ load_module modules/ngx_http_xslt_filter_module.so;
 env AWS_ACCESS_KEY_ID;
 env AWS_SECRET_ACCESS_KEY;
 env AWS_SESSION_TOKEN;
+env S3_SERVICE;
 env S3_BUCKET_NAME;
 env S3_SERVER;
 env S3_SERVER_PORT;

common/etc/nginx/templates/default_s3express.conf.template

@@ -0,0 +1,319 @@
+js_import /etc/nginx/include/awscredentials.js;
+js_import /etc/nginx/include/s3gateway.js;
+
+
+# We include only the variables needed for the authentication signatures that
+# we plan to use.
+include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_js_vars.conf;
+
+# Extracts only the path from the requested URI. This strips out all query
+# parameters and anchors in order to prevent extraneous data from being sent
+# to S3.
+map $request_uri $uri_full_path {
+    "~^(?P<path>.*?)(\?.*)*$"  $path;
+}
+
+# Remove/replace a portion of request URL (if configured)
+map $uri_full_path $uri_path {
+	"~^$STRIP_LEADING_DIRECTORY_PATH(.*)"  $PREFIX_LEADING_DIRECTORY_PATH$1;
+    default $PREFIX_LEADING_DIRECTORY_PATH$uri_full_path;
+}
+
+map $S3_STYLE $s3_host_hdr {
+    virtual "${S3_BUCKET_NAME}.${S3_SERVER}";
+    path    "${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}";
+    default "${S3_BUCKET_NAME}.${S3_SERVER}";
+}
+
+js_var $indexIsEmpty true;
+js_var $forIndexPage true;
+# This creates the HTTP authentication header to be sent to S3
+js_set $s3auth s3gateway.s3auth;
+js_set $awsSessionToken awscredentials.sessionToken;
+js_set $s3uri s3gateway.s3uri;
+
+server {
+    include /etc/nginx/conf.d/gateway/server_variables.conf;
+
+    # Don't display the NGINX version number because we don't want to reveal
+    # information that could be used to find an exploit.
+    server_tokens off;
+
+    # Uncomment this for a HTTP header that will let you know the cache status
+    # of an object.
+    # add_header X-Cache-Status $upstream_cache_status;
+
+    # Proxy caching configuration. Customize this for your needs.
+    proxy_cache s3_cache;
+    proxy_cache_valid 200 302 ${PROXY_CACHE_VALID_OK};
+    proxy_cache_valid 404 ${PROXY_CACHE_VALID_NOTFOUND};
+    proxy_cache_valid 403 ${PROXY_CACHE_VALID_FORBIDDEN};
+    proxy_cache_methods GET HEAD;
+    # When this is enabled a HEAD request to NGINX will result in a GET
+    # request upstream. Unfortunately, proxy_cache_convert_head has to be
+    # disabled because there is no way for the signatures generation code to
+    # get access to the metadata in the GET request that is sent upstream.
+    proxy_cache_convert_head off;
+    proxy_cache_revalidate on;
+    proxy_cache_background_update on;
+    proxy_cache_lock on;
+    proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
+    proxy_cache_key "$request_method$host$uri";
+
+    # If you need to support proxying range request, refer to this article:
+    # https://www.nginx.com/blog/smart-efficient-byte-range-caching-nginx/
+
+    # Do not proxy the S3 SOAP API. The S3 API has a less-documented feature
+    # where the object name "soap" is used for the SOAP API. We don't allow
+    # access to it.
+    location /soap {
+        return 404;
+    }
+
+    location /health {
+        return 200;
+    }
+
+    location / {
+        # This value is templated in based on the value of $CORS_ENABLED. When
+        # CORS is enabled, acceptable methods are GET, HEAD, and OPTIONS.
+        # Otherwise, they are GET and HEAD.
+        limit_except ${LIMIT_METHODS_TO} {}
+
+        # CORS is implemented by returning the appropriate headers as part of
+        # the response to an OPTIONS request. If you want to customize the
+        # CORS response, the cors.conf.template file can be overwritten and
+        # extended to meet your needs.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        auth_request /aws/credentials/retrieve;
+
+        # Redirect to the proper location based on the client request - either
+        # @s3, @s3PreListing or @error405.
+
+        js_content s3gateway.redirectToS3;
+    }
+
+    location /aws/credentials/retrieve {
+        internal;
+        js_content awscredentials.fetchCredentials;
+
+        include /etc/nginx/conf.d/gateway/js_fetch_trusted_certificate.conf;
+    }
+
+    # This is the primary location that proxies the request to s3
+    # See the included s3_location_common.conf file for all logic
+    location @s3 {
+        include /etc/nginx/conf.d/gateway/s3_location_common.conf;
+    }
+
+    # Same as the primary location above but handling and caching
+    # byte range requests efficiently
+    location @s3_sliced {
+        proxy_cache s3_cache_slices;
+        proxy_cache_valid 200 302 206 ${PROXY_CACHE_VALID_OK};
+        proxy_cache_key "$request_method$host$uri$slice_range";
+
+        slice             ${PROXY_CACHE_SLICE_SIZE};
+        proxy_set_header   Range $slice_range;
+        include /etc/nginx/conf.d/gateway/s3_location_common.conf;
+    }
+
+    location @s3PreListing {
+        # We include only the headers needed for the authentication signatures that
+        # we plan to use.
+        include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf;
+
+        # The CORS configuration needs to be imported in several places in order for
+        # it to be applied within different contexts.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        # Don't allow any headers from the client - we don't want them messing
+        # with S3 at all.
+        proxy_pass_request_headers off;
+
+        # Enable passing of the server name through TLS Server Name Indication extension.
+        proxy_ssl_server_name on;
+        proxy_ssl_name ${S3_BUCKET_NAME}.${S3_SERVER};
+
+        # Set the Authorization header to the AWS Signatures credentials
+        proxy_set_header Authorization $s3auth;
+        proxy_set_header X-Amz-Security-Token $awsSessionToken;
+
+        # We set the host as the bucket name to inform the S3 API of the bucket
+        proxy_set_header Host $s3_host_hdr;
+
+        # Use keep alive connections in order to improve performance
+        proxy_http_version 1.1;
+        proxy_set_header Connection '';
+
+        # We strip off all of the AWS specific headers from the server so that
+        # there is nothing identifying the object as having originated in an
+        # object store.
+        js_header_filter s3gateway.editHeaders;
+
+        # Apply XSL transformation to the XML returned from S3 directory listing
+        # results such that we can output an HTML directory contents list.
+        xslt_stylesheet /etc/nginx/include/listing.xsl;
+        xslt_string_param rootPath '${DIRECTORY_LISTING_PATH_PREFIX}';
+        xslt_types application/xml;
+
+        # We apply an output filter to the XML input received from S3 before it
+        # is passed to XSLT in order to determine if the resource is not a valid
+        # S3 directory. If it isn't a valid directory, we do a dirty hack to
+        # corrupt the contents of the XML causing the XSLT to fail and thus
+        # nginx to return a 404 to the client. If you don't care about empty
+        # directory listings for invalid directories, remove this.
+        js_body_filter s3gateway.filterListResponse;
+
+        # Catch all errors from S3 and sanitize them so that the user can't
+        # gain intelligence about the S3 bucket being proxied.
+        proxy_intercept_errors on;
+
+        # Comment out this line to receive the error messages returned by S3
+        error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+
+        js_content s3gateway.loadContent;
+        include /etc/nginx/conf.d/gateway/s3listing_location.conf;
+    }
+
+    location @s3Directory {
+        # We include only the headers needed for the authentication signatures that
+        # we plan to use.
+        include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf;
+
+        # Necessary for determining the correct URI to construct.
+        set $forIndexPage false;
+
+        # The CORS configuration needs to be imported in several places in order for
+        # it to be applied within different contexts.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        # Don't allow any headers from the client - we don't want them messing
+        # with S3 at all.
+        proxy_pass_request_headers off;
+
+        # Enable passing of the server name through TLS Server Name Indication extension.
+        proxy_ssl_server_name on;
+        proxy_ssl_name ${S3_BUCKET_NAME}.${S3_SERVER};
+
+        # Set the Authorization header to the AWS Signatures credentials
+        proxy_set_header Authorization $s3auth;
+        proxy_set_header X-Amz-Security-Token $awsSessionToken;
+
+        # We set the host as the bucket name to inform the S3 API of the bucket
+        proxy_set_header Host $s3_host_hdr;
+
+        # Use keep alive connections in order to improve performance
+        proxy_http_version 1.1;
+        proxy_set_header Connection '';
+
+        # We strip off all of the AWS specific headers from the server so that
+        # there is nothing identifying the object as having originated in an
+        # object store.
+        js_header_filter s3gateway.editHeaders;
+
+        # Apply XSL transformation to the XML returned from S3 directory listing
+        # results such that we can output an HTML directory contents list.
+        xslt_stylesheet /etc/nginx/include/listing.xsl;
+        xslt_string_param rootPath '${DIRECTORY_LISTING_PATH_PREFIX}';
+        xslt_types application/xml;
+
+        # We apply an output filter to the XML input received from S3 before it
+        # is passed to XSLT in order to determine if the resource is not a valid
+        # S3 directory. If it isn't a valid directory, we do a dirty hack to
+        # corrupt the contents of the XML causing the XSLT to fail and thus
+        # nginx to return a 404 to the client. If you don't care about empty
+        # directory listings for invalid directories, remove this.
+        js_body_filter s3gateway.filterListResponse;
+
+        # Catch all errors from S3 and sanitize them so that the user can't
+        # gain intelligence about the S3 bucket being proxied.
+        proxy_intercept_errors on;
+
+        # Comment out this line to receive the error messages returned by S3
+        error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+
+        proxy_pass  ${S3_SERVER_PROTO}://storage_urls$s3Uri;
+        include /etc/nginx/conf.d/gateway/s3listing_location.conf;
+    }
+
+    location ~ /index.html$ {
+        # Configuration for handling locations ending with /index.html
+
+        # Necessary for determining the correct URI to construct.
+        set $forIndexPage true;
+
+        # We include only the headers needed for the authentication signatures that
+        # we plan to use.
+        include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf;
+
+        # The CORS configuration needs to be imported in several places in order for
+        # it to be applied within different contexts.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        # Don't allow any headers from the client - we don't want them messing
+        # with S3 at all.
+        proxy_pass_request_headers off;
+
+        # Enable passing of the server name through TLS Server Name Indication extension.
+        proxy_ssl_server_name on;
+        proxy_ssl_name ${S3_BUCKET_NAME}.${S3_SERVER};
+
+        # Set the Authorization header to the AWS Signatures credentials
+        proxy_set_header Authorization $s3auth;
+        proxy_set_header X-Amz-Security-Token $awsSessionToken;
+
+        # We set the host as the bucket name to inform the S3 API of the bucket
+        proxy_set_header Host $s3_host_hdr;
+
+        # Use keep alive connections in order to improve performance
+        proxy_http_version 1.1;
+        proxy_set_header Connection '';
+
+        # We strip off all of the AWS specific headers from the server so that
+        # there is nothing identifying the object as having originated in an
+        # object store.
+        js_header_filter s3gateway.editHeaders;
+
+        # Catch all errors from S3 and sanitize them so that the user can't
+        # gain intelligence about the S3 bucket being proxied.
+        proxy_intercept_errors on;
+
+        # Comment out this line to receive the error messages returned by S3
+        error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+
+        proxy_pass  ${S3_SERVER_PROTO}://storage_urls$s3uri;
+        include /etc/nginx/conf.d/gateway/s3listing_location.conf;
+    }
+
+    location @error404 {
+        # The CORS configuration needs to be imported in several places in order for
+        # it to be applied within different contexts.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        return 404;
+    }
+
+    location @trailslashControl {
+        # Checks if requesting a folder without trailing slash, and return 302
+        # appending a slash to it when using for static site hosting.
+        js_content s3gateway.trailslashControl;
+    }
+
+    location @trailslash {
+        # 302 to request without slashes
+        # Adding a ? to the end of the replacement param in `rewrite` prevents it from
+        # appending the query string.
+        rewrite ^ $scheme://$http_host$uri/$is_args$query_string? redirect;
+    }
+
+    # Provide a hint to the client on 405 errors of the acceptable request methods
+    error_page 405 @error405;
+    location @error405 {
+        add_header Allow "${LIMIT_METHODS_TO_CSV}" always;
+        return 405;
+    }
+
+    include /etc/nginx/conf.d/gateway/s3_server.conf;
+}