Support custom S3 endpoint host_base setting (in .s3cfg)

Allow the user to specify an endpoint other than s3.amazonaws.com.
This can be set using ~/.s3cfg's host_base setting (only; we ignore
host_bucket); when there's a blessed setting key for .aws/credentials,
we'll support it there too (perhaps endpoint_url; cf aws/aws-cli#1270).

Fixes (part of) #436.

NEWS

@@ -11,6 +11,9 @@ Noteworthy changes in release 1.4
 * New errmod_cal(), probaln_glocal(), sam_cap_mapq(), and sam_prob_realn()
   functions, previously internal to SAMtools, have been added to HTSlib.
 
+* S3 file access now also supports the "host_base" setting in the
+  $HOME/.s3cfg configuration file.
+
 * Data URLs ("data:,text") now follow the standard RFC 2397 format and may
   be base64-encoded (when written as "data:;base64,text") or may include
   percent-encoded characters.  HTSlib's previous over-simplified "data:text"

hfile_libcurl.c

@@ -1,6 +1,6 @@
 /*  hfile_libcurl.c -- libcurl backend for low-level file streams.
 
-    Copyright (C) 2015, 2016 Genome Research Ltd.
+    Copyright (C) 2015-2017 Genome Research Ltd.
 
     Author: John Marshall <[email protected]>
 
@@ -866,6 +866,7 @@ add_s3_settings(hFILE_libcurl *fp, const char *s3url, kstring_t *message)
     kstring_t profile = { 0, 0, NULL };
     kstring_t id = { 0, 0, NULL };
     kstring_t secret = { 0, 0, NULL };
+    kstring_t host_base = { 0, 0, NULL };
     kstring_t token = { 0, 0, NULL };
     kstring_t token_hdr = { 0, 0, NULL };
     kstring_t auth_hdr = { 0, 0, NULL };
@@ -924,17 +925,6 @@ add_s3_settings(hFILE_libcurl *fp, const char *s3url, kstring_t *message)
         else kputs("default", &profile);
     }
 
-    // Use virtual hosted-style access if possible, otherwise path-style.
-    if (is_dns_compliant(bucket, path)) {
-        kputsn(bucket, path - bucket, &url);
-        kputs(".s3.amazonaws.com", &url);
-    }
-    else {
-        kputs("s3.amazonaws.com/", &url);
-        kputsn(bucket, path - bucket, &url);
-    }
-    kputs(path, &url);
-
     if (id.l == 0) {
         const char *v = getenv("AWS_SHARED_CREDENTIALS_FILE");
         parse_ini(v? v : "~/.aws/credentials", profile.s,
@@ -943,10 +933,26 @@ add_s3_settings(hFILE_libcurl *fp, const char *s3url, kstring_t *message)
     }
     if (id.l == 0)
         parse_ini("~/.s3cfg", profile.s, "access_key", &id,
-                  "secret_key", &secret, "access_token", &token, NULL);
+                  "secret_key", &secret, "access_token", &token,
+                  "host_base", &host_base, NULL);
     if (id.l == 0)
         parse_simple("~/.awssecret", &id, &secret);
 
+    if (host_base.l == 0)
+        kputs("s3.amazonaws.com", &host_base);
+    // Use virtual hosted-style access if possible, otherwise path-style.
+    if (is_dns_compliant(bucket, path)) {
+        kputsn(bucket, path - bucket, &url);
+        kputc('.', &url);
+        kputs(host_base.s, &url);
+    }
+    else {
+        kputs(host_base.s, &url);
+        kputc('/', &url);
+        kputsn(bucket, path - bucket, &url);
+    }
+    kputs(path, &url);
+
     if (token.l > 0) {
         kputs("x-amz-security-token:", message);
         kputs(token.s, message);
@@ -989,6 +995,7 @@ add_s3_settings(hFILE_libcurl *fp, const char *s3url, kstring_t *message)
     free(profile.s);
     free(id.s);
     free(secret.s);
+    free(host_base.s);
     free(token.s);
     free(token_hdr.s);
     free(auth_hdr.s);