feat: support s3, r2, minio storage type

zmh-program · zmh-program · commit a50be1b69f56 · 2024-04-22T13:13:26.000+08:00
diff --git a/README.md b/README.md
@@ -85,14 +85,36 @@ Response
      - set env `LOCAL_STORAGE_DOMAIN` to your deployment domain (e.g. `LOCAL_STORAGE_DOMAIN=http://blob-service.onrender.com`)
      - if you are using Docker, you need to mount volume `/static` to the host (e.g. `-v /path/to/static:/static`)
      
-3. 🚀 AWS S3 Storage
+3. 🚀 AWS S3
    - [ ] **Payment Storage Cost**
    - [x] Support Direct URL Access
    - [x] China Mainland User Friendly
    - Config:
      - set env `STORAGE_TYPE` to `s3` (e.g. `STORAGE_TYPE=s3`)
-     - set env `AWS_ACCESS_KEY_ID` to your AWS Access Key ID
-     - set env `AWS_SECRET_ACCESS_KEY` to your AWS Secret Access Key
-     - set env `AWS_S3_BUCKET` to your AWS S3 Bucket Name
-     - set env `AWS_S3_REGION` to your AWS S3 Region
-     - set env `AWS_S3_DOMAIN` to your AWS S3 Domain (e.g. `https://s3.amazonaws.com`)
+     - set env `S3_ACCESS_KEY` to your AWS Access Key ID
+     - set env `S3_SECRET_KEY` to your AWS Secret Access Key
+     - set env `S3_BUCKET` to your AWS S3 Bucket Name
+     - set env `S3_REGION` to your AWS S3 Region
+
+4. 🔔 Cloudflare R2
+   - [x] **Free Storage Quota ([10GB Storage](https://developers.cloudflare.com/r2/pricing/))**
+   - [x] Support Direct URL Access
+   - Config *(S3 Compatible)*:
+     - set env `STORAGE_TYPE` to `s3` (e.g. `STORAGE_TYPE=s3`)
+     - set env `S3_ACCESS_KEY` to your Cloudflare R2 Access Key ID
+     - set env `S3_SECRET_KEY` to your Cloudflare R2 Secret Access Key
+     - set env `S3_BUCKET` to your Cloudflare R2 Bucket Name
+     - set env `S3_DOMAIN` to your Cloudflare R2 Domain Name (e.g. `https://<account-id>.r2.cloudflarestorage.com`)
+     - set env `S3_DIRECT_URL_DOMAIN` to your Cloudflare R2 Public URL Access Domain Name ([Open Public URL Access](https://developers.cloudflare.com/r2/buckets/public-buckets/), e.g. `https://pub-xxx.r2.dev`)
+
+5. 📦 Min IO
+    - [x] **Self Hosted**
+    - [x] Reliable & Flexible Storage
+    - Config *(S3 Compatible)*:
+      - set env `STORAGE_TYPE` to `s3` (e.g. `STORAGE_TYPE=s3`)
+      - set env `S3_SIGN_VERSION` to `s3v4` (e.g. `S3_SIGN_VERSION=s3v4`)
+      - set env `S3_ACCESS_KEY` to your Min IO Access Key ID
+      - set env `S3_SECRET_KEY` to your Min IO Secret Access Key
+      - set env `S3_BUCKET` to your Min IO Bucket Name
+      - set env `S3_DOMAIN` to your Min IO Domain Name (e.g. `https://oss.example.com`)
+      - *[Optional] If you are using CDN, you can set `S3_DIRECT_URL_DOMAIN` to your Min IO Public URL Access Domain Name (e.g. `https://cdn-hk.example.com`)*
diff --git a/config.py b/config.py
@@ -12,3 +12,15 @@
 
 STORAGE_TYPE = environ.get("STORAGE_TYPE", "common").lower()  # Storage Type
 LOCAL_STORAGE_DOMAIN = environ.get("LOCAL_STORAGE_DOMAIN", "").rstrip("/")  # Local Storage Domain
+
+S3_BUCKET = environ.get("S3_BUCKET", "")  # S3 Bucket
+S3_ACCESS_KEY = environ.get("S3_ACCESS_KEY", "")  # S3 Access Key
+S3_SECRET_KEY = environ.get("S3_SECRET_KEY", "")  # S3 Secret Key
+S3_REGION = environ.get("S3_REGION", "")  # S3 Region
+S3_DOMAIN = environ.get("S3_DOMAIN", "").rstrip("/")  # S3 Domain (Optional)
+S3_DIRECT_URL_DOMAIN = environ.get("S3_DIRECT_URL_DOMAIN", "").rstrip("/")  # S3 Direct/Proxy URL Domain (Optional)
+S3_SIGN_VERSION = environ.get("S3_SIGN_VERSION", None)  # S3 Sign Version
+
+S3_API = S3_DOMAIN or f"https://{S3_BUCKET}.s3.{S3_REGION}.amazonaws.com"  # S3 API
+S3_SPACE = S3_DIRECT_URL_DOMAIN or S3_API  # S3 Image URL Domain
+
diff --git a/favicon.ico b/favicon.ico
diff --git a/index.html b/index.html
@@ -3,6 +3,13 @@
 <head>
     <meta charset="UTF-8">
     <title>Chat Nio Blob Service</title>
+    <link rel="icon" href="/favicon.ico">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="Chat Nio Blob Service">
+    <meta name="author" content="Deeptrain Community">
+    <meta name="keywords" content="Chat Nio Blob Service">
+    <meta name="theme-color" content="#ffffff">
+    <meta itemprop="image" content="/favicon.ico">
     <link href="https://open.lightxi.com/fonts/Open-Sans" rel="stylesheet">
     <style>
     html, body {
@@ -13,6 +20,11 @@
         overflow-x: hidden;
         overflow-y: auto;
         background: hsl(var(--background));
+        scrollbar-width: none;
+    }
+
+    ::-webkit-scrollbar {
+        width: 0;
     }
 
     * {
@@ -97,8 +109,16 @@
         color: hsl(var(--text-secondary));
     }
 
+    .drop-label {
+        display: flex;
+        flex-direction: row;
+        align-items: center;
+        justify-content: center;
+    }
+
     .output-text {
         width: 100%;
+        min-height: 60px;
         border: 1px solid hsl(var(--border));
         border-radius: var(--radius);
         padding: 1rem;
@@ -108,13 +128,58 @@
         word-break: break-all;
         word-wrap: break-word;
     }
+
+    .loading {
+        width: 0.75rem;
+        height: 0.75rem;
+        border-radius: 50%;
+        border: 0.1rem solid hsl(var(--text-secondary));
+        border-top-color: hsl(var(--background));
+        animation: spin 1s linear infinite;
+        margin-right: 0.5rem;
+        display: inline-block;
+    }
+
+    .link {
+        position: absolute;
+        top: 0;
+        right: 0;
+        margin: 1.5rem;
+        cursor: pointer;
+        outline: none;
+        text-decoration: none;
+    }
+
+    .link svg {
+        width: 1.25rem;
+        height: 1.25rem;
+        color: hsl(var(--text));
+    }
+
+    .hidden {
+        display: none;
+    }
+
+    @keyframes spin {
+        to {
+            transform: rotate(360deg);
+        }
+    }
 </style>
 </head>
 <body>
+<a href="https://github.com/Deeptrain-Community/chatnio-blob-service" target="_blank" class="link">
+    <svg viewBox="0 0 438.549 438.549">
+        <path fill="currentColor" d="M409.132 114.573c-19.608-33.596-46.205-60.194-79.798-79.8-33.598-19.607-70.277-29.408-110.063-29.408-39.781 0-76.472 9.804-110.063 29.408-33.596 19.605-60.192 46.204-79.8 79.8C9.803 148.168 0 184.854 0 224.63c0 47.78 13.94 90.745 41.827 128.906 27.884 38.164 63.906 64.572 108.063 79.227 5.14.954 8.945.283 11.419-1.996 2.475-2.282 3.711-5.14 3.711-8.562 0-.571-.049-5.708-.144-15.417a2549.81 2549.81 0 01-.144-25.406l-6.567 1.136c-4.187.767-9.469 1.092-15.846 1-6.374-.089-12.991-.757-19.842-1.999-6.854-1.231-13.229-4.086-19.13-8.559-5.898-4.473-10.085-10.328-12.56-17.556l-2.855-6.57c-1.903-4.374-4.899-9.233-8.992-14.559-4.093-5.331-8.232-8.945-12.419-10.848l-1.999-1.431c-1.332-.951-2.568-2.098-3.711-3.429-1.142-1.331-1.997-2.663-2.568-3.997-.572-1.335-.098-2.43 1.427-3.289 1.525-.859 4.281-1.276 8.28-1.276l5.708.853c3.807.763 8.516 3.042 14.133 6.851 5.614 3.806 10.229 8.754 13.846 14.842 4.38 7.806 9.657 13.754 15.846 17.847 6.184 4.093 12.419 6.136 18.699 6.136 6.28 0 11.704-.476 16.274-1.423 4.565-.952 8.848-2.383 12.847-4.285 1.713-12.758 6.377-22.559 13.988-29.41-10.848-1.14-20.601-2.857-29.264-5.14-8.658-2.286-17.605-5.996-26.835-11.14-9.235-5.137-16.896-11.516-22.985-19.126-6.09-7.614-11.088-17.61-14.987-29.979-3.901-12.374-5.852-26.648-5.852-42.826 0-23.035 7.52-42.637 22.557-58.817-7.044-17.318-6.379-36.732 1.997-58.24 5.52-1.715 13.706-.428 24.554 3.853 10.85 4.283 18.794 7.952 23.84 10.994 5.046 3.041 9.089 5.618 12.135 7.708 17.705-4.947 35.976-7.421 54.818-7.421s37.117 2.474 54.823 7.421l10.849-6.849c7.419-4.57 16.18-8.758 26.262-12.565 10.088-3.805 17.802-4.853 23.134-3.138 8.562 21.509 9.325 40.922 2.279 58.24 15.036 16.18 22.559 35.787 22.559 58.817 0 16.178-1.958 30.497-5.853 42.966-3.9 12.471-8.941 22.457-15.125 29.979-6.191 7.521-13.901 13.85-23.131 18.986-9.232 5.14-18.182 8.85-26.84 11.136-8.662 2.286-18.415 4.004-29.263 5.146 9.894 8.562 14.842 22.077 14.842 40.539v60.237c0 3.422 1.19 6.279 3.572 8.562 2.379 2.279 6.136 2.95 11.276 1.995 44.163-14.653 80.185-41.062 108.068-79.226 27.88-38.161 41.825-81.126 41.825-128.906-.01-39.771-9.818-76.454-29.414-110.049z"></path>
+    </svg>
+</a>
 <div class="wrapper">
     <label for="file" class="drop-window">
         <input type="file" id="file" style="display: none">
-        <span>Click to upload a file</span>
+        <span class="drop-label">
+            <span class="loading hidden"></span>
+            Click to upload a file
+        </span>
     </label>
     <div id="output" class="output-text"></div>
 </div>
@@ -140,15 +205,25 @@
         }
     }
 
-    input.addEventListener('change', async (e) => {
-        const file = e.target.files[0];
+    async function handler(file) {
+        const loading = document.querySelector('.loading');
+        loading.classList.remove('hidden');
         const content = await post(file);
         let raw = content.replace(/\\n/g, '\n');
         if (raw.startsWith("data:image") && raw.length > 1000) {
             raw = `${raw.slice(0, 1000)}... (truncated ${raw.length - 1000} characters)`;
         }
 
         output.innerText = raw;
+        loading.classList.add('hidden');
+    }
+
+    input.addEventListener('change', async e => {
+        const file = e.target.files[0];
+        if (!file) return;
+
+        await handler(file);
+        input.value = '';
     });
 </script>
 </body>
diff --git a/main.py b/main.py
@@ -22,6 +22,11 @@ def root():
     return FileResponse("index.html", media_type="text/html")
 
 
+@app.get("/favicon.ico")
+def favicon():
+    return FileResponse("favicon.ico")
+
+
 @app.post("/upload")
 async def upload(file: UploadFile = File(...)):
     """Accepts file and returns its contents."""
diff --git a/requirements.txt b/requirements.txt
@@ -8,5 +8,8 @@ python-pptx
 openpyxl
 xlrd
 
+boto3
+botocore
 azure-cognitiveservices-speech
+
 cryptography
diff --git a/store/local.py b/store/local.py
@@ -1,15 +1,12 @@
-from datetime import datetime
-
 from fastapi import UploadFile
-
 from config import LOCAL_STORAGE_DOMAIN
-from util import md5_encode
+from store.utils import store_filename
 
 
 async def process_local(file: UploadFile) -> str:
-    """Process image and return its base64 url."""
+    """Process image and return its direct url."""
 
-    filename = md5_encode(file.filename + datetime.now().isoformat())
+    filename = store_filename(file.filename)
     path = f"static/{filename}"
 
     with open(path, "wb") as f:
diff --git a/store/s3.py b/store/s3.py
@@ -0,0 +1,53 @@
+from fastapi import UploadFile
+import boto3
+from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError
+from botocore.client import Config
+
+from store.utils import store_filename
+from config import (
+    S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY, S3_REGION, S3_API, S3_DOMAIN, S3_SPACE, S3_SIGN_VERSION,
+)
+
+
+def create_s3_client():
+    config = Config(signature_version=S3_SIGN_VERSION) if S3_SIGN_VERSION else None
+    if S3_DOMAIN and len(S3_DOMAIN) > 0:
+        # Cloudflare R2 Storage
+        return boto3.client(
+            "s3",
+            aws_access_key_id=S3_ACCESS_KEY,
+            aws_secret_access_key=S3_SECRET_KEY,
+            endpoint_url=S3_API,
+            config=config,
+        )
+
+    return boto3.client(
+        "s3",
+        region_name=S3_REGION,
+        aws_access_key_id=S3_ACCESS_KEY,
+        aws_secret_access_key=S3_SECRET_KEY,
+        config=config,
+    )
+
+
+async def process_s3(file: UploadFile) -> str:
+    """Process image and return its s3 url."""
+
+    filename = store_filename(file.filename)
+
+    try:
+        client = create_s3_client()
+        client.upload_fileobj(
+            file.file,
+            S3_BUCKET,
+            filename,
+            ExtraArgs={"ACL": "public-read"},
+        )
+
+        return f"{S3_SPACE}/{filename}"
+    except (NoCredentialsError, PartialCredentialsError) as e:
+        raise ValueError(f"AWS credentials not found: {e}")
+    except ClientError as e:
+        raise ValueError(f"AWS S3 error: {e}")
+    except Exception as e:
+        raise ValueError(f"S3 error: {e}")
diff --git a/store/store.py b/store/store.py
@@ -4,10 +4,12 @@
 
 from store.common import process_base64
 from store.local import process_local
+from store.s3 import process_s3
 
 IMAGE_HANDLERS = {
     "common": process_base64,
     "local": process_local,
+    "s3": process_s3,
 }
 
 
diff --git a/store/utils.py b/store/utils.py
@@ -0,0 +1,8 @@
+from datetime import datetime
+from util import md5_encode
+
+
+def store_filename(filename: str) -> str:
+    """Store filename."""
+    suffix = filename.split(".")[-1] if "." in filename else "jpg"
+    return md5_encode(filename + datetime.now().isoformat()) + "." + suffix

Original file line number	Diff line number	Diff line change
`@@ -4,10 +4,12 @@`
`4`	`4`
`5`	`5`	`from store.common import process_base64`
`6`	`6`	`from store.local import process_local`
	`7`	`+from store.s3 import process_s3`
`7`	`8`
`8`	`9`	`IMAGE_HANDLERS = {`
`9`	`10`	`"common": process_base64,`
`10`	`11`	`"local": process_local,`
	`12`	`+ "s3": process_s3,`
`11`	`13`	`}`
`12`	`14`
`13`	`15`