Some perf flags exposed via env vars: (#590)

* Disabled http2 due to http2 streaming in net/http2 performance issues golang/go#37373
* RCLONE_S3_UPLOAD_CONCURRENCY, RCLONE_S3_CHUNK_SIZE to improve parallelism and download efficiency

Signed-off-by: mzardab <[email protected]>

Author: moadz

resources/operations/rclone-bucket-replicate/job-template.yaml

@@ -74,8 +74,8 @@ objects:
                 - ${SOURCE_ENDPOINT}:${SOURCE_BUCKET}
                 - ${TARGET_ENDPOINT}:${TARGET_BUCKET}
                 - '--rc'
-                - '-v'
                 - '--rc-enable-metrics'
                 - '--rc-addr=:10902'
+                - '--s3-disable-http2' # Needed due to https://github.com/golang/go/issues/37373
           restartPolicy: Never
 

resources/operations/rclone-bucket-replicate/rclone-config-template.yaml

@@ -16,11 +16,15 @@ parameters:
   - name: RCLONE_STATS_INTERVAL
     value: '30s'
   - name: RCLONE_LOG_LEVEL
-    value: '1'
+    value: '0'
   - name: RCLONE_PARALLEL_TRANSFERS
     value: '4'
   - name: RCLONE_PARALLEL_FILE_CHECKERS
     value: '8'
+  - name: RCLONE_S3_UPLOAD_CONCURRENCY
+    value: '8'
+  - name: RCLONE_S3_CHUNK_SIZE
+    value: '64M'
 
 objects:
   - apiVersion: v1
@@ -42,3 +46,7 @@ objects:
       # Checkers do the equality checking of files during a sync
       RCLONE_CHECKERS: ${RCLONE_PARALLEL_FILE_CHECKERS}
 
+      # AWS Client Specific Parameters
+      RCLONE_S3_UPLOAD_CONCURRENCY: ${RCLONE_S3_UPLOAD_CONCURRENCY}
+      RCLONE_S3_CHUNK_SIZE: ${RCLONE_S3_CHUNK_SIZE}
+