ci: Refactor sync-test-datasets workflow to use s5cmd for S3 sync

- Replace AWS CLI with s5cmd for improved performance and efficiency in syncing branches to S3.
- Add caching for s5cmd installation to speed up workflow execution.
- Update branch checkout process to use actions/checkout for better handling of specific branches.

Author: edmundmiller

.github/workflows/sync-test-datasets.yml

@@ -55,29 +55,53 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-1
 
-      - name: Sync branch ${{ matrix.branch }} to S3
+      - name: Checkout specific branch
+        uses: actions/checkout@v4
+        with:
+          repository: nf-core/test-datasets
+          ref: ${{ matrix.branch }}
+          path: test-datasets-branch
+
+      - name: Cache s5cmd
+        id: cache-s5cmd
+        uses: actions/cache@v4
+        with:
+          path: ~/.local/bin/s5cmd
+          key: s5cmd-${{ runner.os }}-latest
+
+      - name: Install s5cmd
+        if: steps.cache-s5cmd.outputs.cache-hit != 'true'
         run: |
-          echo "Syncing branch: ${{ matrix.branch }}"
+          mkdir -p ~/.local/bin
+          curl -L https://github.com/peak/s5cmd/releases/latest/download/s5cmd_$(uname)_amd64.tar.gz | tar -xz
+          mv s5cmd ~/.local/bin/
+          chmod +x ~/.local/bin/s5cmd
 
-          # Clone only the specific branch
-          git clone --single-branch --branch "${{ matrix.branch }}" \
-            https://github.com/nf-core/test-datasets.git test-datasets-branch
+      - name: Add s5cmd to PATH
+        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 
+      - name: Sync branch ${{ matrix.branch }} to S3
+        run: |
+          echo "Syncing branch: ${{ matrix.branch }} using s5cmd"
           cd test-datasets-branch
-
-          # Sync to S3 with branch prefix
-          aws s3 sync ./ "s3://nf-core-test-datasets/${{ matrix.branch }}/" \
+          
+          # Use s5cmd for faster, more efficient uploads
+          # --delete flag removes files in S3 that don't exist locally
+          # --exclude patterns exclude git files
+          # --storage-class sets the S3 storage class
+          # --numworkers increases parallelism for faster uploads
+          s5cmd \
+            --numworkers 10 \
+            --retry-count 3 \
+            sync \
             --delete \
             --exclude ".git/*" \
             --exclude ".github/*" \
-            --storage-class STANDARD_IA
-
+            --storage-class STANDARD_IA \
+            ./ "s3://nf-core-test-datasets/${{ matrix.branch }}/"
+          
           echo "Completed sync for branch: ${{ matrix.branch }}"
 
-          # Clean up
-          cd ..
-          rm -rf test-datasets-branch
-
   update-metadata:
     needs: [discover-branches, sync-branches]
     runs-on: ubuntu-latest