Merge pull request nf-core#2578 from mashehu/self-hosted-sync

Run sync on self-hosted runners

.github/workflows/create-lint-wf.yml

@@ -52,82 +52,97 @@ jobs:
         with:
           node-version: "20"
 
-      - name: Install Prettier
-        run: npm install -g prettier
-
-      # Install the editorconfig linting tools
-      - name: Install editorconfig-checker
-        run: npm install -g editorconfig-checker
+      - name: Install Prettier and editorconfig-checker
+        run: npm install -g prettier editorconfig-checker
 
       # Build a pipeline from the template
       - name: nf-core create
-        run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
+        run: |
+          mkdir create-lint-wf && cd create-lint-wf
+          export NXF_WORK=$(pwd)
+          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
 
       # Try syncing it before we change anything
       - name: nf-core sync
         run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/
+        working-directory: create-lint-wf
 
       # Build a module from the template
       - name: nf-core modules create
         run: nf-core --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta
+        working-directory: create-lint-wf
 
       # Run code style linting
       - name: Run Prettier --check
-        run: prettier --check nf-core-testpipeline
+        run: prettier --check create-lint-wf/nf-core-testpipeline
 
       - name: Run ECLint check
         run: editorconfig-checker -exclude README.md $(find nf-core-testpipeline/.* -type f | grep -v '.git\|.py\|md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile')
+        working-directory: create-lint-wf
 
       # Update modules to the latest version
       - name: nf-core modules update
         run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview
+        working-directory: create-lint-wf
 
       # Remove TODO statements
       - name: remove TODO
         run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
+        working-directory: create-lint-wf
 
       # Replace zenodo.XXXXXX to pass readme linting
       - name: replace zenodo.XXXXXX
         run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
+        working-directory: create-lint-wf
 
       # Run nf-core linting
       - name: nf-core lint
         run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned
+        working-directory: create-lint-wf
 
       # Run the other nf-core commands
       - name: nf-core list
         run: nf-core --log-file log.txt list
+        working-directory: create-lint-wf
 
       # - name: nf-core licences
       #   run: nf-core --log-file log.txt licences nf-core-testpipeline
 
       - name: nf-core schema
         run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts
+        working-directory: create-lint-wf
 
       - name: nf-core bump-version
         run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1
+        working-directory: create-lint-wf
 
       - name: nf-core lint in release mode
         run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release
+        working-directory: create-lint-wf
 
       - name: nf-core modules install
         run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force
+        working-directory: create-lint-wf
 
       - name: nf-core modules install gitlab
         run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/
+        working-directory: create-lint-wf
 
       - name: nf-core modules list local
         run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/
+        working-directory: create-lint-wf
 
       - name: nf-core modules list remote
         run: nf-core --log-file log.txt modules list remote
+        working-directory: create-lint-wf
 
       - name: nf-core modules list remote gitlab
         run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote
+        working-directory: create-lint-wf
 
       - name: Upload log file artifact
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
           name: nf-core-log-file
-          path: log.txt
+          path: create-lint-wf/log.txt

.github/workflows/create-test-lint-wf-template.yml

@@ -66,75 +66,86 @@ jobs:
       # Create template files
       - name: Create template skip all (except github)
         run: |
-          printf "prefix: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > template_skip_all.yml
+          mkdir create-test-lint-wf
+          export NXF_WORK=$(pwd)
+          printf "prefix: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > create-test-lint-wf/template_skip_all.yml
 
       - name: Create template skip github_badges
         run: |
-          printf "prefix: my-prefix\nskip: github_badges" > template_skip_github_badges.yml
+          printf "prefix: my-prefix\nskip: github_badges" > create-test-lint-wf/template_skip_github_badges.yml
 
       - name: Create template skip igenomes
         run: |
-          printf "prefix: my-prefix\nskip: igenomes" > template_skip_igenomes.yml
+          printf "prefix: my-prefix\nskip: igenomes" > create-test-lint-wf/template_skip_igenomes.yml
 
       - name: Create template skip ci
         run: |
-          printf "prefix: my-prefix\nskip: ci" > template_skip_ci.yml
+          printf "prefix: my-prefix\nskip: ci" > create-test-lint-wf/template_skip_ci.yml
 
       - name: Create template skip nf_core_configs
         run: |
-          printf "prefix: my-prefix\nskip: nf_core_configs" > template_skip_nf_core_configs.yml
+          printf "prefix: my-prefix\nskip: nf_core_configs" > create-test-lint-wf/template_skip_nf_core_configs.yml
 
       # Create a pipeline from the template
       - name: create a pipeline from the template ${{ matrix.TEMPLATE }}
         run: |
+          cd create-test-lint-wf
           nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }}
 
       - name: run the pipeline
         run: |
+          cd create-test-lint-wf
           nextflow run my-prefix-testpipeline -profile test,docker --outdir ./results
 
       # Remove results folder before linting
       - name: remove results folder
         run: |
-          rm -rf ./results
+          rm -rf create-test-lint-wf/results
 
       # Try syncing it before we change anything
       - name: nf-core sync
-        run: nf-core --log-file log.txt sync --dir my-prefix-testpipeline/
+        run: nf-core --log-file log.txt sync --dir create-test-lint-wf/my-prefix-testpipeline/
 
       # Run code style linting
       - name: Run Prettier --check
-        run: prettier --check my-prefix-testpipeline
+        run: prettier --check create-test-lint-wf/my-prefix-testpipeline
 
       - name: Run ECLint check
         run: editorconfig-checker -exclude README.md $(find my-prefix-testpipeline/.* -type f | grep -v '.git\|.py\|md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile')
+        working-directory: create-test-lint-wf
 
       # Remove TODO statements
       - name: remove TODO
         run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
+        working-directory: create-test-lint-wf
 
       # Replace zenodo.XXXXXX to pass readme linting
       - name: replace zenodo.XXXXXX
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
+        working-directory: create-test-lint-wf
 
       # Run nf-core linting
       - name: nf-core lint
         run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned
+        working-directory: create-test-lint-wf
 
       # Run bump-version
       - name: nf-core bump-version
         run: nf-core --log-file log.txt bump-version --dir my-prefix-testpipeline/ 1.1
+        working-directory: create-test-lint-wf
 
       # Run nf-core linting in release mode
       - name: nf-core lint in release mode
         run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned --release
+        working-directory: create-test-lint-wf
 
       - name: Tar files
         run: tar -cvf artifact_files.tar log.txt template_skip*.yml
+        working-directory: create-test-lint-wf
 
       - name: Upload log file artifact
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
           name: nf-core-log-file
-          path: artifact_files.tar
+          path: create-test-lint-wf/artifact_files.tar

.github/workflows/create-test-wf.yml

@@ -46,6 +46,8 @@ jobs:
 
       - name: Run nf-core/tools
         run: |
+          mkdir create-test-wf && cd create-test-wf
+          export NXF_WORK=$(pwd)
           nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
           nextflow run nf-core-testpipeline -profile test,docker --outdir ./results
 
@@ -54,4 +56,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: nf-core-log-file
-          path: log.txt
+          path: create-test-wf/log.txt

.github/workflows/sync.yml

@@ -31,7 +31,7 @@ jobs:
 
   sync:
     needs: get-pipelines
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     strategy:
       matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}}
       fail-fast: false

tests/test_utils.py

@@ -175,7 +175,7 @@ def test_get_repo_releases_branches_nf_core(self):
     def test_get_repo_releases_branches_not_nf_core(self):
         wfs = nf_core.list.Workflows()
         wfs.get_remote_workflows()
-        pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("ewels/MultiQC", wfs)
+        pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("MultiQC/MultiQC", wfs)
         for r in wf_releases:
             if r.get("tag_name") == "v1.10":
                 break