[Devops] Cleaning CI workflow files and dependencies (ourownstory#1547)

* move tests ci.yml to tests.yml

* clean up workflow dependencies

* narrow dependencies

* continue separation

* lock

* update workflow names

* fix plotting tests needing ipykernel

* add kaleido to main requirements

* update tests workflow name

.github/workflows/linters.yml

@@ -85,6 +85,6 @@ jobs:
         if: steps.poetry-cache.outputs.cache-hit != 'true'
         uses: snok/install-poetry@v1
       - name: Install dependencies
-        run: poetry install --no-interaction --with=docs,pyright
+        run: poetry install --no-interaction --with=linters --without=dev,docs,pytest,metrics
       - name: pyright
         uses: jakebailey/pyright-action@v1

.github/workflows/metrics.yml

@@ -1,4 +1,4 @@
-name: model
+name: Measure Model Performance
 on:
   push:
     paths: # Run only when files under neuralprophet/ are modified
@@ -32,9 +32,9 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
       - name: Install Dependencies
-        run: poetry install --no-interaction --no-root --without docs
+        run: poetry install --no-interaction --no-root --with=pytest,metrics --without=dev,docs,linters
       - name: Install Project
-        run: poetry install --no-interaction --without docs
+        run: poetry install --no-interaction --with=pytest,metrics --without=dev,docs,linters
       - name: Train model
         run: poetry run pytest tests/test_model_performance.py -n 1 --durations=0
       - name: Download metrics from main

.github/workflows/test-docs.yml

@@ -0,0 +1,33 @@
+name: Test build docs
+on:
+  push:
+    branches:
+      - main
+      - website
+  pull_request:
+    branches:
+      - main
+      - website
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v2
+      - name: Setup Pandoc
+        uses: r-lib/actions/setup-pandoc@v2
+      - name: Setup Requirements
+        run: poetry install  --no-interaction --with=docs --without=dev,pytest,metrics,linters
+      - name: Build with Sphinx
+        run: poetry run sphinx-build docs/source _site

.github/workflows/ci.yml → .github/workflows/tests.yml

@@ -1,25 +1,26 @@
-name: tests
+name: Pytest and Coverage
 on:
   push:
     branches:
       - main
-      - develop
   pull_request:
     branches:
       - main
-      - develop
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  build:
+  tests:
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.12"]
+        include:
+          - os: "ubuntu-22.04"
+            python-version: "3.10"
     env:
       POETRY_VIRTUALENVS_CREATE: false
     steps:
@@ -32,9 +33,9 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
       - name: Install dependencies
-        run: poetry install --no-root --no-interaction --without docs
+        run: poetry install --no-root --no-interaction --with=dev,pytest --without=docs,metrics,linters
       - name: Install Project
-        run: poetry install --no-interaction --without docs
+        run: poetry install --no-interaction --with=dev,pytest --without=docs,metrics,linters
       - name: Pytest
         run: poetry run pytest tests -v -n auto --durations=0 --cov=./ --cov-report=xml
       - name: Upload coverage to codecov
@@ -44,21 +45,3 @@ jobs:
           files: ./coverage.xml
           name: Coverage Report with codecov overview
           verbose: true
-
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.12"
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
-      - name: Setup Pandoc
-        uses: r-lib/actions/setup-pandoc@v2
-      - name: Setup Requirements
-        run: poetry install  --no-interaction --with docs
-      - name: Build with Sphinx
-        run: poetry run sphinx-build docs/source _site

.github/workflows/website.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Setup Poetry
         uses: snok/install-poetry@v1
       - name: Setup Requirements
-        run: poetry install --with=docs
+        run: poetry install --with=docs --without=dev,pytest,metrics,linters
       - name: Build with Sphinx
         run: poetry run sphinx-build docs/source _site
       - name: Upload artifact

pyproject.toml

@@ -26,30 +26,31 @@ torch = "^2.0.0"
 pytorch-lightning = "^1.9.4" # TODO: move to ^2.0.0
 tensorboard = "^2.11.2"
 torchmetrics = "^1.0.0"
-captum = ">=0.6.0"
 holidays = ">=0.41"
+captum = ">=0.6.0"
 matplotlib = "^3.5.3"
 plotly = "^5.13.1"
-nbformat = "^5.8.0"
+kaleido = "0.2.1" # required for plotly static image export
 plotly-resampler = { version = "^0.9.2", python = "<3.12", optional = true }
 livelossplot = { version = ">=0.5.5", optional = true }
 
 [tool.poetry.extras]
 plotly-resampler = ["plotly-resampler"]
 live = ["livelossplot"]
 
-[tool.poetry.group.dev.dependencies]
-black = { extras = ["jupyter"], version = "^24.1.0" }
-flake8 = "^7.0.0"
-isort = "^5.12.0"
+[tool.poetry.group.dev.dependencies] # For dev involving notebooks
+ipykernel = "^6.29.2"
+nbformat = "^5.8.0"
+
+[tool.poetry.group.pytest] # pytest dev setup and CI
+optional = true
+
+[tool.poetry.group.pytest.dependencies] 
 pytest = "^8.0.0"
 pytest-xdist = "^3.1.0"
 pytest-cov = "^4.1.0"
-kaleido = "0.2.1" # required for plotly static image export
-tabulate = "^0.9.0" # Used in model metrics CI only; md export for github-actions bot
-ipykernel = "^6.29.2"
 
-[tool.poetry.group.docs]
+[tool.poetry.group.docs] # for building docs; and testing docs CI
 optional = true
 
 [tool.poetry.group.docs.dependencies]
@@ -60,11 +61,20 @@ sphinx = "^7.0.0"
 sphinx-fontawesome = "^0.0.6"
 furo = "^2024.1.29"
 
-[tool.poetry.group.pyright]
+[tool.poetry.group.metrics] # for metrics CI
 optional = true
 
-[tool.poetry.group.pyright.dependencies]
+[tool.poetry.group.metrics.dependencies]
+tabulate = "^0.9.0" # Used in model metrics CI only; md export for github-actions bot
+
+[tool.poetry.group.linters] # for linters CI
+optional = true
+
+[tool.poetry.group.linters.dependencies]
+black = { extras = ["jupyter"], version = "^24.1.0" }
+isort = "^5.12.0"
 pandas-stubs = "^2.0"
+flake8 = "^7.0.0"
 
 [build-system]
 requires = ["poetry-core"]