Merge branch 'docs_finetuning_callback_example' of https://github.com/SkafteNicki/pytorch-lightning into docs_finetuning_callback_example

SkafteNicki · SkafteNicki · commit e7ef57f02d96 · 2025-09-16T07:12:59.000+02:00
diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml
@@ -47,7 +47,7 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "pytorch.yml / Lit Job (nvidia/cuda:12.1.1-runtime-ubuntu22.04, pytorch, 3.10, A100_X_2)"
+      - "pytorch.yml / Lit Job (nvidia/cuda:12.1.1-runtime-ubuntu22.04, pytorch, 3.10, L4_X_2)"
       - "pytorch.yml / Lit Job (nvidia/cuda:12.6.3-runtime-ubuntu22.04, lightning, 3.12, L4_X_2)"
       - "pytorch.yml / Lit Job (nvidia/cuda:12.6.3-runtime-ubuntu22.04, pytorch, 3.12, L4_X_2)"
 
@@ -148,7 +148,7 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "fabric.yml / Lit Job (nvidia/cuda:12.1.1-runtime-ubuntu22.04, fabric, 3.10, A100_X_2)"
+      - "fabric.yml / Lit Job (nvidia/cuda:12.1.1-runtime-ubuntu22.04, fabric, 3.10, L4_X_2)"
       - "fabric.yml / Lit Job (nvidia/cuda:12.6.3-runtime-ubuntu22.04, fabric, 3.12, L4_X_2)"
       - "fabric.yml / Lit Job (nvidia/cuda:12.6.3-runtime-ubuntu22.04, lightning, 3.12, L4_X_2)"
 
diff --git a/.github/markdown-links-config.json b/.github/markdown-links-config.json
@@ -23,8 +23,8 @@
       }
     }
   ],
-  "timeout": "20s",
+  "timeout": "30s",
   "retryOn429": true,
-  "retryCount": 5,
-  "fallbackRetryDelay": "20s"
+  "retryCount": 10,
+  "fallbackRetryDelay": "10s"
 }
diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml
@@ -67,26 +67,33 @@ jobs:
       - uses: actions/checkout@v5
         with:
           ref: ${{ inputs.checkout }}
+          token: ${{ secrets.GITHUB_TOKEN }}
           # only Pytorch has/uses notebooks
           submodules: ${{ matrix.pkg-name == 'pytorch' }}
           lfs: ${{ matrix.pkg-name == 'pytorch' }}
-      - uses: actions/setup-python@v6
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@v6
         with:
           python-version: "3.10"
+          # TODO: Avoid activating environment like this
+          # see: https://github.com/astral-sh/setup-uv/tree/v6/?tab=readme-ov-file#activate-environment
+          activate-environment: true
+          enable-cache: true
 
       - name: List notebooks
         if: ${{ matrix.pkg-name == 'pytorch' }}
         working-directory: _notebooks/
         run: |
-          pip install -q py-tree
+          uv pip install -q py-tree
           py-tree .notebooks/
           ls -lhR .notebooks/
 
       - name: Pull sphinx template
         run: |
-          pip install -q -r requirements/ci.txt
+          uv pip install -q -r requirements/ci.txt
           aws s3 sync --no-sign-request s3://sphinx-packages/ ${PYPI_LOCAL_DIR}
-          pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR}
+          uv pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR}
 
       - name: pip wheels cache
         uses: actions/cache/restore@v4
@@ -100,25 +107,29 @@ jobs:
         run: |
           sudo apt-get update --fix-missing
           sudo apt-get install -y pandoc
+
       - name: Install package & dependencies
         timeout-minutes: 20
         run: |
           mkdir -p ${PYPI_CACHE_DIR}  # in case cache was not hit
           ls -lh ${PYPI_CACHE_DIR}
-          pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
+          uv pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
             -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} --extra-index-url="${TORCH_URL}"
-          pip list
+          uv pip list
+
       - name: Install req. for Notebooks/tutorials
         if: matrix.pkg-name == 'pytorch'
         timeout-minutes: 10
-        run: pip install -q -r _notebooks/.actions/requires.txt
+        run: uv pip install -q -r _notebooks/.actions/requires.txt
 
       - name: Full build for deployment
         if: github.event_name != 'pull_request'
         run: echo "DOCS_FETCH_ASSETS=1" >> $GITHUB_ENV
+
       - name: Build without warnings
         if: github.event_name != 'workflow_dispatch'
         run: echo "BUILD_SPHINX_OPTS=-W --keep-going" >> $GITHUB_ENV
+
       - name: Make ${{ matrix.target }}
         working-directory: ./docs/source-${{ matrix.pkg-name }}
         # allow failing link check and doctest if you run with dispatch
@@ -128,6 +139,7 @@ jobs:
       - name: Keep artifact
         if: github.event_name == 'pull_request'
         run: echo "ARTIFACT_DAYS=7" >> $GITHUB_ENV
+
       - name: Upload built docs
         if: ${{ matrix.target == 'html' }}
         uses: actions/upload-artifact@v4
diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "_notebooks"]
 	path = _notebooks
-	url = https://github.com/Lightning-AI/lightning-tutorials.git
+	url = https://github.com/Lightning-AI/tutorials.git
 	branch = publication
diff --git a/.lightning/workflows/benchmark.yml b/.lightning/workflows/benchmark.yml
@@ -22,12 +22,13 @@ env:
   RUN_ONLY_CUDA_TESTS: "1"
 
 run: |
-  # Install Python and UV
-  apt-get update -qq --fix-missing
+  echo "Installing dependencies"
+  apt-get update -qq --fix-missing -o=Dpkg::Use-Pty=0 &> /dev/null
   apt-get install -q -y software-properties-common curl
-  # Add deadsnakes PPA for newer Python versions if needed
+  echo "Add deadsnakes PPA for newer Python versions if needed"
   add-apt-repository ppa:deadsnakes/ppa -y
-  apt-get update -qq --fix-missing
+  apt-get update -qq --fix-missing -o=Dpkg::Use-Pty=0 &> /dev/null
+  echo "Install Python ${python_version} and other dependencies"
   apt-get install -q -y --no-install-recommends --allow-downgrades --allow-change-held-packages \
         build-essential \
         pkg-config \
@@ -36,23 +37,25 @@ run: |
         libopenmpi-dev \
         openmpi-bin
 
+  echo "Install Python ${python_version} and UV"
   apt-get install -y python${python_version} python${python_version}-venv python${python_version}-dev
   ln -sf /usr/bin/python${python_version} /usr/bin/python
   curl -LsSf https://astral.sh/uv/install.sh | sh
 
-  # Source the environment and ensure UV is in PATH
+  echo "Source the environment and ensure UV is in PATH"
   [ -f "$HOME/.local/bin/env" ] && . "$HOME/.local/bin/env"
   export PATH="$HOME/.local/bin:$PATH"
   source $HOME/.cargo/env 2>/dev/null || true
   export PATH="$HOME/.cargo/bin:$PATH"
 
-  # Verify UV installation
+  echo "Verify UV installation"
   command -v uv || (echo "UV not found in PATH" && exit 1)
   # Create and activate a local uv virtual environment
   uv venv .venv -p "/usr/bin/python${python_version}" || uv venv .venv -p "python${python_version}" || uv venv .venv
   . .venv/bin/activate
   hash -r
 
+  echo "Show system information"
   whereis nvidia
   nvidia-smi
   python --version
@@ -68,26 +71,26 @@ run: |
   CUDA_VERSION_MM="${CUDA_VERSION_M_M//./}"  # "126"
   export UV_TORCH_BACKEND=cu${CUDA_VERSION_MM}
 
-  # Adjust tests
+  echo "Adjust tests"
   uv pip install -q -r .actions/requirements.txt
   python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
     --source_import="lightning.fabric,lightning.pytorch" \
     --target_import="lightning_fabric,pytorch_lightning"
 
-  # Install package
+  echo "Install package"
   uv pip install ".[dev]"
 
   # Env details
   python requirements/collect_env_details.py
   python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
 
   cd tests/
-  # Testing: benchmarks
+  echo "Testing: benchmarks"
   export PL_RUNNING_BENCHMARKS=1
   python -m pytest parity_${PACKAGE_NAME} -v --durations=0
   export PL_RUNNING_BENCHMARKS=0
 
-  # Testing: fabric standalone tasks
+  echo "Testing: fabric standalone tasks"
   export PL_RUN_STANDALONE_TESTS=1
   if [ "${PACKAGE_NAME}" == "fabric" ]; then
     cd parity_fabric/
diff --git a/.lightning/workflows/fabric.yml b/.lightning/workflows/fabric.yml
@@ -12,7 +12,7 @@ parametrize:
     - image: "nvidia/cuda:12.1.1-runtime-ubuntu22.04"
       PACKAGE_NAME: "fabric"
       python_version: "3.10"
-      machine: "A100_X_2"
+      machine: "L4_X_2"
     - image: "nvidia/cuda:12.6.3-runtime-ubuntu22.04"
       PACKAGE_NAME: "fabric"
       python_version: "3.12"
@@ -37,12 +37,13 @@ env:
   RUN_ONLY_CUDA_TESTS: "1"
 
 run: |
-  # Install Python and UV
-  apt-get update -qq --fix-missing
+  echo "Installing dependencies"
+  apt-get update -qq --fix-missing -o=Dpkg::Use-Pty=0 &> /dev/null
   apt-get install -q -y software-properties-common curl
-  # Add deadsnakes PPA for newer Python versions if needed
+  echo "Add deadsnakes PPA for newer Python versions if needed"
   add-apt-repository ppa:deadsnakes/ppa -y
-  apt-get update -qq --fix-missing
+  apt-get update -qq --fix-missing -o=Dpkg::Use-Pty=0 &> /dev/null
+  echo "Install Python ${python_version} and other dependencies"
   apt-get install -q -y --no-install-recommends --allow-downgrades --allow-change-held-packages \
         build-essential \
         pkg-config \
@@ -54,23 +55,25 @@ run: |
         libnccl2 \
         libnccl-dev
 
+  echo "Install Python ${python_version} and UV"
   apt-get install -y python${python_version} python${python_version}-venv python${python_version}-dev
   ln -sf /usr/bin/python${python_version} /usr/bin/python
   curl -LsSf https://astral.sh/uv/install.sh | sh
 
-  # Source the environment and ensure UV is in PATH
+  echo "Source the environment and ensure UV is in PATH"
   [ -f "$HOME/.local/bin/env" ] && . "$HOME/.local/bin/env"
   export PATH="$HOME/.local/bin:$PATH"
   source $HOME/.cargo/env 2>/dev/null || true
   export PATH="$HOME/.cargo/bin:$PATH"
 
-  # Verify UV installation
+  echo "Verify UV installation"
   command -v uv || (echo "UV not found in PATH" && exit 1)
   # Create and activate a local uv virtual environment
   uv venv .venv -p "/usr/bin/python${python_version}" || uv venv .venv -p "python${python_version}" || uv venv .venv
   . .venv/bin/activate
   hash -r
 
+  echo "Show system information"
   whereis nvidia
   nvidia-smi
   python --version
@@ -98,7 +101,7 @@ run: |
     uv pip install "cython<3.0" wheel  # for compatibility
   fi
 
-  # install the base so we can adjust other packages
+  echo "Install the base so we can adjust other packages"
   uv pip install .
   echo "Adjust torch versions in requirements files"
   PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
@@ -119,6 +122,7 @@ run: |
       --target_import="lightning_fabric"
   fi
 
+  echo "Install package with [${PACKAGE_NAME}] extras"
   extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
   uv pip install ".[${extra}dev]" --upgrade
 
diff --git a/.lightning/workflows/pytorch.yml b/.lightning/workflows/pytorch.yml
@@ -12,7 +12,7 @@ parametrize:
     - image: "nvidia/cuda:12.1.1-runtime-ubuntu22.04"
       PACKAGE_NAME: "pytorch"
       python_version: "3.10"
-      machine: "A100_X_2"
+      machine: "L4_X_2"
     - image: "nvidia/cuda:12.6.3-runtime-ubuntu22.04"
       PACKAGE_NAME: "pytorch"
       python_version: "3.12"
@@ -37,12 +37,13 @@ env:
   RUN_ONLY_CUDA_TESTS: "1"
 
 run: |
-  # Install Python and UV
-  apt-get update -qq --fix-missing
+  echo "Installing dependencies"
+  apt-get update -qq --fix-missing -o=Dpkg::Use-Pty=0 &> /dev/null
   apt-get install -q -y software-properties-common curl
-  # Add deadsnakes PPA for newer Python versions if needed
+  echo "Add deadsnakes PPA for newer Python versions if needed"
   add-apt-repository ppa:deadsnakes/ppa -y
-  apt-get update -qq --fix-missing
+  apt-get update -qq --fix-missing -o=Dpkg::Use-Pty=0 &> /dev/null
+  echo "Install Python ${python_version} and other dependencies"
   apt-get install -q -y --no-install-recommends --allow-downgrades --allow-change-held-packages \
         build-essential \
         pkg-config \
@@ -54,23 +55,25 @@ run: |
         libnccl2 \
         libnccl-dev
 
+  echo "Install Python ${python_version} and UV"
   apt-get install -y python${python_version} python${python_version}-venv python${python_version}-dev
   ln -sf /usr/bin/python${python_version} /usr/bin/python
   curl -LsSf https://astral.sh/uv/install.sh | sh
 
-  # Source the environment and ensure UV is in PATH
+  echo "Source the environment and ensure UV is in PATH"
   [ -f "$HOME/.local/bin/env" ] && . "$HOME/.local/bin/env"
   export PATH="$HOME/.local/bin:$PATH"
   source $HOME/.cargo/env 2>/dev/null || true
   export PATH="$HOME/.cargo/bin:$PATH"
 
-  # Verify UV installation
+  echo "Verify UV installation"
   command -v uv || (echo "UV not found in PATH" && exit 1)
   # Create and activate a local uv virtual environment
   uv venv .venv -p "/usr/bin/python${python_version}" || uv venv .venv -p "python${python_version}" || uv venv .venv
   . .venv/bin/activate
   hash -r
 
+  echo "Show system information"
   whereis nvidia
   nvidia-smi
   python --version
@@ -98,7 +101,7 @@ run: |
     uv pip install "cython<3.0" wheel  # for compatibility
   fi
 
-  # install the base so we can adjust other packages
+  echo "Install the base so we can adjust other packages"
   uv pip install .
   echo "Adjust torch versions in requirements files"
   PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
@@ -119,9 +122,11 @@ run: |
       --target_import="lightning_fabric,pytorch_lightning"
   fi
 
+  echo "Install package"
   extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
   uv pip install -e ".[${extra}dev]" --upgrade
 
+  echo "Ensure only a single package is installed"
   if [ "${PACKAGE_NAME}" == "pytorch" ]; then
     echo "uninstall lightning to have just single package"
     uv pip uninstall lightning
diff --git a/docs/source-pytorch/common/checkpointing_basic.rst b/docs/source-pytorch/common/checkpointing_basic.rst
@@ -58,12 +58,45 @@ Lightning automatically saves a checkpoint for you in your current working direc
     # simply by using the Trainer you get automatic checkpointing
     trainer = Trainer()
 
-To change the checkpoint path use the `default_root_dir` argument:
+
+Checkpoint save location
+========================
+
+The location where checkpoints are saved depends on whether you have configured a logger:
+
+**Without a logger**, checkpoints are saved to the ``default_root_dir``:
+
+.. code-block:: python
+
+    # saves checkpoints to 'some/path/checkpoints/'
+    trainer = Trainer(default_root_dir="some/path/", logger=False)
+
+**With a logger**, checkpoints are saved to the logger's directory, **not** to ``default_root_dir``:
 
 .. code-block:: python
 
-    # saves checkpoints to 'some/path/' at every epoch end
-    trainer = Trainer(default_root_dir="some/path/")
+    from lightning.pytorch.loggers import CSVLogger
+
+    # checkpoints will be saved to 'logs/my_experiment/version_0/checkpoints/'
+    # NOT to 'some/path/checkpoints/'
+    trainer = Trainer(
+        default_root_dir="some/path/",  # This will be ignored for checkpoints!
+        logger=CSVLogger("logs", "my_experiment")
+    )
+
+To explicitly control the checkpoint location when using a logger, use the
+:class:`~lightning.pytorch.callbacks.ModelCheckpoint` callback:
+
+.. code-block:: python
+
+    from lightning.pytorch.callbacks import ModelCheckpoint
+
+    # explicitly set checkpoint directory
+    checkpoint_callback = ModelCheckpoint(dirpath="my/custom/checkpoint/path/")
+    trainer = Trainer(
+        logger=CSVLogger("logs", "my_experiment"),
+        callbacks=[checkpoint_callback]
+    )
 
 
 ----
diff --git a/docs/source-pytorch/common/evaluation_intermediate.rst b/docs/source-pytorch/common/evaluation_intermediate.rst
diff --git a/requirements/typing.txt b/requirements/typing.txt

Original file line number	Diff line number	Diff line change
`@@ -23,8 +23,8 @@`
`23`	`23`	`}`
`24`	`24`	`}`
`25`	`25`	`],`
`26`		`- "timeout": "20s",`
	`26`	`+ "timeout": "30s",`
`27`	`27`	`"retryOn429": true,`
`28`		`- "retryCount": 5,`
`29`		`- "fallbackRetryDelay": "20s"`
	`28`	`+ "retryCount": 10,`
	`29`	`+ "fallbackRetryDelay": "10s"`
`30`	`30`	`}`