Release vn8.1

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 # PR Summary
 
-Sci/Tech Reviewer: <!-- SR id, filled when known -->
+Sci/Tech Reviewer: <!-- SR id, filled by author when ready for review (e.g. @octocat) -->
 Code Reviewer: <!-- CR id, filled by SSD -->
 
 <!-- To be completed by the developer -->

.github/scripts/create_jules_version_docs.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------
+# (C) Crown copyright Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+# -----------------------------------------------------------------------------
+"""
+Build versioned Jules docs
+Expects gh to be available
+Expects to be run in an environment that can build the Jules docs
+"""
+
+import argparse
+import json
+import logging
+import subprocess
+import shutil
+import time
+from pathlib import Path
+from shlex import split
+
+logger = logging.getLogger(__name__)
+
+
+def run_command(
+    command: str,
+    cwd: Path = None,
+) -> subprocess.CompletedProcess:
+    """
+    Run a subprocess command and return the result object
+    Inputs:
+        - command, str with command to run
+    Outputs:
+        - result object from subprocess.run
+    """
+
+    logger.debug(f"Running Command: '{command}'")
+    command = split(command)
+    result = subprocess.run(
+        command,
+        capture_output=True,
+        text=True,
+        timeout=300,
+        shell=False,
+        check=False,
+        cwd=cwd,
+    )
+    if result.returncode:
+        print(result.stdout, end="\n\n\n")
+        raise RuntimeError(
+            f"[FAIL] Issue found running command {command}\n\n{result.stderr}"
+        )
+    return result
+
+
+def get_releases() -> list[str]:
+    """
+    Use gh to get a list of releases in Jules
+    remove git_migration release and sort by version number
+    """
+    result = run_command("gh release list -R MetOffice/jules --json tagName,createdAt")
+    releases = json.loads(result.stdout)
+    time_format = "%Y-%m-%dT%H:%M:%SZ"
+    releases = sorted(
+        releases,
+        reverse=True,
+        key=lambda x: time.mktime(time.strptime(x["createdAt"], time_format)),
+    )
+    releases = [x["tagName"] for x in releases]
+    # Remove git_migration release as docs didn't exist at that point
+    releases.remove("git_migration")
+    logger.info(f"Releases: {releases}")
+    return releases
+
+
+def build_jules_docs(
+    ref: str, name: str, jules: Path, artifact: Path, output: Path, force: bool = False
+) -> None:
+    """
+    Checkout a git ref and build Jules docs at that ref
+    Copy built html to the output directory with subdirectory "name"
+    """
+
+    if not force and artifact:
+        artifact_version = artifact / name
+        if artifact_version.exists():
+            logger.info(f"Copying from artifact for version {name}")
+            shutil.copytree(artifact_version, output / name)
+            return
+
+    # Checkout git ref
+    run_command(f"git -C {jules} checkout {ref}")
+
+    # Build Jules Docs
+    logger.info(f"Building docs for ref {ref}")
+    run_command("make clean html", cwd=jules / "doc")
+
+    # Copy Built docs to output
+    logger.info(f"Copying built docs to output for ref {ref}")
+    shutil.copytree(jules / "doc" / "build" / "html", output / name)
+
+
+def edit_index(index_file_path: Path, releases: list[str]) -> None:
+    """
+    Edit the index.html to point at the different releases
+    """
+    logger.info("Updating template index file")
+
+    releases.append("latest")
+
+    lines = index_file_path.read_text()
+    lines = lines.split("\n")
+
+    for i, line in enumerate(lines):
+        if "LOCATION FOR AUTOMATIC UPDATING" in line:
+            index = i + 1
+
+    for release in releases:
+        vn = f'                <li><a href="{release}/index.html">{release}</a></li>'
+        lines.insert(index, vn)
+
+    with open(index_file_path, "w") as f:
+        for line in lines:
+            f.write(f"{line}\n")
+
+
+def parse_args() -> argparse.Namespace:
+    """
+    Parse Command line arguments
+    """
+
+    parser = argparse.ArgumentParser(description="Build versioned jules docs")
+    parser.add_argument(
+        "-j",
+        "--jules",
+        default=Path("."),
+        type=Path,
+        help="Path to Jules clone (toplevel). Needs to have the full history to build "
+        "all Jules versions",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        default=Path().home() / "jules_docs",
+        type=Path,
+        help="Output directory for builds to be stored in",
+    )
+    parser.add_argument(
+        "-a",
+        "--artifact",
+        default=None,
+        type=Path,
+        help="Path to an existing build directory to copy old versions from. Expected "
+        "to be from a github artifact",
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        default="WARNING",
+        help="Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    """
+    Main Function
+    """
+
+    args = parse_args()
+    logging.basicConfig(level=args.log, format="%(levelname)s: %(message)s")
+
+    # Ensue output directory is empty and then move template html file
+    if args.output.exists():
+        logger.warning("Removing existing output directory")
+        shutil.rmtree(args.output)
+    args.output.mkdir(parents=True)
+    index_file_path = args.output / "index.html"
+    shutil.copy(
+        Path(__file__).parent.resolve() / "template_index.html", index_file_path
+    )
+
+    # Get a list of releases
+    releases = get_releases()
+
+    # Build Docs for latest using main branch
+    build_jules_docs("main", "latest", args.jules, args.artifact, args.output, True)
+
+    # Build or Copy docs for releases
+    for release in releases:
+        build_jules_docs(release, release, args.jules, args.artifact, args.output)
+
+    # Edit template index.html
+    edit_index(index_file_path, releases)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/template_index.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Joint UK Land Environment Simulator Documentation</title>
+        <style>
+            body { font-family : sans-serif; padding-left : 20px; }
+            .content { padding-left : 20px; }
+        </style>
+    </head>
+
+    <body>
+        <h1>Joint UK Land Environment Simulator (JULES) Documentation</h1>
+
+        <div class="content">
+            <h2>User Guide</h2>
+            <ul>
+                <!--LOCATION FOR AUTOMATIC UPDATING-->
+            </ul>
+        </div>
+    </body>
+</html>

.github/workflows/track-review-project.yaml

@@ -0,0 +1,17 @@
+name: Track Review Project
+
+on:
+  workflow_run:
+    workflows: [Trigger Review Project]
+    types:
+      - completed
+
+permissions:
+  actions: read
+  contents: read
+  pull-requests: write
+
+jobs:
+  track_review_project:
+    uses: MetOffice/growss/.github/workflows/track-review-project.yaml@main
+    secrets: inherit

.github/workflows/trigger-project-workflow.yaml

@@ -0,0 +1,17 @@
+name: Trigger Review Project
+
+on:
+  pull_request_target:
+    types: ["opened", "synchronize", "reopened", "edited", "review_requested", "review_request_removed", "closed"]
+  pull_request_review:
+  pull_request_review_comment:
+
+permissions:
+  actions: read
+  contents: read
+  pull-requests: write
+
+jobs:
+  trigger_project_workflow:
+    uses: MetOffice/growss/.github/workflows/trigger-project-workflow.yaml@main
+    secrets: inherit

.github/workflows/user-guide.yaml

@@ -8,6 +8,8 @@
 name: User Guide
 
 on:
+  release:
+    types: [published]
   push:
     branches:
       - main
@@ -19,6 +21,7 @@ on:
   workflow_dispatch:
 
 permissions:
+  actions: read
   contents: read
   pages: write
   id-token: write
@@ -27,13 +30,31 @@ jobs:
   build-and-deploy:
     runs-on: ubuntu-latest
     timeout-minutes: 5
+    outputs:
+      should-deploy: ${{ steps.check-deploy.outputs.should-deploy }}
     env:
       PYTHON_VRSN: '3.13'
-      VENV_PATH: 'doc/.venv'
+      JULES_PATH: jules
+      ARTIFACT_PATH: artifact
+      OUTPUT_PATH: jules_docs
+      VENV_PATH: jules/doc/.venv
 
     steps:
+      - name: Check deployment conditions
+        id: check-deploy
+        run: |
+          if [[ "${{ github.repository }}" == "MetOffice/jules" && "${{ github.ref_name }}" == "main" && ("${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "merge_group" || "${{ github.event_name }}" == "release" || "${{ github.event_name }}" == "workflow_dispatch") ]]; then
+            echo "should-deploy=true" >> $GITHUB_OUTPUT
+          else
+            echo "should-deploy=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Checkout repository
         uses: actions/checkout@v6
+        with:
+          path: ${{ env.JULES_PATH }}
+          # Get all refs to build historic versions
+          fetch-depth: 0
 
       - name: Setup uv with Python ${{ env.PYTHON_VRSN }}
         uses: astral-sh/setup-uv@v7
@@ -50,34 +71,66 @@ jobs:
             ${{ runner.os }}-uv-
 
       - name: Install dependencies
-        working-directory: ./doc
+        working-directory: ${{ env.JULES_PATH }}/doc
         run: uv sync
 
       - name: Lint Sphinx docs
-        working-directory: ./doc
+        working-directory: ${{ env.JULES_PATH }}/doc
         run: uv run sphinx-lint source
 
-      - name: Build HTML docs
-        working-directory: ./doc
+      - name: Build HTML docs for testing
+        if: steps.check-deploy.outputs.should-deploy == 'false'
+        working-directory: ${{ env.JULES_PATH }}/doc
         run: uv run make clean html
 
+      - name: Download Artifact
+        if: steps.check-deploy.outputs.should-deploy == 'true'
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Get workflow id of last run to upload an artifact
+          run_id=$(gh run list -R MetOffice/jules -b main -w user-guide.yaml --limit 2 --json databaseId | jq -r '.[1].databaseId')
+
+          # Download tarred artifact
+          gh run download -R MetOffice/jules $run_id -n github-pages
+
+          # Create output directory and untar
+          tar -xf artifact.tar --one-top-level=artifact
+
+
+      - name: Build HTML docs for deployment
+        if: steps.check-deploy.outputs.should-deploy == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          source ${{ env.VENV_PATH }}/bin/activate
+          if [ -d ${{ env.ARTIFACT_PATH }} ]; then
+            echo "Using Artifact"
+            python3 ${{ env.JULES_PATH }}/.github/scripts/create_jules_version_docs.py -j ${{ env.JULES_PATH }} -a ${{ env.ARTIFACT_PATH }} -o ${{ env.OUTPUT_PATH }} -l INFO
+          else
+            echo "Not Using Artifact"
+            python3 ${{ env.JULES_PATH }}/.github/scripts/create_jules_version_docs.py -j jules -o ${{ env.OUTPUT_PATH }} -l INFO
+          fi
+
       - name: Minimize uv cache
         run: uv cache prune --ci
 
       # -- Deploy to GitHub Pages only on push to upstream main
       - name: Setup GitHub Pages
-        if: ${{ github.ref_name == 'main' && (github.event_name == 'push' || github.event_name == 'merge_group') }}
+        if: steps.check-deploy.outputs.should-deploy == 'true'
         uses: actions/configure-pages@v5
 
       - name: Upload artifact to GitHub Pages
-        if: ${{ github.ref_name == 'main' && (github.event_name == 'push' || github.event_name == 'merge_group') }}
+        if: steps.check-deploy.outputs.should-deploy == 'true'
         uses: actions/upload-pages-artifact@v4
         with:
           name: github-pages
-          path: doc/build/html
-          retention-days: 1
+          path: ${{ env.OUTPUT_PATH }}
+          # Use maximum retention days to avoid rebuilding all versions if possible
+          retention-days: 90
 
       - name: Deploy to GitHub Pages
         id: deployment
-        if: ${{ github.ref_name == 'main' && (github.event_name == 'push' || github.event_name == 'merge_group') }}
+        if: steps.check-deploy.outputs.should-deploy == 'true'
         uses: actions/deploy-pages@v4