Fix Windows UTF-8 encoding errors across entire backend (251 instances)

.github/workflows/beta-release.yml

@@ -97,13 +97,21 @@ jobs:
       - name: Install Rust toolchain (for building native Python packages)
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Cache pip wheel cache (for compiled packages like real_ladybug)
+        uses: actions/cache@v4
+        with:
+          path: ~/Library/Caches/pip
+          key: pip-wheel-${{ runner.os }}-x64-${{ hashFiles('apps/backend/requirements.txt') }}
+          restore-keys: |
+            pip-wheel-${{ runner.os }}-x64-
+
       - name: Cache bundled Python
         uses: actions/cache@v4
         with:
           path: apps/frontend/python-runtime
-          key: python-bundle-${{ runner.os }}-x64-3.12.8-rust
+          key: python-bundle-${{ runner.os }}-x64-3.12.8-rust-${{ hashFiles('apps/backend/requirements.txt') }}
           restore-keys: |
-            python-bundle-${{ runner.os }}-x64-
+            python-bundle-${{ runner.os }}-x64-3.12.8-rust-
 
       - name: Build application
         run: cd apps/frontend && npm run build
@@ -181,13 +189,21 @@ jobs:
       - name: Install dependencies
         run: cd apps/frontend && npm ci
 
+      - name: Cache pip wheel cache
+        uses: actions/cache@v4
+        with:
+          path: ~/Library/Caches/pip
+          key: pip-wheel-${{ runner.os }}-arm64-${{ hashFiles('apps/backend/requirements.txt') }}
+          restore-keys: |
+            pip-wheel-${{ runner.os }}-arm64-
+
       - name: Cache bundled Python
         uses: actions/cache@v4
         with:
           path: apps/frontend/python-runtime
-          key: python-bundle-${{ runner.os }}-arm64-3.12.8
+          key: python-bundle-${{ runner.os }}-arm64-3.12.8-${{ hashFiles('apps/backend/requirements.txt') }}
           restore-keys: |
-            python-bundle-${{ runner.os }}-arm64-
+            python-bundle-${{ runner.os }}-arm64-3.12.8-
 
       - name: Build application
         run: cd apps/frontend && npm run build
@@ -265,13 +281,21 @@ jobs:
       - name: Install dependencies
         run: cd apps/frontend && npm ci
 
+      - name: Cache pip wheel cache
+        uses: actions/cache@v4
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: pip-wheel-${{ runner.os }}-x64-${{ hashFiles('apps/backend/requirements.txt') }}
+          restore-keys: |
+            pip-wheel-${{ runner.os }}-x64-
+
       - name: Cache bundled Python
         uses: actions/cache@v4
         with:
           path: apps/frontend/python-runtime
-          key: python-bundle-${{ runner.os }}-x64-3.12.8
+          key: python-bundle-${{ runner.os }}-x64-3.12.8-${{ hashFiles('apps/backend/requirements.txt') }}
           restore-keys: |
-            python-bundle-${{ runner.os }}-x64-
+            python-bundle-${{ runner.os }}-x64-3.12.8-
 
       - name: Build application
         run: cd apps/frontend && npm run build
@@ -335,13 +359,21 @@ jobs:
           flatpak install -y --user flathub org.freedesktop.Platform//25.08 org.freedesktop.Sdk//25.08
           flatpak install -y --user flathub org.electronjs.Electron2.BaseApp//25.08
 
+      - name: Cache pip wheel cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-wheel-${{ runner.os }}-x64-${{ hashFiles('apps/backend/requirements.txt') }}
+          restore-keys: |
+            pip-wheel-${{ runner.os }}-x64-
+
       - name: Cache bundled Python
         uses: actions/cache@v4
         with:
           path: apps/frontend/python-runtime
-          key: python-bundle-${{ runner.os }}-x64-3.12.8
+          key: python-bundle-${{ runner.os }}-x64-3.12.8-${{ hashFiles('apps/backend/requirements.txt') }}
           restore-keys: |
-            python-bundle-${{ runner.os }}-x64-
+            python-bundle-${{ runner.os }}-x64-3.12.8-
 
       - name: Build application
         run: cd apps/frontend && npm run build

.github/workflows/prepare-release.yml

@@ -1,8 +1,10 @@
 name: Prepare Release
 
 # Triggers when code is pushed to main (e.g., merging develop → main)
-# If package.json version is newer than the latest tag, creates a new tag
-# which then triggers the release.yml workflow
+# If package.json version is newer than the latest tag:
+# 1. Validates CHANGELOG.md has an entry for this version (FAILS if missing)
+# 2. Extracts release notes from CHANGELOG.md
+# 3. Creates a new tag which triggers release.yml
 
 on:
   push:
@@ -67,8 +69,122 @@ jobs:
             echo "⏭️ No release needed (package version not newer than latest tag)"
           fi
 
-      - name: Create and push tag
+      # CRITICAL: Validate CHANGELOG.md has entry for this version BEFORE creating tag
+      - name: Validate and extract changelog
         if: steps.check.outputs.should_release == 'true'
+        id: changelog
+        run: |
+          VERSION="${{ steps.check.outputs.new_version }}"
+          CHANGELOG_FILE="CHANGELOG.md"
+
+          echo "🔍 Validating CHANGELOG.md for version $VERSION..."
+
+          if [ ! -f "$CHANGELOG_FILE" ]; then
+            echo "::error::CHANGELOG.md not found! Please create CHANGELOG.md with release notes."
+            exit 1
+          fi
+
+          # Extract changelog section for this version
+          # Looks for "## X.Y.Z" header and captures until next "## " or "---" or end
+          CHANGELOG_CONTENT=$(awk -v ver="$VERSION" '
+            BEGIN { found=0; content="" }
+            /^## / {
+              if (found) exit
+              # Match version at start of header (e.g., "## 2.7.3 -" or "## 2.7.3")
+              if ($2 == ver || $2 ~ "^"ver"[[:space:]]*-") {
+                found=1
+                # Skip the header line itself, we will add our own
+                next
+              }
+            }
+            /^---$/ { if (found) exit }
+            found { content = content $0 "\n" }
+            END {
+              if (!found) {
+                print "NOT_FOUND"
+                exit 1
+              }
+              # Trim leading/trailing whitespace
+              gsub(/^[[:space:]]+|[[:space:]]+$/, "", content)
+              print content
+            }
+          ' "$CHANGELOG_FILE")
+
+          if [ "$CHANGELOG_CONTENT" = "NOT_FOUND" ] || [ -z "$CHANGELOG_CONTENT" ]; then
+            echo ""
+            echo "::error::═══════════════════════════════════════════════════════════════════════"
+            echo "::error::  CHANGELOG VALIDATION FAILED"
+            echo "::error::═══════════════════════════════════════════════════════════════════════"
+            echo "::error::"
+            echo "::error::  Version $VERSION not found in CHANGELOG.md!"
+            echo "::error::"
+            echo "::error::  Before releasing, please update CHANGELOG.md with an entry like:"
+            echo "::error::"
+            echo "::error::    ## $VERSION - Your Release Title"
+            echo "::error::"
+            echo "::error::    ### ✨ New Features"
+            echo "::error::    - Feature description"
+            echo "::error::"
+            echo "::error::    ### 🐛 Bug Fixes"
+            echo "::error::    - Fix description"
+            echo "::error::"
+            echo "::error::═══════════════════════════════════════════════════════════════════════"
+            echo ""
+
+            # Also add to job summary for visibility
+            echo "## ❌ Release Blocked: Missing Changelog" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "Version **$VERSION** was not found in CHANGELOG.md." >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "### How to fix:" >> $GITHUB_STEP_SUMMARY
+            echo "1. Update CHANGELOG.md with release notes for version $VERSION" >> $GITHUB_STEP_SUMMARY
+            echo "2. Commit and push the changes" >> $GITHUB_STEP_SUMMARY
+            echo "3. The release will automatically retry" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "### Expected format:" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`markdown" >> $GITHUB_STEP_SUMMARY
+            echo "## $VERSION - Release Title" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "### ✨ New Features" >> $GITHUB_STEP_SUMMARY
+            echo "- Feature description" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "### 🐛 Bug Fixes" >> $GITHUB_STEP_SUMMARY
+            echo "- Fix description" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+            exit 1
+          fi
+
+          echo "✅ Found changelog entry for version $VERSION"
+          echo ""
+          echo "--- Extracted Release Notes ---"
+          echo "$CHANGELOG_CONTENT"
+          echo "--- End Release Notes ---"
+
+          # Save changelog to file for artifact upload
+          echo "$CHANGELOG_CONTENT" > changelog-extract.md
+
+          # Also save to output (for short changelogs)
+          # Using heredoc for multiline output
+          {
+            echo "content<<CHANGELOG_EOF"
+            echo "$CHANGELOG_CONTENT"
+            echo "CHANGELOG_EOF"
+          } >> $GITHUB_OUTPUT
+
+          echo "changelog_valid=true" >> $GITHUB_OUTPUT
+
+      # Upload changelog as artifact for release.yml to use
+      - name: Upload changelog artifact
+        if: steps.check.outputs.should_release == 'true' && steps.changelog.outputs.changelog_valid == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: changelog-${{ steps.check.outputs.new_version }}
+          path: changelog-extract.md
+          retention-days: 1
+
+      - name: Create and push tag
+        if: steps.check.outputs.should_release == 'true' && steps.changelog.outputs.changelog_valid == 'true'
         run: |
           VERSION="${{ steps.check.outputs.new_version }}"
           TAG="v$VERSION"
@@ -85,17 +201,19 @@ jobs:
 
       - name: Summary
         run: |
-          if [ "${{ steps.check.outputs.should_release }}" = "true" ]; then
+          if [ "${{ steps.check.outputs.should_release }}" = "true" ] && [ "${{ steps.changelog.outputs.changelog_valid }}" = "true" ]; then
             echo "## 🚀 Release Triggered" >> $GITHUB_STEP_SUMMARY
             echo "" >> $GITHUB_STEP_SUMMARY
             echo "**Version:** v${{ steps.check.outputs.new_version }}" >> $GITHUB_STEP_SUMMARY
             echo "" >> $GITHUB_STEP_SUMMARY
+            echo "✅ Changelog validated and extracted from CHANGELOG.md" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
             echo "The release workflow has been triggered and will:" >> $GITHUB_STEP_SUMMARY
             echo "1. Build binaries for all platforms" >> $GITHUB_STEP_SUMMARY
-            echo "2. Generate changelog from PRs" >> $GITHUB_STEP_SUMMARY
+            echo "2. Use changelog from CHANGELOG.md" >> $GITHUB_STEP_SUMMARY
             echo "3. Create GitHub release" >> $GITHUB_STEP_SUMMARY
             echo "4. Update README with new version" >> $GITHUB_STEP_SUMMARY
-          else
+          elif [ "${{ steps.check.outputs.should_release }}" = "false" ]; then
             echo "## ⏭️ No Release Needed" >> $GITHUB_STEP_SUMMARY
             echo "" >> $GITHUB_STEP_SUMMARY
             echo "**Package version:** ${{ steps.package.outputs.version }}" >> $GITHUB_STEP_SUMMARY