diff --git a/.github/workflows/template.hbs b/.github/workflows/template.hbs
new file mode 100644
index 0000000..78d9486
--- /dev/null
+++ b/.github/workflows/template.hbs
@@ -0,0 +1 @@
+[❌ {{github.job}} failed]({{github.buildUrl}})
diff --git a/.github/workflows/test-repositories.yml b/.github/workflows/test-repositories.yml
index 8b29fff..6316682 100644
--- a/.github/workflows/test-repositories.yml
+++ b/.github/workflows/test-repositories.yml
@@ -8,13 +8,7 @@
 
 name: Generate OpenAPI in Repositories
 
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-      - master
-      - stable*
+on: pull_request
 
 permissions:
   contents: read
@@ -43,7 +37,7 @@ jobs:
           - "nextcloud/tables"
           - "nextcloud/terms_of_service"
 
-    name: openapi-repositories
+    name: ${{ matrix.repositories }}
 
     steps:
       - name: Checkout
@@ -85,19 +79,14 @@ jobs:
       - name: Show changes of the API for assistance
         working-directory: temp-repository/
         run: |
-            git status
-            git --no-pager diff
+          git status
+          git --no-pager diff
 
-  summary:
-    permissions:
-      contents: none
-    runs-on: ubuntu-latest
-    needs: openapi-repositories
-
-    if: always()
-
-    name: openapi-repositories-summary
-
-    steps:
-      - name: Summary status
-        run: if ${{ needs.openapi-repositories.result != 'success' && needs.openapi-repositories.result != 'skipped' }}; then exit 1; fi
+      - name: Generate Markdown Summary
+        uses: Ma11hewThomas/github-markdown-builder@v1
+        if: failure()
+        with:
+          template-file-path: './.github/workflows/template.hbs'
+          pull-request: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.COMMAND_BOT_PAT }}
diff --git a/REUSE.toml b/REUSE.toml
index d1be7ce..f16f519 100644
--- a/REUSE.toml
+++ b/REUSE.toml
@@ -22,3 +22,9 @@ path = ["tests/openapi-administration.json", "tests/openapi-ex_app.json", "tests
 precedence = "aggregate"
 SPDX-FileCopyrightText = "2024 Nextcloud GmbH and Nextcloud contributors"
 SPDX-License-Identifier = "AGPL-3.0-or-later"
+
+[[annotations]]
+path = [".github/workflows/template.hbs"]
+precedence = "aggregate"
+SPDX-FileCopyrightText = "2025 Nextcloud GmbH and Nextcloud contributors"
+SPDX-License-Identifier = "AGPL-3.0-or-later"