Fix eval workflows (#1228)

* Fix eval workflows

* Address review: secure two-phase PR comment & byte-based truncation

- skill-check.yml: Revert to pull_request trigger (read-only token).
  Remove PR comment posting; upload results as artifact instead.
- skill-check-comment.yml: New workflow_run-triggered workflow that
  downloads the artifact and posts/updates the PR comment with
  write permissions, without ever checking out PR code.
- skill-quality-report.yml: Replace character-based truncation with
  byte-based (Buffer.byteLength) limit. Shrink <details> sections
  structurally before falling back to hard byte-trim, keeping
  markdown rendering intact.

.github/workflows/skill-check-comment.yml

@@ -0,0 +1,109 @@
+name: Skill Validator — PR Comment
+
+# Posts results from the "Skill Validator — PR Gate" workflow.
+# Runs with write permissions but never checks out PR code,
+# so it is safe for fork PRs.
+
+on:
+  workflow_run:
+    workflows: ["Skill Validator — PR Gate"]
+    types: [completed]
+
+permissions:
+  pull-requests: write
+  actions: read # needed to download artifacts
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download results artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: skill-validator-results
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+
+      - name: Post PR comment with results
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+
+            const total = parseInt(fs.readFileSync('total.txt', 'utf8').trim(), 10);
+            if (total === 0) {
+              console.log('No skills/agents were checked — skipping comment.');
+              return;
+            }
+
+            const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim(), 10);
+            const exitCode = fs.readFileSync('exit-code.txt', 'utf8').trim();
+            const skillCount = parseInt(fs.readFileSync('skill-count.txt', 'utf8').trim(), 10);
+            const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10);
+            const totalChecked = skillCount + agentCount;
+
+            const marker = '<!-- skill-validator-results -->';
+            const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
+
+            // Count errors, warnings, advisories from output
+            const errorCount = (output.match(/\bError\b/gi) || []).length;
+            const warningCount = (output.match(/\bWarning\b/gi) || []).length;
+            const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
+
+            let statusLine;
+            if (errorCount > 0) {
+              statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
+            } else if (warningCount > 0) {
+              statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
+            } else {
+              statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
+            }
+
+            const body = [
+              marker,
+              '## 🔍 Skill Validator Results',
+              '',
+              statusLine,
+              '',
+              '<details>',
+              '<summary>Full output</summary>',
+              '',
+              '```',
+              output,
+              '```',
+              '',
+              '</details>',
+              '',
+              exitCode !== '0'
+                ? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
+                : '',
+            ].join('\n');
+
+            // Find existing comment with our marker
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            });
+
+            const existing = comments.find(c => c.body.includes(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              console.log(`Updated existing comment ${existing.id}`);
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+              console.log('Created new PR comment');
+            }

.github/workflows/skill-check.yml

@@ -12,8 +12,6 @@ on:
 
 permissions:
   contents: read
-  pull-requests: write
-  issues: write
 
 jobs:
   skill-check:
@@ -135,82 +133,27 @@ jobs:
 
           echo "$OUTPUT"
 
-      # ── Post / update PR comment ──────────────────────────────────
+      # ── Upload results for the commenting workflow ────────────────
-      - name: Post PR comment with results
+      - name: Save metadata
-        if: steps.detect.outputs.total != '0'
+        if: always()
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        run: |
+          mkdir -p sv-results
+          echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt
+          echo "${{ steps.detect.outputs.total }}" > sv-results/total.txt
+          echo "${{ steps.detect.outputs.skill_count }}" > sv-results/skill-count.txt
+          echo "${{ steps.detect.outputs.agent_count }}" > sv-results/agent-count.txt
+          echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt
+          if [ -f sv-output.txt ]; then
+            cp sv-output.txt sv-results/sv-output.txt
+          fi
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
         with:
-          script: |
+          name: skill-validator-results
-            const fs = require('fs');
+          path: sv-results/
-
+          retention-days: 1
-            const marker = '<!-- skill-validator-results -->';
-            const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
-            const exitCode = '${{ steps.check.outputs.exit_code }}';
-            const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10);
-            const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10);
-            const totalChecked = skillCount + agentCount;
-
-            // Count errors, warnings, advisories from output
-            const errorCount = (output.match(/\bError\b/gi) || []).length;
-            const warningCount = (output.match(/\bWarning\b/gi) || []).length;
-            const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
-
-            let statusLine;
-            if (errorCount > 0) {
-              statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
-            } else if (warningCount > 0) {
-              statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
-            } else {
-              statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
-            }
-
-            const body = [
-              marker,
-              '## 🔍 Skill Validator Results',
-              '',
-              statusLine,
-              '',
-              '<details>',
-              '<summary>Full output</summary>',
-              '',
-              '```',
-              output,
-              '```',
-              '',
-              '</details>',
-              '',
-              exitCode !== '0'
-                ? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
-                : '',
-            ].join('\n');
-
-            // Find existing comment with our marker
-            const { data: comments } = await github.rest.issues.listComments({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              per_page: 100,
-            });
-
-            const existing = comments.find(c => c.body.includes(marker));
-
-            if (existing) {
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: existing.id,
-                body,
-              });
-              console.log(`Updated existing comment ${existing.id}`);
-            } else {
-              await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.issue.number,
-                body,
-              });
-              console.log('Created new PR comment');
-            }
 
       - name: Post skip notice if no skills changed
         if: steps.detect.outputs.total == '0'

.github/workflows/skill-quality-report.yml

@@ -248,7 +248,51 @@ jobs:
 
             core.setOutput('title', title);
             core.setOutput('body_file', 'report-body.md');
-            fs.writeFileSync('report-body.md', body);
+
+            // GitHub Issues/Discussions enforce a body size limit on the
+            // UTF-8 payload (~65536 bytes). Use byte-based limits and prefer
+            // shrinking verbose <details> sections to keep markdown valid.
+            const MAX_BODY_BYTES = 65000; // leave some margin
+
+            function shrinkDetailsSections(markdown) {
+              return markdown.replace(
+                /<details([\s\S]*?)>[\s\S]*?<\/details>/g,
+                (match, attrs) => {
+                  const placeholder = '\n<summary>Details truncated</summary>\n\n' +
+                    "> Full output was truncated to fit GitHub's body size limit. " +
+                    'See the workflow run for complete output.\n';
+                  return `<details${attrs}>${placeholder}</details>`;
+                }
+              );
+            }
+
+            function trimToByteLimit(str, maxBytes) {
+              const buf = Buffer.from(str, 'utf8');
+              if (buf.length <= maxBytes) return str;
+              // Slice bytes and decode, which safely handles multi-byte chars
+              return buf.slice(0, maxBytes).toString('utf8').replace(/\uFFFD$/, '');
+            }
+
+            const truncNote = '\n\n> **Note:** Output was truncated to fit GitHub\'s body size limit. See the [workflow run](https://github.com/' + context.repo.owner + '/' + context.repo.repo + '/actions/workflows/skill-quality-report.yml) for full output.\n';
+            const truncNoteBytes = Buffer.byteLength(truncNote, 'utf8');
+
+            let finalBody = body;
+
+            if (Buffer.byteLength(finalBody, 'utf8') > MAX_BODY_BYTES) {
+              // First try: collapse <details> sections to reduce size
+              finalBody = shrinkDetailsSections(finalBody);
+            }
+
+            if (Buffer.byteLength(finalBody, 'utf8') > MAX_BODY_BYTES) {
+              // Last resort: hard byte-trim + truncation note
+              finalBody = trimToByteLimit(finalBody, MAX_BODY_BYTES - truncNoteBytes);
+            }
+
+            if (Buffer.byteLength(finalBody, 'utf8') < Buffer.byteLength(body, 'utf8')) {
+              finalBody += truncNote;
+            }
+
+            fs.writeFileSync('report-body.md', finalBody);
 
       # ── Create Discussion (preferred) or Issue (fallback) ────────
       - name: Create Discussion