Awesome/awesome-copilot
1
0
Fork 0
mirror of https://github.com/github/awesome-copilot.git synced 2026-04-11 10:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix eval workflows (#1228)

Browse Source 
* Fix eval workflows

* Address review: secure two-phase PR comment & byte-based truncation

- skill-check.yml: Revert to pull_request trigger (read-only token).
  Remove PR comment posting; upload results as artifact instead.
- skill-check-comment.yml: New workflow_run-triggered workflow that
  downloads the artifact and posts/updates the PR comment with
  write permissions, without ever checking out PR code.
- skill-quality-report.yml: Replace character-based truncation with
  byte-based (Buffer.byteLength) limit. Shrink <details> sections
  structurally before falling back to hard byte-trim, keeping
  markdown rendering intact.
This commit is contained in:
Jan Krivanek
2026-03-31 01:47:54 +02:00
committed by GitHub
parent 784f373c5f
commit 1c6002448d
3 changed files with 174 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
.github/workflows/skill-check-comment.yml vendored Normal file
View File

@@ -0,0 +1,109 @@
name: Skill Validator — PR Comment
# Posts results from the "Skill Validator — PR Gate" workflow.
# Runs with write permissions but never checks out PR code,
# so it is safe for fork PRs.
on:
workflow_run:
workflows: ["Skill Validator — PR Gate"]
types: [completed]
permissions:
pull-requests: write
actions: read # needed to download artifacts
jobs:
comment:
runs-on: ubuntu-latest
if: github.event.workflow_run.event == 'pull_request'
steps:
- name: Download results artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: skill-validator-results
run-id: ${{ github.event.workflow_run.id }}
github-token: ${{ github.token }}
- name: Post PR comment with results
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
script: |
const fs = require('fs');
const total = parseInt(fs.readFileSync('total.txt', 'utf8').trim(), 10);
if (total === 0) {
console.log('No skills/agents were checked — skipping comment.');
return;
}
const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim(), 10);
const exitCode = fs.readFileSync('exit-code.txt', 'utf8').trim();
const skillCount = parseInt(fs.readFileSync('skill-count.txt', 'utf8').trim(), 10);
const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10);
const totalChecked = skillCount + agentCount;
const marker = '<!-- skill-validator-results -->';
const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
// Count errors, warnings, advisories from output
const errorCount = (output.match(/\bError\b/gi) || []).length;
const warningCount = (output.match(/\bWarning\b/gi) || []).length;
const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
let statusLine;
if (errorCount > 0) {
statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
} else if (warningCount > 0) {
statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
} else {
statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
}
const body = [
marker,
'## 🔍 Skill Validator Results',
'',
statusLine,
'',
'<details>',
'<summary>Full output</summary>',
'',
'```',
output,
'```',
'',
'</details>',
'',
exitCode !== '0'
? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
: '',
].join('\n');
// Find existing comment with our marker
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
per_page: 100,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
console.log(`Updated existing comment ${existing.id}`);
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body,
});
console.log('Created new PR comment');
}

97
.github/workflows/skill-check.yml vendored
View File

@@ -12,8 +12,6 @@ on:
permissions:
contents: read
pull-requests: write
issues: write
jobs:
skill-check:
@@ -135,82 +133,27 @@ jobs:
echo "$OUTPUT"
# ── Post / update PR comment ──────────────────────────────────
- name: Post PR comment with results
if: steps.detect.outputs.total != '0'
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
# ── Upload results for the commenting workflow ────────────────
- name: Save metadata
if: always()
run: |
mkdir -p sv-results
echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt
echo "${{ steps.detect.outputs.total }}" > sv-results/total.txt
echo "${{ steps.detect.outputs.skill_count }}" > sv-results/skill-count.txt
echo "${{ steps.detect.outputs.agent_count }}" > sv-results/agent-count.txt
echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt
if [ -f sv-output.txt ]; then
cp sv-output.txt sv-results/sv-output.txt
fi
- name: Upload results
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
with:
script: |
const fs = require('fs');
const marker = '<!-- skill-validator-results -->';
const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
const exitCode = '${{ steps.check.outputs.exit_code }}';
const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10);
const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10);
const totalChecked = skillCount + agentCount;
// Count errors, warnings, advisories from output
const errorCount = (output.match(/\bError\b/gi) || []).length;
const warningCount = (output.match(/\bWarning\b/gi) || []).length;
const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
let statusLine;
if (errorCount > 0) {
statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
} else if (warningCount > 0) {
statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
} else {
statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
}
const body = [
marker,
'## 🔍 Skill Validator Results',
'',
statusLine,
'',
'<details>',
'<summary>Full output</summary>',
'',
'```',
output,
'```',
'',
'</details>',
'',
exitCode !== '0'
? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
: '',
].join('\n');
// Find existing comment with our marker
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
per_page: 100,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
console.log(`Updated existing comment ${existing.id}`);
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body,
});
console.log('Created new PR comment');
}
name: skill-validator-results
path: sv-results/
retention-days: 1
- name: Post skip notice if no skills changed
if: steps.detect.outputs.total == '0'

46
.github/workflows/skill-quality-report.yml vendored
View File

@@ -248,7 +248,51 @@ jobs:
core.setOutput('title', title);
core.setOutput('body_file', 'report-body.md');
fs.writeFileSync('report-body.md', body);
// GitHub Issues/Discussions enforce a body size limit on the
// UTF-8 payload (~65536 bytes). Use byte-based limits and prefer
// shrinking verbose <details> sections to keep markdown valid.
const MAX_BODY_BYTES = 65000; // leave some margin
function shrinkDetailsSections(markdown) {
return markdown.replace(
/<details([\s\S]*?)>[\s\S]*?<\/details>/g,
(match, attrs) => {
const placeholder = '\n<summary>Details truncated</summary>\n\n' +
"> Full output was truncated to fit GitHub's body size limit. " +
'See the workflow run for complete output.\n';
return `<details${attrs}>${placeholder}</details>`;
}
);
}
function trimToByteLimit(str, maxBytes) {
const buf = Buffer.from(str, 'utf8');
if (buf.length <= maxBytes) return str;
// Slice bytes and decode, which safely handles multi-byte chars
return buf.slice(0, maxBytes).toString('utf8').replace(/\uFFFD$/, '');
}
const truncNote = '\n\n> **Note:** Output was truncated to fit GitHub\'s body size limit. See the [workflow run](https://github.com/' + context.repo.owner + '/' + context.repo.repo + '/actions/workflows/skill-quality-report.yml) for full output.\n';
const truncNoteBytes = Buffer.byteLength(truncNote, 'utf8');
let finalBody = body;
if (Buffer.byteLength(finalBody, 'utf8') > MAX_BODY_BYTES) {
// First try: collapse <details> sections to reduce size
finalBody = shrinkDetailsSections(finalBody);
}
if (Buffer.byteLength(finalBody, 'utf8') > MAX_BODY_BYTES) {
// Last resort: hard byte-trim + truncation note
finalBody = trimToByteLimit(finalBody, MAX_BODY_BYTES - truncNoteBytes);
}
if (Buffer.byteLength(finalBody, 'utf8') < Buffer.byteLength(body, 'utf8')) {
finalBody += truncNote;
}
fs.writeFileSync('report-body.md', finalBody);
# ── Create Discussion (preferred) or Issue (fallback) ────────
- name: Create Discussion