From 85d690908b2897693c798da4671b253535222d00 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 30 Mar 2026 03:02:11 +0200 Subject: [PATCH] Add static eval via skill-validator (#1195) * Add static eval via skill-validator * Add issues: write permission for PR comment posting --- .github/workflows/skill-check.yml | 217 ++++++++++++++ .github/workflows/skill-quality-report.yml | 320 +++++++++++++++++++++ 2 files changed, 537 insertions(+) create mode 100644 .github/workflows/skill-check.yml create mode 100644 .github/workflows/skill-quality-report.yml diff --git a/.github/workflows/skill-check.yml b/.github/workflows/skill-check.yml new file mode 100644 index 00000000..cab66e3f --- /dev/null +++ b/.github/workflows/skill-check.yml @@ -0,0 +1,217 @@ +name: Skill Validator — PR Gate + +on: + pull_request: + branches: [staged] + types: [opened, synchronize, reopened] + paths: + - "skills/**" + - "agents/**" + - "plugins/**/skills/**" + - "plugins/**/agents/**" + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + skill-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + + # ── Download & cache skill-validator ────────────────────────── + - name: Get cache key date + id: cache-date + run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" + + - name: Restore skill-validator from cache + id: cache-sv + uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: .skill-validator + key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + restore-keys: | + skill-validator-linux-x64- + + - name: Download skill-validator + if: steps.cache-sv.outputs.cache-hit != 'true' + run: | + mkdir -p .skill-validator + curl -fsSL \ + "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \ + -o .skill-validator/skill-validator-linux-x64.tar.gz + tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator + rm .skill-validator/skill-validator-linux-x64.tar.gz + chmod +x .skill-validator/skill-validator + + - name: Save skill-validator to cache + if: steps.cache-sv.outputs.cache-hit != 'true' + uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: .skill-validator + key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + + # ── Detect changed skills & agents ──────────────────────────── + - name: Detect changed skills and agents + id: detect + run: | + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + # Extract unique skill directories that were touched + SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^skills/[^/]+' | sort -u || true) + + # Extract agent files that were touched + AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^agents/[^/]+\.agent\.md$' | sort -u || true) + + # Extract plugin skill directories + PLUGIN_SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/skills/[^/]+' | sort -u || true) + + # Extract plugin agent files + PLUGIN_AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/agents/[^/]+\.agent\.md$' | sort -u || true) + + # Build CLI arguments for --skills + SKILL_ARGS="" + for dir in $SKILL_DIRS $PLUGIN_SKILL_DIRS; do + if [ -d "$dir" ]; then + SKILL_ARGS="$SKILL_ARGS $dir" + fi + done + + # Build CLI arguments for --agents + AGENT_ARGS="" + for f in $AGENT_FILES $PLUGIN_AGENT_FILES; do + if [ -f "$f" ]; then + AGENT_ARGS="$AGENT_ARGS $f" + fi + done + + SKILL_COUNT=$(echo "$SKILL_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0) + AGENT_COUNT=$(echo "$AGENT_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0) + TOTAL=$((SKILL_COUNT + AGENT_COUNT)) + + echo "skill_args=$SKILL_ARGS" >> "$GITHUB_OUTPUT" + echo "agent_args=$AGENT_ARGS" >> "$GITHUB_OUTPUT" + echo "total=$TOTAL" >> "$GITHUB_OUTPUT" + echo "skill_count=$SKILL_COUNT" >> "$GITHUB_OUTPUT" + echo "agent_count=$AGENT_COUNT" >> "$GITHUB_OUTPUT" + + echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check." + + # ── Run skill-validator check ───────────────────────────────── + - name: Run skill-validator check + id: check + if: steps.detect.outputs.total != '0' + run: | + SKILL_ARGS="${{ steps.detect.outputs.skill_args }}" + AGENT_ARGS="${{ steps.detect.outputs.agent_args }}" + + CMD=".skill-validator/skill-validator check --verbose" + + if [ -n "$SKILL_ARGS" ]; then + CMD="$CMD --skills $SKILL_ARGS" + fi + + if [ -n "$AGENT_ARGS" ]; then + CMD="$CMD --agents $AGENT_ARGS" + fi + + echo "Running: $CMD" + + # Capture output; don't fail the workflow (warn-only mode) + set +e + OUTPUT=$($CMD 2>&1) + EXIT_CODE=$? + set -e + + echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" + + # Save output to file (multi-line safe) + echo "$OUTPUT" > sv-output.txt + + echo "$OUTPUT" + + # ── Post / update PR comment ────────────────────────────────── + - name: Post PR comment with results + if: steps.detect.outputs.total != '0' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const fs = require('fs'); + + const marker = ''; + const output = fs.readFileSync('sv-output.txt', 'utf8').trim(); + const exitCode = '${{ steps.check.outputs.exit_code }}'; + const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10); + const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10); + const totalChecked = skillCount + agentCount; + + // Count errors, warnings, advisories from output + const errorCount = (output.match(/\bError\b/gi) || []).length; + const warningCount = (output.match(/\bWarning\b/gi) || []).length; + const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length; + + let statusLine; + if (errorCount > 0) { + statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`; + } else if (warningCount > 0) { + statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`; + } else { + statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`; + } + + const body = [ + marker, + '## 🔍 Skill Validator Results', + '', + statusLine, + '', + '
', + 'Full output', + '', + '```', + output, + '```', + '', + '
', + '', + exitCode !== '0' + ? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.' + : '', + ].join('\n'); + + // Find existing comment with our marker + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + + const existing = comments.find(c => c.body.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + console.log(`Updated existing comment ${existing.id}`); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + console.log('Created new PR comment'); + } + + - name: Post skip notice if no skills changed + if: steps.detect.outputs.total == '0' + run: echo "No skill or agent files changed in this PR — skipping validation." diff --git a/.github/workflows/skill-quality-report.yml b/.github/workflows/skill-quality-report.yml new file mode 100644 index 00000000..df12b00c --- /dev/null +++ b/.github/workflows/skill-quality-report.yml @@ -0,0 +1,320 @@ +name: Skill Quality Report — Nightly Scan + +on: + schedule: + - cron: "0 3 * * *" # 3:00 AM UTC daily + workflow_dispatch: # allow manual trigger + +permissions: + contents: read + discussions: write + issues: write # fallback if Discussions are not enabled + +jobs: + nightly-scan: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 # full history for git-log author fallback + + # ── Download & cache skill-validator ────────────────────────── + - name: Get cache key date + id: cache-date + run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" + + - name: Restore skill-validator from cache + id: cache-sv + uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: .skill-validator + key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + restore-keys: | + skill-validator-linux-x64- + + - name: Download skill-validator + if: steps.cache-sv.outputs.cache-hit != 'true' + run: | + mkdir -p .skill-validator + curl -fsSL \ + "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \ + -o .skill-validator/skill-validator-linux-x64.tar.gz + tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator + rm .skill-validator/skill-validator-linux-x64.tar.gz + chmod +x .skill-validator/skill-validator + + - name: Save skill-validator to cache + if: steps.cache-sv.outputs.cache-hit != 'true' + uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: .skill-validator + key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + + # ── Run full scan ───────────────────────────────────────────── + - name: Run skill-validator check on all skills + id: check-skills + run: | + set +e + set -o pipefail + .skill-validator/skill-validator check \ + --skills ./skills \ + --verbose \ + 2>&1 | tee sv-skills-output.txt + echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + set +o pipefail + set -e + + - name: Run skill-validator check on all agents + id: check-agents + run: | + set +e + set -o pipefail + AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null | tr '\n' ' ') + if [ -n "$AGENT_FILES" ]; then + .skill-validator/skill-validator check \ + --agents $AGENT_FILES \ + --verbose \ + 2>&1 | tee sv-agents-output.txt + echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + else + echo "No agent files found." + echo "" > sv-agents-output.txt + echo "exit_code=0" >> "$GITHUB_OUTPUT" + fi + set +o pipefail + set -e + + # ── Build report with author attribution ────────────────────── + - name: Build quality report + id: report + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const { execSync } = require('child_process'); + + // ── Parse CODEOWNERS ────────────────────────────────── + function parseCodeowners() { + const map = new Map(); + try { + const raw = fs.readFileSync('CODEOWNERS', 'utf8'); + for (const line of raw.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const parts = trimmed.split(/\s+/); + if (parts.length >= 2) { + const filePath = parts[0].replace(/^\//, '').replace(/\/$/, ''); + const owners = parts.slice(1).filter(p => p.startsWith('@')); + if (owners.length > 0) { + map.set(filePath, owners); + } + } + } + } catch (e) { + console.log('Could not parse CODEOWNERS:', e.message); + } + return map; + } + + // ── Resolve author for a path ───────────────────────── + function resolveAuthor(resourcePath, codeowners) { + // CODEOWNERS semantics: last matching rule wins. + // Also treat "*" as a match-all default rule. + let matchedOwners = null; + for (const [pattern, owners] of codeowners) { + if ( + pattern === '*' || + resourcePath === pattern || + resourcePath.startsWith(pattern + '/') + ) { + matchedOwners = owners; + } + } + if (matchedOwners && matchedOwners.length > 0) { + return matchedOwners.join(', '); + } + // Fallback: git log + try { + const author = execSync( + `git log --format='%aN' --follow -1 -- "${resourcePath}"`, + { encoding: 'utf8' } + ).trim(); + return author || 'unknown'; + } catch { + return 'unknown'; + } + } + + // ── Parse skill-validator output ────────────────────── + // The output is a text report; we preserve it as-is and + // augment it with author info in the summary. + const skillsOutput = fs.readFileSync('sv-skills-output.txt', 'utf8').trim(); + const agentsOutput = fs.existsSync('sv-agents-output.txt') + ? fs.readFileSync('sv-agents-output.txt', 'utf8').trim() + : ''; + + const codeowners = parseCodeowners(); + + // Count findings + const combined = skillsOutput + '\n' + agentsOutput; + const errorCount = (combined.match(/\bError\b/gi) || []).length; + const warningCount = (combined.match(/\bWarning\b/gi) || []).length; + const advisoryCount = (combined.match(/\bAdvisory\b/gi) || []).length; + + // Count total skills & agents checked + let skillDirs = []; + try { + skillDirs = fs.readdirSync('skills', { withFileTypes: true }) + .filter(d => d.isDirectory()) + .map(d => d.name); + } catch {} + + let agentFiles = []; + try { + agentFiles = fs.readdirSync('agents') + .filter(f => f.endsWith('.agent.md')); + } catch {} + + // ── Build author-attributed summary ─────────────────── + // Extract per-resource blocks from output. The validator + // prints skill names as headers — we annotate them with + // the resolved owner. + function annotateWithAuthors(output, kind) { + if (!output) return '_No findings._'; + const lines = output.split('\n'); + const annotated = []; + for (const line of lines) { + // Skill names appear as headers, e.g. "## skill-name" or "skill-name:" + const headerMatch = line.match(/^(?:#{1,3}\s+)?([a-z0-9][a-z0-9-]+(?:\.[a-z0-9.-]+)?)\b/); + if (headerMatch) { + const name = headerMatch[1]; + const resourcePath = kind === 'skill' + ? `skills/${name}` + : `agents/${name}.agent.md`; + const author = resolveAuthor(resourcePath, codeowners); + annotated.push(`${line} — ${author}`); + } else { + annotated.push(line); + } + } + return annotated.join('\n'); + } + + const today = new Date().toISOString().split('T')[0]; + + const title = `Skill Quality Report — ${today}`; + + const body = [ + `# ${title}`, + '', + `**${skillDirs.length} skills** and **${agentFiles.length} agents** scanned.`, + '', + `| Severity | Count |`, + `|----------|-------|`, + `| ⛔ Errors | ${errorCount} |`, + `| ⚠️ Warnings | ${warningCount} |`, + `| ℹ️ Advisories | ${advisoryCount} |`, + '', + '---', + '', + '## Skills', + '', + '
', + 'Full skill-validator output for skills', + '', + '```', + annotateWithAuthors(skillsOutput, 'skill'), + '```', + '', + '
', + '', + '## Agents', + '', + '
', + 'Full skill-validator output for agents', + '', + '```', + annotateWithAuthors(agentsOutput, 'agent'), + '```', + '', + '
', + '', + '---', + '', + `_Generated by the [Skill Validator nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`, + ].join('\n'); + + core.setOutput('title', title); + core.setOutput('body_file', 'report-body.md'); + fs.writeFileSync('report-body.md', body); + + # ── Create Discussion (preferred) or Issue (fallback) ──────── + - name: Create Discussion + id: create-discussion + continue-on-error: true + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const fs = require('fs'); + const title = '${{ steps.report.outputs.title }}'.replace(/'/g, "\\'"); + const body = fs.readFileSync('report-body.md', 'utf8'); + + // Find the "Skill Quality Reports" category + const categoriesResult = await github.graphql(` + query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + id + discussionCategories(first: 25) { + nodes { id name } + } + } + } + `, { + owner: context.repo.owner, + repo: context.repo.repo, + }); + + const repo = categoriesResult.repository; + const categories = repo.discussionCategories.nodes; + const category = categories.find(c => + c.name === 'Skill Quality Reports' + ); + + if (!category) { + core.setFailed('Discussion category "Skill Quality Reports" not found. Falling back to issue.'); + return; + } + + await github.graphql(` + mutation($repoId: ID!, $categoryId: ID!, $title: String!, $body: String!) { + createDiscussion(input: { + repositoryId: $repoId, + categoryId: $categoryId, + title: $title, + body: $body + }) { + discussion { url } + } + } + `, { + repoId: repo.id, + categoryId: category.id, + title: title, + body: body, + }); + + console.log('Discussion created successfully.'); + + - name: Fallback — Create Issue + if: steps.create-discussion.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + run: | + # Create label if it doesn't exist (ignore errors if it already exists) + gh label create "skill-quality" --description "Automated skill quality reports" --color "d4c5f9" 2>/dev/null || true + gh issue create \ + --title "${{ steps.report.outputs.title }}" \ + --body-file report-body.md \ + --label "skill-quality"