Add static eval via skill-validator (#1195)

* Add static eval via skill-validator

* Add issues: write permission for PR comment posting

.github/workflows/skill-check.yml

@@ -0,0 +1,217 @@
+name: Skill Validator — PR Gate
+
+on:
+  pull_request:
+    branches: [staged]
+    types: [opened, synchronize, reopened]
+    paths:
+      - "skills/**"
+      - "agents/**"
+      - "plugins/**/skills/**"
+      - "plugins/**/agents/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  skill-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0
+
+      # ── Download & cache skill-validator ──────────────────────────
+      - name: Get cache key date
+        id: cache-date
+        run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
+
+      - name: Restore skill-validator from cache
+        id: cache-sv
+        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+        with:
+          path: .skill-validator
+          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+          restore-keys: |
+            skill-validator-linux-x64-
+
+      - name: Download skill-validator
+        if: steps.cache-sv.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p .skill-validator
+          curl -fsSL \
+            "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
+            -o .skill-validator/skill-validator-linux-x64.tar.gz
+          tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
+          rm .skill-validator/skill-validator-linux-x64.tar.gz
+          chmod +x .skill-validator/skill-validator
+
+      - name: Save skill-validator to cache
+        if: steps.cache-sv.outputs.cache-hit != 'true'
+        uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+        with:
+          path: .skill-validator
+          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+
+      # ── Detect changed skills & agents ────────────────────────────
+      - name: Detect changed skills and agents
+        id: detect
+        run: |
+          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
+
+          # Extract unique skill directories that were touched
+          SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^skills/[^/]+' | sort -u || true)
+
+          # Extract agent files that were touched
+          AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^agents/[^/]+\.agent\.md$' | sort -u || true)
+
+          # Extract plugin skill directories
+          PLUGIN_SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/skills/[^/]+' | sort -u || true)
+
+          # Extract plugin agent files
+          PLUGIN_AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/agents/[^/]+\.agent\.md$' | sort -u || true)
+
+          # Build CLI arguments for --skills
+          SKILL_ARGS=""
+          for dir in $SKILL_DIRS $PLUGIN_SKILL_DIRS; do
+            if [ -d "$dir" ]; then
+              SKILL_ARGS="$SKILL_ARGS $dir"
+            fi
+          done
+
+          # Build CLI arguments for --agents
+          AGENT_ARGS=""
+          for f in $AGENT_FILES $PLUGIN_AGENT_FILES; do
+            if [ -f "$f" ]; then
+              AGENT_ARGS="$AGENT_ARGS $f"
+            fi
+          done
+
+          SKILL_COUNT=$(echo "$SKILL_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0)
+          AGENT_COUNT=$(echo "$AGENT_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0)
+          TOTAL=$((SKILL_COUNT + AGENT_COUNT))
+
+          echo "skill_args=$SKILL_ARGS" >> "$GITHUB_OUTPUT"
+          echo "agent_args=$AGENT_ARGS" >> "$GITHUB_OUTPUT"
+          echo "total=$TOTAL" >> "$GITHUB_OUTPUT"
+          echo "skill_count=$SKILL_COUNT" >> "$GITHUB_OUTPUT"
+          echo "agent_count=$AGENT_COUNT" >> "$GITHUB_OUTPUT"
+
+          echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check."
+
+      # ── Run skill-validator check ─────────────────────────────────
+      - name: Run skill-validator check
+        id: check
+        if: steps.detect.outputs.total != '0'
+        run: |
+          SKILL_ARGS="${{ steps.detect.outputs.skill_args }}"
+          AGENT_ARGS="${{ steps.detect.outputs.agent_args }}"
+
+          CMD=".skill-validator/skill-validator check --verbose"
+
+          if [ -n "$SKILL_ARGS" ]; then
+            CMD="$CMD --skills $SKILL_ARGS"
+          fi
+
+          if [ -n "$AGENT_ARGS" ]; then
+            CMD="$CMD --agents $AGENT_ARGS"
+          fi
+
+          echo "Running: $CMD"
+
+          # Capture output; don't fail the workflow (warn-only mode)
+          set +e
+          OUTPUT=$($CMD 2>&1)
+          EXIT_CODE=$?
+          set -e
+
+          echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
+
+          # Save output to file (multi-line safe)
+          echo "$OUTPUT" > sv-output.txt
+
+          echo "$OUTPUT"
+
+      # ── Post / update PR comment ──────────────────────────────────
+      - name: Post PR comment with results
+        if: steps.detect.outputs.total != '0'
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+
+            const marker = '<!-- skill-validator-results -->';
+            const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
+            const exitCode = '${{ steps.check.outputs.exit_code }}';
+            const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10);
+            const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10);
+            const totalChecked = skillCount + agentCount;
+
+            // Count errors, warnings, advisories from output
+            const errorCount = (output.match(/\bError\b/gi) || []).length;
+            const warningCount = (output.match(/\bWarning\b/gi) || []).length;
+            const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
+
+            let statusLine;
+            if (errorCount > 0) {
+              statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
+            } else if (warningCount > 0) {
+              statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
+            } else {
+              statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
+            }
+
+            const body = [
+              marker,
+              '## 🔍 Skill Validator Results',
+              '',
+              statusLine,
+              '',
+              '<details>',
+              '<summary>Full output</summary>',
+              '',
+              '```',
+              output,
+              '```',
+              '',
+              '</details>',
+              '',
+              exitCode !== '0'
+                ? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
+                : '',
+            ].join('\n');
+
+            // Find existing comment with our marker
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              per_page: 100,
+            });
+
+            const existing = comments.find(c => c.body.includes(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              console.log(`Updated existing comment ${existing.id}`);
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+              console.log('Created new PR comment');
+            }
+
+      - name: Post skip notice if no skills changed
+        if: steps.detect.outputs.total == '0'
+        run: echo "No skill or agent files changed in this PR — skipping validation."

.github/workflows/skill-quality-report.yml

@@ -0,0 +1,320 @@
+name: Skill Quality Report — Nightly Scan
+
+on:
+  schedule:
+    - cron: "0 3 * * *" # 3:00 AM UTC daily
+  workflow_dispatch: # allow manual trigger
+
+permissions:
+  contents: read
+  discussions: write
+  issues: write # fallback if Discussions are not enabled
+
+jobs:
+  nightly-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0 # full history for git-log author fallback
+
+      # ── Download & cache skill-validator ──────────────────────────
+      - name: Get cache key date
+        id: cache-date
+        run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
+
+      - name: Restore skill-validator from cache
+        id: cache-sv
+        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+        with:
+          path: .skill-validator
+          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+          restore-keys: |
+            skill-validator-linux-x64-
+
+      - name: Download skill-validator
+        if: steps.cache-sv.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p .skill-validator
+          curl -fsSL \
+            "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
+            -o .skill-validator/skill-validator-linux-x64.tar.gz
+          tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
+          rm .skill-validator/skill-validator-linux-x64.tar.gz
+          chmod +x .skill-validator/skill-validator
+
+      - name: Save skill-validator to cache
+        if: steps.cache-sv.outputs.cache-hit != 'true'
+        uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+        with:
+          path: .skill-validator
+          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+
+      # ── Run full scan ─────────────────────────────────────────────
+      - name: Run skill-validator check on all skills
+        id: check-skills
+        run: |
+          set +e
+          set -o pipefail
+          .skill-validator/skill-validator check \
+            --skills ./skills \
+            --verbose \
+            2>&1 | tee sv-skills-output.txt
+          echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+          set +o pipefail
+          set -e
+
+      - name: Run skill-validator check on all agents
+        id: check-agents
+        run: |
+          set +e
+          set -o pipefail
+          AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null | tr '\n' ' ')
+          if [ -n "$AGENT_FILES" ]; then
+            .skill-validator/skill-validator check \
+              --agents $AGENT_FILES \
+              --verbose \
+              2>&1 | tee sv-agents-output.txt
+            echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+          else
+            echo "No agent files found."
+            echo "" > sv-agents-output.txt
+            echo "exit_code=0" >> "$GITHUB_OUTPUT"
+          fi
+          set +o pipefail
+          set -e
+
+      # ── Build report with author attribution ──────────────────────
+      - name: Build quality report
+        id: report
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+            const { execSync } = require('child_process');
+
+            // ── Parse CODEOWNERS ──────────────────────────────────
+            function parseCodeowners() {
+              const map = new Map();
+              try {
+                const raw = fs.readFileSync('CODEOWNERS', 'utf8');
+                for (const line of raw.split('\n')) {
+                  const trimmed = line.trim();
+                  if (!trimmed || trimmed.startsWith('#')) continue;
+                  const parts = trimmed.split(/\s+/);
+                  if (parts.length >= 2) {
+                    const filePath = parts[0].replace(/^\//, '').replace(/\/$/, '');
+                    const owners = parts.slice(1).filter(p => p.startsWith('@'));
+                    if (owners.length > 0) {
+                      map.set(filePath, owners);
+                    }
+                  }
+                }
+              } catch (e) {
+                console.log('Could not parse CODEOWNERS:', e.message);
+              }
+              return map;
+            }
+
+            // ── Resolve author for a path ─────────────────────────
+            function resolveAuthor(resourcePath, codeowners) {
+              // CODEOWNERS semantics: last matching rule wins.
+              // Also treat "*" as a match-all default rule.
+              let matchedOwners = null;
+              for (const [pattern, owners] of codeowners) {
+                if (
+                  pattern === '*' ||
+                  resourcePath === pattern ||
+                  resourcePath.startsWith(pattern + '/')
+                ) {
+                  matchedOwners = owners;
+                }
+              }
+              if (matchedOwners && matchedOwners.length > 0) {
+                return matchedOwners.join(', ');
+              }
+              // Fallback: git log
+              try {
+                const author = execSync(
+                  `git log --format='%aN' --follow -1 -- "${resourcePath}"`,
+                  { encoding: 'utf8' }
+                ).trim();
+                return author || 'unknown';
+              } catch {
+                return 'unknown';
+              }
+            }
+
+            // ── Parse skill-validator output ──────────────────────
+            // The output is a text report; we preserve it as-is and
+            // augment it with author info in the summary.
+            const skillsOutput = fs.readFileSync('sv-skills-output.txt', 'utf8').trim();
+            const agentsOutput = fs.existsSync('sv-agents-output.txt')
+              ? fs.readFileSync('sv-agents-output.txt', 'utf8').trim()
+              : '';
+
+            const codeowners = parseCodeowners();
+
+            // Count findings
+            const combined = skillsOutput + '\n' + agentsOutput;
+            const errorCount = (combined.match(/\bError\b/gi) || []).length;
+            const warningCount = (combined.match(/\bWarning\b/gi) || []).length;
+            const advisoryCount = (combined.match(/\bAdvisory\b/gi) || []).length;
+
+            // Count total skills & agents checked
+            let skillDirs = [];
+            try {
+              skillDirs = fs.readdirSync('skills', { withFileTypes: true })
+                .filter(d => d.isDirectory())
+                .map(d => d.name);
+            } catch {}
+
+            let agentFiles = [];
+            try {
+              agentFiles = fs.readdirSync('agents')
+                .filter(f => f.endsWith('.agent.md'));
+            } catch {}
+
+            // ── Build author-attributed summary ───────────────────
+            // Extract per-resource blocks from output.  The validator
+            // prints skill names as headers — we annotate them with
+            // the resolved owner.
+            function annotateWithAuthors(output, kind) {
+              if (!output) return '_No findings._';
+              const lines = output.split('\n');
+              const annotated = [];
+              for (const line of lines) {
+                // Skill names appear as headers, e.g. "## skill-name" or "skill-name:"
+                const headerMatch = line.match(/^(?:#{1,3}\s+)?([a-z0-9][a-z0-9-]+(?:\.[a-z0-9.-]+)?)\b/);
+                if (headerMatch) {
+                  const name = headerMatch[1];
+                  const resourcePath = kind === 'skill'
+                    ? `skills/${name}`
+                    : `agents/${name}.agent.md`;
+                  const author = resolveAuthor(resourcePath, codeowners);
+                  annotated.push(`${line}  — ${author}`);
+                } else {
+                  annotated.push(line);
+                }
+              }
+              return annotated.join('\n');
+            }
+
+            const today = new Date().toISOString().split('T')[0];
+
+            const title = `Skill Quality Report — ${today}`;
+
+            const body = [
+              `# ${title}`,
+              '',
+              `**${skillDirs.length} skills** and **${agentFiles.length} agents** scanned.`,
+              '',
+              `| Severity | Count |`,
+              `|----------|-------|`,
+              `| ⛔ Errors | ${errorCount} |`,
+              `| ⚠️ Warnings | ${warningCount} |`,
+              `| ℹ️ Advisories | ${advisoryCount} |`,
+              '',
+              '---',
+              '',
+              '## Skills',
+              '',
+              '<details>',
+              '<summary>Full skill-validator output for skills</summary>',
+              '',
+              '```',
+              annotateWithAuthors(skillsOutput, 'skill'),
+              '```',
+              '',
+              '</details>',
+              '',
+              '## Agents',
+              '',
+              '<details>',
+              '<summary>Full skill-validator output for agents</summary>',
+              '',
+              '```',
+              annotateWithAuthors(agentsOutput, 'agent'),
+              '```',
+              '',
+              '</details>',
+              '',
+              '---',
+              '',
+              `_Generated by the [Skill Validator nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`,
+            ].join('\n');
+
+            core.setOutput('title', title);
+            core.setOutput('body_file', 'report-body.md');
+            fs.writeFileSync('report-body.md', body);
+
+      # ── Create Discussion (preferred) or Issue (fallback) ────────
+      - name: Create Discussion
+        id: create-discussion
+        continue-on-error: true
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+            const title = '${{ steps.report.outputs.title }}'.replace(/'/g, "\\'");
+            const body = fs.readFileSync('report-body.md', 'utf8');
+
+            // Find the "Skill Quality Reports" category
+            const categoriesResult = await github.graphql(`
+              query($owner: String!, $repo: String!) {
+                repository(owner: $owner, name: $repo) {
+                  id
+                  discussionCategories(first: 25) {
+                    nodes { id name }
+                  }
+                }
+              }
+            `, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            });
+
+            const repo = categoriesResult.repository;
+            const categories = repo.discussionCategories.nodes;
+            const category = categories.find(c =>
+              c.name === 'Skill Quality Reports'
+            );
+
+            if (!category) {
+              core.setFailed('Discussion category "Skill Quality Reports" not found. Falling back to issue.');
+              return;
+            }
+
+            await github.graphql(`
+              mutation($repoId: ID!, $categoryId: ID!, $title: String!, $body: String!) {
+                createDiscussion(input: {
+                  repositoryId: $repoId,
+                  categoryId: $categoryId,
+                  title: $title,
+                  body: $body
+                }) {
+                  discussion { url }
+                }
+              }
+            `, {
+              repoId: repo.id,
+              categoryId: category.id,
+              title: title,
+              body: body,
+            });
+
+            console.log('Discussion created successfully.');
+
+      - name: Fallback — Create Issue
+        if: steps.create-discussion.outcome == 'failure'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Create label if it doesn't exist (ignore errors if it already exists)
+          gh label create "skill-quality" --description "Automated skill quality reports" --color "d4c5f9" 2>/dev/null || true
+          gh issue create \
+            --title "${{ steps.report.outputs.title }}" \
+            --body-file report-body.md \
+            --label "skill-quality"