diff --git a/.github/workflows/skill-check-comment.yml b/.github/workflows/skill-check-comment.yml index 7c27c243..483d7906 100644 --- a/.github/workflows/skill-check-comment.yml +++ b/.github/workflows/skill-check-comment.yml @@ -1,12 +1,12 @@ -name: Skill Validator — PR Comment +name: Vally Lint — PR Comment -# Posts results from the "Skill Validator — PR Gate" workflow. +# Posts results from the "Vally Lint — PR Gate" workflow. # Runs with write permissions but never checks out PR code, # so it is safe for fork PRs. on: workflow_run: - workflows: ["Skill Validator — PR Gate"] + workflows: ["Vally Lint — PR Gate"] types: [completed] permissions: @@ -22,7 +22,7 @@ jobs: - name: Download results artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: skill-validator-results + name: vally-lint-results run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} @@ -34,11 +34,11 @@ jobs: const managedLabels = { 'skill-check-warning': { color: 'FBCA04', - description: 'Skill validator reported warnings' + description: 'Vally lint reported warnings' }, 'skill-check-error': { color: 'B60205', - description: 'Skill validator reported errors' + description: 'Vally lint reported errors' } }; @@ -105,9 +105,9 @@ jobs: const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10); const totalChecked = skillCount + agentCount; - const marker = ''; - const rawOutput = fs.existsSync('sv-output.txt') - ? fs.readFileSync('sv-output.txt', 'utf8') + const marker = ''; + const rawOutput = fs.existsSync('vally-output.txt') + ? fs.readFileSync('vally-output.txt', 'utf8') : ''; const output = rawOutput.replace(/\x1b\[[0-9;]*m/g, '').trim(); @@ -171,7 +171,7 @@ jobs: ]; const findingsTable = summaryLines.length === 0 - ? ['_No findings were emitted by the validator._'] + ? ['_No findings were emitted by the linter._'] : [ '| Level | Finding |', '|---|---|', @@ -190,7 +190,7 @@ jobs: const body = [ marker, - '## 🔍 Skill Validator Results', + '## 🔍 Vally Lint Results', '', `**${verdict}**`, '', @@ -203,16 +203,16 @@ jobs: ...findingsTable, '', '
', - 'Full validator output', + 'Full linter output', '', '```text', - output || 'No validator output captured.', + output || 'No linter output captured.', '```', '', '
', '', exitCode !== '0' - ? '> **Note:** The validator returned a non-zero exit code. Please review the findings above before merge.' + ? '> **Note:** Vally lint returned a non-zero exit code. Please review the findings above before merge.' : '', ].join('\n'); diff --git a/.github/workflows/skill-check.yml b/.github/workflows/skill-check.yml index 7948fc86..879a7c4d 100644 --- a/.github/workflows/skill-check.yml +++ b/.github/workflows/skill-check.yml @@ -1,4 +1,4 @@ -name: Skill Validator — PR Gate +name: Vally Lint — PR Gate on: pull_request: @@ -22,37 +22,10 @@ jobs: with: fetch-depth: 0 - # ── Download & cache skill-validator ────────────────────────── - - name: Get cache key date - id: cache-date - run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" - - - name: Restore skill-validator from cache - id: cache-sv - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - name: Setup Node.js + uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0 with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} - restore-keys: | - skill-validator-linux-x64- - - - name: Download skill-validator - if: steps.cache-sv.outputs.cache-hit != 'true' - run: | - mkdir -p .skill-validator - curl -fsSL \ - "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \ - -o .skill-validator/skill-validator-linux-x64.tar.gz - tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator - rm .skill-validator/skill-validator-linux-x64.tar.gz - chmod +x .skill-validator/skill-validator - - - name: Save skill-validator to cache - if: steps.cache-sv.outputs.cache-hit != 'true' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + node-version: 20 # ── Detect changed skills & agents ──────────────────────────── - name: Detect changed skills and agents @@ -111,8 +84,8 @@ jobs: echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check." - # ── Run skill-validator check ───────────────────────────────── - - name: Run skill-validator check + # ── Run vally lint check ─────────────────────────────────────── + - name: Run vally lint check id: check if: steps.detect.outputs.total != '0' env: @@ -134,53 +107,58 @@ jobs: done <<< "$AGENT_FILES_RAW" fi - CMD=(.skill-validator/skill-validator check --verbose) + EXIT_CODE=0 + : > vally-output.txt - if [ ${#SKILL_DIRS[@]} -gt 0 ]; then - CMD+=(--skills "${SKILL_DIRS[@]}") + if [ ${#SKILL_DIRS[@]} -eq 0 ] && [ ${#AGENT_FILES[@]} -eq 0 ]; then + echo "No skills or agents to validate." | tee -a vally-output.txt fi + for skill_dir in "${SKILL_DIRS[@]}"; do + echo "### Linting ${skill_dir}" | tee -a vally-output.txt + set +e + OUTPUT=$(npx --yes @microsoft/vally-cli lint "$skill_dir" --verbose 2>&1) + CMD_EXIT=$? + set -e + echo "$OUTPUT" | tee -a vally-output.txt + echo "" >> vally-output.txt + + if [ "$CMD_EXIT" -ne 0 ]; then + EXIT_CODE=1 + fi + done + if [ ${#AGENT_FILES[@]} -gt 0 ]; then - CMD+=(--agents "${AGENT_FILES[@]}") + { + echo "### Agent files detected (not linted by vally)" + echo "ℹ️ Vally currently lints SKILL.md content. Agent files were detected but skipped:" + printf '%s\n' "${AGENT_FILES[@]}" + echo "" + } | tee -a vally-output.txt fi - printf 'Running: ' - printf '%q ' "${CMD[@]}" - echo - - # Capture output; don't fail the workflow (warn-only mode) - set +e - OUTPUT=$("${CMD[@]}" 2>&1) - EXIT_CODE=$? - set -e - echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" - # Save output to file (multi-line safe) - echo "$OUTPUT" > sv-output.txt - - echo "$OUTPUT" - # ── Upload results for the commenting workflow ──────────────── - name: Save metadata if: always() run: | - mkdir -p sv-results - echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt - echo "${{ steps.detect.outputs.total }}" > sv-results/total.txt - echo "${{ steps.detect.outputs.skill_count }}" > sv-results/skill-count.txt - echo "${{ steps.detect.outputs.agent_count }}" > sv-results/agent-count.txt - echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt - if [ -f sv-output.txt ]; then - cp sv-output.txt sv-results/sv-output.txt + mkdir -p vally-results + echo "${{ github.event.pull_request.number }}" > vally-results/pr-number.txt + echo "${{ steps.detect.outputs.total }}" > vally-results/total.txt + echo "${{ steps.detect.outputs.skill_count }}" > vally-results/skill-count.txt + echo "${{ steps.detect.outputs.agent_count }}" > vally-results/agent-count.txt + echo "${{ steps.check.outputs.exit_code }}" > vally-results/exit-code.txt + if [ -f vally-output.txt ]; then + cp vally-output.txt vally-results/vally-output.txt fi - name: Upload results if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: - name: skill-validator-results - path: sv-results/ + name: vally-lint-results + path: vally-results/ retention-days: 1 - name: Post skip notice if no skills changed diff --git a/.github/workflows/skill-quality-report.yml b/.github/workflows/skill-quality-report.yml index 75db829d..70a3209b 100644 --- a/.github/workflows/skill-quality-report.yml +++ b/.github/workflows/skill-quality-report.yml @@ -19,71 +19,37 @@ jobs: with: fetch-depth: 0 # full history for git-log author fallback - # ── Download & cache skill-validator ────────────────────────── - - name: Get cache key date - id: cache-date - run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" - - - name: Restore skill-validator from cache - id: cache-sv - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - name: Setup Node.js + uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0 with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} - restore-keys: | - skill-validator-linux-x64- - - - name: Download skill-validator - if: steps.cache-sv.outputs.cache-hit != 'true' - run: | - mkdir -p .skill-validator - curl -fsSL \ - "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \ - -o .skill-validator/skill-validator-linux-x64.tar.gz - tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator - rm .skill-validator/skill-validator-linux-x64.tar.gz - chmod +x .skill-validator/skill-validator - - - name: Save skill-validator to cache - if: steps.cache-sv.outputs.cache-hit != 'true' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + node-version: 20 # ── Run full scan ───────────────────────────────────────────── - - name: Run skill-validator check on all skills + - name: Run vally lint on all skills id: check-skills run: | set +e set -o pipefail - .skill-validator/skill-validator check \ - --skills ./skills \ - --verbose \ - 2>&1 | tee sv-skills-output.txt + npx --yes @microsoft/vally-cli lint ./skills --verbose 2>&1 | tee vally-skills-output.txt echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" set +o pipefail set -e - - name: Run skill-validator check on all agents + - name: Note agent scan status id: check-agents run: | - set +e - set -o pipefail AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null | tr '\n' ' ') if [ -n "$AGENT_FILES" ]; then - .skill-validator/skill-validator check \ - --agents $AGENT_FILES \ - --verbose \ - 2>&1 | tee sv-agents-output.txt - echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + { + echo "ℹ️ Vally currently lints SKILL.md content." + echo "ℹ️ Agent files are detected but excluded from this scan:" + echo "$AGENT_FILES" + } > vally-agents-output.txt else echo "No agent files found." - echo "" > sv-agents-output.txt + echo "" > vally-agents-output.txt echo "exit_code=0" >> "$GITHUB_OUTPUT" fi - set +o pipefail - set -e # ── Build report with author attribution ────────────────────── - name: Build quality report @@ -147,18 +113,18 @@ jobs: } } - // ── Parse skill-validator output ────────────────────── + // ── Parse vally lint output ─────────────────────────── // The output is a text report; we preserve it as-is and // augment it with author info in the summary. - const skillsOutput = fs.readFileSync('sv-skills-output.txt', 'utf8').trim(); - const agentsOutput = fs.existsSync('sv-agents-output.txt') - ? fs.readFileSync('sv-agents-output.txt', 'utf8').trim() + const skillsOutput = fs.readFileSync('vally-skills-output.txt', 'utf8').trim(); + const agentsOutput = fs.existsSync('vally-agents-output.txt') + ? fs.readFileSync('vally-agents-output.txt', 'utf8').trim() : ''; const codeowners = parseCodeowners(); // Count findings - // The skill-validator uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories + // Vally lint uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories const combined = skillsOutput + '\n' + agentsOutput; const errorCount = (combined.match(/❌/g) || []).length; const warningCount = (combined.match(/⚠/g) || []).length; @@ -179,7 +145,7 @@ jobs: } catch {} // ── Build author-attributed summary ─────────────────── - // Extract per-resource blocks from output. The validator + // Extract per-resource blocks from output. The linter // prints skill names as headers — we annotate them with // the resolved owner. function annotateWithAuthors(output, kind) { @@ -238,10 +204,10 @@ jobs: `| ℹ️ Advisories | ${advisoryCount} |`, '', '---', ]; - const footer = `\n---\n\n_Generated by the [Skill Validator nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`; + const footer = `\n---\n\n_Generated by the [Vally lint nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`; - const skillsBlock = makeDetailsBlock('Skills', 'Full skill-validator output for skills', annotatedSkills); - const agentsBlock = makeDetailsBlock('Agents', 'Full skill-validator output for agents', annotatedAgents); + const skillsBlock = makeDetailsBlock('Skills', 'Full vally lint output for skills', annotatedSkills); + const agentsBlock = makeDetailsBlock('Agents', 'Agent scan notes', annotatedAgents); // Try full inline body first const fullBody = summaryLines.join('\n') + '\n\n' + skillsBlock + '\n\n' + agentsBlock + footer;