mirror of
https://github.com/github/awesome-copilot.git
synced 2026-07-14 10:01:06 +00:00
Aaron Powell
28c3a14af4
* Switch skill CI checks to vally lint Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Adding Vally to allowed words * case sensitivity * Migrate external plugin quality gates from skill-validator to vally lint Replace the downloaded skill-validator binary with px @microsoft/vally-cli lint in the external plugin quality gates pipeline: - Remove downloadSkillValidator() and SKILL_VALIDATOR_ARCHIVE_URL constant - Replace uildSkillValidatorArgs() + unSkillValidatorGate() with uildVallyLintArgs() + unVallyLintGate() that run px vally-cli lint per resolved skill directory (falling back to the full plugin root when no specific skill paths can be resolved from plugin.json) - Rename result keys skill_validator_status / skill_validator_output to ally_lint_status / ally_lint_output throughout both ng/external-plugin-quality-gates.mjs and ng/external-plugin-intake.mjs - Update PR comment markdown to show 'vally lint' instead of 'skill-validator' - Update CONTRIBUTING.md prose references accordingly Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use @microsoft/vally library directly instead of vally-cli subprocess Replace the npx-spawned vally-cli process with a direct call to the @microsoft/vally core library in the external plugin quality gates scripts: - Add @microsoft/vally as a devDependency in package.json - Import runLint and LintConsoleReporter from @microsoft/vally - Replace runVallyLintGate() process spawn with async API call: - runLint({ rootPath }) returns structured LintResults - LintConsoleReporter with a Writable capture stream collects text output without printing to stdout - Make runExternalPluginQualityGates() async (propagated to runExternalPluginPrQualityGates() and both main entry points) - Use Promise.all in runExternalPluginPrQualityGates() for parallel plugin checks - Fix remaining skill_validator_status reference in pr-quality-gates summary string (now vally-lint=...) and YAML workflow table header - Add 'npm install @microsoft/vally' step to both calling workflows This removes a layer of indirection (Node -> npx -> CLI -> library) and replaces it with a direct in-process library call, which is faster, more reliable, and gives structured access to lint results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
440 lines
18 KiB
YAML
440 lines
18 KiB
YAML
name: Skill Quality Report — Nightly Scan
|
||
|
||
on:
|
||
schedule:
|
||
- cron: "0 3 * * *" # 3:00 AM UTC daily
|
||
workflow_dispatch: # allow manual trigger
|
||
|
||
permissions:
|
||
contents: read
|
||
discussions: write
|
||
issues: write # fallback if Discussions are not enabled
|
||
|
||
jobs:
|
||
nightly-scan:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||
with:
|
||
fetch-depth: 0 # full history for git-log author fallback
|
||
|
||
- name: Setup Node.js
|
||
uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0
|
||
with:
|
||
node-version: 20
|
||
|
||
# ── Run full scan ─────────────────────────────────────────────
|
||
- name: Run vally lint on all skills
|
||
id: check-skills
|
||
run: |
|
||
set +e
|
||
set -o pipefail
|
||
npx --yes @microsoft/vally-cli lint ./skills --verbose 2>&1 | tee vally-skills-output.txt
|
||
echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
||
set +o pipefail
|
||
set -e
|
||
|
||
- name: Note agent scan status
|
||
id: check-agents
|
||
run: |
|
||
AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null | tr '\n' ' ')
|
||
if [ -n "$AGENT_FILES" ]; then
|
||
{
|
||
echo "ℹ️ Vally currently lints SKILL.md content."
|
||
echo "ℹ️ Agent files are detected but excluded from this scan:"
|
||
echo "$AGENT_FILES"
|
||
} > vally-agents-output.txt
|
||
else
|
||
echo "No agent files found."
|
||
echo "" > vally-agents-output.txt
|
||
echo "exit_code=0" >> "$GITHUB_OUTPUT"
|
||
fi
|
||
|
||
# ── Build report with author attribution ──────────────────────
|
||
- name: Build quality report
|
||
id: report
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
|
||
with:
|
||
script: |
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const { execSync } = require('child_process');
|
||
|
||
// ── Parse CODEOWNERS ──────────────────────────────────
|
||
function parseCodeowners() {
|
||
const map = new Map();
|
||
try {
|
||
const raw = fs.readFileSync('CODEOWNERS', 'utf8');
|
||
for (const line of raw.split('\n')) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||
const parts = trimmed.split(/\s+/);
|
||
if (parts.length >= 2) {
|
||
const filePath = parts[0].replace(/^\//, '').replace(/\/$/, '');
|
||
const owners = parts.slice(1).filter(p => p.startsWith('@'));
|
||
if (owners.length > 0) {
|
||
map.set(filePath, owners);
|
||
}
|
||
}
|
||
}
|
||
} catch (e) {
|
||
console.log('Could not parse CODEOWNERS:', e.message);
|
||
}
|
||
return map;
|
||
}
|
||
|
||
// ── Resolve author for a path ─────────────────────────
|
||
function resolveAuthor(resourcePath, codeowners) {
|
||
// CODEOWNERS semantics: last matching rule wins.
|
||
// Also treat "*" as a match-all default rule.
|
||
let matchedOwners = null;
|
||
for (const [pattern, owners] of codeowners) {
|
||
if (
|
||
pattern === '*' ||
|
||
resourcePath === pattern ||
|
||
resourcePath.startsWith(pattern + '/')
|
||
) {
|
||
matchedOwners = owners;
|
||
}
|
||
}
|
||
if (matchedOwners && matchedOwners.length > 0) {
|
||
return matchedOwners.join(', ');
|
||
}
|
||
// Fallback: git log
|
||
try {
|
||
const author = execSync(
|
||
`git log --format='%aN' --follow -1 -- "${resourcePath}"`,
|
||
{ encoding: 'utf8' }
|
||
).trim();
|
||
return author || 'unknown';
|
||
} catch {
|
||
return 'unknown';
|
||
}
|
||
}
|
||
|
||
// ── Parse vally lint output ───────────────────────────
|
||
// The output is a text report; we preserve it as-is and
|
||
// augment it with author info in the summary.
|
||
const skillsOutput = fs.readFileSync('vally-skills-output.txt', 'utf8').trim();
|
||
const agentsOutput = fs.existsSync('vally-agents-output.txt')
|
||
? fs.readFileSync('vally-agents-output.txt', 'utf8').trim()
|
||
: '';
|
||
|
||
const codeowners = parseCodeowners();
|
||
|
||
// Count findings
|
||
// Vally lint uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories
|
||
const combined = skillsOutput + '\n' + agentsOutput;
|
||
const errorCount = (combined.match(/❌/g) || []).length;
|
||
const warningCount = (combined.match(/⚠/g) || []).length;
|
||
const advisoryCount = (combined.match(/ℹ\uFE0F?/g) || []).length;
|
||
|
||
// Count total skills & agents checked
|
||
let skillDirs = [];
|
||
try {
|
||
skillDirs = fs.readdirSync('skills', { withFileTypes: true })
|
||
.filter(d => d.isDirectory())
|
||
.map(d => d.name);
|
||
} catch {}
|
||
|
||
let agentFiles = [];
|
||
try {
|
||
agentFiles = fs.readdirSync('agents')
|
||
.filter(f => f.endsWith('.agent.md'));
|
||
} catch {}
|
||
|
||
// ── Build author-attributed summary ───────────────────
|
||
// Extract per-resource blocks from output. The linter
|
||
// prints skill names as headers — we annotate them with
|
||
// the resolved owner.
|
||
function annotateWithAuthors(output, kind) {
|
||
if (!output) return '_No findings._';
|
||
const lines = output.split('\n');
|
||
const annotated = [];
|
||
for (const line of lines) {
|
||
// Skill names appear as headers, e.g. "## skill-name" or "skill-name:"
|
||
const headerMatch = line.match(/^(?:#{1,3}\s+)?([a-z0-9][a-z0-9-]+(?:\.[a-z0-9.-]+)?)\b/);
|
||
if (headerMatch) {
|
||
const name = headerMatch[1];
|
||
const resourcePath = kind === 'skill'
|
||
? `skills/${name}`
|
||
: `agents/${name}.agent.md`;
|
||
const author = resolveAuthor(resourcePath, codeowners);
|
||
annotated.push(`${line} — ${author}`);
|
||
} else {
|
||
annotated.push(line);
|
||
}
|
||
}
|
||
return annotated.join('\n');
|
||
}
|
||
|
||
const today = new Date().toISOString().split('T')[0];
|
||
|
||
const title = `Skill Quality Report — ${today}`;
|
||
|
||
const annotatedSkills = annotateWithAuthors(skillsOutput, 'skill');
|
||
const annotatedAgents = annotateWithAuthors(agentsOutput, 'agent');
|
||
|
||
// ── Body size management ──────────────────────────────
|
||
// GitHub body limit is ~65536 UTF-8 bytes for both
|
||
// Discussions and Issues. When the full report fits, we
|
||
// inline everything. When it doesn't, the body gets a
|
||
// compact summary and the verbose sections are written to
|
||
// separate files that get posted as follow-up comments.
|
||
const MAX_BYTES = 65000; // leave margin
|
||
|
||
function makeDetailsBlock(heading, summary, content) {
|
||
return [
|
||
`## ${heading}`, '',
|
||
'<details>',
|
||
`<summary>${summary}</summary>`, '',
|
||
'```', content, '```', '',
|
||
'</details>',
|
||
].join('\n');
|
||
}
|
||
|
||
const summaryLines = [
|
||
`# ${title}`, '',
|
||
`**${skillDirs.length} skills** and **${agentFiles.length} agents** scanned.`, '',
|
||
'| Severity | Count |',
|
||
'|----------|-------|',
|
||
`| ⛔ Errors | ${errorCount} |`,
|
||
`| ⚠️ Warnings | ${warningCount} |`,
|
||
`| ℹ️ Advisories | ${advisoryCount} |`, '',
|
||
'---',
|
||
];
|
||
const footer = `\n---\n\n_Generated by the [Vally lint nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`;
|
||
|
||
const skillsBlock = makeDetailsBlock('Skills', 'Full vally lint output for skills', annotatedSkills);
|
||
const agentsBlock = makeDetailsBlock('Agents', 'Agent scan notes', annotatedAgents);
|
||
|
||
// Try full inline body first
|
||
const fullBody = summaryLines.join('\n') + '\n\n' + skillsBlock + '\n\n' + agentsBlock + footer;
|
||
|
||
const commentParts = []; // overflow comment files
|
||
|
||
let finalBody;
|
||
if (Buffer.byteLength(fullBody, 'utf8') <= MAX_BYTES) {
|
||
finalBody = fullBody;
|
||
} else {
|
||
// Details won't fit inline — move them to follow-up comments
|
||
const bodyNote = '\n\n> **Note:** Detailed output is posted in the comments below (too large for the discussion body).\n';
|
||
finalBody = summaryLines.join('\n') + bodyNote + footer;
|
||
|
||
// Split each section into ≤65 KB chunks
|
||
function chunkContent(label, content) {
|
||
const prefix = `## ${label}\n\n\`\`\`\n`;
|
||
const suffix = '\n```';
|
||
const overhead = Buffer.byteLength(prefix + suffix, 'utf8');
|
||
const budget = MAX_BYTES - overhead;
|
||
|
||
const buf = Buffer.from(content, 'utf8');
|
||
if (buf.length <= budget) {
|
||
return [prefix + content + suffix];
|
||
}
|
||
const parts = [];
|
||
let offset = 0;
|
||
let partNum = 1;
|
||
while (offset < buf.length) {
|
||
const slice = buf.slice(offset, offset + budget).toString('utf8');
|
||
// Remove trailing replacement char from mid-codepoint cut
|
||
const clean = slice.replace(/\uFFFD$/, '');
|
||
const hdr = `## ${label} (part ${partNum})\n\n\`\`\`\n`;
|
||
parts.push(hdr + clean + suffix);
|
||
offset += Buffer.byteLength(clean, 'utf8');
|
||
partNum++;
|
||
}
|
||
return parts;
|
||
}
|
||
|
||
commentParts.push(...chunkContent('Skills', annotatedSkills));
|
||
commentParts.push(...chunkContent('Agents', annotatedAgents));
|
||
}
|
||
|
||
core.setOutput('title', title);
|
||
core.setOutput('body_file', 'report-body.md');
|
||
|
||
fs.writeFileSync('report-body.md', finalBody);
|
||
|
||
// Write overflow comment parts as numbered files
|
||
for (let i = 0; i < commentParts.length; i++) {
|
||
fs.writeFileSync(`report-comment-${i}.md`, commentParts[i]);
|
||
}
|
||
core.setOutput('comment_count', String(commentParts.length));
|
||
|
||
- name: Close old report discussions
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
|
||
with:
|
||
script: |
|
||
const RETENTION_DAYS = 5;
|
||
const CATEGORY_NAME = 'Skill Quality Reports';
|
||
const TITLE_PREFIX = 'Skill Quality Report — ';
|
||
|
||
const cutoff = new Date();
|
||
cutoff.setUTCDate(cutoff.getUTCDate() - RETENTION_DAYS);
|
||
|
||
let after = null;
|
||
let closedCount = 0;
|
||
let reachedCutoff = false;
|
||
|
||
while (!reachedCutoff) {
|
||
const result = await github.graphql(`
|
||
query($owner: String!, $repo: String!, $after: String) {
|
||
repository(owner: $owner, name: $repo) {
|
||
discussions(first: 100, after: $after, orderBy: { field: CREATED_AT, direction: ASC }) {
|
||
nodes {
|
||
id
|
||
title
|
||
createdAt
|
||
closed
|
||
url
|
||
category {
|
||
name
|
||
}
|
||
}
|
||
pageInfo {
|
||
hasNextPage
|
||
endCursor
|
||
}
|
||
}
|
||
}
|
||
}
|
||
`, {
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
after,
|
||
});
|
||
|
||
const discussions = result.repository.discussions;
|
||
|
||
for (const discussion of discussions.nodes) {
|
||
if (new Date(discussion.createdAt) >= cutoff) {
|
||
reachedCutoff = true;
|
||
break;
|
||
}
|
||
|
||
if (discussion.category?.name !== CATEGORY_NAME) continue;
|
||
if (!discussion.title.startsWith(TITLE_PREFIX)) continue;
|
||
if (discussion.closed) continue;
|
||
|
||
await github.graphql(`
|
||
mutation($discussionId: ID!) {
|
||
closeDiscussion(input: { discussionId: $discussionId }) {
|
||
discussion {
|
||
id
|
||
}
|
||
}
|
||
}
|
||
`, { discussionId: discussion.id });
|
||
|
||
closedCount++;
|
||
console.log(`Closed old report discussion: ${discussion.url}`);
|
||
}
|
||
|
||
if (reachedCutoff || !discussions.pageInfo.hasNextPage) {
|
||
break;
|
||
}
|
||
|
||
after = discussions.pageInfo.endCursor;
|
||
}
|
||
|
||
console.log(`Closed ${closedCount} report discussion(s) older than ${RETENTION_DAYS} days.`);
|
||
|
||
# ── Create Discussion (preferred) or Issue (fallback) ────────
|
||
- name: Create Discussion
|
||
id: create-discussion
|
||
continue-on-error: true
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
|
||
with:
|
||
script: |
|
||
const fs = require('fs');
|
||
const title = '${{ steps.report.outputs.title }}'.replace(/'/g, "\\'");
|
||
const body = fs.readFileSync('report-body.md', 'utf8');
|
||
const commentCount = parseInt('${{ steps.report.outputs.comment_count }}' || '0', 10);
|
||
|
||
// Find the "Skill Quality Reports" category
|
||
const categoriesResult = await github.graphql(`
|
||
query($owner: String!, $repo: String!) {
|
||
repository(owner: $owner, name: $repo) {
|
||
id
|
||
discussionCategories(first: 25) {
|
||
nodes { id name }
|
||
}
|
||
}
|
||
}
|
||
`, {
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
});
|
||
|
||
const repo = categoriesResult.repository;
|
||
const categories = repo.discussionCategories.nodes;
|
||
const category = categories.find(c =>
|
||
c.name === 'Skill Quality Reports'
|
||
);
|
||
|
||
if (!category) {
|
||
core.setFailed('Discussion category "Skill Quality Reports" not found. Falling back to issue.');
|
||
return;
|
||
}
|
||
|
||
const result = await github.graphql(`
|
||
mutation($repoId: ID!, $categoryId: ID!, $title: String!, $body: String!) {
|
||
createDiscussion(input: {
|
||
repositoryId: $repoId,
|
||
categoryId: $categoryId,
|
||
title: $title,
|
||
body: $body
|
||
}) {
|
||
discussion { id url }
|
||
}
|
||
}
|
||
`, {
|
||
repoId: repo.id,
|
||
categoryId: category.id,
|
||
title: title,
|
||
body: body,
|
||
});
|
||
|
||
const discussionId = result.createDiscussion.discussion.id;
|
||
console.log(`Discussion created: ${result.createDiscussion.discussion.url}`);
|
||
|
||
// Post overflow detail comments
|
||
for (let i = 0; i < commentCount; i++) {
|
||
const commentBody = fs.readFileSync(`report-comment-${i}.md`, 'utf8');
|
||
await github.graphql(`
|
||
mutation($discussionId: ID!, $body: String!) {
|
||
addDiscussionComment(input: {
|
||
discussionId: $discussionId,
|
||
body: $body
|
||
}) {
|
||
comment { id }
|
||
}
|
||
}
|
||
`, { discussionId, body: commentBody });
|
||
console.log(`Posted detail comment ${i + 1}/${commentCount}`);
|
||
}
|
||
|
||
- name: Fallback — Create Issue
|
||
if: steps.create-discussion.outcome == 'failure'
|
||
env:
|
||
GH_TOKEN: ${{ github.token }}
|
||
run: |
|
||
# Create label if it doesn't exist (ignore errors if it already exists)
|
||
gh label create "skill-quality" --description "Automated skill quality reports" --color "d4c5f9" 2>/dev/null || true
|
||
ISSUE_URL=$(gh issue create \
|
||
--title "${{ steps.report.outputs.title }}" \
|
||
--body-file report-body.md \
|
||
--label "skill-quality")
|
||
echo "Created issue: $ISSUE_URL"
|
||
|
||
# Post overflow detail comments on the issue
|
||
COMMENT_COUNT=${{ steps.report.outputs.comment_count }}
|
||
for i in $(seq 0 $(( ${COMMENT_COUNT:-0} - 1 ))); do
|
||
if [ -f "report-comment-${i}.md" ]; then
|
||
gh issue comment "$ISSUE_URL" --body-file "report-comment-${i}.md"
|
||
echo "Posted detail comment $((i+1))/${COMMENT_COUNT}"
|
||
fi
|
||
done
|