mirror of
https://github.com/github/awesome-copilot.git
synced 2026-07-14 01:51:02 +00:00
Switch skill CI validation workflows to vally lint (#2030)
* Switch skill CI checks to vally lint Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Adding Vally to allowed words * case sensitivity * Migrate external plugin quality gates from skill-validator to vally lint Replace the downloaded skill-validator binary with px @microsoft/vally-cli lint in the external plugin quality gates pipeline: - Remove downloadSkillValidator() and SKILL_VALIDATOR_ARCHIVE_URL constant - Replace uildSkillValidatorArgs() + unSkillValidatorGate() with uildVallyLintArgs() + unVallyLintGate() that run px vally-cli lint per resolved skill directory (falling back to the full plugin root when no specific skill paths can be resolved from plugin.json) - Rename result keys skill_validator_status / skill_validator_output to ally_lint_status / ally_lint_output throughout both ng/external-plugin-quality-gates.mjs and ng/external-plugin-intake.mjs - Update PR comment markdown to show 'vally lint' instead of 'skill-validator' - Update CONTRIBUTING.md prose references accordingly Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use @microsoft/vally library directly instead of vally-cli subprocess Replace the npx-spawned vally-cli process with a direct call to the @microsoft/vally core library in the external plugin quality gates scripts: - Add @microsoft/vally as a devDependency in package.json - Import runLint and LintConsoleReporter from @microsoft/vally - Replace runVallyLintGate() process spawn with async API call: - runLint({ rootPath }) returns structured LintResults - LintConsoleReporter with a Writable capture stream collects text output without printing to stdout - Make runExternalPluginQualityGates() async (propagated to runExternalPluginPrQualityGates() and both main entry points) - Use Promise.all in runExternalPluginPrQualityGates() for parallel plugin checks - Fix remaining skill_validator_status reference in pr-quality-gates summary string (now vally-lint=...) and YAML workflow table header - Add 'npm install @microsoft/vally' step to both calling workflows This removes a layer of indirection (Node -> npx -> CLI -> library) and replaces it with a direct in-process library call, which is faster, more reliable, and gives structured access to lint results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Aaron Powell
GitHub
@@ -423,11 +423,11 @@ export function parseMarkReadyForReviewCommand(body) {
|
||||
function normalizeQualityGateResult(rawResult) {
|
||||
const defaults = {
|
||||
overall_status: "not_run",
|
||||
skill_validator_status: "not_run",
|
||||
vally_lint_status: "not_run",
|
||||
smoke_status: "not_run",
|
||||
failure_class: "none",
|
||||
summary: "",
|
||||
skill_validator_output: "",
|
||||
vally_lint_output: "",
|
||||
smoke_output: "",
|
||||
};
|
||||
|
||||
@@ -442,7 +442,7 @@ function normalizeQualityGateResult(rawResult) {
|
||||
}
|
||||
|
||||
function buildQualityGatesCommentSection(qualityResult) {
|
||||
const skillState = qualityResult.skill_validator_status || "not_run";
|
||||
const vallyState = qualityResult.vally_lint_status || "not_run";
|
||||
const smokeState = qualityResult.smoke_status || "not_run";
|
||||
const summaryText = String(qualityResult.summary || "").trim() || "_No quality gate details were provided._";
|
||||
|
||||
@@ -451,21 +451,21 @@ function buildQualityGatesCommentSection(qualityResult) {
|
||||
"",
|
||||
"| Gate | Status |",
|
||||
"|---|---|",
|
||||
`| skill-validator | ${skillState} |`,
|
||||
`| vally lint | ${vallyState} |`,
|
||||
`| install smoke test | ${smokeState} |`,
|
||||
"",
|
||||
summaryText,
|
||||
];
|
||||
|
||||
const skillOutput = String(qualityResult.skill_validator_output || "").trim();
|
||||
if (skillOutput) {
|
||||
const vallyOutput = String(qualityResult.vally_lint_output || "").trim();
|
||||
if (vallyOutput) {
|
||||
sections.push(
|
||||
"",
|
||||
"<details>",
|
||||
"<summary>skill-validator output</summary>",
|
||||
"<summary>vally lint output</summary>",
|
||||
"",
|
||||
"```text",
|
||||
skillOutput,
|
||||
vallyOutput,
|
||||
"```",
|
||||
"",
|
||||
"</details>",
|
||||
|
||||
@@ -66,27 +66,27 @@ function aggregateResultStatus(pluginResults) {
|
||||
};
|
||||
}
|
||||
|
||||
export function runExternalPluginPrQualityGates(plugins) {
|
||||
export async function runExternalPluginPrQualityGates(plugins) {
|
||||
if (!Array.isArray(plugins)) {
|
||||
throw new Error("plugins must be an array");
|
||||
}
|
||||
|
||||
const checkedPlugins = plugins.map((plugin) => {
|
||||
const quality = runExternalPluginQualityGates(plugin);
|
||||
const checkedPlugins = await Promise.all(plugins.map(async (plugin) => {
|
||||
const quality = await runExternalPluginQualityGates(plugin);
|
||||
return {
|
||||
name: plugin?.name ?? "unknown",
|
||||
source: plugin?.source ?? {},
|
||||
source_tree_url: buildSourceTreeUrl(plugin),
|
||||
quality,
|
||||
};
|
||||
});
|
||||
}));
|
||||
|
||||
const aggregate = aggregateResultStatus(checkedPlugins);
|
||||
const summary = checkedPlugins.length === 0
|
||||
? "No changed external plugin entries were detected in plugins/external.json."
|
||||
: checkedPlugins
|
||||
.map((entry) =>
|
||||
`- ${entry.name}: skill-validator=${entry.quality.skill_validator_status}, install-smoke=${entry.quality.smoke_status}, overall=${entry.quality.overall_status}`
|
||||
`- ${entry.name}: vally-lint=${entry.quality.vally_lint_status}, install-smoke=${entry.quality.smoke_status}, overall=${entry.quality.overall_status}`
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
@@ -120,6 +120,6 @@ if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
}
|
||||
|
||||
const plugins = JSON.parse(args["plugins-json"]);
|
||||
const result = runExternalPluginPrQualityGates(plugins);
|
||||
const result = await runExternalPluginPrQualityGates(plugins);
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
}
|
||||
|
||||
@@ -3,10 +3,11 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { Writable } from "stream";
|
||||
import { spawnSync } from "child_process";
|
||||
import { runLint, LintConsoleReporter } from "@microsoft/vally";
|
||||
|
||||
const MAX_OUTPUT_LENGTH = 12000;
|
||||
const SKILL_VALIDATOR_ARCHIVE_URL = "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz";
|
||||
|
||||
const INFRA_ERROR_PATTERNS = [
|
||||
/\b401\b/,
|
||||
@@ -132,35 +133,10 @@ function cloneSubmissionRepository(workDir, plugin) {
|
||||
return repoDir;
|
||||
}
|
||||
|
||||
function downloadSkillValidator(workDir) {
|
||||
const validatorDir = path.join(workDir, "skill-validator");
|
||||
ensureDirectory(validatorDir);
|
||||
const archivePath = path.join(validatorDir, "skill-validator-linux-x64.tar.gz");
|
||||
|
||||
const download = runCommand("curl", ["-fsSL", SKILL_VALIDATOR_ARCHIVE_URL, "-o", archivePath]);
|
||||
if (download.exitCode !== 0) {
|
||||
throw new Error(`Failed to download skill-validator: ${download.output}`);
|
||||
}
|
||||
|
||||
const untar = runCommand("tar", ["-xzf", archivePath, "-C", validatorDir]);
|
||||
if (untar.exitCode !== 0) {
|
||||
throw new Error(`Failed to extract skill-validator: ${untar.output}`);
|
||||
}
|
||||
|
||||
const binaryPath = path.join(validatorDir, "skill-validator");
|
||||
if (!fs.existsSync(binaryPath)) {
|
||||
throw new Error("skill-validator binary was not found after extraction");
|
||||
}
|
||||
|
||||
runCommand("chmod", ["+x", binaryPath]);
|
||||
return binaryPath;
|
||||
}
|
||||
|
||||
// Ordered list of candidate locations for plugin.json, from most to least specific.
|
||||
// The skill-validator --plugin mode expects plugin.json at the plugin root, but
|
||||
// both the Copilot CLI and many external repos use nested conventions. We read the
|
||||
// manifest ourselves so skill/agent paths can be resolved from the plugin root
|
||||
// consistently, regardless of where the manifest lives.
|
||||
// Both the Copilot CLI and many external repos use nested conventions. We read the
|
||||
// manifest ourselves so skill paths can be resolved from the plugin root consistently,
|
||||
// regardless of where the manifest lives.
|
||||
// NOTE: Keep in sync with EXTERNAL_PLUGIN_ROOT_MANIFEST_PATHS in external-plugin-validation.mjs
|
||||
const PLUGIN_JSON_CANDIDATES = [
|
||||
[".github", "plugin", "plugin.json"],
|
||||
@@ -178,72 +154,66 @@ function findPluginJson(pluginRoot) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildSkillValidatorArgs(pluginRoot) {
|
||||
function buildVallyLintArgs(pluginRoot) {
|
||||
const pluginJsonPath = findPluginJson(pluginRoot);
|
||||
if (!pluginJsonPath) {
|
||||
// No recognised plugin.json location found — let the validator fail with its
|
||||
// own diagnostic (covers exotic layouts and surfaces the real error to submitters).
|
||||
return ["check", "--verbose", "--plugin", pluginRoot];
|
||||
// No recognised plugin.json location — lint the whole plugin root and let
|
||||
// vally surface the real error to the submitter.
|
||||
return [pluginRoot];
|
||||
}
|
||||
|
||||
let pluginJson;
|
||||
try {
|
||||
pluginJson = JSON.parse(fs.readFileSync(pluginJsonPath, "utf8"));
|
||||
} catch {
|
||||
// Malformed plugin.json — let the validator surface the parse error.
|
||||
return ["check", "--verbose", "--plugin", pluginRoot];
|
||||
// Malformed plugin.json — fall back to linting the full root.
|
||||
return [pluginRoot];
|
||||
}
|
||||
|
||||
const args = ["check", "--verbose"];
|
||||
|
||||
// Paths in plugin.json are relative to the plugin root regardless of where
|
||||
// plugin.json itself lives. Use [].concat() to accept both string and array values.
|
||||
// Collect skill directory paths from plugin.json.
|
||||
const skillPaths = [].concat(pluginJson.skills ?? [])
|
||||
.map((s) => path.resolve(pluginRoot, s))
|
||||
.filter((p) => fs.existsSync(p));
|
||||
|
||||
// Agent entries may be directory paths or explicit file paths; normalise to directories
|
||||
// so AgentDiscovery.DiscoverAgentsInDirectory can discover agents within them.
|
||||
// Deduplicate in case multiple file entries share the same parent directory.
|
||||
const agentPaths = [...new Set(
|
||||
[].concat(pluginJson.agents ?? [])
|
||||
.map((a) => {
|
||||
const resolved = path.resolve(pluginRoot, a);
|
||||
if (fs.existsSync(resolved) && fs.statSync(resolved).isFile()) {
|
||||
return path.dirname(resolved);
|
||||
}
|
||||
return resolved;
|
||||
})
|
||||
.filter((p) => fs.existsSync(p))
|
||||
)];
|
||||
.filter((p) => fs.existsSync(p) && fs.statSync(p).isDirectory());
|
||||
|
||||
if (skillPaths.length > 0) {
|
||||
args.push("--skills", ...skillPaths);
|
||||
}
|
||||
if (agentPaths.length > 0) {
|
||||
args.push("--agents", ...agentPaths);
|
||||
return skillPaths;
|
||||
}
|
||||
|
||||
if (skillPaths.length === 0 && agentPaths.length === 0) {
|
||||
// plugin.json found but no resolvable skills/agents — fall back to --plugin so the
|
||||
// validator can surface the specific validation error to the submitter.
|
||||
return ["check", "--verbose", "--plugin", pluginRoot];
|
||||
}
|
||||
|
||||
return args;
|
||||
// No resolvable skill directories — lint the full plugin root so vally can
|
||||
// surface the specific validation error to the submitter.
|
||||
return [pluginRoot];
|
||||
}
|
||||
|
||||
function runSkillValidatorGate(workDir, pluginRoot) {
|
||||
async function runVallyLintGate(pluginRoot) {
|
||||
try {
|
||||
const validatorBinary = downloadSkillValidator(workDir);
|
||||
const args = buildSkillValidatorArgs(pluginRoot);
|
||||
const check = runCommand(validatorBinary, args);
|
||||
const targets = buildVallyLintArgs(pluginRoot);
|
||||
|
||||
if (check.exitCode === 0) {
|
||||
return { status: "pass", output: check.output };
|
||||
let combinedOutput = "";
|
||||
let anyFailure = false;
|
||||
|
||||
for (const target of targets) {
|
||||
const chunks = [];
|
||||
const captureStream = new Writable({
|
||||
write(chunk, _encoding, callback) {
|
||||
chunks.push(chunk.toString());
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
const result = await runLint({ rootPath: target });
|
||||
const reporter = new LintConsoleReporter({ verbose: true, stream: captureStream });
|
||||
await reporter.report(result);
|
||||
|
||||
combinedOutput += chunks.join("") + "\n";
|
||||
if (!result.passed) {
|
||||
anyFailure = true;
|
||||
}
|
||||
}
|
||||
|
||||
return { status: "fail", output: check.output };
|
||||
return {
|
||||
status: anyFailure ? "fail" : "pass",
|
||||
output: truncateOutput(combinedOutput),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
status: "infra_error",
|
||||
@@ -358,15 +328,15 @@ function toFailureClass(overallStatus) {
|
||||
return "none";
|
||||
}
|
||||
|
||||
export function runExternalPluginQualityGates(plugin) {
|
||||
export async function runExternalPluginQualityGates(plugin) {
|
||||
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "external-plugin-quality-"));
|
||||
const result = {
|
||||
overall_status: "not_run",
|
||||
skill_validator_status: "not_run",
|
||||
vally_lint_status: "not_run",
|
||||
smoke_status: "not_run",
|
||||
failure_class: "none",
|
||||
summary: "",
|
||||
skill_validator_output: "",
|
||||
vally_lint_output: "",
|
||||
smoke_output: "",
|
||||
};
|
||||
|
||||
@@ -376,7 +346,7 @@ export function runExternalPluginQualityGates(plugin) {
|
||||
const pluginRoot = normalizedPluginPath ? path.join(repoDir, normalizedPluginPath) : repoDir;
|
||||
|
||||
if (!fs.existsSync(pluginRoot) || !fs.statSync(pluginRoot).isDirectory()) {
|
||||
result.skill_validator_status = "fail";
|
||||
result.vally_lint_status = "fail";
|
||||
result.smoke_status = "fail";
|
||||
result.overall_status = "fail";
|
||||
result.failure_class = "submitter_fixes";
|
||||
@@ -384,18 +354,18 @@ export function runExternalPluginQualityGates(plugin) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const skillResult = runSkillValidatorGate(workDir, pluginRoot);
|
||||
result.skill_validator_status = skillResult.status;
|
||||
result.skill_validator_output = skillResult.output;
|
||||
const vallyResult = await runVallyLintGate(pluginRoot);
|
||||
result.vally_lint_status = vallyResult.status;
|
||||
result.vally_lint_output = vallyResult.output;
|
||||
|
||||
const smokeResult = runInstallSmokeGate(workDir, plugin);
|
||||
result.smoke_status = smokeResult.status;
|
||||
result.smoke_output = smokeResult.output;
|
||||
|
||||
result.overall_status = toOverallStatus(result.skill_validator_status, result.smoke_status);
|
||||
result.overall_status = toOverallStatus(result.vally_lint_status, result.smoke_status);
|
||||
result.failure_class = toFailureClass(result.overall_status);
|
||||
result.summary = [
|
||||
`- skill-validator: ${result.skill_validator_status}`,
|
||||
`- vally lint: ${result.vally_lint_status}`,
|
||||
`- install smoke test: ${result.smoke_status}`,
|
||||
`- overall: ${result.overall_status}`,
|
||||
].join("\n");
|
||||
@@ -405,7 +375,7 @@ export function runExternalPluginQualityGates(plugin) {
|
||||
result.overall_status = "infra_error";
|
||||
result.failure_class = "infra";
|
||||
result.summary = truncateOutput(error.message);
|
||||
result.skill_validator_output = truncateOutput(error.stack || error.message);
|
||||
result.vally_lint_output = truncateOutput(error.stack || error.message);
|
||||
return result;
|
||||
} finally {
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
@@ -434,6 +404,6 @@ if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
}
|
||||
|
||||
const plugin = JSON.parse(args["plugin-json"]);
|
||||
const result = runExternalPluginQualityGates(plugin);
|
||||
const result = await runExternalPluginQualityGates(plugin);
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user