#!/usr/bin/env node import fs from "fs"; import os from "os"; import path from "path"; import { Writable } from "stream"; import { spawnSync } from "child_process"; import { runLint, LintConsoleReporter } from "@microsoft/vally"; const MAX_OUTPUT_LENGTH = 12000; const INFRA_ERROR_PATTERNS = [ /\b401\b/, /\b403\b/, /authentication (required|failed|error)/, /unauthenticated/, /unauthorized/, /not logged in/, /please (log in|authenticate|sign in)/, /invalid (access |auth )?token/, /credentials? (are )?expired/, /dns.*(resolve|lookup|fail)/, /network.*unreachable/, /connection (refused|reset)/, /\btimeout\b/, /enotfound/, /econnrefused/, /etimedout/, ]; function truncateOutput(value) { const normalized = String(value ?? "").replace(/\x1b\[[0-9;]*m/g, "").trim(); if (normalized.length <= MAX_OUTPUT_LENGTH) { return normalized; } return `${normalized.slice(0, MAX_OUTPUT_LENGTH)}\n...output truncated...`; } function runCommand(command, args, options = {}) { const result = spawnSync(command, args, { encoding: "utf8", ...options, }); return { exitCode: typeof result.status === "number" ? result.status : 1, stdout: truncateOutput(result.stdout), stderr: truncateOutput(result.stderr), output: truncateOutput(`${result.stdout ?? ""}\n${result.stderr ?? ""}`), error: result.error ? String(result.error.message ?? result.error) : "", }; } function normalizePluginPath(pluginPath) { if (!pluginPath || pluginPath === "/") { return ""; } const normalized = String(pluginPath).trim().replace(/^\/+|\/+$/g, ""); if (!normalized) { return ""; } if (normalized.includes("..") || normalized.includes("\\")) { throw new Error(`Invalid plugin path "${pluginPath}"`); } return normalized; } function resolveFetchSpec(pluginSource) { if (pluginSource.sha) { return pluginSource.sha; } if (!pluginSource.ref) { throw new Error("source.ref or source.sha is required for quality gates"); } const ref = String(pluginSource.ref).trim(); if (!ref) { throw new Error("source.ref or source.sha is required for quality gates"); } if (ref.startsWith("refs/")) { return ref; } return ref; } function classifySmokeFailure(output) { const normalized = String(output ?? "").toLowerCase(); if (INFRA_ERROR_PATTERNS.some((pattern) => pattern.test(normalized))) { return "infra_error"; } return "fail"; } function ensureDirectory(dirPath) { fs.mkdirSync(dirPath, { recursive: true }); } function cloneSubmissionRepository(workDir, plugin) { const repoDir = path.join(workDir, "submission"); ensureDirectory(repoDir); const sourceRepo = plugin.source?.repo; const fetchSpec = resolveFetchSpec(plugin.source ?? {}); const init = runCommand("git", ["init", "-q"], { cwd: repoDir }); if (init.exitCode !== 0) { throw new Error(`git init failed: ${init.output}`); } const addRemote = runCommand("git", ["remote", "add", "origin", `https://github.com/${sourceRepo}.git`], { cwd: repoDir }); if (addRemote.exitCode !== 0) { throw new Error(`git remote add failed: ${addRemote.output}`); } const fetch = runCommand("git", ["fetch", "--depth=1", "origin", fetchSpec], { cwd: repoDir }); if (fetch.exitCode !== 0) { throw new Error(`git fetch failed for ${fetchSpec}: ${fetch.output}`); } const checkout = runCommand("git", ["checkout", "--detach", "FETCH_HEAD"], { cwd: repoDir }); if (checkout.exitCode !== 0) { throw new Error(`git checkout failed: ${checkout.output}`); } return { repoDir, fetchSpec, }; } // Ordered list of candidate locations for plugin.json, from most to least specific. // Both the Copilot CLI and many external repos use nested conventions. We read the // manifest ourselves so skill paths can be resolved from the plugin root consistently, // regardless of where the manifest lives. // NOTE: Keep in sync with EXTERNAL_PLUGIN_ROOT_MANIFEST_PATHS in external-plugin-validation.mjs const PLUGIN_JSON_CANDIDATES = [ [".github", "plugin", "plugin.json"], [".plugin", "plugin.json"], ["plugin.json"], ]; function toPosixPath(...segments) { return segments .filter((segment) => segment !== undefined && segment !== null && String(segment).length > 0) .map((segment) => String(segment).replace(/\\/g, "/")) .join("/"); } function findPluginJson(pluginRoot) { for (const segments of PLUGIN_JSON_CANDIDATES) { const candidate = path.join(pluginRoot, ...segments); if (fs.existsSync(candidate)) { return candidate; } } return null; } function buildVallyLintArgs(pluginRoot) { const pluginJsonPath = findPluginJson(pluginRoot); if (!pluginJsonPath) { // No recognised plugin.json location — lint the whole plugin root and let // vally surface the real error to the submitter. return [pluginRoot]; } let pluginJson; try { pluginJson = JSON.parse(fs.readFileSync(pluginJsonPath, "utf8")); } catch { // Malformed plugin.json — fall back to linting the full root. return [pluginRoot]; } // Collect skill directory paths from plugin.json. const skillPaths = [].concat(pluginJson.skills ?? []) .map((s) => path.resolve(pluginRoot, s)) .filter((p) => fs.existsSync(p) && fs.statSync(p).isDirectory()); if (skillPaths.length > 0) { return skillPaths; } // No resolvable skill directories — lint the full plugin root so vally can // surface the specific validation error to the submitter. return [pluginRoot]; } async function runVallyLintGate(pluginRoot) { try { const targets = buildVallyLintArgs(pluginRoot); let combinedOutput = ""; let anyFailure = false; for (const target of targets) { const chunks = []; const captureStream = new Writable({ write(chunk, _encoding, callback) { chunks.push(chunk.toString()); callback(); }, }); const result = await runLint({ rootPath: target }); const reporter = new LintConsoleReporter({ verbose: true, stream: captureStream }); await reporter.report(result); combinedOutput += chunks.join("") + "\n"; if (!result.passed) { anyFailure = true; } } return { status: anyFailure ? "fail" : "pass", output: truncateOutput(combinedOutput), }; } catch (error) { return { status: "infra_error", output: truncateOutput(error.message), }; } } function buildEphemeralMarketplace(workDir, plugin) { const marketplaceDir = path.join(workDir, "marketplace"); ensureDirectory(marketplaceDir); const marketplace = { name: "external-plugin-intake", metadata: { description: "Temporary marketplace for external plugin intake smoke tests", version: "1.0.0", pluginRoot: ".", }, owner: { name: "awesome-copilot-intake", email: "noreply@github.com", }, plugins: [plugin], }; fs.writeFileSync(path.join(marketplaceDir, "marketplace.json"), `${JSON.stringify(marketplace, null, 2)}\n`); return marketplaceDir; } function runInstallSmokeGate(workDir, plugin) { if (runCommand("bash", ["-lc", "command -v copilot"]).exitCode !== 0) { return { status: "infra_error", output: "copilot CLI is not available on this runner.", }; } try { const homeDir = path.join(workDir, "copilot-home"); ensureDirectory(homeDir); const marketplaceDir = buildEphemeralMarketplace(workDir, plugin); const env = { ...process.env, HOME: homeDir, XDG_CONFIG_HOME: path.join(homeDir, ".config"), XDG_CACHE_HOME: path.join(homeDir, ".cache"), XDG_DATA_HOME: path.join(homeDir, ".local", "share"), }; const marketplaceAdd = runCommand("copilot", ["plugin", "marketplace", "add", marketplaceDir], { env }); if (marketplaceAdd.exitCode !== 0) { const status = classifySmokeFailure(marketplaceAdd.output); return { status, output: marketplaceAdd.output }; } const install = runCommand("copilot", ["plugin", "install", `${plugin.name}@external-plugin-intake`], { env }); if (install.exitCode !== 0) { const status = classifySmokeFailure(install.output); return { status, output: install.output }; } const installedPluginPath = path.join(homeDir, ".copilot", "installed-plugins", "external-plugin-intake", plugin.name); if (!fs.existsSync(installedPluginPath)) { return { status: "fail", output: `Plugin installed but install directory was not found at ${installedPluginPath}`, }; } const pluginManifestPath = findPluginJson(installedPluginPath); if (!pluginManifestPath) { return { status: "fail", output: `Plugin installed but no plugin.json was found in any recognized location under ${installedPluginPath}`, }; } return { status: "pass", output: `Install smoke test succeeded. Verified ${pluginManifestPath}.`, }; } catch (error) { return { status: "infra_error", output: truncateOutput(error.message), }; } } function isMissingPathAtLocator(output) { const normalized = String(output ?? "").toLowerCase(); return ( normalized.includes("does not exist in") || normalized.includes("exists on disk, but not in") || (normalized.includes("path '") && normalized.includes("not in")) ); } function fetchLocatorIntoRepo(repoDir, locator) { const result = runCommand("git", ["fetch", "--depth=1", "origin", locator], { cwd: repoDir }); if (result.exitCode === 0) { return { status: "pass", output: "", }; } const status = classifySmokeFailure(result.output); return { status, output: `git fetch failed for "${locator}": ${result.output}`, }; } function readPluginManifestAtLocator(repoDir, locator, normalizedPluginPath) { const manifestCandidates = PLUGIN_JSON_CANDIDATES.map((segments) => toPosixPath(normalizedPluginPath, ...segments) ); for (const manifestPath of manifestCandidates) { const showResult = runCommand("git", ["show", `${locator}:${manifestPath}`], { cwd: repoDir }); if (showResult.exitCode === 0) { const rawShow = spawnSync("git", ["show", `${locator}:${manifestPath}`], { cwd: repoDir, encoding: "utf8" }); const rawStdout = String(rawShow.stdout ?? ""); try { return { kind: "found", manifestPath, manifest: JSON.parse(rawStdout), }; } catch (error) { return { kind: "invalid", manifestPath, message: `Invalid JSON in "${manifestPath}" at "${locator}": ${error.message}`, }; } } if (isMissingPathAtLocator(showResult.output)) { continue; } return { kind: "infra_error", message: `Unable to read "${manifestPath}" at "${locator}": ${showResult.output}`, }; } return { kind: "not_found", message: `No plugin.json found at "${locator}". Expected one of: ${manifestCandidates.join(", ")}`, }; } function runVersionMatchGate(repoDir, plugin, primaryFetchSpec) { const expectedVersion = String(plugin?.version ?? "").trim(); const normalizedPluginPath = normalizePluginPath(plugin?.source?.path || "/"); const locators = [plugin?.source?.ref, plugin?.source?.sha] .filter((value) => typeof value === "string" && value.trim().length > 0) .map((value) => value.trim()) .filter((value, index, values) => values.indexOf(value) === index); if (locators.length === 0) { return { status: "not_run", output: "Version match gate skipped because neither source.ref nor source.sha was provided.", }; } const messages = []; let hasFailure = false; let hasInfraError = false; for (const locator of locators) { if (locator !== primaryFetchSpec) { const fetchResult = fetchLocatorIntoRepo(repoDir, locator); if (fetchResult.status === "fail") { hasFailure = true; messages.push(`- ${locator}: ${fetchResult.output}`); continue; } if (fetchResult.status === "infra_error") { hasInfraError = true; messages.push(`- ${locator}: ${fetchResult.output}`); continue; } } const manifestResult = readPluginManifestAtLocator(repoDir, locator, normalizedPluginPath); if (manifestResult.kind === "not_found" || manifestResult.kind === "invalid") { hasFailure = true; messages.push(`- ${locator}: ${manifestResult.message}`); continue; } if (manifestResult.kind === "infra_error") { hasInfraError = true; messages.push(`- ${locator}: ${manifestResult.message}`); continue; } const actualVersion = String(manifestResult.manifest?.version ?? "").trim(); if (!actualVersion) { hasFailure = true; messages.push(`- ${locator}: "${manifestResult.manifestPath}" is missing a non-empty "version" field.`); continue; } if (actualVersion !== expectedVersion) { hasFailure = true; messages.push( `- ${locator}: external.json version "${expectedVersion}" does not match "${manifestResult.manifestPath}" version "${actualVersion}".` ); continue; } messages.push(`- ${locator}: matched version "${expectedVersion}" at "${manifestResult.manifestPath}".`); } if (hasFailure) { return { status: "fail", output: messages.join("\n"), }; } if (hasInfraError) { return { status: "infra_error", output: messages.join("\n"), }; } return { status: "pass", output: messages.join("\n"), }; } function toOverallStatus(states) { if (states.includes("infra_error")) { return "infra_error"; } if (states.includes("fail")) { return "fail"; } if (states.every((state) => state === "not_run")) { return "not_run"; } return "pass"; } function toFailureClass(overallStatus) { if (overallStatus === "infra_error") { return "infra"; } if (overallStatus === "fail") { return "submitter_fixes"; } return "none"; } export async function runExternalPluginQualityGates(plugin) { const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "external-plugin-quality-")); const result = { overall_status: "not_run", vally_lint_status: "not_run", smoke_status: "not_run", version_match_status: "not_run", failure_class: "none", summary: "", vally_lint_output: "", smoke_output: "", version_match_output: "", }; try { const { repoDir, fetchSpec } = cloneSubmissionRepository(workDir, plugin); const normalizedPluginPath = normalizePluginPath(plugin.source?.path || "/"); const pluginRoot = normalizedPluginPath ? path.join(repoDir, normalizedPluginPath) : repoDir; if (!fs.existsSync(pluginRoot) || !fs.statSync(pluginRoot).isDirectory()) { result.vally_lint_status = "fail"; result.smoke_status = "fail"; result.version_match_status = "fail"; result.overall_status = "fail"; result.failure_class = "submitter_fixes"; result.summary = `Plugin path "${plugin.source?.path || "/"}" was not found in the submitted repository snapshot.`; result.version_match_output = result.summary; return result; } const versionMatchResult = runVersionMatchGate(repoDir, plugin, fetchSpec); result.version_match_status = versionMatchResult.status; result.version_match_output = versionMatchResult.output; const vallyResult = await runVallyLintGate(pluginRoot); result.vally_lint_status = vallyResult.status; result.vally_lint_output = vallyResult.output; const smokeResult = runInstallSmokeGate(workDir, plugin); result.smoke_status = smokeResult.status; result.smoke_output = smokeResult.output; result.overall_status = toOverallStatus([result.vally_lint_status, result.smoke_status, result.version_match_status]); result.failure_class = toFailureClass(result.overall_status); result.summary = [ `- vally lint: ${result.vally_lint_status}`, `- install smoke test: ${result.smoke_status}`, `- version match: ${result.version_match_status}`, `- overall: ${result.overall_status}`, ].join("\n"); return result; } catch (error) { result.overall_status = "infra_error"; result.failure_class = "infra"; result.summary = truncateOutput(error.message); result.vally_lint_output = truncateOutput(error.stack || error.message); return result; } finally { fs.rmSync(workDir, { recursive: true, force: true }); } } function parseCliArgs(argv) { const args = {}; for (let index = 0; index < argv.length; index += 1) { const key = argv[index]; if (!key.startsWith("--")) { continue; } args[key.slice(2)] = argv[index + 1]; index += 1; } return args; } if (import.meta.url === `file://${process.argv[1]}`) { const args = parseCliArgs(process.argv.slice(2)); if (!args["plugin-json"]) { console.error("Usage: node ./eng/external-plugin-quality-gates.mjs --plugin-json ''"); process.exit(1); } const plugin = JSON.parse(args["plugin-json"]); const result = await runExternalPluginQualityGates(plugin); process.stdout.write(`${JSON.stringify(result)}\n`); }