chore: publish from staged

2026-05-29 18:11:45 +00:00 · 2026-05-28 05:50:33 +00:00
parent a1ebf8a4f2
commit 53230940c5
10 changed files with 933 additions and 49 deletions
@@ -11,6 +11,10 @@ export const EXTERNAL_PLUGIN_INTAKE_LABELS = Object.freeze({
    color: "0E8A16",
    description: "Submission passed intake validation and is ready for maintainer review",
  },
+  "requires-submitter-fixes": {
+    color: "D93F0B",
+    description: "Submission has quality-gate findings that submitter must fix before maintainer review",
+  },
  approved: {
    color: "1D76DB",
    description: "Submission was approved by a maintainer",
@@ -25,6 +29,7 @@ const EXTERNAL_PLUGIN_INTAKE_SYNC_LABELS = Object.freeze([
  "external-plugin",
  "awaiting-review",
  "ready-for-review",
+  "requires-submitter-fixes",
  "rejected",
 ]);

@@ -138,9 +143,14 @@ export async function applyExternalPluginIntakeEvaluation({
  issueNumber,
  evaluation,
 }) {
-  const desiredLabels = evaluation.valid
-    ? new Set(["external-plugin", "ready-for-review"])
-    : new Set(["external-plugin", "rejected"]);
+  const state = evaluation.intakeState ?? (evaluation.valid ? "ready-for-review" : "rejected");
+  const desiredLabelsByState = {
+    "ready-for-review": new Set(["external-plugin", "ready-for-review"]),
+    "requires-submitter-fixes": new Set(["external-plugin", "requires-submitter-fixes"]),
+    "awaiting-review": new Set(["external-plugin", "awaiting-review"]),
+    rejected: new Set(["external-plugin", "rejected"]),
+  };
+  const desiredLabels = desiredLabelsByState[state] ?? desiredLabelsByState.rejected;

  await syncExternalPluginIntakeLabels({
    github,
@@ -9,10 +9,15 @@ import { readExternalPlugins, validateExternalPlugin } from "./external-plugin-v
 export const ISSUE_FORM_MARKER = "<!-- external-plugin-submission -->";
 export const EXTERNAL_PLUGIN_INTAKE_COMMENT_MARKER = "<!-- external-plugin-intake -->";
 export const RERUN_INTAKE_COMMAND = "/rerun-intake";
+export const MARK_READY_FOR_REVIEW_COMMAND = "/mark-ready-for-review";
 const RERUN_INTAKE_COMMAND_PATTERN = new RegExp(
  `^\\s*${RERUN_INTAKE_COMMAND.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`,
  "m",
 );
+const MARK_READY_FOR_REVIEW_COMMAND_PATTERN = new RegExp(
+  `^\\s*${MARK_READY_FOR_REVIEW_COMMAND.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`,
+  "m",
+);
 const PLUGINS_DIR = path.join(ROOT_FOLDER, "plugins");

 // Each entry is a Set of equivalent checklist item texts (new + legacy aliases).
@@ -318,6 +323,168 @@ export function parseRerunIntakeCommand(body) {
  return RERUN_INTAKE_COMMAND_PATTERN.test(String(body ?? ""));
 }

+export function parseMarkReadyForReviewCommand(body) {
+  const text = String(body ?? "");
+  if (!MARK_READY_FOR_REVIEW_COMMAND_PATTERN.test(text)) {
+    return undefined;
+  }
+
+  const commandLine = text.split(/\r?\n/).find((line) => MARK_READY_FOR_REVIEW_COMMAND_PATTERN.test(line));
+  const reason = commandLine?.replace(MARK_READY_FOR_REVIEW_COMMAND_PATTERN, "").trim();
+
+  return {
+    command: MARK_READY_FOR_REVIEW_COMMAND,
+    reason: reason || undefined,
+  };
+}
+
+function normalizeQualityGateResult(rawResult) {
+  const defaults = {
+    overall_status: "not_run",
+    skill_validator_status: "not_run",
+    smoke_status: "not_run",
+    failure_class: "none",
+    summary: "",
+    skill_validator_output: "",
+    smoke_output: "",
+  };
+
+  if (!rawResult || typeof rawResult !== "object" || Array.isArray(rawResult)) {
+    return defaults;
+  }
+
+  return {
+    ...defaults,
+    ...rawResult,
+  };
+}
+
+function buildQualityGatesCommentSection(qualityResult) {
+  const skillState = qualityResult.skill_validator_status || "not_run";
+  const smokeState = qualityResult.smoke_status || "not_run";
+  const summaryText = String(qualityResult.summary || "").trim() || "_No quality gate details were provided._";
+
+  const sections = [
+    "### Quality gate summary",
+    "",
+    "| Gate | Status |",
+    "|---|---|",
+    `| skill-validator | ${skillState} |`,
+    `| install smoke test | ${smokeState} |`,
+    "",
+    summaryText,
+  ];
+
+  const skillOutput = String(qualityResult.skill_validator_output || "").trim();
+  if (skillOutput) {
+    sections.push(
+      "",
+      "<details>",
+      "<summary>skill-validator output</summary>",
+      "",
+      "```text",
+      skillOutput,
+      "```",
+      "",
+      "</details>",
+    );
+  }
+
+  const smokeOutput = String(qualityResult.smoke_output || "").trim();
+  if (smokeOutput) {
+    sections.push(
+      "",
+      "<details>",
+      "<summary>Install smoke test output</summary>",
+      "",
+      "```text",
+      smokeOutput,
+      "```",
+      "",
+      "</details>",
+    );
+  }
+
+  return sections.join("\n");
+}
+
+function getIntakeStateFromQualityResult(baseResult, qualityResult) {
+  if (!baseResult.valid) {
+    return "rejected";
+  }
+
+  if (qualityResult.failure_class === "submitter_fixes") {
+    return "requires-submitter-fixes";
+  }
+
+  if (qualityResult.failure_class === "infra") {
+    return "awaiting-review";
+  }
+
+  return "ready-for-review";
+}
+
+function buildMergedIntakeComment(baseResult, qualityResult) {
+  if (!baseResult.valid) {
+    return baseResult.commentBody;
+  }
+
+  const marker = baseResult.commentMarker ?? EXTERNAL_PLUGIN_INTAKE_COMMENT_MARKER;
+  const qualitySection = buildQualityGatesCommentSection(qualityResult);
+
+  const intro =
+    qualityResult.failure_class === "submitter_fixes"
+      ? "## ⚠️ External plugin intake requires submitter fixes"
+      : qualityResult.failure_class === "infra"
+        ? "## ⚠️ External plugin intake could not complete quality checks"
+        : "## ✅ External plugin intake passed";
+
+  const statusLine =
+    qualityResult.failure_class === "submitter_fixes"
+      ? "This submission passed metadata validation, but quality gates found issues that must be fixed before it can move to maintainer review. Update the issue details or source plugin and then comment `/rerun-intake`."
+      : qualityResult.failure_class === "infra"
+        ? "This submission passed metadata validation, but the automated quality checks hit an infrastructure issue. A maintainer should rerun intake or use the explicit override command after review."
+        : "This submission passed automated intake validation and quality checks and is ready for maintainer review.";
+
+  return [
+    marker,
+    intro,
+    "",
+    statusLine,
+    "",
+    `- **Plugin:** ${baseResult.plugin?.name ?? "unknown"}`,
+    `- **Repository:** ${baseResult.plugin?.repository ?? "unknown"}`,
+    baseResult.plugin?.source?.ref ? `- **Ref:** ${baseResult.plugin.source.ref}` : undefined,
+    baseResult.plugin?.source?.sha ? `- **SHA:** ${baseResult.plugin.source.sha}` : undefined,
+    "",
+    qualitySection,
+    "",
+    "### Canonical external.json payload",
+    "",
+    "```json",
+    JSON.stringify(baseResult.plugin ?? {}, null, 2),
+    "```",
+    baseResult.warnings?.length
+      ? ["", "### Warnings", "", ...baseResult.warnings.map((warning) => `- ${warning}`)].join("\n")
+      : "",
+  ].filter(Boolean).join("\n");
+}
+
+export function applyQualityGateResult(baseEvaluation, qualityGateResult) {
+  const baseResult = typeof baseEvaluation === "string" ? JSON.parse(baseEvaluation) : baseEvaluation;
+  const qualityResult = normalizeQualityGateResult(
+    typeof qualityGateResult === "string" ? JSON.parse(qualityGateResult) : qualityGateResult,
+  );
+  const intakeState = getIntakeStateFromQualityResult(baseResult, qualityResult);
+
+  return {
+    ...baseResult,
+    qualityGates: qualityResult,
+    intakeState,
+    commentBody: buildMergedIntakeComment(baseResult, qualityResult),
+  };
+}
+
 export async function evaluateExternalPluginIssue({ issue, token } = {}) {
  const issueBody = issue?.body ?? "";
  const parsed = parseExternalPluginIssueBody(issueBody);
@@ -403,6 +570,7 @@ export async function evaluateExternalPluginIssue({ issue, token } = {}) {

  return {
    valid,
+    intakeState: valid ? "ready-for-review" : "rejected",
    markerPresent: parsed.markerPresent,
    errors: dedupedErrors,
    warnings: dedupedWarnings,
@@ -0,0 +1,355 @@
+#!/usr/bin/env node
+
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { spawnSync } from "child_process";
+
+const MAX_OUTPUT_LENGTH = 12000;
+const SKILL_VALIDATOR_ARCHIVE_URL = "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz";
+
+const INFRA_ERROR_PATTERNS = [
+  /\b401\b/,
+  /\b403\b/,
+  /authentication (required|failed|error)/,
+  /unauthenticated/,
+  /unauthorized/,
+  /not logged in/,
+  /please (log in|authenticate|sign in)/,
+  /invalid (access |auth )?token/,
+  /credentials? (are )?expired/,
+  /dns.*(resolve|lookup|fail)/,
+  /network.*unreachable/,
+  /connection (refused|reset)/,
+  /\btimeout\b/,
+  /enotfound/,
+  /econnrefused/,
+  /etimedout/,
+];
+
+function truncateOutput(value) {
+  const normalized = String(value ?? "").replace(/\x1b\[[0-9;]*m/g, "").trim();
+  if (normalized.length <= MAX_OUTPUT_LENGTH) {
+    return normalized;
+  }
+
+  return `${normalized.slice(0, MAX_OUTPUT_LENGTH)}\n...output truncated...`;
+}
+
+function runCommand(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    ...options,
+  });
+
+  return {
+    exitCode: typeof result.status === "number" ? result.status : 1,
+    stdout: truncateOutput(result.stdout),
+    stderr: truncateOutput(result.stderr),
+    output: truncateOutput(`${result.stdout ?? ""}\n${result.stderr ?? ""}`),
+    error: result.error ? String(result.error.message ?? result.error) : "",
+  };
+}
+
+function normalizePluginPath(pluginPath) {
+  if (!pluginPath || pluginPath === "/") {
+    return "";
+  }
+
+  const normalized = String(pluginPath).trim().replace(/^\/+|\/+$/g, "");
+  if (!normalized) {
+    return "";
+  }
+
+  if (normalized.includes("..") || normalized.includes("\\")) {
+    throw new Error(`Invalid plugin path "${pluginPath}"`);
+  }
+
+  return normalized;
+}
+
+function resolveFetchSpec(pluginSource) {
+  if (pluginSource.sha) {
+    return pluginSource.sha;
+  }
+
+  if (!pluginSource.ref) {
+    throw new Error("source.ref or source.sha is required for quality gates");
+  }
+
+  const ref = String(pluginSource.ref).trim();
+  if (!ref) {
+    throw new Error("source.ref or source.sha is required for quality gates");
+  }
+
+  if (ref.startsWith("refs/")) {
+    return ref;
+  }
+
+  return ref;
+}
+
+function classifySmokeFailure(output) {
+  const normalized = String(output ?? "").toLowerCase();
+  if (INFRA_ERROR_PATTERNS.some((pattern) => pattern.test(normalized))) {
+    return "infra_error";
+  }
+
+  return "fail";
+}
+
+function ensureDirectory(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function cloneSubmissionRepository(workDir, plugin) {
+  const repoDir = path.join(workDir, "submission");
+  ensureDirectory(repoDir);
+
+  const sourceRepo = plugin.source?.repo;
+  const fetchSpec = resolveFetchSpec(plugin.source ?? {});
+
+  const init = runCommand("git", ["init", "-q"], { cwd: repoDir });
+  if (init.exitCode !== 0) {
+    throw new Error(`git init failed: ${init.output}`);
+  }
+
+  const addRemote = runCommand("git", ["remote", "add", "origin", `https://github.com/${sourceRepo}.git`], { cwd: repoDir });
+  if (addRemote.exitCode !== 0) {
+    throw new Error(`git remote add failed: ${addRemote.output}`);
+  }
+
+  const fetch = runCommand("git", ["fetch", "--depth=1", "origin", fetchSpec], { cwd: repoDir });
+  if (fetch.exitCode !== 0) {
+    throw new Error(`git fetch failed for ${fetchSpec}: ${fetch.output}`);
+  }
+
+  const checkout = runCommand("git", ["checkout", "--detach", "FETCH_HEAD"], { cwd: repoDir });
+  if (checkout.exitCode !== 0) {
+    throw new Error(`git checkout failed: ${checkout.output}`);
+  }
+
+  return repoDir;
+}
+
+function downloadSkillValidator(workDir) {
+  const validatorDir = path.join(workDir, "skill-validator");
+  ensureDirectory(validatorDir);
+  const archivePath = path.join(validatorDir, "skill-validator-linux-x64.tar.gz");
+
+  const download = runCommand("curl", ["-fsSL", SKILL_VALIDATOR_ARCHIVE_URL, "-o", archivePath]);
+  if (download.exitCode !== 0) {
+    throw new Error(`Failed to download skill-validator: ${download.output}`);
+  }
+
+  const untar = runCommand("tar", ["-xzf", archivePath, "-C", validatorDir]);
+  if (untar.exitCode !== 0) {
+    throw new Error(`Failed to extract skill-validator: ${untar.output}`);
+  }
+
+  const binaryPath = path.join(validatorDir, "skill-validator");
+  if (!fs.existsSync(binaryPath)) {
+    throw new Error("skill-validator binary was not found after extraction");
+  }
+
+  runCommand("chmod", ["+x", binaryPath]);
+  return binaryPath;
+}
+
+function runSkillValidatorGate(workDir, pluginRoot) {
+  try {
+    const validatorBinary = downloadSkillValidator(workDir);
+    const check = runCommand(validatorBinary, ["check", "--verbose", "--plugin", pluginRoot]);
+
+    if (check.exitCode === 0) {
+      return { status: "pass", output: check.output };
+    }
+
+    return { status: "fail", output: check.output };
+  } catch (error) {
+    return {
+      status: "infra_error",
+      output: truncateOutput(error.message),
+    };
+  }
+}
+
+function buildEphemeralMarketplace(workDir, plugin) {
+  const marketplaceDir = path.join(workDir, "marketplace");
+  ensureDirectory(marketplaceDir);
+
+  const marketplace = {
+    name: "external-plugin-intake",
+    metadata: {
+      description: "Temporary marketplace for external plugin intake smoke tests",
+      version: "1.0.0",
+      pluginRoot: ".",
+    },
+    owner: {
+      name: "awesome-copilot-intake",
+      email: "noreply@github.com",
+    },
+    plugins: [plugin],
+  };
+
+  fs.writeFileSync(path.join(marketplaceDir, "marketplace.json"), `${JSON.stringify(marketplace, null, 2)}\n`);
+  return marketplaceDir;
+}
+
+function runInstallSmokeGate(workDir, plugin) {
+  if (runCommand("bash", ["-lc", "command -v copilot"]).exitCode !== 0) {
+    return {
+      status: "infra_error",
+      output: "copilot CLI is not available on this runner.",
+    };
+  }
+
+  try {
+    const homeDir = path.join(workDir, "copilot-home");
+    ensureDirectory(homeDir);
+    const marketplaceDir = buildEphemeralMarketplace(workDir, plugin);
+
+    const env = {
+      ...process.env,
+      HOME: homeDir,
+      XDG_CONFIG_HOME: path.join(homeDir, ".config"),
+      XDG_CACHE_HOME: path.join(homeDir, ".cache"),
+      XDG_DATA_HOME: path.join(homeDir, ".local", "share"),
+    };
+
+    const marketplaceAdd = runCommand("copilot", ["plugin", "marketplace", "add", marketplaceDir], { env });
+    if (marketplaceAdd.exitCode !== 0) {
+      const status = classifySmokeFailure(marketplaceAdd.output);
+      return { status, output: marketplaceAdd.output };
+    }
+
+    const install = runCommand("copilot", ["plugin", "install", `${plugin.name}@external-plugin-intake`], { env });
+    if (install.exitCode !== 0) {
+      const status = classifySmokeFailure(install.output);
+      return { status, output: install.output };
+    }
+
+    const installedPluginPath = path.join(homeDir, ".copilot", "installed-plugins", "external-plugin-intake", plugin.name);
+    const pluginManifestPath = path.join(installedPluginPath, ".github", "plugin", "plugin.json");
+    if (!fs.existsSync(installedPluginPath) || !fs.existsSync(pluginManifestPath)) {
+      return {
+        status: "fail",
+        output: `Plugin installed but expected files were missing at ${installedPluginPath}`,
+      };
+    }
+
+    return {
+      status: "pass",
+      output: `Install smoke test succeeded. Verified ${pluginManifestPath}.`,
+    };
+  } catch (error) {
+    return {
+      status: "infra_error",
+      output: truncateOutput(error.message),
+    };
+  }
+}
+
+function toOverallStatus(skillStatus, smokeStatus) {
+  const states = [skillStatus, smokeStatus];
+  if (states.includes("infra_error")) {
+    return "infra_error";
+  }
+  if (states.includes("fail")) {
+    return "fail";
+  }
+  if (states.every((state) => state === "not_run")) {
+    return "not_run";
+  }
+  return "pass";
+}
+
+function toFailureClass(overallStatus) {
+  if (overallStatus === "infra_error") {
+    return "infra";
+  }
+  if (overallStatus === "fail") {
+    return "submitter_fixes";
+  }
+  return "none";
+}
+
+export function runExternalPluginQualityGates(plugin) {
+  const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "external-plugin-quality-"));
+  const result = {
+    overall_status: "not_run",
+    skill_validator_status: "not_run",
+    smoke_status: "not_run",
+    failure_class: "none",
+    summary: "",
+    skill_validator_output: "",
+    smoke_output: "",
+  };
+
+  try {
+    const repoDir = cloneSubmissionRepository(workDir, plugin);
+    const normalizedPluginPath = normalizePluginPath(plugin.source?.path || "/");
+    const pluginRoot = normalizedPluginPath ? path.join(repoDir, normalizedPluginPath) : repoDir;
+
+    if (!fs.existsSync(pluginRoot) || !fs.statSync(pluginRoot).isDirectory()) {
+      result.skill_validator_status = "fail";
+      result.smoke_status = "fail";
+      result.overall_status = "fail";
+      result.failure_class = "submitter_fixes";
+      result.summary = `Plugin path "${plugin.source?.path || "/"}" was not found in the submitted repository snapshot.`;
+      return result;
+    }
+
+    const skillResult = runSkillValidatorGate(workDir, pluginRoot);
+    result.skill_validator_status = skillResult.status;
+    result.skill_validator_output = skillResult.output;
+
+    const smokeResult = runInstallSmokeGate(workDir, plugin);
+    result.smoke_status = smokeResult.status;
+    result.smoke_output = smokeResult.output;
+
+    result.overall_status = toOverallStatus(result.skill_validator_status, result.smoke_status);
+    result.failure_class = toFailureClass(result.overall_status);
+    result.summary = [
+      `- skill-validator: ${result.skill_validator_status}`,
+      `- install smoke test: ${result.smoke_status}`,
+      `- overall: ${result.overall_status}`,
+    ].join("\n");
+
+    return result;
+  } catch (error) {
+    result.overall_status = "infra_error";
+    result.failure_class = "infra";
+    result.summary = truncateOutput(error.message);
+    result.skill_validator_output = truncateOutput(error.stack || error.message);
+    return result;
+  } finally {
+    fs.rmSync(workDir, { recursive: true, force: true });
+  }
+}
+
+function parseCliArgs(argv) {
+  const args = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const key = argv[index];
+    if (!key.startsWith("--")) {
+      continue;
+    }
+
+    args[key.slice(2)] = argv[index + 1];
+    index += 1;
+  }
+  return args;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const args = parseCliArgs(process.argv.slice(2));
+  if (!args["plugin-json"]) {
+    console.error("Usage: node ./eng/external-plugin-quality-gates.mjs --plugin-json '<json>'");
+    process.exit(1);
+  }
+
+  const plugin = JSON.parse(args["plugin-json"]);
+  const result = runExternalPluginQualityGates(plugin);
+  process.stdout.write(`${JSON.stringify(result)}\n`);
+}