Add version parity checks to external plugin quality gates (#2307)

* Add external plugin version-match gate

Enforce external.json version matching against remote plugin.json for source ref and/or sha in shared quality gates, and surface the new gate status/output in intake and PR workflows.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 8aa3e98d-1873-4cab-8866-1b2efd0f24ad

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

.github/workflows/external-plugin-command-router.yml

@@ -746,8 +746,9 @@ jobs:
               if (process.env.QUALITY_JOB_RESULT === 'failure' || process.env.QUALITY_JOB_RESULT === 'cancelled') {
                 qualityResult = {
                   overall_status: 'infra_error',
-                  skill_validator_status: 'infra_error',
+                  vally_lint_status: 'infra_error',
                   smoke_status: 'infra_error',
+                  version_match_status: 'infra_error',
                   failure_class: 'infra',
                   summary: 'Quality-gate workflow failed unexpectedly. Re-run intake to retry.',
                 };
@@ -756,8 +757,9 @@ jobs:
               } else {
                 qualityResult = {
                   overall_status: 'infra_error',
-                  skill_validator_status: 'infra_error',
+                  vally_lint_status: 'infra_error',
                   smoke_status: 'infra_error',
+                  version_match_status: 'infra_error',
                   failure_class: 'infra',
                   summary: 'Quality-gate workflow did not return results. Re-run intake to retry.',
                 };

.github/workflows/external-plugin-intake.yml

@@ -113,8 +113,9 @@ jobs:
               if (process.env.QUALITY_JOB_RESULT === 'failure' || process.env.QUALITY_JOB_RESULT === 'cancelled') {
                 qualityResult = {
                   overall_status: 'infra_error',
-                  skill_validator_status: 'infra_error',
+                  vally_lint_status: 'infra_error',
                   smoke_status: 'infra_error',
+                  version_match_status: 'infra_error',
                   failure_class: 'infra',
                   summary: 'Quality-gate workflow failed unexpectedly. Re-run intake to retry.',
                 };
@@ -123,8 +124,9 @@ jobs:
               } else {
                 qualityResult = {
                   overall_status: 'infra_error',
-                  skill_validator_status: 'infra_error',
+                  vally_lint_status: 'infra_error',
                   smoke_status: 'infra_error',
+                  version_match_status: 'infra_error',
                   failure_class: 'infra',
                   summary: 'Quality-gate workflow did not return results. Re-run intake to retry.',
                 };

.github/workflows/external-plugin-pr-quality-gates.yml

@@ -157,6 +157,7 @@ jobs:
                 overall_status: 'infra_error',
                 failure_class: 'infra',
                 checked_plugins: [],
+                version_match_status: 'infra_error',
                 summary: 'External plugin PR change detection failed unexpectedly. Re-run this workflow.',
               };
             } else if (shouldRun) {
@@ -165,6 +166,7 @@ jobs:
                   overall_status: 'infra_error',
                   failure_class: 'infra',
                   checked_plugins: [],
+                  version_match_status: 'infra_error',
                   summary: 'External plugin PR quality checks failed unexpectedly. Re-run this workflow.',
                 };
               } else if (process.env.QUALITY_RESULT_JSON) {
@@ -174,6 +176,7 @@ jobs:
                   overall_status: 'infra_error',
                   failure_class: 'infra',
                   checked_plugins: [],
+                  version_match_status: 'infra_error',
                   summary: 'External plugin PR quality checks did not return a result payload.',
                 };
               }
@@ -242,14 +245,15 @@ jobs:
                   const sourceUrl = String(entry?.source_tree_url || '');
                   const locator = String(entry?.source?.sha || entry?.source?.ref || 'repository');
                   const sourceCell = sourceUrl ? `[${locator}](${sourceUrl})` : locator;
-                  return `| ${name} | ${quality.vally_lint_status || 'not_run'} | ${quality.smoke_status || 'not_run'} | ${quality.overall_status || 'not_run'} | ${sourceCell} |`;
+                  return `| ${name} | ${quality.vally_lint_status || 'not_run'} | ${quality.smoke_status || 'not_run'} | ${quality.version_match_status || 'not_run'} | ${quality.overall_status || 'not_run'} | ${sourceCell} |`;
                 })
-              : ['| _none_ | not_run | not_run | not_run | _n/a_ |'];
+              : ['| _none_ | not_run | not_run | not_run | not_run | _n/a_ |'];
             const failureDetails = checkedPlugins.flatMap((entry) => {
               const name = String(entry?.name || 'unknown');
               const quality = entry?.quality || {};
               const shouldShowVally = quality.vally_lint_status === 'fail' || quality.vally_lint_status === 'infra_error' || String(quality.vally_lint_output || '').trim().length > 0;
               const shouldShowSmoke = quality.smoke_status === 'fail' || quality.smoke_status === 'infra_error' || String(quality.smoke_output || '').trim().length > 0;
+              const shouldShowVersionMatch = quality.version_match_status === 'fail' || quality.version_match_status === 'infra_error' || String(quality.version_match_output || '').trim().length > 0;
 
               const details = [];
               if (shouldShowVally) {
@@ -258,6 +262,9 @@ jobs:
               if (shouldShowSmoke) {
                 details.push(formatGateOutput(name, 'install smoke test', quality.smoke_status, quality.smoke_output));
               }
+              if (shouldShowVersionMatch) {
+                details.push(formatGateOutput(name, 'version match', quality.version_match_status, quality.version_match_output));
+              }
               return details;
             });
 
@@ -270,8 +277,8 @@ jobs:
               '',
               '### Per-plugin quality summary',
               '',
-              '| Plugin | vally lint | install smoke test | overall | source tree |',
-              '|---|---|---|---|---|',
+              '| Plugin | vally lint | install smoke test | version match | overall | source tree |',
+              '|---|---|---|---|---|---|',
               ...rows,
               '',
               ...(failureDetails.length > 0

eng/external-plugin-intake.mjs

@@ -425,10 +425,12 @@ function normalizeQualityGateResult(rawResult) {
     overall_status: "not_run",
     vally_lint_status: "not_run",
     smoke_status: "not_run",
+    version_match_status: "not_run",
     failure_class: "none",
     summary: "",
     vally_lint_output: "",
     smoke_output: "",
+    version_match_output: "",
   };
 
   if (!rawResult || typeof rawResult !== "object" || Array.isArray(rawResult)) {
@@ -444,6 +446,7 @@ function normalizeQualityGateResult(rawResult) {
 function buildQualityGatesCommentSection(qualityResult) {
   const vallyState = qualityResult.vally_lint_status || "not_run";
   const smokeState = qualityResult.smoke_status || "not_run";
+  const versionMatchState = qualityResult.version_match_status || "not_run";
   const summaryText = String(qualityResult.summary || "").trim() || "_No quality gate details were provided._";
 
   const sections = [
@@ -453,6 +456,7 @@ function buildQualityGatesCommentSection(qualityResult) {
     "|---|---|",
     `| vally lint | ${vallyState} |`,
     `| install smoke test | ${smokeState} |`,
+    `| version match | ${versionMatchState} |`,
     "",
     summaryText,
   ];
@@ -487,6 +491,21 @@ function buildQualityGatesCommentSection(qualityResult) {
     );
   }
 
+  const versionMatchOutput = String(qualityResult.version_match_output || "").trim();
+  if (versionMatchOutput) {
+    sections.push(
+      "",
+      "<details>",
+      "<summary>Version match output</summary>",
+      "",
+      "```text",
+      versionMatchOutput,
+      "```",
+      "",
+      "</details>",
+    );
+  }
+
   return sections.join("\n");
 }
 

eng/external-plugin-pr-quality-gates.mjs

@@ -86,7 +86,7 @@ export async function runExternalPluginPrQualityGates(plugins) {
     ? "No changed external plugin entries were detected in plugins/external.json."
     : checkedPlugins
       .map((entry) =>
-        `- ${entry.name}: vally-lint=${entry.quality.vally_lint_status}, install-smoke=${entry.quality.smoke_status}, overall=${entry.quality.overall_status}`
+        `- ${entry.name}: vally-lint=${entry.quality.vally_lint_status}, install-smoke=${entry.quality.smoke_status}, version-match=${entry.quality.version_match_status}, overall=${entry.quality.overall_status}`
       )
       .join("\n");
 

eng/external-plugin-quality-gates.mjs

@@ -130,7 +130,10 @@ function cloneSubmissionRepository(workDir, plugin) {
     throw new Error(`git checkout failed: ${checkout.output}`);
   }
 
-  return repoDir;
+  return {
+    repoDir,
+    fetchSpec,
+  };
 }
 
 // Ordered list of candidate locations for plugin.json, from most to least specific.
@@ -144,6 +147,13 @@ const PLUGIN_JSON_CANDIDATES = [
   ["plugin.json"],
 ];
 
+function toPosixPath(...segments) {
+  return segments
+    .filter((segment) => segment !== undefined && segment !== null && String(segment).length > 0)
+    .map((segment) => String(segment).replace(/\\/g, "/"))
+    .join("/");
+}
+
 function findPluginJson(pluginRoot) {
   for (const segments of PLUGIN_JSON_CANDIDATES) {
     const candidate = path.join(pluginRoot, ...segments);
@@ -304,8 +314,160 @@ function runInstallSmokeGate(workDir, plugin) {
   }
 }
 
-function toOverallStatus(skillStatus, smokeStatus) {
-  const states = [skillStatus, smokeStatus];
+function isMissingPathAtLocator(output) {
+  const normalized = String(output ?? "").toLowerCase();
+  return (
+    normalized.includes("does not exist in") ||
+    normalized.includes("exists on disk, but not in") ||
+    (normalized.includes("path '") && normalized.includes("not in"))
+  );
+}
+
+function fetchLocatorIntoRepo(repoDir, locator) {
+  const result = runCommand("git", ["fetch", "--depth=1", "origin", locator], { cwd: repoDir });
+  if (result.exitCode === 0) {
+    return {
+      status: "pass",
+      output: "",
+    };
+  }
+
+  const status = classifySmokeFailure(result.output);
+  return {
+    status,
+    output: `git fetch failed for "${locator}": ${result.output}`,
+  };
+}
+
+function readPluginManifestAtLocator(repoDir, locator, normalizedPluginPath) {
+  const manifestCandidates = PLUGIN_JSON_CANDIDATES.map((segments) =>
+    toPosixPath(normalizedPluginPath, ...segments)
+  );
+
+  for (const manifestPath of manifestCandidates) {
+    const showResult = runCommand("git", ["show", `${locator}:${manifestPath}`], { cwd: repoDir });
+    if (showResult.exitCode === 0) {
+      const rawShow = spawnSync("git", ["show", `${locator}:${manifestPath}`], { cwd: repoDir, encoding: "utf8" });
+      const rawStdout = String(rawShow.stdout ?? "");
+
+      try {
+        return {
+          kind: "found",
+          manifestPath,
+          manifest: JSON.parse(rawStdout),
+        };
+      } catch (error) {
+        return {
+          kind: "invalid",
+          manifestPath,
+          message: `Invalid JSON in "${manifestPath}" at "${locator}": ${error.message}`,
+        };
+      }
+    }
+
+    if (isMissingPathAtLocator(showResult.output)) {
+      continue;
+    }
+
+    return {
+      kind: "infra_error",
+      message: `Unable to read "${manifestPath}" at "${locator}": ${showResult.output}`,
+    };
+  }
+
+  return {
+    kind: "not_found",
+    message: `No plugin.json found at "${locator}". Expected one of: ${manifestCandidates.join(", ")}`,
+  };
+}
+
+function runVersionMatchGate(repoDir, plugin, primaryFetchSpec) {
+  const expectedVersion = String(plugin?.version ?? "").trim();
+  const normalizedPluginPath = normalizePluginPath(plugin?.source?.path || "/");
+  const locators = [plugin?.source?.ref, plugin?.source?.sha]
+    .filter((value) => typeof value === "string" && value.trim().length > 0)
+    .map((value) => value.trim())
+    .filter((value, index, values) => values.indexOf(value) === index);
+
+  if (locators.length === 0) {
+    return {
+      status: "not_run",
+      output: "Version match gate skipped because neither source.ref nor source.sha was provided.",
+    };
+  }
+
+  const messages = [];
+  let hasFailure = false;
+  let hasInfraError = false;
+
+  for (const locator of locators) {
+    if (locator !== primaryFetchSpec) {
+      const fetchResult = fetchLocatorIntoRepo(repoDir, locator);
+      if (fetchResult.status === "fail") {
+        hasFailure = true;
+        messages.push(`- ${locator}: ${fetchResult.output}`);
+        continue;
+      }
+
+      if (fetchResult.status === "infra_error") {
+        hasInfraError = true;
+        messages.push(`- ${locator}: ${fetchResult.output}`);
+        continue;
+      }
+    }
+
+    const manifestResult = readPluginManifestAtLocator(repoDir, locator, normalizedPluginPath);
+    if (manifestResult.kind === "not_found" || manifestResult.kind === "invalid") {
+      hasFailure = true;
+      messages.push(`- ${locator}: ${manifestResult.message}`);
+      continue;
+    }
+
+    if (manifestResult.kind === "infra_error") {
+      hasInfraError = true;
+      messages.push(`- ${locator}: ${manifestResult.message}`);
+      continue;
+    }
+
+    const actualVersion = String(manifestResult.manifest?.version ?? "").trim();
+    if (!actualVersion) {
+      hasFailure = true;
+      messages.push(`- ${locator}: "${manifestResult.manifestPath}" is missing a non-empty "version" field.`);
+      continue;
+    }
+
+    if (actualVersion !== expectedVersion) {
+      hasFailure = true;
+      messages.push(
+        `- ${locator}: external.json version "${expectedVersion}" does not match "${manifestResult.manifestPath}" version "${actualVersion}".`
+      );
+      continue;
+    }
+
+    messages.push(`- ${locator}: matched version "${expectedVersion}" at "${manifestResult.manifestPath}".`);
+  }
+
+  if (hasFailure) {
+    return {
+      status: "fail",
+      output: messages.join("\n"),
+    };
+  }
+
+  if (hasInfraError) {
+    return {
+      status: "infra_error",
+      output: messages.join("\n"),
+    };
+  }
+
+  return {
+    status: "pass",
+    output: messages.join("\n"),
+  };
+}
+
+function toOverallStatus(states) {
   if (states.includes("infra_error")) {
     return "infra_error";
   }
@@ -334,26 +496,34 @@ export async function runExternalPluginQualityGates(plugin) {
     overall_status: "not_run",
     vally_lint_status: "not_run",
     smoke_status: "not_run",
+    version_match_status: "not_run",
     failure_class: "none",
     summary: "",
     vally_lint_output: "",
     smoke_output: "",
+    version_match_output: "",
   };
 
   try {
-    const repoDir = cloneSubmissionRepository(workDir, plugin);
+    const { repoDir, fetchSpec } = cloneSubmissionRepository(workDir, plugin);
     const normalizedPluginPath = normalizePluginPath(plugin.source?.path || "/");
     const pluginRoot = normalizedPluginPath ? path.join(repoDir, normalizedPluginPath) : repoDir;
 
     if (!fs.existsSync(pluginRoot) || !fs.statSync(pluginRoot).isDirectory()) {
       result.vally_lint_status = "fail";
       result.smoke_status = "fail";
+      result.version_match_status = "fail";
       result.overall_status = "fail";
       result.failure_class = "submitter_fixes";
       result.summary = `Plugin path "${plugin.source?.path || "/"}" was not found in the submitted repository snapshot.`;
+      result.version_match_output = result.summary;
       return result;
     }
 
+    const versionMatchResult = runVersionMatchGate(repoDir, plugin, fetchSpec);
+    result.version_match_status = versionMatchResult.status;
+    result.version_match_output = versionMatchResult.output;
+
     const vallyResult = await runVallyLintGate(pluginRoot);
     result.vally_lint_status = vallyResult.status;
     result.vally_lint_output = vallyResult.output;
@@ -362,11 +532,12 @@ export async function runExternalPluginQualityGates(plugin) {
     result.smoke_status = smokeResult.status;
     result.smoke_output = smokeResult.output;
 
-    result.overall_status = toOverallStatus(result.vally_lint_status, result.smoke_status);
+    result.overall_status = toOverallStatus([result.vally_lint_status, result.smoke_status, result.version_match_status]);
     result.failure_class = toFailureClass(result.overall_status);
     result.summary = [
       `- vally lint: ${result.vally_lint_status}`,
       `- install smoke test: ${result.smoke_status}`,
+      `- version match: ${result.version_match_status}`,
       `- overall: ${result.overall_status}`,
     ].join("\n");