mirror of
https://github.com/github/awesome-copilot.git
synced 2026-10-12 14:54:16 +00:00
Aaron PowellandCopilot
04440f4f78
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 88f4f5b8-b61a-4f5b-845f-dcc8c82ce55e
1057 lines
51 KiB
JavaScript
1057 lines
51 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { test } from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
import * as yaml from "js-yaml";
|
|
import {
|
|
canRunPrCommand,
|
|
classifyFailure,
|
|
classifyRisk,
|
|
computeState,
|
|
evaluateApprovals,
|
|
gateConclusion,
|
|
evaluateCheck,
|
|
evaluateSubmission,
|
|
globToRegExp,
|
|
latestRunsByWorkflow,
|
|
loadGateConfig,
|
|
normalizeRouting,
|
|
publishGateCheck,
|
|
GATE_CHECK_EXTERNAL_ID,
|
|
parsePrCommand,
|
|
renderStatusComment,
|
|
rerunChecks,
|
|
resolvePullRequestForWorkflowRun,
|
|
riskLabelSignature,
|
|
runPrCommand,
|
|
validatePrCommandRequest,
|
|
PR_COMMAND_SCHEMA,
|
|
sanitize,
|
|
selectApplicableChecks,
|
|
STATUS_MARKER,
|
|
summarizeChecks,
|
|
syncPullRequestStatus,
|
|
touchedDomains,
|
|
} from "./submission-gate.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const config = loadGateConfig(repoRoot);
|
|
const tiers = config.tiers;
|
|
|
|
const file = (filename, extra = {}) => ({ filename, status: "modified", additions: 1, deletions: 1, changes: 2, patch: "+x\n-y", ...extra });
|
|
const review = (login, state, submitted_at = "2026-09-29T10:00:00Z", extra = {}) => ({
|
|
user: { login, type: "User" },
|
|
state,
|
|
submitted_at,
|
|
...extra,
|
|
});
|
|
|
|
// --- globs -----------------------------------------------------------------
|
|
|
|
test("globToRegExp follows GitHub path filter semantics", () => {
|
|
assert.ok(globToRegExp("skills/**").test("skills/a/SKILL.md"));
|
|
assert.ok(globToRegExp("*.js").test("index.js"));
|
|
assert.ok(!globToRegExp("*.js").test("eng/index.js"));
|
|
assert.ok(globToRegExp("**/mcp.json").test("mcp.json"));
|
|
assert.ok(globToRegExp("**/mcp.json").test("plugins/x/mcp.json"));
|
|
assert.ok(globToRegExp("plugins/**/skills/**").test("plugins/p/skills/s/SKILL.md"));
|
|
assert.ok(!globToRegExp("docs/**").test("docsx/a.md"));
|
|
});
|
|
|
|
// --- config drift ------------------------------------------------------------
|
|
|
|
test("check path and branch filters mirror their workflow triggers", () => {
|
|
for (const check of config.gate.checks.filter((candidate) => candidate.workflow)) {
|
|
const workflowPath = path.join(repoRoot, ".github", "workflows", check.workflow);
|
|
assert.ok(fs.existsSync(workflowPath), `${check.id}: ${check.workflow} does not exist`);
|
|
const workflow = yaml.load(fs.readFileSync(workflowPath, "utf8"));
|
|
const on = workflow.on ?? workflow[true];
|
|
const trigger = on?.pull_request;
|
|
assert.ok(on && Object.prototype.hasOwnProperty.call(on, "pull_request"), `${check.id}: ${check.workflow} must trigger on pull_request`);
|
|
// A workflow without a path filter reports on every PR, so the gate may narrow where it applies.
|
|
if (trigger?.paths) {
|
|
assert.deepEqual([...(check.paths || [])].sort(), [...trigger.paths].sort(), `${check.id}: paths drifted from ${check.workflow}`);
|
|
}
|
|
// validate-agentic-workflows-pr.yml rejects any `.github/**` change, so it can't trigger on its own file.
|
|
const selfPathExempt = ["validate-submission-gate.yml", "validate-agentic-workflows-pr.yml"];
|
|
if (check.paths && !selfPathExempt.includes(check.workflow)) {
|
|
assert.ok(
|
|
check.paths.includes(`.github/workflows/${check.workflow}`),
|
|
`${check.id}: paths should include its own workflow file`
|
|
);
|
|
}
|
|
assert.deepEqual(check.branches || [], trigger?.branches || [], `${check.id}: branches drifted from ${check.workflow}`);
|
|
}
|
|
});
|
|
|
|
test("every check has a unique id and a workflow or check name", () => {
|
|
const ids = new Set();
|
|
for (const check of config.gate.checks) {
|
|
assert.ok(check.id && !ids.has(check.id), `duplicate or missing id ${check.id}`);
|
|
ids.add(check.id);
|
|
assert.ok(check.workflow || check.check_name, `${check.id} needs workflow or check_name`);
|
|
}
|
|
assert.ok(config.gate.checks.some((check) => check.check_name === "canvas-smoke-test" && check.optional));
|
|
});
|
|
|
|
test("selectApplicableChecks honors paths and base branch", () => {
|
|
const ids = (files, base = "main") => selectApplicableChecks(config.gate.checks, files, base).map((check) => check.id);
|
|
const docs = ids([file("docs/README.skills.md")]);
|
|
assert.ok(docs.includes("line-endings"));
|
|
assert.ok(docs.includes("readme"));
|
|
assert.ok(!docs.includes("skill-validation"));
|
|
assert.ok(!docs.includes("canvas-smoke-test"));
|
|
|
|
const skill = ids([file("skills/foo/SKILL.md", { status: "added" })]);
|
|
assert.ok(skill.includes("skill-validation"));
|
|
assert.ok(skill.includes("risk-scan"));
|
|
|
|
const canvas = ids([file("extensions/board/extension.mjs")]);
|
|
assert.ok(canvas.includes("canvas-extension-validation"));
|
|
assert.ok(canvas.includes("canvas-smoke-test"));
|
|
|
|
const renamed = ids([file("docs/x.md", { status: "renamed", previous_filename: "skills/x/SKILL.md" })]);
|
|
assert.ok(renamed.includes("skill-validation"), "previous filename counts for path filters");
|
|
|
|
const otherBase = ids([file("skills/foo/SKILL.md")], "staged");
|
|
assert.ok(!otherBase.includes("skill-validation"));
|
|
assert.ok(otherBase.includes("contributor-reputation"));
|
|
});
|
|
|
|
// --- risk tiers ---------------------------------------------------------------
|
|
|
|
test("documentation and generated output are low risk", () => {
|
|
const result = classifyRisk({
|
|
files: [file("docs/README.skills.md", { changes: 400 }), file(".github/plugin/marketplace.json", { changes: 90 })],
|
|
tiers,
|
|
});
|
|
assert.equal(result.tier, "low");
|
|
});
|
|
|
|
test("generated plugin marketplace output is owned by the plugin domain", () => {
|
|
assert.deepEqual(touchedDomains([file(".github/plugin/marketplace.json")], tiers.domains), [
|
|
{ id: "plugin", pools: ["plugin"] },
|
|
]);
|
|
});
|
|
|
|
test("small modification of an existing resource is low risk", () => {
|
|
const result = classifyRisk({ files: [file("skills/foo/SKILL.md", { changes: 10 })], tiers });
|
|
assert.equal(result.tier, "low");
|
|
});
|
|
|
|
test("large update or new resource is medium risk", () => {
|
|
assert.equal(classifyRisk({ files: [file("skills/foo/SKILL.md", { changes: 300 })], tiers }).tier, "medium");
|
|
assert.equal(
|
|
classifyRisk({ files: [file("agents/new.agent.md", { status: "added", changes: 5 })], tiers }).tier,
|
|
"medium"
|
|
);
|
|
assert.equal(
|
|
classifyRisk({
|
|
files: [file("extensions/board/extension.mjs", { status: "added", patch: "+export default {}\n+const x = 1;" })],
|
|
tiers,
|
|
}).tier,
|
|
"medium"
|
|
);
|
|
});
|
|
|
|
test("repository workflows, scripts, MCP config, and policy files are high risk", () => {
|
|
for (const name of [
|
|
".github/workflows/ci.yml",
|
|
".github/CODEOWNERS",
|
|
".github/review-routing.yml",
|
|
".github/risk-tiers.yml",
|
|
"hooks/x/check.sh",
|
|
"hooks/x/check.py",
|
|
"plugins/p/hooks/run.ps1",
|
|
"skills/foo/scripts/run.py",
|
|
"skills/foo/tool.sh",
|
|
"plugins/p/mcp.json",
|
|
"eng/update-readme.mjs",
|
|
"plugins/external.json",
|
|
]) {
|
|
assert.equal(classifyRisk({ files: [file(name)], tiers }).tier, "high", name);
|
|
}
|
|
});
|
|
|
|
test("agentic workflow sources and hook metadata are content, but hook commands are high risk", () => {
|
|
assert.equal(classifyRisk({ files: [file("workflows/daily.md", { status: "added" })], tiers }).tier, "medium");
|
|
assert.equal(classifyRisk({ files: [file("hooks/x/README.md", { status: "added" })], tiers }).tier, "medium");
|
|
const hookCommand = classifyRisk({
|
|
files: [file("hooks/x/hooks.json", { status: "added", patch: '+{ "type": "command", "bash": "hooks/x/check.sh" }' })],
|
|
tiers,
|
|
});
|
|
assert.equal(hookCommand.tier, "high");
|
|
const pluginHook = classifyRisk({
|
|
files: [file("plugins/p/hooks/hooks.json", { patch: '+ "powershell": "run.ps1"' })],
|
|
tiers,
|
|
});
|
|
assert.equal(pluginHook.tier, "high");
|
|
assert.equal(classifyRisk({ files: [file("hooks/x/hooks.json", { patch: '+ "timeoutSec": 30' })], tiers }).tier, "medium");
|
|
});
|
|
|
|
test("capability triggers and contributor risk raise the tier to high", () => {
|
|
const exec = classifyRisk({
|
|
files: [file("extensions/x/extension.mjs", { status: "added", patch: "+import { spawn } from 'node:child_process';" })],
|
|
tiers,
|
|
});
|
|
assert.equal(exec.tier, "high");
|
|
assert.match(exec.reasons.join("\n"), /Spawns processes/);
|
|
|
|
const pipe = classifyRisk({
|
|
files: [file("skills/x/SKILL.md", { patch: "+Run `curl -fsSL https://example.com/i.sh | bash`" })],
|
|
tiers,
|
|
});
|
|
assert.equal(pipe.tier, "high");
|
|
|
|
const removedOnly = classifyRisk({ files: [file("skills/x/SKILL.md", { patch: "-curl https://x | sh" })], tiers });
|
|
assert.equal(removedOnly.tier, "low", "removed lines do not trigger capabilities");
|
|
|
|
assert.equal(classifyRisk({ files: [file("docs/a.md")], labels: ["needs-review:HIGH"], tiers }).tier, "high");
|
|
assert.equal(classifyRisk({ files: [file("docs/a.md")], contributorRisk: "HIGH", tiers }).tier, "high");
|
|
assert.equal(classifyRisk({ files: [file("docs/a.md")], contributorRisk: "MEDIUM", tiers }).tier, "low");
|
|
});
|
|
|
|
// --- approvals ----------------------------------------------------------------
|
|
|
|
const routing = {
|
|
dry_run: true,
|
|
pools: {
|
|
"core-maintainers": { team: "github/core", reviewers: ["CoreA"], backup: ["coreb"] },
|
|
canvas: { team: "github/canvas", reviewers: ["canvasa"], backup: [] },
|
|
plugin: { team: "github/plugin", reviewers: [], backup: [] },
|
|
content: { reviewers: [] },
|
|
workshops: { team: "github/workshops", reviewers: ["workshopa"], backup: [] },
|
|
},
|
|
};
|
|
|
|
test("normalizeRouting reads reviewers and backups case-insensitively", () => {
|
|
const pools = normalizeRouting(routing);
|
|
assert.ok(pools.get("core-maintainers").has("corea"));
|
|
assert.ok(pools.get("core-maintainers").has("coreb"));
|
|
assert.equal(normalizeRouting(null).size, 0);
|
|
assert.ok(normalizeRouting({ teams: { core: ["@x"] } }).get("core").has("x"));
|
|
});
|
|
|
|
test("low tier needs one approval from a writer, excluding the author", () => {
|
|
const permissions = new Map([["alice", "write"], ["author", "write"], ["rando", "read"]]);
|
|
const base = { tier: "low", tiers, author: "author", permissions, files: [file("docs/a.md")] };
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("author", "APPROVED")] }).satisfied, false);
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("rando", "APPROVED")] }).satisfied, false);
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("alice", "APPROVED")] }).satisfied, true);
|
|
});
|
|
|
|
test("latest review state wins and changes requested blocks", () => {
|
|
const permissions = new Map([["alice", "write"], ["bob", "write"]]);
|
|
const base = { tier: "low", tiers, author: "author", permissions, files: [file("docs/a.md")] };
|
|
const dismissed = evaluateApprovals({
|
|
...base,
|
|
reviews: [review("alice", "APPROVED", "2026-09-29T10:00:00Z"), review("alice", "DISMISSED", "2026-09-29T11:00:00Z")],
|
|
});
|
|
assert.equal(dismissed.satisfied, false);
|
|
const commentAfterApproval = evaluateApprovals({
|
|
...base,
|
|
reviews: [review("alice", "APPROVED", "2026-09-29T10:00:00Z"), review("alice", "COMMENTED", "2026-09-29T11:00:00Z")],
|
|
});
|
|
assert.equal(commentAfterApproval.satisfied, true);
|
|
const blocked = evaluateApprovals({
|
|
...base,
|
|
reviews: [review("alice", "APPROVED"), review("bob", "CHANGES_REQUESTED")],
|
|
});
|
|
assert.equal(blocked.satisfied, false);
|
|
assert.deepEqual(blocked.changesRequestedBy, ["bob"]);
|
|
});
|
|
|
|
test("medium tier requires a domain reviewer when the pool is staffed", () => {
|
|
const permissions = new Map([
|
|
["alice", "write"],
|
|
["canvasa", "write"],
|
|
["workshopa", "write"],
|
|
]);
|
|
const files = [file("extensions/x/extension.mjs", { status: "added" })];
|
|
const base = { tier: "medium", tiers, author: "author", permissions, routing, files };
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("alice", "APPROVED")] }).satisfied, false);
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("canvasa", "APPROVED")] }).satisfied, true);
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("corea", "APPROVED")] }).satisfied, true, "core counts as domain");
|
|
|
|
const unstaffed = evaluateApprovals({
|
|
...base,
|
|
files: [file("plugins/p/README.md", { status: "added" })],
|
|
reviews: [review("alice", "APPROVED")],
|
|
});
|
|
assert.equal(unstaffed.satisfied, true, "falls back to any writer when the domain pool is empty");
|
|
assert.ok(unstaffed.notes.length > 0);
|
|
|
|
const workshop = evaluateApprovals({
|
|
...base,
|
|
files: [file("website/src/content/docs/learning-hub/copilot-workshops/lesson.md", { status: "added" })],
|
|
reviews: [review("workshopa", "APPROVED")],
|
|
});
|
|
assert.equal(workshop.satisfied, true);
|
|
|
|
const noRouting = evaluateApprovals({ ...base, routing: null, reviews: [review("alice", "APPROVED")] });
|
|
assert.equal(noRouting.satisfied, true);
|
|
});
|
|
|
|
test("high tier requires one approval from a core maintainer", () => {
|
|
const permissions = new Map([["alice", "write"], ["bob", "write"], ["corea", "write"], ["admin1", "admin"]]);
|
|
const files = [file(".github/workflows/x.yml")];
|
|
const base = { tier: "high", tiers, author: "author", permissions, routing, files };
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("corea", "APPROVED")] }).satisfied, true);
|
|
assert.equal(evaluateApprovals({ ...base, reviews: [review("alice", "APPROVED"), review("bob", "APPROVED")] }).satisfied, false);
|
|
|
|
const fallback = { ...base, routing: null };
|
|
assert.equal(evaluateApprovals({ ...fallback, reviews: [review("alice", "APPROVED")] }).satisfied, false);
|
|
assert.equal(evaluateApprovals({ ...fallback, reviews: [review("admin1", "APPROVED")] }).satisfied, true);
|
|
});
|
|
|
|
test("gateConclusion reports a PR that only awaits review as action_required", () => {
|
|
const clean = { contributionFailures: [], infrastructureFailures: [], pending: [] };
|
|
const waiting = { satisfied: false, changesRequestedBy: [] };
|
|
assert.equal(gateConclusion({ passed: true, automation: clean, approvals: { satisfied: true, changesRequestedBy: [] } }), "success");
|
|
assert.equal(gateConclusion({ passed: false, automation: clean, approvals: waiting }), "action_required");
|
|
assert.equal(gateConclusion({ passed: false, automation: clean, approvals: { ...waiting, changesRequestedBy: ["bob"] } }), "failure");
|
|
assert.equal(gateConclusion({ passed: false, automation: { ...clean, contributionFailures: [{}] }, approvals: waiting }), "failure");
|
|
});
|
|
|
|
// --- checks -------------------------------------------------------------------
|
|
|
|
const readmeCheck = config.gate.checks.find((check) => check.id === "readme");
|
|
const infraSteps = config.gate.infrastructure_steps;
|
|
const failedJob = (stepName) => [
|
|
{
|
|
name: "job",
|
|
conclusion: "failure",
|
|
steps: [
|
|
{ name: "Set up job", conclusion: "success" },
|
|
{ name: stepName, conclusion: "failure" },
|
|
],
|
|
},
|
|
];
|
|
|
|
test("failed steps are classified as contribution or infrastructure failures", () => {
|
|
assert.equal(classifyFailure(readmeCheck, failedJob("Fail workflow if files need updating"), infraSteps).category, "contribution");
|
|
assert.equal(classifyFailure(readmeCheck, failedJob("Install dependencies"), infraSteps).category, "infrastructure");
|
|
assert.equal(classifyFailure(readmeCheck, failedJob("Checkout code"), infraSteps).category, "infrastructure");
|
|
assert.equal(classifyFailure(readmeCheck, failedJob("Something new"), infraSteps).category, "contribution");
|
|
assert.equal(classifyFailure(readmeCheck, [{ name: "job", conclusion: "timed_out", steps: [] }], infraSteps).category, "infrastructure");
|
|
assert.equal(classifyFailure({ failure_kind: "infrastructure" }, [], infraSteps).category, "infrastructure");
|
|
assert.equal(classifyFailure(readmeCheck, [], infraSteps).category, "infrastructure");
|
|
});
|
|
|
|
test("evaluateCheck maps run states to gate outcomes", () => {
|
|
const check = { id: "x", title: "X", hint: "fix it" };
|
|
assert.equal(evaluateCheck(check, { found: false }).outcome, "pending");
|
|
assert.equal(evaluateCheck(check, { found: false }, { gaveUp: true }).category, "infrastructure");
|
|
assert.equal(evaluateCheck({ ...check, optional: true }, { found: false }, { gaveUp: true }).outcome, "skipped");
|
|
assert.equal(evaluateCheck(check, { found: true, status: "in_progress" }).outcome, "pending");
|
|
assert.equal(evaluateCheck(check, { found: true, status: "in_progress" }, { gaveUp: true }).category, "infrastructure");
|
|
assert.equal(evaluateCheck(check, { found: true, status: "completed", conclusion: "success" }).outcome, "pass");
|
|
const skipped = evaluateCheck(check, { found: true, status: "completed", conclusion: "skipped" });
|
|
assert.equal(skipped.outcome, "failure", "a skipped required check validated nothing");
|
|
assert.equal(skipped.category, "infrastructure");
|
|
const skippedOk = { found: true, status: "completed", conclusion: "skipped" };
|
|
assert.equal(evaluateCheck({ ...check, allow_skip: true }, skippedOk).outcome, "skipped");
|
|
assert.equal(evaluateCheck({ ...check, optional: true }, skippedOk).outcome, "skipped");
|
|
assert.equal(evaluateCheck({ ...check, required: false }, skippedOk).outcome, "skipped");
|
|
assert.equal(config.gate.checks.find((c) => c.id === "contributor-reputation").allow_skip, true, "skips for bot authors");
|
|
assert.equal(evaluateCheck(check, { found: true, status: "completed", conclusion: "cancelled" }).category, "infrastructure");
|
|
assert.equal(evaluateCheck(check, { found: true, status: "completed", conclusion: "action_required" }).category, "infrastructure");
|
|
const advisory = evaluateCheck({ ...check, required: false, failure_kind: "infrastructure" }, { found: true, status: "completed", conclusion: "failure" });
|
|
assert.equal(advisory.required, false);
|
|
assert.equal(summarizeChecks([advisory]).warnings.length, 1);
|
|
assert.equal(summarizeChecks([advisory]).infrastructureFailures.length, 0, "advisory failures never block");
|
|
});
|
|
|
|
test("state machine precedence", () => {
|
|
const approvals = { changesRequestedBy: [], satisfied: false, reviewers: [] };
|
|
const automation = (overrides = {}) => ({ contributionFailures: [], pending: [], infrastructureFailures: [], ...overrides });
|
|
assert.equal(computeState({ automation: automation({ contributionFailures: [{}], pending: [{}] }), approvals }), "requires-submitter-fixes");
|
|
assert.equal(computeState({ automation: automation(), approvals: { ...approvals, changesRequestedBy: ["x"] } }), "requires-submitter-fixes");
|
|
assert.equal(computeState({ automation: automation({ pending: [{}] }), approvals: { ...approvals, satisfied: true } }), "awaiting-automation");
|
|
assert.equal(computeState({ automation: automation({ infrastructureFailures: [{}] }), approvals }), "awaiting-automation");
|
|
assert.equal(computeState({ automation: automation(), approvals: { ...approvals, satisfied: true } }), "approved");
|
|
assert.equal(computeState({ automation: automation(), approvals: { ...approvals, reviewers: ["a"] } }), "review-in-progress");
|
|
assert.equal(computeState({ automation: automation(), approvals }), "ready-for-review");
|
|
});
|
|
|
|
test("latestRunsByWorkflow keeps the newest PR-triggered run per workflow", () => {
|
|
const latest = latestRunsByWorkflow([
|
|
{ id: 1, path: ".github/workflows/a.yml", event: "pull_request", created_at: "2026-09-29T10:00:00Z" },
|
|
{ id: 2, path: ".github/workflows/a.yml", event: "pull_request", created_at: "2026-09-29T11:00:00Z" },
|
|
{ id: 3, path: ".github/workflows/b.yml", event: "push", created_at: "2026-09-29T12:00:00Z" },
|
|
]);
|
|
assert.equal(latest.get("a.yml").id, 2);
|
|
assert.ok(!latest.has("b.yml"));
|
|
});
|
|
|
|
// --- commands and rendering ------------------------------------------------------
|
|
|
|
test("parsePrCommand only accepts supported commands at the start of the comment", () => {
|
|
assert.deepEqual(parsePrCommand("/rerun-checks"), { command: "rerun-checks" });
|
|
assert.deepEqual(parsePrCommand("/Request-Review please\nthanks"), { command: "request-review" });
|
|
assert.equal(parsePrCommand("\n/rerun-checks"), null, "matches the workflow's startsWith filter");
|
|
assert.equal(parsePrCommand(" /rerun-checks"), null);
|
|
assert.equal(parsePrCommand("please /rerun-checks"), null);
|
|
assert.equal(parsePrCommand("/rerun-checksx"), null);
|
|
assert.equal(parsePrCommand(""), null);
|
|
});
|
|
|
|
test("canRunPrCommand allows the author and writers only", () => {
|
|
assert.ok(canRunPrCommand({ commenter: "Author", prAuthor: "author", permission: "read" }));
|
|
assert.ok(canRunPrCommand({ commenter: "m", prAuthor: "author", permission: "maintain" }));
|
|
assert.ok(!canRunPrCommand({ commenter: "x", prAuthor: "author", permission: "triage" }));
|
|
assert.ok(!canRunPrCommand({ commenter: "", prAuthor: "", permission: "read" }));
|
|
});
|
|
|
|
test("sanitize neutralizes mentions, HTML, and table breaks", () => {
|
|
assert.equal(sanitize("@team <b>|x"), "@\u200bteam <b>\\|x");
|
|
assert.equal(sanitize("a".repeat(10), 5).length, 5);
|
|
});
|
|
|
|
// --- orchestration with a fake GitHub client --------------------------------------
|
|
|
|
function fakeGithub({ pr, files = [], reviews = [], runs = [], jobs = {}, checkRuns = [], comments = [], reactions = [], permissions = {} }) {
|
|
const calls = [];
|
|
const record = (name, fn) => async (params) => {
|
|
calls.push({ name, params });
|
|
return fn(params);
|
|
};
|
|
const rest = {
|
|
pulls: {
|
|
get: record("pulls.get", async ({ pull_number }) => ({ data: { ...pr, number: pull_number } })),
|
|
listFiles: "pulls.listFiles",
|
|
listReviews: "pulls.listReviews",
|
|
list: "pulls.list",
|
|
},
|
|
actions: {
|
|
listWorkflowRunsForRepo: "actions.listWorkflowRunsForRepo",
|
|
listJobsForWorkflowRun: "actions.listJobsForWorkflowRun",
|
|
getJobForWorkflowRun: record("actions.getJobForWorkflowRun", async () => {
|
|
throw Object.assign(new Error("nope"), { status: 404 });
|
|
}),
|
|
reRunWorkflowFailedJobs: record("actions.reRunWorkflowFailedJobs", async () => ({})),
|
|
reRunWorkflow: record("actions.reRunWorkflow", async () => ({})),
|
|
createWorkflowDispatch: record("actions.createWorkflowDispatch", async () => ({})),
|
|
},
|
|
checks: {
|
|
listForRef: record("checks.listForRef", async ({ check_name }) => ({
|
|
data: { check_runs: checkRuns.filter((checkRun) => !check_name || checkRun.name === check_name) },
|
|
})),
|
|
create: record("checks.create", async () => ({ data: { id: 999 } })),
|
|
update: record("checks.update", async () => ({ data: {} })),
|
|
},
|
|
repos: {
|
|
getCollaboratorPermissionLevel: record("repos.getCollaboratorPermissionLevel", async ({ username }) => ({
|
|
data: { permission: permissions[username] || "read" },
|
|
})),
|
|
},
|
|
issues: {
|
|
addLabels: record("issues.addLabels", async () => ({})),
|
|
removeLabel: record("issues.removeLabel", async () => ({})),
|
|
listComments: "issues.listComments",
|
|
updateComment: record("issues.updateComment", async () => ({})),
|
|
createComment: record("issues.createComment", async () => ({})),
|
|
getComment: record("issues.getComment", async ({ comment_id }) => {
|
|
const found = comments.find((comment) => comment.id === comment_id);
|
|
if (!found) throw Object.assign(new Error("Not Found"), { status: 404 });
|
|
return { data: found };
|
|
}),
|
|
},
|
|
reactions: {
|
|
listForIssueComment: "reactions.listForIssueComment",
|
|
createForIssueComment: record("reactions.createForIssueComment", async () => ({})),
|
|
},
|
|
};
|
|
const pages = {
|
|
"pulls.listFiles": files,
|
|
"pulls.listReviews": reviews,
|
|
"pulls.list": [pr],
|
|
"actions.listWorkflowRunsForRepo": runs,
|
|
"issues.listComments": comments,
|
|
"reactions.listForIssueComment": reactions,
|
|
};
|
|
return {
|
|
calls,
|
|
rest,
|
|
paginate: async (method, params) => {
|
|
calls.push({ name: method, params });
|
|
if (method === "actions.listJobsForWorkflowRun") return jobs[params.run_id] || [];
|
|
return pages[method] || [];
|
|
},
|
|
};
|
|
}
|
|
|
|
const basePr = {
|
|
number: 7,
|
|
state: "open",
|
|
head: { sha: "a".repeat(40), ref: "feature", repo: { full_name: "fork/awesome-copilot" } },
|
|
base: { ref: "main", repo: { full_name: "github/awesome-copilot" } },
|
|
user: { login: "author" },
|
|
labels: [{ name: "review-due:2026-10-01" }, { name: "merge-risk:high" }],
|
|
requested_reviewers: [{ login: "alice" }],
|
|
requested_teams: [],
|
|
};
|
|
|
|
const run = (id, workflow, conclusion, status = "completed") => ({
|
|
id,
|
|
path: `.github/workflows/${workflow}`,
|
|
name: workflow,
|
|
event: "pull_request",
|
|
status,
|
|
conclusion,
|
|
created_at: "2026-09-29T10:00:00Z",
|
|
html_url: `https://example.test/runs/${id}`,
|
|
});
|
|
|
|
test("evaluateSubmission reaches approved when checks pass and approvals exist", async () => {
|
|
const github = fakeGithub({
|
|
pr: basePr,
|
|
files: [file("docs/a.md")],
|
|
reviews: [review("alice", "APPROVED")],
|
|
permissions: { alice: "write" },
|
|
runs: [
|
|
run(1, "check-line-endings.yml", "success"),
|
|
run(90, "codespell.yml", "success"),
|
|
run(2, "validate-readme.yml", "success"),
|
|
run(3, "contributor-check.yml", "success"),
|
|
run(4, "pr-duplicate-check.lock.yml", "failure"),
|
|
run(5, "pr-quality-signal.lock.yml", "skipped"),
|
|
],
|
|
});
|
|
const evaluation = await evaluateSubmission(github, {
|
|
owner: "github",
|
|
repo: "awesome-copilot",
|
|
pullNumber: 7,
|
|
config,
|
|
readContributorRisk: () => "LOW",
|
|
});
|
|
assert.equal(evaluation.risk.tier, "low");
|
|
assert.equal(evaluation.state, "approved");
|
|
assert.equal(evaluation.passed, true);
|
|
assert.equal(evaluation.automation.warnings.length, 1, "duplicate-scan infra failure is advisory");
|
|
assert.equal(evaluation.reviewAssignment.due, "2026-10-01");
|
|
|
|
await syncPullRequestStatus(github, { owner: "github", repo: "awesome-copilot", evaluation });
|
|
const added = github.calls.find((call) => call.name === "issues.addLabels");
|
|
assert.deepEqual(added.params.labels.sort(), ["approved", "merge-risk:low"]);
|
|
const removed = github.calls.filter((call) => call.name === "issues.removeLabel").map((call) => call.params.name);
|
|
assert.deepEqual(removed, ["merge-risk:high"]);
|
|
const created = github.calls.find((call) => call.name === "issues.createComment");
|
|
assert.ok(created.params.body.startsWith(STATUS_MARKER));
|
|
});
|
|
|
|
test("evaluateSubmission separates contribution and infrastructure failures", async () => {
|
|
const github = fakeGithub({
|
|
pr: basePr,
|
|
files: [file("skills/x/SKILL.md", { status: "added" })],
|
|
runs: [
|
|
run(1, "check-line-endings.yml", "success"),
|
|
run(90, "codespell.yml", "success"),
|
|
run(2, "validate-readme.yml", "failure"),
|
|
run(3, "contributor-check.yml", "success"),
|
|
run(4, "validate-skills.yml", "failure"),
|
|
run(5, "skill-check.yml", "success"),
|
|
run(6, "pr-risk-scan.yml", "cancelled"),
|
|
],
|
|
jobs: {
|
|
2: failedJob("Fail workflow if files need updating"),
|
|
4: failedJob("Install dependencies"),
|
|
},
|
|
});
|
|
const evaluation = await evaluateSubmission(github, {
|
|
owner: "github",
|
|
repo: "awesome-copilot",
|
|
pullNumber: 7,
|
|
config,
|
|
finalized: true,
|
|
readContributorRisk: () => null,
|
|
});
|
|
assert.deepEqual(evaluation.automation.contributionFailures.map((r) => r.id), ["readme"]);
|
|
assert.deepEqual(
|
|
evaluation.automation.infrastructureFailures.map((r) => r.id).sort(),
|
|
["risk-scan", "skill-validation"]
|
|
);
|
|
assert.equal(evaluation.state, "requires-submitter-fixes");
|
|
const body = renderStatusComment(evaluation);
|
|
assert.match(body, /Contribution failure/);
|
|
assert.match(body, /Infrastructure failure/);
|
|
assert.match(body, /npm start/);
|
|
assert.match(body, /\/rerun-checks/);
|
|
assert.match(body, /2026-10-01/);
|
|
});
|
|
|
|
test("evaluateSubmission waits for pending checks in gate mode", async () => {
|
|
let clock = 0;
|
|
let polls = 0;
|
|
const runs = [
|
|
run(1, "check-line-endings.yml", null, "in_progress"),
|
|
run(90, "codespell.yml", "success"),
|
|
run(3, "contributor-check.yml", "success"),
|
|
run(4, "pr-duplicate-check.lock.yml", "success"),
|
|
run(5, "pr-quality-signal.lock.yml", "success"),
|
|
];
|
|
const github = fakeGithub({ pr: basePr, files: [file("LICENSE")], runs });
|
|
const originalPaginate = github.paginate;
|
|
github.paginate = async (method, params) => {
|
|
if (method === "actions.listWorkflowRunsForRepo") {
|
|
polls += 1;
|
|
if (polls >= 2) runs[0] = run(1, "check-line-endings.yml", "success");
|
|
}
|
|
return originalPaginate(method, params);
|
|
};
|
|
const evaluation = await evaluateSubmission(github, {
|
|
owner: "github",
|
|
repo: "awesome-copilot",
|
|
pullNumber: 7,
|
|
config,
|
|
wait: true,
|
|
now: () => clock,
|
|
sleep: async (ms) => {
|
|
clock += ms;
|
|
},
|
|
readContributorRisk: () => null,
|
|
});
|
|
assert.equal(polls, 2);
|
|
assert.equal(evaluation.automation.pending.length, 0);
|
|
assert.equal(evaluation.state, "ready-for-review");
|
|
});
|
|
|
|
test("unreported required checks become infrastructure failures after the grace period", async () => {
|
|
let clock = 0;
|
|
const github = fakeGithub({ pr: basePr, files: [file("LICENSE")], runs: [run(3, "contributor-check.yml", "success")] });
|
|
const evaluation = await evaluateSubmission(github, {
|
|
owner: "github",
|
|
repo: "awesome-copilot",
|
|
pullNumber: 7,
|
|
config,
|
|
wait: true,
|
|
now: () => clock,
|
|
sleep: async (ms) => {
|
|
clock += ms;
|
|
},
|
|
readContributorRisk: () => null,
|
|
});
|
|
const lineEndings = evaluation.automation.results.find((result) => result.id === "line-endings");
|
|
assert.equal(lineEndings.category, "infrastructure");
|
|
assert.ok(clock >= config.gate.wait.report_grace_minutes * 60_000);
|
|
assert.ok(clock < config.gate.wait.timeout_minutes * 60_000);
|
|
});
|
|
|
|
test("rerunChecks re-runs failed workflows and the gate but skips approval-gated runs", async () => {
|
|
const github = fakeGithub({
|
|
pr: basePr,
|
|
runs: [
|
|
run(1, "validate-readme.yml", "failure"),
|
|
run(2, "check-line-endings.yml", "success"),
|
|
run(3, "contributor-check.yml", "action_required"),
|
|
run(4, "submission-gate.yml", "failure"),
|
|
run(5, "pr-risk-scan.yml", null, "in_progress"),
|
|
],
|
|
});
|
|
const result = await rerunChecks(github, { owner: "github", repo: "awesome-copilot", headSha: basePr.head.sha });
|
|
assert.deepEqual(result.rerun, ["validate-readme.yml", "submission-gate.yml"]);
|
|
assert.equal(result.skipped.length, 1);
|
|
assert.ok(github.calls.some((call) => call.name === "actions.reRunWorkflow" && call.params.run_id === 4));
|
|
});
|
|
|
|
test("syncPullRequestStatus leaves state labels to external plugin intake and updates the comment in place", async () => {
|
|
const pr = { ...basePr, labels: [{ name: "external-plugin" }, { name: "ready-for-review" }] };
|
|
const github = fakeGithub({
|
|
pr,
|
|
comments: [
|
|
{ id: 1, user: { login: "someone" }, body: STATUS_MARKER },
|
|
{ id: 2, user: { login: "github-actions[bot]" }, body: `${STATUS_MARKER}\nold` },
|
|
],
|
|
});
|
|
const evaluation = {
|
|
pr,
|
|
headSha: pr.head.sha,
|
|
labels: ["external-plugin", "ready-for-review"],
|
|
risk: { tier: "high", reasons: ["`plugins/external.json` is a high-risk path"] },
|
|
automation: summarizeChecks([]),
|
|
approvals: { required: 2, requirement: "2 approvals", approvers: [], changesRequestedBy: [], reviewers: [], missing: ["2 more approval(s)"], notes: [], satisfied: false },
|
|
state: "awaiting-automation",
|
|
reviewAssignment: { users: [], teams: [], due: null },
|
|
};
|
|
await syncPullRequestStatus(github, { owner: "github", repo: "awesome-copilot", evaluation });
|
|
assert.deepEqual(github.calls.find((call) => call.name === "issues.addLabels").params.labels, ["merge-risk:high"]);
|
|
assert.ok(!github.calls.some((call) => call.name === "issues.removeLabel"));
|
|
const updated = github.calls.find((call) => call.name === "issues.updateComment");
|
|
assert.equal(updated.params.comment_id, 2);
|
|
assert.ok(!github.calls.some((call) => call.name === "issues.createComment"));
|
|
});
|
|
|
|
test("resolvePullRequestForWorkflowRun requires an exact head match", async () => {
|
|
const github = fakeGithub({ pr: basePr });
|
|
const workflowRun = {
|
|
head_sha: basePr.head.sha,
|
|
head_branch: "feature",
|
|
head_repository: { full_name: "fork/awesome-copilot" },
|
|
pull_requests: [],
|
|
};
|
|
assert.equal(await resolvePullRequestForWorkflowRun(github, { owner: "github", repo: "awesome-copilot", workflowRun }), 7);
|
|
assert.equal(
|
|
await resolvePullRequestForWorkflowRun(github, {
|
|
owner: "github",
|
|
repo: "awesome-copilot",
|
|
workflowRun: { ...workflowRun, head_sha: "b".repeat(40) },
|
|
}),
|
|
null
|
|
);
|
|
});
|
|
|
|
// --- review hardening ------------------------------------------------------------
|
|
|
|
test("text files without a scannable diff or a truncated file list fail closed to high", () => {
|
|
const noPatch = classifyRisk({ files: [file("extensions/x/extension.mjs", { patch: undefined })], tiers });
|
|
assert.equal(noPatch.tier, "high");
|
|
assert.match(noPatch.reasons.join("\n"), /no diff available/);
|
|
assert.equal(classifyRisk({ files: [file("skills/x/guide.pdf", { patch: undefined, status: "added" })], tiers }).tier, "medium");
|
|
assert.equal(classifyRisk({ files: [file("docs/a.md", { patch: undefined })], tiers }).tier, "low", "outside capability scope");
|
|
assert.equal(classifyRisk({ files: [file("docs/a.md")], tiers, incompleteFiles: true }).tier, "high");
|
|
});
|
|
|
|
test("low tier requires an approval from an owner of the changed resource", () => {
|
|
const permissions = new Map([["alice", "write"], ["canvasa", "write"], ["corea", "write"]]);
|
|
const base = { tier: "low", tiers, author: "author", permissions, routing };
|
|
const canvas = { ...base, files: [file("extensions/x/README.md")] };
|
|
assert.equal(evaluateApprovals({ ...canvas, reviews: [review("alice", "APPROVED")] }).satisfied, false);
|
|
assert.equal(evaluateApprovals({ ...canvas, reviews: [review("canvasa", "APPROVED")] }).satisfied, true);
|
|
const docs = { ...base, files: [file("docs/a.md")] };
|
|
assert.equal(evaluateApprovals({ ...docs, reviews: [review("alice", "APPROVED")] }).satisfied, false, "core pools own docs");
|
|
assert.equal(evaluateApprovals({ ...docs, reviews: [review("corea", "APPROVED")] }).satisfied, true);
|
|
const unstaffed = evaluateApprovals({ ...base, files: [file("skills/x/SKILL.md")], reviews: [review("alice", "APPROVED")] });
|
|
assert.equal(unstaffed.satisfied, true, "unstaffed content pool falls back to any writer");
|
|
});
|
|
|
|
test("approvals fail closed when a reviewer's permission can't be read", async () => {
|
|
const github = fakeGithub({ pr: basePr, files: [file("docs/a.md")], reviews: [review("alice", "APPROVED", undefined, { author_association: "COLLABORATOR" })] });
|
|
github.rest.repos.getCollaboratorPermissionLevel = async () => {
|
|
throw Object.assign(new Error("forbidden"), { status: 403 });
|
|
};
|
|
const evaluation = await evaluateSubmission(github, { owner: "github", repo: "awesome-copilot", pullNumber: 7, config, finalized: true, readContributorRisk: () => null });
|
|
assert.deepEqual(evaluation.approvals.approvers, []);
|
|
});
|
|
|
|
test("evaluateSubmission blocks when GitHub truncates the changed file list", async () => {
|
|
const github = fakeGithub({ pr: { ...basePr, changed_files: 3001 }, files: [file("docs/a.md")] });
|
|
const evaluation = await evaluateSubmission(github, { owner: "github", repo: "awesome-copilot", pullNumber: 7, config, finalized: true, readContributorRisk: () => null });
|
|
assert.equal(evaluation.risk.tier, "high");
|
|
const truncated = evaluation.automation.infrastructureFailures.find((result) => result.id === "changed-files");
|
|
assert.ok(truncated);
|
|
assert.equal(evaluation.passed, false);
|
|
});
|
|
|
|
test("evaluateSubmission is stale when the head differs from the expected or re-read head", async () => {
|
|
const expected = await evaluateSubmission(fakeGithub({ pr: basePr }), {
|
|
owner: "github", repo: "awesome-copilot", pullNumber: 7, config, expectedHeadSha: "b".repeat(40), readContributorRisk: () => null,
|
|
});
|
|
assert.equal(expected.stale, true);
|
|
|
|
const github = fakeGithub({ pr: basePr, files: [file("docs/a.md")] });
|
|
let gets = 0;
|
|
github.rest.pulls.get = async ({ pull_number }) => {
|
|
gets += 1;
|
|
return { data: { ...basePr, number: pull_number, head: { ...basePr.head, sha: gets === 1 ? basePr.head.sha : "c".repeat(40) } } };
|
|
};
|
|
const moved = await evaluateSubmission(github, { owner: "github", repo: "awesome-copilot", pullNumber: 7, config, readContributorRisk: () => null });
|
|
assert.equal(moved.stale, true);
|
|
assert.notEqual(moved.passed, true);
|
|
});
|
|
|
|
test("a submission-gate check not published by the writer is flagged as tampering", async () => {
|
|
const github = fakeGithub({
|
|
pr: basePr,
|
|
files: [file("docs/a.md")],
|
|
reviews: [review("alice", "APPROVED")],
|
|
permissions: { alice: "write" },
|
|
checkRuns: [{ id: 5, name: "submission-gate", external_id: "", html_url: "https://example.test/impostor" }],
|
|
runs: [run(1, "check-line-endings.yml", "success"), run(90, "codespell.yml", "success"), run(2, "validate-readme.yml", "success"), run(3, "contributor-check.yml", "success")],
|
|
});
|
|
const evaluation = await evaluateSubmission(github, { owner: "github", repo: "awesome-copilot", pullNumber: 7, config, finalized: true, readContributorRisk: () => null });
|
|
assert.equal(evaluation.risk.tier, "high");
|
|
assert.ok(evaluation.automation.contributionFailures.some((result) => result.id === "gate-integrity"));
|
|
assert.equal(evaluation.passed, false);
|
|
|
|
await publishGateCheck(github, { owner: "github", repo: "awesome-copilot", evaluation });
|
|
const created = github.calls.find((call) => call.name === "checks.create");
|
|
assert.equal(created.params.name, "submission-gate");
|
|
assert.equal(created.params.external_id, GATE_CHECK_EXTERNAL_ID);
|
|
assert.equal(created.params.conclusion, "failure");
|
|
});
|
|
|
|
test("publishGateCheck updates the writer's check run and reports pending as in progress", async () => {
|
|
const ours = { id: 42, name: "submission-gate", external_id: GATE_CHECK_EXTERNAL_ID };
|
|
const github = fakeGithub({ pr: basePr, checkRuns: [ours] });
|
|
const evaluation = {
|
|
headSha: basePr.head.sha,
|
|
state: "approved",
|
|
passed: true,
|
|
risk: { tier: "low" },
|
|
automation: summarizeChecks([]),
|
|
approvals: { satisfied: true, missing: [] },
|
|
};
|
|
await publishGateCheck(github, { owner: "github", repo: "awesome-copilot", evaluation, detailsUrl: "https://example.test/run" });
|
|
const updated = github.calls.find((call) => call.name === "checks.update");
|
|
assert.equal(updated.params.check_run_id, 42);
|
|
assert.equal(updated.params.conclusion, "success");
|
|
assert.equal(updated.params.details_url, "https://example.test/run");
|
|
|
|
const pending = { ...evaluation, state: "awaiting-automation", passed: false, automation: summarizeChecks([{ id: "x", title: "X", required: true, outcome: "pending" }]) };
|
|
const github2 = fakeGithub({ pr: basePr });
|
|
await publishGateCheck(github2, { owner: "github", repo: "awesome-copilot", evaluation: pending });
|
|
const created = github2.calls.find((call) => call.name === "checks.create");
|
|
assert.equal(created.params.status, "in_progress");
|
|
assert.equal(created.params.conclusion, undefined);
|
|
});
|
|
|
|
test("syncPullRequestStatus does not write when the head or reviews changed", async () => {
|
|
const evaluation = {
|
|
pr: basePr,
|
|
headSha: "b".repeat(40),
|
|
labels: [],
|
|
reviewsSignature: "",
|
|
risk: { tier: "low", reasons: [] },
|
|
automation: summarizeChecks([]),
|
|
approvals: { required: 1, requirement: "1 approval", approvers: [], changesRequestedBy: [], reviewers: [], missing: [], notes: [], satisfied: true },
|
|
state: "approved",
|
|
passed: true,
|
|
reviewAssignment: { users: [], teams: [], due: null },
|
|
};
|
|
const moved = fakeGithub({ pr: basePr });
|
|
assert.deepEqual(await syncPullRequestStatus(moved, { owner: "github", repo: "awesome-copilot", evaluation, publishCheck: true }), {
|
|
updated: false,
|
|
reason: "head-changed",
|
|
});
|
|
assert.ok(!moved.calls.some((call) => /addLabels|removeLabel|Comment|checks\.(create|update)/.test(call.name)));
|
|
|
|
const reviewed = fakeGithub({ pr: basePr, reviews: [review("alice", "CHANGES_REQUESTED")] });
|
|
const result = await syncPullRequestStatus(reviewed, {
|
|
owner: "github",
|
|
repo: "awesome-copilot",
|
|
evaluation: { ...evaluation, headSha: basePr.head.sha },
|
|
publishCheck: true,
|
|
});
|
|
assert.equal(result.reason, "reviews-changed");
|
|
assert.ok(!reviewed.calls.some((call) => /addLabels|checks\.create/.test(call.name)));
|
|
});
|
|
|
|
test("syncPullRequestStatus publishes the gate check even when label or comment writes fail", async () => {
|
|
const evaluation = {
|
|
pr: basePr,
|
|
headSha: basePr.head.sha,
|
|
labels: [],
|
|
reviewsSignature: "",
|
|
risk: { tier: "low", reasons: [] },
|
|
automation: summarizeChecks([]),
|
|
approvals: { required: 1, requirement: "1 approval", approvers: [], changesRequestedBy: [], reviewers: [], missing: ["1 more approval"], notes: [], satisfied: false },
|
|
state: "review-in-progress",
|
|
passed: false,
|
|
reviewAssignment: { users: [], teams: [], due: null },
|
|
};
|
|
const github = fakeGithub({ pr: basePr });
|
|
github.rest.issues.addLabels = async () => {
|
|
throw Object.assign(new Error("Resource not accessible"), { status: 403 });
|
|
};
|
|
github.rest.issues.createComment = async () => {
|
|
throw Object.assign(new Error("Server Error"), { status: 502 });
|
|
};
|
|
await assert.rejects(
|
|
syncPullRequestStatus(github, { owner: "github", repo: "awesome-copilot", evaluation, publishCheck: true }),
|
|
/labels: Resource not accessible; status comment: Server Error/
|
|
);
|
|
assert.ok(github.calls.some((call) => call.name === "checks.create"), "check is published before label and comment writes");
|
|
});
|
|
|
|
test("a successful contributor check with an unreadable artifact fails closed", async () => {
|
|
const runs = [
|
|
run(1, "check-line-endings.yml", "success"),
|
|
run(90, "codespell.yml", "success"),
|
|
run(2, "validate-readme.yml", "success"),
|
|
run(3, "contributor-check.yml", "success"),
|
|
];
|
|
const approved = { reviews: [review("alice", "APPROVED")], permissions: { alice: "write" } };
|
|
const options = { owner: "github", repo: "awesome-copilot", pullNumber: 7, config, finalized: true, readContributorRisk: () => null };
|
|
|
|
const lost = await evaluateSubmission(
|
|
fakeGithub({ pr: basePr, files: [file("docs/a.md")], runs, jobs: { 3: [{ name: "pr-check", conclusion: "success" }] }, ...approved }),
|
|
options
|
|
);
|
|
assert.deepEqual(lost.automation.infrastructureFailures.map((r) => r.id), ["contributor-risk-signal"]);
|
|
assert.equal(lost.passed, false);
|
|
|
|
const skipped = await evaluateSubmission(
|
|
fakeGithub({ pr: basePr, files: [file("docs/a.md")], runs, jobs: { 3: [{ name: "pr-check", conclusion: "skipped" }] }, ...approved }),
|
|
options
|
|
);
|
|
assert.equal(skipped.automation.infrastructureFailures.length, 0, "a skipped PR job carries no signal");
|
|
assert.equal(skipped.passed, true);
|
|
});
|
|
|
|
test("evaluation and final write are discarded when the base branch or risk labels change", async () => {
|
|
const github = fakeGithub({ pr: basePr, files: [file("docs/a.md")] });
|
|
let gets = 0;
|
|
github.rest.pulls.get = async ({ pull_number }) => {
|
|
gets += 1;
|
|
return { data: { ...basePr, number: pull_number, base: { ...basePr.base, ref: gets === 1 ? "main" : "staged" } } };
|
|
};
|
|
const retargeted = await evaluateSubmission(github, { owner: "github", repo: "awesome-copilot", pullNumber: 7, config, readContributorRisk: () => null });
|
|
assert.equal(retargeted.stale, true);
|
|
|
|
const evaluation = await evaluateSubmission(fakeGithub({ pr: basePr, files: [file("docs/a.md")] }), {
|
|
owner: "github", repo: "awesome-copilot", pullNumber: 7, config, finalized: true, readContributorRisk: () => null,
|
|
});
|
|
assert.equal(evaluation.baseRef, "main");
|
|
assert.equal(evaluation.riskLabelSignature, "");
|
|
|
|
const rebased = fakeGithub({ pr: { ...basePr, base: { ...basePr.base, ref: "staged" } } });
|
|
assert.equal((await syncPullRequestStatus(rebased, { owner: "github", repo: "awesome-copilot", evaluation })).reason, "base-changed");
|
|
|
|
const flagged = fakeGithub({ pr: { ...basePr, labels: [...basePr.labels, { name: "needs-review:HIGH" }] } });
|
|
assert.equal((await syncPullRequestStatus(flagged, { owner: "github", repo: "awesome-copilot", evaluation })).reason, "labels-changed");
|
|
assert.ok(!flagged.calls.some((call) => /addLabels|removeLabel|Comment|checks\./.test(call.name)));
|
|
|
|
assert.equal(riskLabelSignature(["b", "needs-review:HIGH", "a"], ["needs-review:HIGH"]), "needs-review:HIGH");
|
|
});
|
|
|
|
test("publishGateCheck corrects copies of the writer's check that claim success for a failing evaluation", async () => {
|
|
const older = { id: 10, name: "submission-gate", external_id: GATE_CHECK_EXTERNAL_ID, conclusion: "failure" };
|
|
const forged = { id: 11, name: "submission-gate", external_id: GATE_CHECK_EXTERNAL_ID, conclusion: "success" };
|
|
const github = fakeGithub({ pr: basePr, checkRuns: [older, forged] });
|
|
const evaluation = {
|
|
headSha: basePr.head.sha,
|
|
state: "review-in-progress",
|
|
passed: false,
|
|
risk: { tier: "high" },
|
|
automation: summarizeChecks([]),
|
|
approvals: { satisfied: false, missing: ["1 more approval"] },
|
|
};
|
|
const logs = [];
|
|
await publishGateCheck(github, { owner: "github", repo: "awesome-copilot", evaluation, log: (line) => logs.push(line) });
|
|
const updates = github.calls.filter((call) => call.name === "checks.update");
|
|
assert.deepEqual(updates.map((call) => call.params.check_run_id), [11]);
|
|
assert.equal(updates[0].params.conclusion, "action_required");
|
|
assert.ok(logs.some((line) => /Correcting 1/.test(line)));
|
|
});
|
|
|
|
test("validatePrCommandRequest only accepts well-formed requests from the triggering run", () => {
|
|
const request = { schema_version: PR_COMMAND_SCHEMA, pr_number: 7, comment_id: 55, run_id: "123" };
|
|
assert.deepEqual(validatePrCommandRequest(request, { workflowRunId: 123 }), { prNumber: 7, commentId: 55 });
|
|
assert.throws(() => validatePrCommandRequest({ ...request, run_id: "124" }, { workflowRunId: 123 }), /run_id/);
|
|
assert.throws(() => validatePrCommandRequest({ ...request, pr_number: "7" }, { workflowRunId: 123 }), /pr_number/);
|
|
assert.throws(() => validatePrCommandRequest({ ...request, comment_id: 0 }, { workflowRunId: 123 }), /comment_id/);
|
|
assert.throws(() => validatePrCommandRequest({ ...request, schema_version: "x" }, { workflowRunId: 123 }), /schema_version/);
|
|
assert.throws(() => validatePrCommandRequest(null, { workflowRunId: 123 }), /not an object/);
|
|
});
|
|
|
|
const commandComment = (body, login = "author", extra = {}) => ({
|
|
id: 55,
|
|
body,
|
|
user: { login, type: "User" },
|
|
issue_url: "https://api.github.com/repos/github/awesome-copilot/issues/7",
|
|
...extra,
|
|
});
|
|
const commandOptions = { owner: "github", repo: "awesome-copilot", prNumber: 7, commentId: 55, config };
|
|
|
|
test("runPrCommand re-reads the comment, PR, and permission before writing", async () => {
|
|
const request = fakeGithub({ pr: basePr, comments: [commandComment("/request-review")] });
|
|
assert.deepEqual(await runPrCommand(request, commandOptions), { status: "handled", command: "request-review" });
|
|
assert.deepEqual(request.calls.find((call) => call.name === "issues.addLabels").params.labels, ["needs-reviewer"]);
|
|
const dispatched = request.calls.find((call) => call.name === "actions.createWorkflowDispatch");
|
|
assert.equal(dispatched.params.workflow_id, "review-routing.yml");
|
|
assert.deepEqual(dispatched.params.inputs, { pr_number: "7" });
|
|
assert.ok(request.calls.some((call) => call.name === "reactions.createForIssueComment"));
|
|
assert.ok(request.calls.some((call) => call.name === "issues.createComment"));
|
|
const order = request.calls.map((call) => (call.name === "reactions.createForIssueComment" ? `reaction:${call.params.content}` : call.name));
|
|
assert.ok(order.indexOf("reaction:eyes") < order.indexOf("issues.addLabels"), "claim marker is written before acting");
|
|
assert.ok(order.indexOf("reaction:rocket") > order.indexOf("issues.createComment"), "done marker is written after the reply");
|
|
|
|
const rerun = fakeGithub({ pr: basePr, comments: [commandComment("/rerun-checks", "maint")], permissions: { maint: "maintain" } });
|
|
assert.equal((await runPrCommand(rerun, commandOptions)).command, "rerun-checks");
|
|
assert.equal(rerun.calls.find((call) => call.name === "actions.createWorkflowDispatch").params.workflow_id, "submission-gate-writer.yml");
|
|
});
|
|
|
|
test("runPrCommand ignores outsiders, closed PRs, edits, replays, and mismatched comments", async () => {
|
|
const writes = /addLabels|createComment|createForIssueComment|createWorkflowDispatch|reRun/;
|
|
|
|
const outsider = fakeGithub({ pr: basePr, comments: [commandComment("/rerun-checks", "stranger")] });
|
|
assert.equal((await runPrCommand(outsider, commandOptions)).status, "ignored");
|
|
assert.ok(!outsider.calls.some((call) => writes.test(call.name)));
|
|
|
|
const closed = fakeGithub({ pr: { ...basePr, state: "closed" }, comments: [commandComment("/rerun-checks")] });
|
|
assert.equal((await runPrCommand(closed, commandOptions)).reason, "PR is not open");
|
|
|
|
const edited = fakeGithub({ pr: basePr, comments: [commandComment("thanks!")] });
|
|
assert.equal((await runPrCommand(edited, commandOptions)).reason, "no supported command");
|
|
|
|
const replay = fakeGithub({
|
|
pr: basePr,
|
|
comments: [commandComment("/request-review")],
|
|
reactions: [
|
|
{ content: "eyes", user: { login: "github-actions[bot]" } },
|
|
{ content: "rocket", user: { login: "github-actions[bot]" } },
|
|
],
|
|
});
|
|
assert.equal((await runPrCommand(replay, commandOptions)).reason, "already handled");
|
|
assert.ok(!replay.calls.some((call) => writes.test(call.name)));
|
|
|
|
const elsewhere = fakeGithub({
|
|
pr: basePr,
|
|
comments: [commandComment("/request-review", "author", { issue_url: "https://api.github.com/repos/github/awesome-copilot/issues/70" })],
|
|
});
|
|
await assert.rejects(runPrCommand(elsewhere, commandOptions), /does not belong/);
|
|
});
|
|
test("runPrCommand retries a command that was claimed but never completed", async () => {
|
|
const claimedOnly = fakeGithub({
|
|
pr: basePr,
|
|
comments: [commandComment("/request-review")],
|
|
reactions: [{ content: "eyes", user: { login: "github-actions[bot]" } }],
|
|
});
|
|
assert.equal((await runPrCommand(claimedOnly, commandOptions)).status, "handled");
|
|
|
|
const failing = fakeGithub({ pr: basePr, comments: [commandComment("/request-review")] });
|
|
failing.rest.issues.createComment = async () => {
|
|
throw Object.assign(new Error("boom"), { status: 502 });
|
|
};
|
|
await assert.rejects(runPrCommand(failing, commandOptions), /boom/);
|
|
const reactionsWritten = failing.calls.filter((call) => call.name === "reactions.createForIssueComment").map((call) => call.params.content);
|
|
assert.deepEqual(reactionsWritten, ["eyes"], "a failed run leaves only the claim marker, so it can be retried");
|
|
});
|
|
|
|
test("advisory checks that are still pending do not hold the gate", () => {
|
|
const advisory = evaluateCheck({ id: "quality", required: false }, { found: true, status: "in_progress" });
|
|
const required = evaluateCheck({ id: "build" }, { found: true, status: "completed", conclusion: "success" });
|
|
const automation = summarizeChecks([advisory, required]);
|
|
assert.equal(automation.pending.length, 0);
|
|
assert.equal(automation.advisoryPending.length, 1);
|
|
const approvals = { satisfied: true, reviewers: [], changesRequestedBy: [] };
|
|
assert.equal(computeState({ automation, approvals }), "approved");
|
|
|
|
const requiredPending = summarizeChecks([evaluateCheck({ id: "build" }, { found: true, status: "queued" })]);
|
|
assert.equal(computeState({ automation: requiredPending, approvals }), "awaiting-automation");
|
|
});
|
|
|
|
test("loadGateConfig fails closed on an unparsable review-routing.yml but tolerates a missing one", () => {
|
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "gate-config-"));
|
|
try {
|
|
fs.mkdirSync(path.join(dir, ".github"));
|
|
for (const name of ["submission-gate.yml", "risk-tiers.yml"]) {
|
|
fs.copyFileSync(path.join(repoRoot, ".github", name), path.join(dir, ".github", name));
|
|
}
|
|
assert.equal(loadGateConfig(dir).routing, null, "missing routing file falls back");
|
|
|
|
const routingPath = path.join(dir, ".github", "review-routing.yml");
|
|
fs.writeFileSync(routingPath, "pools: [unclosed\n");
|
|
assert.throws(() => loadGateConfig(dir), /review-routing\.yml could not be parsed/);
|
|
|
|
fs.writeFileSync(routingPath, "pools:\n - core\n");
|
|
assert.throws(() => loadGateConfig(dir), /`pools` must be a mapping/);
|
|
|
|
fs.writeFileSync(routingPath, "- just\n- a list\n");
|
|
assert.throws(() => loadGateConfig(dir), /must be a mapping/);
|
|
} finally {
|
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}); |