chore: publish from staged

.github/workflows/external-plugin-approval-command.yml

@@ -6,6 +6,10 @@ on:
   pull_request:
     types: [closed]
 
+concurrency:
+  group: external-plugin-intake-${{ github.event.issue.number }}
+  cancel-in-progress: false
+
 permissions:
   contents: write
   issues: write
@@ -272,6 +276,10 @@ jobs:
                 color: '0E8A16',
                 description: 'Submission passed intake validation and is ready for maintainer review'
               },
+              'requires-submitter-fixes': {
+                color: 'D93F0B',
+                description: 'Submission has quality-gate findings that submitter must fix before maintainer review'
+              },
               'approved': {
                 color: '1D76DB',
                 description: 'Submission was approved by a maintainer'
@@ -490,6 +498,10 @@ jobs:
                 color: '0E8A16',
                 description: 'Submission passed intake validation and is ready for maintainer review'
               },
+              'requires-submitter-fixes': {
+                color: 'D93F0B',
+                description: 'Submission has quality-gate findings that submitter must fix before maintainer review'
+              },
               'approved': {
                 color: '1D76DB',
                 description: 'Submission was approved by a maintainer'
@@ -541,6 +553,7 @@ jobs:
 
             await removeLabel('awaiting-review');
             await removeLabel('ready-for-review');
+            await removeLabel('requires-submitter-fixes');
             await removeLabel('approved');
 
             const marker = '<!-- external-plugin-rejection -->';

.github/workflows/external-plugin-intake.yml

@@ -13,14 +13,40 @@ permissions:
   issues: write
 
 jobs:
-  validate-submission:
+  evaluate-submission:
     runs-on: ubuntu-latest
     if: >-
       contains(github.event.issue.labels.*.name, 'external-plugin') ||
       contains(github.event.issue.body, '<!-- external-plugin-submission -->')
+    outputs:
+      evaluation: ${{ steps.evaluation.outputs.result }}
+      should-sync: ${{ steps.guard.outputs.should-sync }}
+      issue-state: ${{ steps.guard.outputs.issue-state }}
+      issue-action: ${{ steps.guard.outputs.issue-action }}
+      issue-labels: ${{ steps.guard.outputs.issue-labels }}
+      plugin-json: ${{ steps.evaluation.outputs.plugin-json }}
+      valid: ${{ steps.evaluation.outputs.valid }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: staged
+
+      - name: Evaluate issue guard rails
+        id: guard
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const issueState = context.payload.issue.state;
+            const action = context.payload.action;
+            const labels = (context.payload.issue.labels || []).map((label) => label.name);
+            const isApproved = labels.includes('approved');
+            const isClosedWithoutReopen = issueState === 'closed' && action !== 'reopened';
+
+            core.setOutput('issue-state', issueState);
+            core.setOutput('issue-action', action);
+            core.setOutput('issue-labels', JSON.stringify(labels));
+            core.setOutput('should-sync', (!isApproved && !isClosedWithoutReopen) ? 'true' : 'false');
 
       - name: Evaluate submission
         id: evaluation
@@ -34,46 +60,101 @@ jobs:
             echo 'EOF'
           } >> "$GITHUB_OUTPUT"
 
-      - name: Sync labels and comment
+          valid=$(node -e "const data = JSON.parse(process.argv[1]); process.stdout.write(data.valid ? 'true' : 'false');" "$result")
+          plugin=$(node -e "const data = JSON.parse(process.argv[1]); process.stdout.write(JSON.stringify(data.plugin || {}));" "$result")
+          echo "valid=$valid" >> "$GITHUB_OUTPUT"
+          {
+            echo 'plugin-json<<EOF'
+            echo "$plugin"
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+
+  quality-gates:
+    needs: evaluate-submission
+    if: >-
+      needs.evaluate-submission.outputs.should-sync == 'true' &&
+      needs.evaluate-submission.outputs.valid == 'true'
+    uses: ./.github/workflows/external-plugin-quality-gates.yml
+    with:
+      plugin-json: ${{ needs.evaluate-submission.outputs.plugin-json }}
+
+  sync-state:
+    runs-on: ubuntu-latest
+    needs: [evaluate-submission, quality-gates]
+    if: always() && needs.evaluate-submission.outputs.should-sync == 'true'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: staged
+
+      - name: Merge evaluation and sync labels/comments
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         env:
-          RESULT_JSON: ${{ steps.evaluation.outputs.result }}
+          BASE_RESULT_JSON: ${{ needs.evaluate-submission.outputs.evaluation }}
+          BASE_VALID: ${{ needs.evaluate-submission.outputs.valid }}
+          QUALITY_RESULT_JSON: ${{ needs.quality-gates.outputs.quality-result }}
+          QUALITY_JOB_RESULT: ${{ needs.quality-gates.result }}
+          ISSUE_STATE: ${{ needs.evaluate-submission.outputs.issue-state }}
+          ISSUE_LABELS: ${{ needs.evaluate-submission.outputs.issue-labels }}
         with:
           script: |
             const path = require('path');
             const { pathToFileURL } = require('url');
 
+            const intake = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake.mjs')).href);
             const intakeState = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake-state.mjs')).href);
 
-            const result = JSON.parse(process.env.RESULT_JSON);
-            const issueNumber = context.issue.number;
-            const issueState = context.payload.issue.state;
-            const action = context.payload.action;
-            const existingLabelNames = (context.payload.issue.labels || []).map((label) => label.name);
+            const baseResult = JSON.parse(process.env.BASE_RESULT_JSON);
+            let finalResult = baseResult;
 
-            if (existingLabelNames.includes('approved')) {
-              core.info('Issue is already approved; skipping intake synchronization.');
-              return;
-            }
+            if (process.env.BASE_VALID === 'true') {
+              let qualityResult;
+              if (process.env.QUALITY_JOB_RESULT === 'failure' || process.env.QUALITY_JOB_RESULT === 'cancelled') {
+                qualityResult = {
+                  overall_status: 'infra_error',
+                  skill_validator_status: 'infra_error',
+                  smoke_status: 'infra_error',
+                  failure_class: 'infra',
+                  summary: 'Quality-gate workflow failed unexpectedly. Re-run intake to retry.',
+                };
+              } else if (process.env.QUALITY_RESULT_JSON) {
+                qualityResult = JSON.parse(process.env.QUALITY_RESULT_JSON);
+              } else {
+                qualityResult = {
+                  overall_status: 'infra_error',
+                  skill_validator_status: 'infra_error',
+                  smoke_status: 'infra_error',
+                  failure_class: 'infra',
+                  summary: 'Quality-gate workflow did not return results. Re-run intake to retry.',
+                };
+              }
 
-            if (issueState === 'closed' && action !== 'reopened') {
-              core.info('Issue is closed; waiting for reopen before rerunning intake synchronization.');
-              return;
+              finalResult = intake.applyQualityGateResult(baseResult, qualityResult);
             }
 
             await intakeState.applyExternalPluginIntakeEvaluation({
               github,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issueNumber,
-              evaluation: result
+              issueNumber: context.issue.number,
+              evaluation: finalResult
             });
 
-            if (!result.valid && issueState === 'open') {
+            const issueState = process.env.ISSUE_STATE;
+            const labels = new Set(JSON.parse(process.env.ISSUE_LABELS || '[]'));
+            if (finalResult.intakeState === 'rejected' && issueState === 'open') {
               await github.rest.issues.update({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                issue_number: issueNumber,
+                issue_number: context.issue.number,
                 state: 'closed'
               });
+            } else if (finalResult.intakeState !== 'rejected' && issueState === 'closed' && labels.has('rejected')) {
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                state: 'open'
+              });
             }

.github/workflows/external-plugin-mark-ready-command.yml

@@ -0,0 +1,119 @@
+name: External Plugin Mark Ready Command
+
+on:
+  issue_comment:
+    types: [created]
+
+concurrency:
+  group: external-plugin-intake-${{ github.event.issue.number }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  mark-ready:
+    runs-on: ubuntu-latest
+    if: >-
+      !github.event.issue.pull_request &&
+      startsWith(github.event.comment.body, '/mark-ready-for-review')
+    steps:
+      - name: Checkout staged branch
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: staged
+
+      - name: Apply explicit ready-for-review override
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const path = require('path');
+            const { pathToFileURL } = require('url');
+
+            const intake = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake.mjs')).href);
+            const intakeState = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake-state.mjs')).href);
+
+            const parsed = intake.parseMarkReadyForReviewCommand(context.payload.comment.body);
+            if (!parsed) {
+              core.info('No supported /mark-ready-for-review command was found.');
+              return;
+            }
+
+            const actor = context.payload.comment.user?.login;
+            if (!actor || context.payload.comment.user?.type === 'Bot' || actor === 'github-actions[bot]') {
+              core.info('Ignoring command from a bot or unknown actor.');
+              return;
+            }
+
+            const permission = await github.rest.repos.getCollaboratorPermissionLevel({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              username: actor
+            });
+            const hasWriteAccess = ['admin', 'write', 'maintain'].includes(permission.data.permission);
+            if (!hasWriteAccess) {
+              core.info(`Ignoring /mark-ready-for-review because ${actor} does not have write access.`);
+              return;
+            }
+
+            const { data: currentIssue } = await github.rest.issues.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number
+            });
+
+            const labelNames = new Set((currentIssue.labels || []).map((label) => label.name));
+            if (!labelNames.has('external-plugin')) {
+              core.info('Ignoring command because issue is not an external plugin submission.');
+              return;
+            }
+
+            if (labelNames.has('approved')) {
+              core.info('Ignoring command because issue is already approved.');
+              return;
+            }
+
+            if (!labelNames.has('requires-submitter-fixes')) {
+              core.info('Ignoring command because issue is not currently blocked by submitter-fix gates.');
+              return;
+            }
+
+            await intakeState.syncExternalPluginIntakeLabels({
+              github,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issueNumber: context.issue.number,
+              desiredLabels: new Set(['external-plugin', 'ready-for-review'])
+            });
+
+            const marker = '<!-- external-plugin-mark-ready-override -->';
+            const reason = parsed.reason || 'No reason provided.';
+            const body = [
+              marker,
+              '## ✅ External plugin manually moved to ready-for-review',
+              '',
+              `Maintainer **${actor}** used \`${intake.MARK_READY_FOR_REVIEW_COMMAND}\` to move this submission from \`requires-submitter-fixes\` to \`ready-for-review\`.`,
+              '',
+              '### Reason',
+              '',
+              reason
+            ].join('\n');
+
+            await intakeState.upsertExternalPluginIntakeComment({
+              github,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issueNumber: context.issue.number,
+              marker,
+              body
+            });
+
+            if (currentIssue.state === 'closed') {
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                state: 'open'
+              });
+            }

.github/workflows/external-plugin-quality-gates.yml

@@ -0,0 +1,49 @@
+name: External Plugin Quality Gates
+
+on:
+  workflow_call:
+    inputs:
+      plugin-json:
+        description: Canonical plugin payload JSON from intake parsing
+        required: true
+        type: string
+    outputs:
+      quality-result:
+        description: JSON result for quality checks
+        value: ${{ jobs.quality.outputs.quality-result }}
+
+permissions:
+  contents: read
+
+jobs:
+  quality:
+    runs-on: ubuntu-latest
+    outputs:
+      quality-result: ${{ steps.quality.outputs.quality-result }}
+    steps:
+      - name: Checkout staged branch
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: staged
+          persist-credentials: false
+          submodules: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: 22
+
+      - name: Install GitHub Copilot CLI
+        run: npm install -g @github/copilot
+
+      - name: Run external plugin quality gates
+        id: quality
+        env:
+          PLUGIN_JSON: ${{ inputs.plugin-json }}
+        run: |
+          result=$(node ./eng/external-plugin-quality-gates.mjs --plugin-json "$PLUGIN_JSON")
+          {
+            echo 'quality-result<<EOF'
+            echo "$result"
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"

.github/workflows/external-plugin-rerun-intake-command.yml

@@ -13,18 +13,26 @@ permissions:
   issues: write
 
 jobs:
-  handle-command:
+  parse-command:
     runs-on: ubuntu-latest
     if: >-
       !github.event.issue.pull_request &&
       startsWith(github.event.comment.body, '/rerun-intake')
+    outputs:
+      should-run: ${{ steps.evaluate.outputs.should-run }}
+      base-result: ${{ steps.evaluate.outputs.base-result }}
+      valid: ${{ steps.evaluate.outputs.valid }}
+      plugin-json: ${{ steps.evaluate.outputs.plugin-json }}
+      issue-state: ${{ steps.evaluate.outputs.issue-state }}
+      issue-labels: ${{ steps.evaluate.outputs.issue-labels }}
     steps:
       - name: Checkout staged branch
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
           ref: staged
 
-      - name: Re-run external plugin intake
+      - name: Validate command and evaluate intake
+        id: evaluate
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -34,7 +42,8 @@ jobs:
             const { pathToFileURL } = require('url');
 
             const intake = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake.mjs')).href);
-            const intakeState = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake-state.mjs')).href);
+
+            core.setOutput('should-run', 'false');
 
             const commentAuthor = context.payload.comment.user?.login;
             if (!commentAuthor || context.payload.comment.user?.type === 'Bot' || commentAuthor === 'github-actions[bot]') {
@@ -91,34 +100,107 @@ jobs:
               return;
             }
 
-            const evaluation = await intake.evaluateExternalPluginIssue({
+            const baseResult = await intake.evaluateExternalPluginIssue({
               issue: currentIssue,
               token: process.env.GITHUB_TOKEN
             });
 
+            core.setOutput('should-run', 'true');
+            core.setOutput('base-result', JSON.stringify(baseResult));
+            core.setOutput('valid', baseResult.valid ? 'true' : 'false');
+            core.setOutput('plugin-json', JSON.stringify(baseResult.plugin || {}));
+            core.setOutput('issue-state', currentIssue.state);
+            core.setOutput('issue-labels', JSON.stringify([...labelNames]));
+
+  quality-gates:
+    needs: parse-command
+    if: >-
+      needs.parse-command.outputs.should-run == 'true' &&
+      needs.parse-command.outputs.valid == 'true'
+    uses: ./.github/workflows/external-plugin-quality-gates.yml
+    with:
+      plugin-json: ${{ needs.parse-command.outputs.plugin-json }}
+
+  apply-state:
+    runs-on: ubuntu-latest
+    needs: [parse-command, quality-gates]
+    if: always() && needs.parse-command.outputs.should-run == 'true'
+    steps:
+      - name: Checkout staged branch
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: staged
+
+      - name: Apply merged intake evaluation
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        env:
+          BASE_RESULT_JSON: ${{ needs.parse-command.outputs.base-result }}
+          BASE_VALID: ${{ needs.parse-command.outputs.valid }}
+          QUALITY_RESULT_JSON: ${{ needs.quality-gates.outputs.quality-result }}
+          QUALITY_JOB_RESULT: ${{ needs.quality-gates.result }}
+          ISSUE_STATE: ${{ needs.parse-command.outputs.issue-state }}
+          ISSUE_LABELS: ${{ needs.parse-command.outputs.issue-labels }}
+        with:
+          script: |
+            const path = require('path');
+            const { pathToFileURL } = require('url');
+
+            const intake = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake.mjs')).href);
+            const intakeState = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake-state.mjs')).href);
+
+            const baseResult = JSON.parse(process.env.BASE_RESULT_JSON);
+            let finalResult = baseResult;
+
+            if (process.env.BASE_VALID === 'true') {
+              let qualityResult;
+              if (process.env.QUALITY_JOB_RESULT === 'failure' || process.env.QUALITY_JOB_RESULT === 'cancelled') {
+                qualityResult = {
+                  overall_status: 'infra_error',
+                  skill_validator_status: 'infra_error',
+                  smoke_status: 'infra_error',
+                  failure_class: 'infra',
+                  summary: 'Quality-gate workflow failed unexpectedly. Re-run intake to retry.',
+                };
+              } else if (process.env.QUALITY_RESULT_JSON) {
+                qualityResult = JSON.parse(process.env.QUALITY_RESULT_JSON);
+              } else {
+                qualityResult = {
+                  overall_status: 'infra_error',
+                  skill_validator_status: 'infra_error',
+                  smoke_status: 'infra_error',
+                  failure_class: 'infra',
+                  summary: 'Quality-gate workflow did not return results. Re-run intake to retry.',
+                };
+              }
+
+              finalResult = intake.applyQualityGateResult(baseResult, qualityResult);
+            }
+
             await intakeState.applyExternalPluginIntakeEvaluation({
               github,
               owner: context.repo.owner,
               repo: context.repo.repo,
               issueNumber: context.issue.number,
-              evaluation
+              evaluation: finalResult
             });
 
-            if (evaluation.valid && currentIssue.state === 'closed' && labelNames.has('rejected')) {
-              await github.rest.issues.update({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.issue.number,
-                state: 'open'
-              });
-              return;
-            }
-
-            if (!evaluation.valid && currentIssue.state === 'open') {
+            const issueState = process.env.ISSUE_STATE;
+            const labels = new Set(JSON.parse(process.env.ISSUE_LABELS || '[]'));
+            if (finalResult.intakeState === 'rejected' && issueState === 'open') {
               await github.rest.issues.update({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 issue_number: context.issue.number,
                 state: 'closed'
               });
+              return;
+            }
+
+            if (finalResult.intakeState !== 'rejected' && issueState === 'closed' && labels.has('rejected')) {
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                state: 'open'
+              });
             }

AGENTS.md

@@ -166,12 +166,13 @@ When adding a new agent, instruction, skill, hook, workflow, or plugin:
 2. Public external plugin submissions use the external plugin issue workflow documented in [CONTRIBUTING.md](CONTRIBUTING.md#adding-external-plugins)
 3. In v1, only GitHub-hosted plugins are accepted for public submission, using a public repo plus an immutable `ref`, `sha`, or both
 4. The shared validator in `eng/external-plugin-validation.mjs` is the canonical source of truth for external plugin data rules; reuse it instead of duplicating checks in scripts or workflows
-5. Submission issues move through `external-plugin` + `awaiting-review` -> `ready-for-review` -> `approved` or `rejected`
-6. After issue edits, the issue author or a maintainer can comment `/rerun-intake` to re-run automated intake without opening a new submission issue
-7. Maintainers make the decision with `/approve` or `/reject <reason>` issue comments; approved issues are closed and used as the six-month re-review anchor
-8. Approval automation creates or updates the PR against `staged`, updates `plugins/external.json`, and regenerates marketplace outputs
-9. Nightly re-review automation finds closed `external-plugin` + `approved` issues that are at least six months old, applies `re-review-due`, and opens or updates a tracking issue for maintainers
-10. Maintainers complete re-review on the original approved submission issue with `/re-review-keep`, `/re-review-needs-changes`, or `/re-review-remove`; keep resets the issue `closed_at`, and remove opens a PR against `staged`
+5. Submission issues move through `external-plugin` + `awaiting-review` and then either `ready-for-review` or `requires-submitter-fixes` based on automated quality gates
+6. After issue edits, the issue author or a maintainer can comment `/rerun-intake` to re-run automated intake and quality gates without opening a new submission issue
+7. Maintainers can explicitly override a quality-gate blocker with `/mark-ready-for-review [optional reason]`, which moves the issue to `ready-for-review`
+8. Maintainers make the decision with `/approve` or `/reject <reason>` issue comments once the issue is in `ready-for-review`; approved issues are closed and used as the six-month re-review anchor
+9. Approval automation creates or updates the PR against `staged`, updates `plugins/external.json`, and regenerates marketplace outputs
+10. Nightly re-review automation finds closed `external-plugin` + `approved` issues that are at least six months old, applies `re-review-due`, and opens or updates a tracking issue for maintainers
+11. Maintainers complete re-review on the original approved submission issue with `/re-review-keep`, `/re-review-needs-changes`, or `/re-review-remove`; keep resets the issue `closed_at`, and remove opens a PR against `staged`
 
 ### Testing Instructions
 

CONTRIBUTING.md

@@ -230,11 +230,16 @@ The public-submission policy builds on those rules and also requires `license` p
 
 1. **Open an issue** using the external plugin issue form. Automation applies the `external-plugin` and `awaiting-review` labels.
 2. **Automated intake validation** checks that the required fields are present and correctly formatted for a GitHub-hosted plugin. Invalid submissions are closed with a comment explaining what must be fixed before resubmitting.
-3. **Ready for maintainer review**: if the issue passes intake validation, automation removes `awaiting-review` and adds `ready-for-review`.
-4. **Requesting another intake pass**: after updating the issue body, the issue author or a maintainer can comment `/rerun-intake` to re-run automated intake on demand. Open issues still re-trigger intake automatically on edit, but closed rejected issues need `/rerun-intake`.
-5. **Maintainer decision**: a maintainer with write access performs the manual review, then comments `/approve` or `/reject <reason>` on the issue. Commands from non-maintainers are ignored.
-6. **Approval path**: on `/approve`, automation removes `ready-for-review`, adds `approved`, closes the issue, and opens or updates a PR against `staged` that updates `plugins/external.json` and generated marketplace outputs.
-7. **Rejection path**: on `/reject <reason>`, automation removes `ready-for-review`, adds `rejected`, closes the issue, and records the reason in an issue comment. After addressing the feedback, update the same issue and use `/rerun-intake` to re-queue intake.
+3. **Automated quality gates** run after metadata validation:
+   - `skill-validator check --plugin` against the submitted plugin path/ref/sha
+   - install smoke test via Copilot CLI against an ephemeral marketplace entry generated from the submission
+4. **Ready for maintainer review**: if metadata validation and quality gates pass, automation removes `awaiting-review` and adds `ready-for-review`.
+5. **Submitter-fix blocker**: if metadata is valid but quality gates fail, automation applies `requires-submitter-fixes` instead of advancing to human review.
+6. **Requesting another intake pass**: after updating the issue body or source plugin, the issue author or a maintainer can comment `/rerun-intake` to re-run automated intake and quality gates on demand. Open issues still re-trigger intake automatically on edit, but closed rejected issues need `/rerun-intake`.
+7. **Maintainer override path**: a maintainer with write access can comment `/mark-ready-for-review [optional reason]` to explicitly move a `requires-submitter-fixes` issue to `ready-for-review`.
+8. **Maintainer decision**: once in `ready-for-review`, a maintainer with write access performs the manual review, then comments `/approve` or `/reject <reason>` on the issue. Commands from non-maintainers are ignored.
+9. **Approval path**: on `/approve`, automation removes `ready-for-review`, adds `approved`, closes the issue, and opens or updates a PR against `staged` that updates `plugins/external.json` and generated marketplace outputs.
+10. **Rejection path**: on `/reject <reason>`, automation removes `ready-for-review`, adds `rejected`, closes the issue, and records the reason in an issue comment. After addressing the feedback, update the same issue and use `/rerun-intake` to re-queue intake.
 
 ##### Maintainer review responsibilities
 
@@ -251,6 +256,7 @@ Maintainers are responsible for confirming that the submission:
 - `external-plugin`: applied to every public external plugin submission and retained on approved issues so scheduled review automation can find them later
 - `awaiting-review`: initial intake state before automation finishes validating the issue
 - `ready-for-review`: the issue passed automated intake checks and is waiting on a maintainer decision
+- `requires-submitter-fixes`: metadata validation passed but automated quality gates failed; submitter updates are required before human review
 - `approved`: the issue was approved, closed, and can be used as the source of truth for six-month re-review
 - `rejected`: the issue was rejected and closed without being added to the marketplace
 - `re-review-due`: the approved issue reached the six-month review threshold and is waiting on a maintainer re-review decision

eng/external-plugin-intake-state.mjs

@@ -11,6 +11,10 @@ export const EXTERNAL_PLUGIN_INTAKE_LABELS = Object.freeze({
     color: "0E8A16",
     description: "Submission passed intake validation and is ready for maintainer review",
   },
+  "requires-submitter-fixes": {
+    color: "D93F0B",
+    description: "Submission has quality-gate findings that submitter must fix before maintainer review",
+  },
   approved: {
     color: "1D76DB",
     description: "Submission was approved by a maintainer",
@@ -25,6 +29,7 @@ const EXTERNAL_PLUGIN_INTAKE_SYNC_LABELS = Object.freeze([
   "external-plugin",
   "awaiting-review",
   "ready-for-review",
+  "requires-submitter-fixes",
   "rejected",
 ]);
 
@@ -138,9 +143,14 @@ export async function applyExternalPluginIntakeEvaluation({
   issueNumber,
   evaluation,
 }) {
-  const desiredLabels = evaluation.valid
-    ? new Set(["external-plugin", "ready-for-review"])
-    : new Set(["external-plugin", "rejected"]);
+  const state = evaluation.intakeState ?? (evaluation.valid ? "ready-for-review" : "rejected");
+  const desiredLabelsByState = {
+    "ready-for-review": new Set(["external-plugin", "ready-for-review"]),
+    "requires-submitter-fixes": new Set(["external-plugin", "requires-submitter-fixes"]),
+    "awaiting-review": new Set(["external-plugin", "awaiting-review"]),
+    rejected: new Set(["external-plugin", "rejected"]),
+  };
+  const desiredLabels = desiredLabelsByState[state] ?? desiredLabelsByState.rejected;
 
   await syncExternalPluginIntakeLabels({
     github,

eng/external-plugin-intake.mjs

@@ -9,10 +9,15 @@ import { readExternalPlugins, validateExternalPlugin } from "./external-plugin-v
 export const ISSUE_FORM_MARKER = "<!-- external-plugin-submission -->";
 export const EXTERNAL_PLUGIN_INTAKE_COMMENT_MARKER = "<!-- external-plugin-intake -->";
 export const RERUN_INTAKE_COMMAND = "/rerun-intake";
+export const MARK_READY_FOR_REVIEW_COMMAND = "/mark-ready-for-review";
 const RERUN_INTAKE_COMMAND_PATTERN = new RegExp(
   `^\\s*${RERUN_INTAKE_COMMAND.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`,
   "m",
 );
+const MARK_READY_FOR_REVIEW_COMMAND_PATTERN = new RegExp(
+  `^\\s*${MARK_READY_FOR_REVIEW_COMMAND.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`,
+  "m",
+);
 const PLUGINS_DIR = path.join(ROOT_FOLDER, "plugins");
 
 // Each entry is a Set of equivalent checklist item texts (new + legacy aliases).
@@ -318,6 +323,168 @@ export function parseRerunIntakeCommand(body) {
   return RERUN_INTAKE_COMMAND_PATTERN.test(String(body ?? ""));
 }
 
+export function parseMarkReadyForReviewCommand(body) {
+  const text = String(body ?? "");
+  if (!MARK_READY_FOR_REVIEW_COMMAND_PATTERN.test(text)) {
+    return undefined;
+  }
+
+  const commandLine = text.split(/\r?\n/).find((line) => MARK_READY_FOR_REVIEW_COMMAND_PATTERN.test(line));
+  const reason = commandLine?.replace(MARK_READY_FOR_REVIEW_COMMAND_PATTERN, "").trim();
+
+  return {
+    command: MARK_READY_FOR_REVIEW_COMMAND,
+    reason: reason || undefined,
+  };
+}
+
+function normalizeQualityGateResult(rawResult) {
+  const defaults = {
+    overall_status: "not_run",
+    skill_validator_status: "not_run",
+    smoke_status: "not_run",
+    failure_class: "none",
+    summary: "",
+    skill_validator_output: "",
+    smoke_output: "",
+  };
+
+  if (!rawResult || typeof rawResult !== "object" || Array.isArray(rawResult)) {
+    return defaults;
+  }
+
+  return {
+    ...defaults,
+    ...rawResult,
+  };
+}
+
+function buildQualityGatesCommentSection(qualityResult) {
+  const skillState = qualityResult.skill_validator_status || "not_run";
+  const smokeState = qualityResult.smoke_status || "not_run";
+  const summaryText = String(qualityResult.summary || "").trim() || "_No quality gate details were provided._";
+
+  const sections = [
+    "### Quality gate summary",
+    "",
+    "| Gate | Status |",
+    "|---|---|",
+    `| skill-validator | ${skillState} |`,
+    `| install smoke test | ${smokeState} |`,
+    "",
+    summaryText,
+  ];
+
+  const skillOutput = String(qualityResult.skill_validator_output || "").trim();
+  if (skillOutput) {
+    sections.push(
+      "",
+      "<details>",
+      "<summary>skill-validator output</summary>",
+      "",
+      "```text",
+      skillOutput,
+      "```",
+      "",
+      "</details>",
+    );
+  }
+
+  const smokeOutput = String(qualityResult.smoke_output || "").trim();
+  if (smokeOutput) {
+    sections.push(
+      "",
+      "<details>",
+      "<summary>Install smoke test output</summary>",
+      "",
+      "```text",
+      smokeOutput,
+      "```",
+      "",
+      "</details>",
+    );
+  }
+
+  return sections.join("\n");
+}
+
+function getIntakeStateFromQualityResult(baseResult, qualityResult) {
+  if (!baseResult.valid) {
+    return "rejected";
+  }
+
+  if (qualityResult.failure_class === "submitter_fixes") {
+    return "requires-submitter-fixes";
+  }
+
+  if (qualityResult.failure_class === "infra") {
+    return "awaiting-review";
+  }
+
+  return "ready-for-review";
+}
+
+function buildMergedIntakeComment(baseResult, qualityResult) {
+  if (!baseResult.valid) {
+    return baseResult.commentBody;
+  }
+
+  const marker = baseResult.commentMarker ?? EXTERNAL_PLUGIN_INTAKE_COMMENT_MARKER;
+  const qualitySection = buildQualityGatesCommentSection(qualityResult);
+
+  const intro =
+    qualityResult.failure_class === "submitter_fixes"
+      ? "## ⚠️ External plugin intake requires submitter fixes"
+      : qualityResult.failure_class === "infra"
+        ? "## ⚠️ External plugin intake could not complete quality checks"
+        : "## ✅ External plugin intake passed";
+
+  const statusLine =
+    qualityResult.failure_class === "submitter_fixes"
+      ? "This submission passed metadata validation, but quality gates found issues that must be fixed before it can move to maintainer review. Update the issue details or source plugin and then comment `/rerun-intake`."
+      : qualityResult.failure_class === "infra"
+        ? "This submission passed metadata validation, but the automated quality checks hit an infrastructure issue. A maintainer should rerun intake or use the explicit override command after review."
+        : "This submission passed automated intake validation and quality checks and is ready for maintainer review.";
+
+  return [
+    marker,
+    intro,
+    "",
+    statusLine,
+    "",
+    `- **Plugin:** ${baseResult.plugin?.name ?? "unknown"}`,
+    `- **Repository:** ${baseResult.plugin?.repository ?? "unknown"}`,
+    baseResult.plugin?.source?.ref ? `- **Ref:** ${baseResult.plugin.source.ref}` : undefined,
+    baseResult.plugin?.source?.sha ? `- **SHA:** ${baseResult.plugin.source.sha}` : undefined,
+    "",
+    qualitySection,
+    "",
+    "### Canonical external.json payload",
+    "",
+    "```json",
+    JSON.stringify(baseResult.plugin ?? {}, null, 2),
+    "```",
+    baseResult.warnings?.length
+      ? ["", "### Warnings", "", ...baseResult.warnings.map((warning) => `- ${warning}`)].join("\n")
+      : "",
+  ].filter(Boolean).join("\n");
+}
+
+export function applyQualityGateResult(baseEvaluation, qualityGateResult) {
+  const baseResult = typeof baseEvaluation === "string" ? JSON.parse(baseEvaluation) : baseEvaluation;
+  const qualityResult = normalizeQualityGateResult(
+    typeof qualityGateResult === "string" ? JSON.parse(qualityGateResult) : qualityGateResult,
+  );
+  const intakeState = getIntakeStateFromQualityResult(baseResult, qualityResult);
+
+  return {
+    ...baseResult,
+    qualityGates: qualityResult,
+    intakeState,
+    commentBody: buildMergedIntakeComment(baseResult, qualityResult),
+  };
+}
+
 export async function evaluateExternalPluginIssue({ issue, token } = {}) {
   const issueBody = issue?.body ?? "";
   const parsed = parseExternalPluginIssueBody(issueBody);
@@ -403,6 +570,7 @@ export async function evaluateExternalPluginIssue({ issue, token } = {}) {
 
   return {
     valid,
+    intakeState: valid ? "ready-for-review" : "rejected",
     markerPresent: parsed.markerPresent,
     errors: dedupedErrors,
     warnings: dedupedWarnings,

eng/external-plugin-quality-gates.mjs

@@ -0,0 +1,355 @@
+#!/usr/bin/env node
+
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { spawnSync } from "child_process";
+
+const MAX_OUTPUT_LENGTH = 12000;
+const SKILL_VALIDATOR_ARCHIVE_URL = "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz";
+
+const INFRA_ERROR_PATTERNS = [
+  /\b401\b/,
+  /\b403\b/,
+  /authentication (required|failed|error)/,
+  /unauthenticated/,
+  /unauthorized/,
+  /not logged in/,
+  /please (log in|authenticate|sign in)/,
+  /invalid (access |auth )?token/,
+  /credentials? (are )?expired/,
+  /dns.*(resolve|lookup|fail)/,
+  /network.*unreachable/,
+  /connection (refused|reset)/,
+  /\btimeout\b/,
+  /enotfound/,
+  /econnrefused/,
+  /etimedout/,
+];
+
+function truncateOutput(value) {
+  const normalized = String(value ?? "").replace(/\x1b\[[0-9;]*m/g, "").trim();
+  if (normalized.length <= MAX_OUTPUT_LENGTH) {
+    return normalized;
+  }
+
+  return `${normalized.slice(0, MAX_OUTPUT_LENGTH)}\n...output truncated...`;
+}
+
+function runCommand(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    ...options,
+  });
+
+  return {
+    exitCode: typeof result.status === "number" ? result.status : 1,
+    stdout: truncateOutput(result.stdout),
+    stderr: truncateOutput(result.stderr),
+    output: truncateOutput(`${result.stdout ?? ""}\n${result.stderr ?? ""}`),
+    error: result.error ? String(result.error.message ?? result.error) : "",
+  };
+}
+
+function normalizePluginPath(pluginPath) {
+  if (!pluginPath || pluginPath === "/") {
+    return "";
+  }
+
+  const normalized = String(pluginPath).trim().replace(/^\/+|\/+$/g, "");
+  if (!normalized) {
+    return "";
+  }
+
+  if (normalized.includes("..") || normalized.includes("\\")) {
+    throw new Error(`Invalid plugin path "${pluginPath}"`);
+  }
+
+  return normalized;
+}
+
+function resolveFetchSpec(pluginSource) {
+  if (pluginSource.sha) {
+    return pluginSource.sha;
+  }
+
+  if (!pluginSource.ref) {
+    throw new Error("source.ref or source.sha is required for quality gates");
+  }
+
+  const ref = String(pluginSource.ref).trim();
+  if (!ref) {
+    throw new Error("source.ref or source.sha is required for quality gates");
+  }
+
+  if (ref.startsWith("refs/")) {
+    return ref;
+  }
+
+  return ref;
+}
+
+function classifySmokeFailure(output) {
+  const normalized = String(output ?? "").toLowerCase();
+  if (INFRA_ERROR_PATTERNS.some((pattern) => pattern.test(normalized))) {
+    return "infra_error";
+  }
+
+  return "fail";
+}
+
+function ensureDirectory(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function cloneSubmissionRepository(workDir, plugin) {
+  const repoDir = path.join(workDir, "submission");
+  ensureDirectory(repoDir);
+
+  const sourceRepo = plugin.source?.repo;
+  const fetchSpec = resolveFetchSpec(plugin.source ?? {});
+
+  const init = runCommand("git", ["init", "-q"], { cwd: repoDir });
+  if (init.exitCode !== 0) {
+    throw new Error(`git init failed: ${init.output}`);
+  }
+
+  const addRemote = runCommand("git", ["remote", "add", "origin", `https://github.com/${sourceRepo}.git`], { cwd: repoDir });
+  if (addRemote.exitCode !== 0) {
+    throw new Error(`git remote add failed: ${addRemote.output}`);
+  }
+
+  const fetch = runCommand("git", ["fetch", "--depth=1", "origin", fetchSpec], { cwd: repoDir });
+  if (fetch.exitCode !== 0) {
+    throw new Error(`git fetch failed for ${fetchSpec}: ${fetch.output}`);
+  }
+
+  const checkout = runCommand("git", ["checkout", "--detach", "FETCH_HEAD"], { cwd: repoDir });
+  if (checkout.exitCode !== 0) {
+    throw new Error(`git checkout failed: ${checkout.output}`);
+  }
+
+  return repoDir;
+}
+
+function downloadSkillValidator(workDir) {
+  const validatorDir = path.join(workDir, "skill-validator");
+  ensureDirectory(validatorDir);
+  const archivePath = path.join(validatorDir, "skill-validator-linux-x64.tar.gz");
+
+  const download = runCommand("curl", ["-fsSL", SKILL_VALIDATOR_ARCHIVE_URL, "-o", archivePath]);
+  if (download.exitCode !== 0) {
+    throw new Error(`Failed to download skill-validator: ${download.output}`);
+  }
+
+  const untar = runCommand("tar", ["-xzf", archivePath, "-C", validatorDir]);
+  if (untar.exitCode !== 0) {
+    throw new Error(`Failed to extract skill-validator: ${untar.output}`);
+  }
+
+  const binaryPath = path.join(validatorDir, "skill-validator");
+  if (!fs.existsSync(binaryPath)) {
+    throw new Error("skill-validator binary was not found after extraction");
+  }
+
+  runCommand("chmod", ["+x", binaryPath]);
+  return binaryPath;
+}
+
+function runSkillValidatorGate(workDir, pluginRoot) {
+  try {
+    const validatorBinary = downloadSkillValidator(workDir);
+    const check = runCommand(validatorBinary, ["check", "--verbose", "--plugin", pluginRoot]);
+
+    if (check.exitCode === 0) {
+      return { status: "pass", output: check.output };
+    }
+
+    return { status: "fail", output: check.output };
+  } catch (error) {
+    return {
+      status: "infra_error",
+      output: truncateOutput(error.message),
+    };
+  }
+}
+
+function buildEphemeralMarketplace(workDir, plugin) {
+  const marketplaceDir = path.join(workDir, "marketplace");
+  ensureDirectory(marketplaceDir);
+
+  const marketplace = {
+    name: "external-plugin-intake",
+    metadata: {
+      description: "Temporary marketplace for external plugin intake smoke tests",
+      version: "1.0.0",
+      pluginRoot: ".",
+    },
+    owner: {
+      name: "awesome-copilot-intake",
+      email: "noreply@github.com",
+    },
+    plugins: [plugin],
+  };
+
+  fs.writeFileSync(path.join(marketplaceDir, "marketplace.json"), `${JSON.stringify(marketplace, null, 2)}\n`);
+  return marketplaceDir;
+}
+
+function runInstallSmokeGate(workDir, plugin) {
+  if (runCommand("bash", ["-lc", "command -v copilot"]).exitCode !== 0) {
+    return {
+      status: "infra_error",
+      output: "copilot CLI is not available on this runner.",
+    };
+  }
+
+  try {
+    const homeDir = path.join(workDir, "copilot-home");
+    ensureDirectory(homeDir);
+    const marketplaceDir = buildEphemeralMarketplace(workDir, plugin);
+
+    const env = {
+      ...process.env,
+      HOME: homeDir,
+      XDG_CONFIG_HOME: path.join(homeDir, ".config"),
+      XDG_CACHE_HOME: path.join(homeDir, ".cache"),
+      XDG_DATA_HOME: path.join(homeDir, ".local", "share"),
+    };
+
+    const marketplaceAdd = runCommand("copilot", ["plugin", "marketplace", "add", marketplaceDir], { env });
+    if (marketplaceAdd.exitCode !== 0) {
+      const status = classifySmokeFailure(marketplaceAdd.output);
+      return { status, output: marketplaceAdd.output };
+    }
+
+    const install = runCommand("copilot", ["plugin", "install", `${plugin.name}@external-plugin-intake`], { env });
+    if (install.exitCode !== 0) {
+      const status = classifySmokeFailure(install.output);
+      return { status, output: install.output };
+    }
+
+    const installedPluginPath = path.join(homeDir, ".copilot", "installed-plugins", "external-plugin-intake", plugin.name);
+    const pluginManifestPath = path.join(installedPluginPath, ".github", "plugin", "plugin.json");
+    if (!fs.existsSync(installedPluginPath) || !fs.existsSync(pluginManifestPath)) {
+      return {
+        status: "fail",
+        output: `Plugin installed but expected files were missing at ${installedPluginPath}`,
+      };
+    }
+
+    return {
+      status: "pass",
+      output: `Install smoke test succeeded. Verified ${pluginManifestPath}.`,
+    };
+  } catch (error) {
+    return {
+      status: "infra_error",
+      output: truncateOutput(error.message),
+    };
+  }
+}
+
+function toOverallStatus(skillStatus, smokeStatus) {
+  const states = [skillStatus, smokeStatus];
+  if (states.includes("infra_error")) {
+    return "infra_error";
+  }
+  if (states.includes("fail")) {
+    return "fail";
+  }
+  if (states.every((state) => state === "not_run")) {
+    return "not_run";
+  }
+  return "pass";
+}
+
+function toFailureClass(overallStatus) {
+  if (overallStatus === "infra_error") {
+    return "infra";
+  }
+  if (overallStatus === "fail") {
+    return "submitter_fixes";
+  }
+  return "none";
+}
+
+export function runExternalPluginQualityGates(plugin) {
+  const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "external-plugin-quality-"));
+  const result = {
+    overall_status: "not_run",
+    skill_validator_status: "not_run",
+    smoke_status: "not_run",
+    failure_class: "none",
+    summary: "",
+    skill_validator_output: "",
+    smoke_output: "",
+  };
+
+  try {
+    const repoDir = cloneSubmissionRepository(workDir, plugin);
+    const normalizedPluginPath = normalizePluginPath(plugin.source?.path || "/");
+    const pluginRoot = normalizedPluginPath ? path.join(repoDir, normalizedPluginPath) : repoDir;
+
+    if (!fs.existsSync(pluginRoot) || !fs.statSync(pluginRoot).isDirectory()) {
+      result.skill_validator_status = "fail";
+      result.smoke_status = "fail";
+      result.overall_status = "fail";
+      result.failure_class = "submitter_fixes";
+      result.summary = `Plugin path "${plugin.source?.path || "/"}" was not found in the submitted repository snapshot.`;
+      return result;
+    }
+
+    const skillResult = runSkillValidatorGate(workDir, pluginRoot);
+    result.skill_validator_status = skillResult.status;
+    result.skill_validator_output = skillResult.output;
+
+    const smokeResult = runInstallSmokeGate(workDir, plugin);
+    result.smoke_status = smokeResult.status;
+    result.smoke_output = smokeResult.output;
+
+    result.overall_status = toOverallStatus(result.skill_validator_status, result.smoke_status);
+    result.failure_class = toFailureClass(result.overall_status);
+    result.summary = [
+      `- skill-validator: ${result.skill_validator_status}`,
+      `- install smoke test: ${result.smoke_status}`,
+      `- overall: ${result.overall_status}`,
+    ].join("\n");
+
+    return result;
+  } catch (error) {
+    result.overall_status = "infra_error";
+    result.failure_class = "infra";
+    result.summary = truncateOutput(error.message);
+    result.skill_validator_output = truncateOutput(error.stack || error.message);
+    return result;
+  } finally {
+    fs.rmSync(workDir, { recursive: true, force: true });
+  }
+}
+
+function parseCliArgs(argv) {
+  const args = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const key = argv[index];
+    if (!key.startsWith("--")) {
+      continue;
+    }
+
+    args[key.slice(2)] = argv[index + 1];
+    index += 1;
+  }
+  return args;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const args = parseCliArgs(process.argv.slice(2));
+  if (!args["plugin-json"]) {
+    console.error("Usage: node ./eng/external-plugin-quality-gates.mjs --plugin-json '<json>'");
+    process.exit(1);
+  }
+
+  const plugin = JSON.parse(args["plugin-json"]);
+  const result = runExternalPluginQualityGates(plugin);
+  process.stdout.write(`${JSON.stringify(result)}\n`);
+}