chatgpt reviewer test cases + hooks: no child_process, single opt-out

README.md

@@ -147,7 +147,7 @@ anything is sent, created, or deleted.
 Claude Code and Copilot CLI include telemetry adapters, but **`TELEMETRY_ENABLED`
 is `false`**: this package sends no usage events and generates no telemetry
 hooks in client manifests. [docs/telemetry.md](docs/telemetry.md) describes
-the event contract, what would be stored locally when enabled, opt-outs, and
+the event contract, what would be stored locally when enabled, the opt-out, and
 the separate approval required before collection can be enabled. The Arcade MCP
 gateway remains the canonical source for request and tool-call telemetry.
 Adapter behavior is covered by in-process tests;
@@ -203,7 +203,7 @@ Configure paths in `release-please-config.json`. The workflow lives at
 Each release also gets `arcade-chatgpt-{VERSION}.zip`, the package to upload
 in OpenAI's plugin portal for the ChatGPT and Codex directory. It holds the
 portable plugin only (`plugin.json` at the ZIP root, `mcp.json`, `skills/`,
-the logo, and `LICENSE`), with no hooks or other clients' folders. Build it
+the listing images, and `LICENSE`), with no hooks or other clients' folders. Build it
 locally with `npm run package:chatgpt` in `dev/` (writes `dist/` at the repo
 root); the build fails if the `extensions.com.openai.interface` listing breaks
 the portal's limits, such as the 30-character display name and subtitle

dev/scripts/generate-manifests.mjs

@@ -24,10 +24,20 @@ const inPlugin = (file) => `${PLUGIN_DIR}/${file}`;
 
 // One policy URL for both clients. The other OpenAI directory fields below are
 // written into plugin/plugin.json from here too. Identity text (name, descriptions,
-// category, website, logo) stays in plugin/plugin.json.
+// category, website) stays in plugin/plugin.json. Listing images and brandColor
+// are written from the constants below. Cursor's manifest takes only `logo`
+// (without the "./" prefix). Claude has no logo field. Dark variants and
+// brandColor stay on the OpenAI listing; the other clients' schemas have no
+// fields for them.
 const privacyPolicyUrl = "https://www.arcade.dev/privacy-policy/";
 const openaiSupportUrl = "https://support.arcade.dev";
 const openaiTermsUrl = "https://www.arcade.dev/terms-of-service/";
+const OPENAI_LOGO = "./assets/logo.png";
+const OPENAI_LOGO_DARK = "./assets/logo-dark.png";
+const OPENAI_COMPOSER_ICON = "./assets/composer-icon.png";
+const OPENAI_COMPOSER_ICON_DARK = "./assets/composer-icon-dark.png";
+const OPENAI_BRAND_COLOR = "#4708F9";
+const OPENAI_IMAGE_KEYS = ["logo", "logoDark", "composerIcon", "composerIconDark", "brandColor", "brandColorDark"];
 
 const OPENAI_CAPABILITIES = [
   "Read and search email",
@@ -43,56 +53,62 @@ const OPENAI_DEFAULT_PROMPTS = [
   "Which GitHub issues are assigned to me?",
 ];
 
-// Tool names are the MCP forms of Arcade toolkit tools (dots become underscores).
+// Reviewers sign up for their own free Arcade account. Positive prompts call
+// Arcade_SelectTools, then Arcade_UseTool for the toolkit tool named in
+// parentheses (Toolkit.ToolName). Flights, Finance, and YouTube need no user
+// OAuth; fares, prices, and video lists change, so expected_behavior checks
+// the shape of the result. The Math average is the one fixed number. Negative
+// prompts do not mention Arcade, so the app should not be called. Positive
+// descriptions stay within OpenAI's 4000-character limit.
 // demo_recording_url is a real review field and is required before MCP review,
 // but a ZIP may omit it. No recording URL is published here; omitting the field
 // leaves any value already saved in the dashboard.
 const OPENAI_REVIEW = {
   test_cases: {
     positive: [
       {
-        description: "Summarize unread email in the connected Gmail account",
-        prompt: "Summarize my unread email from the last 7 days.",
-        tools_triggered: "Gmail_ListEmails",
-        expected_behavior: "Lists the seeded unread test emails (sender, subject, one-line summary each) from the reviewer Gmail account. No email is changed or marked read.",
+        description: "Find trip-planning tools without running them",
+        prompt: "Use Arcade to find tools that could help me plan a weekend trip to Chicago: flights, hotels and driving directions. Just list what's available, don't run anything yet.",
+        tools_triggered: "Arcade_SelectTools",
+        expected_behavior: "Lists tools for flights, hotels, and driving directions, with a short description of each, and runs none of them. Returns no account names, emails, or connection status. Which tools are offered can vary with the catalog.",
       },
       {
-        description: "Read tomorrow's calendar",
-        prompt: "What's on my calendar tomorrow?",
-        tools_triggered: "GoogleCalendar_ListEvents",
-        expected_behavior: "Returns the seeded recurring test events for tomorrow with times and titles from the reviewer Google Calendar. Nothing is created or changed.",
+        description: "Find a one-way fare and total it for 3 travelers",
+        prompt: "Use Arcade to find the cheapest one-way economy flight from SFO to JFK two weeks from today, then use Arcade's Math tools to calculate the total fare for 3 travelers.",
+        tools_triggered: "Arcade_SelectTools, Arcade_UseTool (GoogleFlights.SearchFlights), Arcade_UseTool (Math.Multiply)",
+        expected_behavior: "Reports the cheapest one-way economy fare it finds from SFO to JFK about two weeks out, with airline, flight number, and times, then the total for 3 travelers. Fares change; the total should equal the reported fare times 3. Nothing is booked.",
       },
       {
-        description: "Find GitHub issues assigned to the user",
-        prompt: "Which open GitHub issues are assigned to me in arcade-review/sample-tracker?",
-        tools_triggered: "Github_ListIssues",
-        expected_behavior: "Lists the seeded open issues assigned to the reviewer GitHub account in the sample repo, with titles and links.",
+        description: "Get the Google Finance summary for NVDA",
+        prompt: "Use Arcade to get the Google Finance stock summary for NVDA.",
+        tools_triggered: "Arcade_SelectTools, Arcade_UseTool (GoogleFinance.GetStockSummary)",
+        expected_behavior: "Returns the Google Finance summary for NVDA, including NVIDIA's latest price in USD and the day's move in value and percent. Prices change; check the ticker and those fields. No order is placed.",
       },
       {
-        description: "Read recent messages in a Slack channel",
-        prompt: "What did people say in #launch-plan on Slack this week?",
-        tools_triggered: "Slack_GetMessages",
-        expected_behavior: "Summarizes the seeded messages in #launch-plan in the review Slack workspace and names who said what. Nothing is posted.",
+        description: "Search YouTube for videos about MCP",
+        prompt: "Use Arcade to search YouTube for videos explaining the Model Context Protocol.",
+        tools_triggered: "Arcade_SelectTools, Arcade_UseTool (Youtube.SearchForVideos)",
+        expected_behavior: "Returns YouTube videos about the Model Context Protocol, each with a title, channel, and link. The list changes; check that the videos are about MCP.",
       },
       {
-        description: "Draft an email and stop for confirmation",
-        prompt: "Draft a reply to the 'Q4 venue options' email saying Thursday works, but don't send it.",
-        tools_triggered: "Gmail_ListEmails, Gmail_WriteDraftReplyEmail",
-        expected_behavior: "Shows the draft text and recipient, asks before saving, then saves it as a Gmail draft. The email is not sent.",
+        description: "Average a list of amounts",
+        prompt: "Use Arcade to average my monthly software bills: $1,249.50, $389.99, $72.25 and $418.26.",
+        tools_triggered: "Arcade_SelectTools, Arcade_UseTool (Math.Avg)",
+        expected_behavior: "Answers $532.50.",
       },
     ],
     negative: [
       {
-        description: "General knowledge question that needs no connected app",
+        description: "General knowledge question that needs no tool",
         prompt: "What's the difference between TCP and UDP?",
       },
       {
-        description: "Request outside any connected app (payments)",
+        description: "Payment request; Arcade connects no payment app and does not move money",
         prompt: "Send $50 to my landlord on Venmo.",
       },
       {
-        description: "Irreversible bulk action without a confirmed target",
-        prompt: "Delete every email in my inbox.",
+        description: "Concept question about a stock ratio, with no quote to fetch",
+        prompt: "Explain what a P/E ratio means when evaluating a stock.",
       },
     ],
   },
@@ -117,11 +133,9 @@ const applyOpenAiDirectory = (openai) => {
     developerName,
     category,
     websiteURL,
-    logo,
-    composerIcon,
     ...rest
   } = openai.interface;
-  for (const key of GENERATED_INTERFACE_KEYS) delete rest[key];
+  for (const key of [...GENERATED_INTERFACE_KEYS, ...OPENAI_IMAGE_KEYS]) delete rest[key];
   openai.interface = {
     displayName,
     shortDescription,
@@ -134,8 +148,11 @@ const applyOpenAiDirectory = (openai) => {
     privacyPolicyURL: privacyPolicyUrl,
     termsOfServiceURL: openaiTermsUrl,
     defaultPrompt: OPENAI_DEFAULT_PROMPTS,
-    logo,
-    composerIcon,
+    logo: OPENAI_LOGO,
+    logoDark: OPENAI_LOGO_DARK,
+    composerIcon: OPENAI_COMPOSER_ICON,
+    composerIconDark: OPENAI_COMPOSER_ICON_DARK,
+    brandColor: OPENAI_BRAND_COLOR,
     ...rest,
   };
   openai.review = OPENAI_REVIEW;
@@ -182,7 +199,7 @@ export const COPIED_FILES = {
  * entry here — buildFiles throws if one is missing.
  */
 export const FILE_SOURCES = {
-  // extensions.com.openai listing URLs, capabilities, defaultPrompt, review, and publication.
+  // extensions.com.openai listing URLs, images, brandColor, capabilities, defaultPrompt, review, and publication.
   [PLUGIN_JSON]: ["dev/scripts/generate-manifests.mjs"],
   // plugin.json and mcp.json supply identity and the gateway URL; VERSION
   // supplies the version; hook-hosts.mjs supplies the hooks manifest path; the
@@ -343,18 +360,15 @@ const buildFiles = (root) => {
     throw new Error(`plugin.json version ${plugin.version} does not match VERSION ${version}`);
   }
   const { url } = readJson(root, MCP_JSON).mcpServers.arcade;
-  const { interface: openaiListing } = plugin.extensions["com.openai"];
   // The ChatGPT directory already lists a plugin named "Arcade" from another
   // company, so the OpenAI listing uses "Arcade.dev". The other clients keep
   // "Arcade".
   const displayName = "Arcade";
-  if (!openaiListing.logo) {
-    throw new Error("plugin.json extensions.com.openai.interface.logo is missing");
-  }
-  // Codex wants the logo path with a leading "./"; Cursor takes it without one.
-  // The Claude marketplace has no logo field. `logo` is relative to the plugin
-  // directory; the repo-root Cursor manifest prefixes it with plugin/.
-  const logo = openaiListing.logo.replace(/^\.\//, "");
+  // Codex and ChatGPT want the logo path with a leading "./" (OPENAI_LOGO).
+  // Cursor takes it without one. The Claude marketplace has no logo field.
+  // `logo` is relative to the plugin directory; the repo-root Cursor manifest
+  // prefixes it with plugin/. Dark icons are not copied onto Cursor or Claude.
+  const logo = OPENAI_LOGO.replace(/^\.\//, "");
   const { name, description, author, homepage, license, keywords, repository } = plugin;
   const identity = { name, description, author, homepage, license, keywords, version };
   const owner = { ...author, email: ACCEPTED_LISTING_OWNER_EMAIL };

dev/scripts/package-chatgpt.mjs

@@ -45,6 +45,21 @@ export const MAX_DEFAULT_PROMPT_LENGTH = 128;
 
 const isMultiline = (value) => /[\r\n\u2028\u2029]/.test(value);
 
+// sRGB relative luminance. OpenAI requires brandColor contrast of at least 2:1
+// against white (https://developers.openai.com/plugins/deploy/submission).
+const linearChannel = (byte) => {
+  const s = byte / 255;
+  return s <= 0.04045 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4;
+};
+
+const contrastAgainstWhite = (hex) => {
+  const n = Number.parseInt(hex.slice(1), 16);
+  const luminance = 0.2126 * linearChannel((n >> 16) & 255)
+    + 0.7152 * linearChannel((n >> 8) & 255)
+    + 0.0722 * linearChannel(n & 255);
+  return 1.05 / (luminance + 0.05);
+};
+
 /** Returns every problem the portal would reject in the OpenAI listing. */
 export const listingProblems = (listing) => {
   const problems = [];
@@ -61,6 +76,18 @@ export const listingProblems = (listing) => {
     problems.push(`category must be one of: ${LISTING_CATEGORIES.join(", ")}`);
   }
   if (!listing.logo) problems.push("logo is required");
+  if (listing.brandColor !== undefined && !/^#[0-9A-F]{6}$/i.test(listing.brandColor)) {
+    problems.push("brandColor must be a #RRGGBB color");
+  } else if (typeof listing.brandColor === "string" && contrastAgainstWhite(listing.brandColor) < 2) {
+    problems.push("brandColor must have at least 2:1 contrast against white");
+  }
+  for (const key of ["logo", "logoDark", "composerIcon", "composerIconDark"]) {
+    const value = listing[key];
+    if (value === undefined) continue;
+    if (typeof value !== "string" || !value.startsWith("./") || value.split("/").includes("..")) {
+      problems.push(`${key} must be a relative path starting with ./`);
+    }
+  }
   // A remote MCP submission ("With MCP") needs all four listing URLs; only a
   // skills-only upload can omit them. This ZIP always ships mcp.json.
   for (const key of LISTING_URLS) {
@@ -114,8 +141,11 @@ const listFiles = (root, dir) =>
 export const packageFiles = (root = ROOT) => {
   const pluginRoot = pluginRootOf(root);
   const plugin = JSON.parse(readFileSync(join(pluginRoot, "plugin.json"), "utf8"));
-  const { logo, composerIcon } = plugin.extensions["com.openai"].interface;
-  const images = [logo, composerIcon].filter(Boolean).map((path) => path.replace(/^\.\//, ""));
+  const listing = plugin.extensions["com.openai"].interface;
+  const images = ["logo", "logoDark", "composerIcon", "composerIconDark"]
+    .map((key) => listing[key])
+    .filter(Boolean)
+    .map((path) => path.replace(/^\.\//, ""));
   return [...new Set(["plugin.json", "mcp.json", "LICENSE", ...listFiles(pluginRoot, "skills"), ...images])];
 };
 

dev/test/check.test.mjs

@@ -20,12 +20,13 @@ test("plugin.json and mcp.json match the vendored Agent Plugins 1.0 schemas", ()
   }
   const openaiListing = plugin.extensions?.["com.openai"]?.interface;
   assert.ok(openaiListing?.displayName, "plugin.json: Codex displayName");
-  for (const key of ["logo", "composerIcon"]) {
+  for (const key of ["logo", "logoDark", "composerIcon", "composerIconDark"]) {
     const file = openaiListing?.[key];
     assert.ok(file && existsSync(path.join(ROOT, "plugin", file)), `plugin.json: Codex ${key} must point at a committed file`);
     // Codex's own plugins all write these paths with a leading "./".
     assert.match(file, /^\.\//, `plugin.json: Codex ${key} must start with ./`);
   }
+  assert.match(openaiListing.brandColor, /^#[0-9A-F]{6}$/i);
 });
 
 test("hand-written files name the gateway from mcp.json", () => {