From 95f3fb3a8f4df2f132e25389a2ee148343c87109 Mon Sep 17 00:00:00 2001 From: MemOS AutoDev Date: Thu, 2 Jul 2026 21:40:02 +0800 Subject: [PATCH] fix: accept JSON5 syntax in openclaw.json (#1543) The memos-local-openclaw plugin failed to patch tools.allow at startup whenever the user's openclaw.json contained // comments, single-quoted strings or trailing commas, because every read site used JSON.parse. Add a tiny shared helper (src/shared/openclaw-config.ts) that parses with json5 (strict JSON is a subset, so nothing regresses) and route all runtime + script read sites through it. The regex-based tools.allow insertion is unchanged for the common double-quoted case; when the regex cannot safely locate the insertion point we now warn asking the user to add "group:plugins" manually instead of silently doing nothing. Adds json5@^2.2.3 to plugin deps and 11 vitest cases covering line and block comments, single-quoted keys/values, trailing commas, unquoted identifier keys, and end-to-end loadOpenClawFallbackConfig on a JSON5 config. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/memos-local-openclaw/index.ts | 12 +- apps/memos-local-openclaw/package.json | 1 + .../scripts/refresh-skill.ts | 5 +- .../scripts/refresh-summaries.ts | 3 +- .../scripts/run-accuracy-test.ts | 3 +- .../scripts/test-agent-isolation.ts | 3 +- .../src/ingest/providers/index.ts | 3 +- .../src/shared/llm-call.ts | 3 +- .../src/shared/openclaw-config.ts | 22 ++ .../memos-local-openclaw/src/viewer/server.ts | 3 +- .../tests/openclaw-json5.test.ts | 190 ++++++++++++++++++ 11 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 apps/memos-local-openclaw/src/shared/openclaw-config.ts create mode 100644 apps/memos-local-openclaw/tests/openclaw-json5.test.ts diff --git a/apps/memos-local-openclaw/index.ts b/apps/memos-local-openclaw/index.ts index 2f46052ee..3ab730732 100644 --- a/apps/memos-local-openclaw/index.ts +++ b/apps/memos-local-openclaw/index.ts @@ -31,6 +31,7 @@ import { SkillInstaller } from "./src/skill/installer"; import { Summarizer } from "./src/ingest/providers"; import { MEMORY_GUIDE_SKILL_MD } from "./src/skill/bundled-memory-guide"; import { Telemetry } from "./src/telemetry"; +import { parseOpenClawConfig } from "./src/shared/openclaw-config"; /** Remove near-duplicate hits based on summary word overlap (>70%). Keeps first (highest-scored) hit. */ @@ -351,12 +352,15 @@ const memosLocalPlugin = { ctx.log.warn(`memos-local: could not write to managed skills dir: ${e}`); } - // Ensure plugin tools are enabled in openclaw.json tools.allow + // Ensure plugin tools are enabled in openclaw.json tools.allow. + // Users routinely author openclaw.json in JSON5 style (line comments, + // trailing commas, mixed quoting); parse tolerantly so we don't crash + // the whole plugin init on config style — see issue #1543. try { const openclawJsonPath = path.join(stateDir, "openclaw.json"); if (fs.existsSync(openclawJsonPath)) { const raw = fs.readFileSync(openclawJsonPath, "utf-8"); - const cfg = JSON.parse(raw); + const cfg = parseOpenClawConfig(raw) as any; const allow: string[] | undefined = cfg?.tools?.allow; if (Array.isArray(allow) && allow.length > 0 && !allow.includes("group:plugins") && !allow.includes("*")) { const lastEntry = JSON.stringify(allow[allow.length - 1]); @@ -367,6 +371,10 @@ const memosLocalPlugin = { if (patched !== raw && patched.includes("group:plugins")) { fs.writeFileSync(openclawJsonPath, patched, "utf-8"); ctx.log.info("memos-local: added 'group:plugins' to tools.allow in openclaw.json"); + } else { + ctx.log.warn( + "memos-local: could not auto-patch tools.allow (likely due to comments or non-standard quoting); please add \"group:plugins\" to tools.allow manually.", + ); } } } diff --git a/apps/memos-local-openclaw/package.json b/apps/memos-local-openclaw/package.json index 0323ee06c..48745704d 100644 --- a/apps/memos-local-openclaw/package.json +++ b/apps/memos-local-openclaw/package.json @@ -51,6 +51,7 @@ "@huggingface/transformers": "^3.8.0", "@sinclair/typebox": "^0.34.48", "better-sqlite3": "^12.10.0", + "json5": "^2.2.3", "posthog-node": "^5.28.0", "puppeteer": "^24.38.0", "semver": "^7.7.4", diff --git a/apps/memos-local-openclaw/scripts/refresh-skill.ts b/apps/memos-local-openclaw/scripts/refresh-skill.ts index cf19802fa..09cddbbb0 100644 --- a/apps/memos-local-openclaw/scripts/refresh-skill.ts +++ b/apps/memos-local-openclaw/scripts/refresh-skill.ts @@ -16,15 +16,16 @@ if (!skillId) { } import * as fs from "fs"; +import { parseOpenClawConfig } from "../src/shared/openclaw-config"; const home = process.env.HOME ?? "/tmp"; const stateDir = `${home}/.openclaw`; const workspaceDir = `${home}/.openclaw/workspace`; -// Read plugin config from openclaw.json +// Read plugin config from openclaw.json (tolerates JSON5 comments / quoting). let pluginConfig: Record | undefined; try { - const oc = JSON.parse(fs.readFileSync(`${stateDir}/openclaw.json`, "utf-8")); + const oc = parseOpenClawConfig(fs.readFileSync(`${stateDir}/openclaw.json`, "utf-8")) as any; pluginConfig = oc?.plugins?.entries?.["memos-local"]?.config; } catch {} diff --git a/apps/memos-local-openclaw/scripts/refresh-summaries.ts b/apps/memos-local-openclaw/scripts/refresh-summaries.ts index 46577c09b..bd83cdf3b 100644 --- a/apps/memos-local-openclaw/scripts/refresh-summaries.ts +++ b/apps/memos-local-openclaw/scripts/refresh-summaries.ts @@ -2,6 +2,7 @@ import Database from "better-sqlite3"; import * as path from "path"; import * as os from "os"; import * as fs from "fs"; +import { parseOpenClawConfig } from "../src/shared/openclaw-config"; const TASK_SUMMARY_PROMPT = `You create a DETAILED task summary from a multi-turn conversation. This summary will be the ONLY record of this conversation, so it must preserve ALL important information. @@ -52,7 +53,7 @@ function parseTitleFromSummary(summary: string): { title: string; body: string } async function main() { const configPath = path.join(os.homedir(), ".openclaw", "openclaw.json"); - const config = JSON.parse(fs.readFileSync(configPath, "utf-8")); + const config = parseOpenClawConfig(fs.readFileSync(configPath, "utf-8")) as any; const memosConfig = config.plugins?.entries?.["memos-local"]?.config ?? config.plugins?.configs?.["memos-local"]?.config; const cfg = memosConfig?.summarizer; diff --git a/apps/memos-local-openclaw/scripts/run-accuracy-test.ts b/apps/memos-local-openclaw/scripts/run-accuracy-test.ts index afb3b6c34..78910f677 100644 --- a/apps/memos-local-openclaw/scripts/run-accuracy-test.ts +++ b/apps/memos-local-openclaw/scripts/run-accuracy-test.ts @@ -21,6 +21,7 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; import { initPlugin, type MemosLocalPlugin } from "../src/index"; +import { parseOpenClawConfig } from "../src/shared/openclaw-config"; // ─── CLI args ─── @@ -38,7 +39,7 @@ function loadConfig() { if (!fs.existsSync(cfgPath)) { throw new Error(`OpenClaw config not found: ${cfgPath}`); } - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseOpenClawConfig(fs.readFileSync(cfgPath, "utf-8")) as any; return raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? {}; } diff --git a/apps/memos-local-openclaw/scripts/test-agent-isolation.ts b/apps/memos-local-openclaw/scripts/test-agent-isolation.ts index f059cb2d9..9dec34dc5 100644 --- a/apps/memos-local-openclaw/scripts/test-agent-isolation.ts +++ b/apps/memos-local-openclaw/scripts/test-agent-isolation.ts @@ -16,6 +16,7 @@ import { SqliteStore } from "../src/storage/sqlite"; import { Embedder } from "../src/embedding"; import { RecallEngine } from "../src/recall/engine"; import { buildContext } from "../src/config"; +import { parseOpenClawConfig } from "../src/shared/openclaw-config"; const RUN_ID = Date.now(); const AGENT_A = "iso-test-alpha"; @@ -61,7 +62,7 @@ async function main() { const home = process.env.HOME ?? process.env.USERPROFILE ?? "/tmp"; const stateDir = path.join(home, ".openclaw"); const cfgPath = path.join(stateDir, "openclaw.json"); - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseOpenClawConfig(fs.readFileSync(cfgPath, "utf-8")) as any; const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? {}; // ── Step 1: Ingest data with different owners ── diff --git a/apps/memos-local-openclaw/src/ingest/providers/index.ts b/apps/memos-local-openclaw/src/ingest/providers/index.ts index b08818520..665b6b7af 100644 --- a/apps/memos-local-openclaw/src/ingest/providers/index.ts +++ b/apps/memos-local-openclaw/src/ingest/providers/index.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import type { SummarizerConfig, SummaryProvider, Logger, OpenClawAPI } from "../../types"; +import { parseOpenClawConfig } from "../../shared/openclaw-config"; import { summarizeOpenAI, summarizeTaskOpenAI, generateTaskTitleOpenAI, judgeNewTopicOpenAI, classifyTopicOpenAI, arbitrateTopicSplitOpenAI, filterRelevantOpenAI, judgeDedupOpenAI, parseFilterResult, parseDedupResult, parseTopicClassifyResult } from "./openai"; import type { FilterResult, DedupResult, TopicClassifyResult } from "./openai"; export type { FilterResult, DedupResult, TopicClassifyResult } from "./openai"; @@ -66,7 +67,7 @@ function loadOpenClawFallbackConfig(log: Logger): SummarizerConfig | undefined { || path.join(process.env.OPENCLAW_STATE_DIR || path.join(home, ".openclaw"), "openclaw.json"); if (!fs.existsSync(cfgPath)) return undefined; - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseOpenClawConfig(fs.readFileSync(cfgPath, "utf-8")) as any; const agentModel: string | undefined = raw?.agents?.defaults?.model?.primary; if (!agentModel) return undefined; diff --git a/apps/memos-local-openclaw/src/shared/llm-call.ts b/apps/memos-local-openclaw/src/shared/llm-call.ts index aa868fc4c..98f276820 100644 --- a/apps/memos-local-openclaw/src/shared/llm-call.ts +++ b/apps/memos-local-openclaw/src/shared/llm-call.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import type { SummarizerConfig, SummaryProvider, Logger, PluginContext, OpenClawAPI } from "../types"; +import { parseOpenClawConfig } from "./openclaw-config"; /** * Resolve a SecretInput (string | SecretRef) to a plain string. @@ -54,7 +55,7 @@ export function loadOpenClawFallbackConfig(log: Logger): SummarizerConfig | unde || path.join(process.env.OPENCLAW_STATE_DIR || path.join(home, ".openclaw"), "openclaw.json"); if (!fs.existsSync(cfgPath)) return undefined; - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseOpenClawConfig(fs.readFileSync(cfgPath, "utf-8")) as any; const agentModel: string | undefined = raw?.agents?.defaults?.model?.primary; if (!agentModel) return undefined; diff --git a/apps/memos-local-openclaw/src/shared/openclaw-config.ts b/apps/memos-local-openclaw/src/shared/openclaw-config.ts new file mode 100644 index 000000000..76e0f2e55 --- /dev/null +++ b/apps/memos-local-openclaw/src/shared/openclaw-config.ts @@ -0,0 +1,22 @@ +import * as fs from "fs"; +import JSON5 from "json5"; + +/** + * Parse an openclaw.json file, accepting JSON5 syntax (line/block comments, + * single-quoted strings, trailing commas, unquoted identifier keys). + * + * Strict JSON is a subset of JSON5, so any file that used to parse with + * `JSON.parse` continues to parse here. + * + * We centralize this in one helper so every openclaw.json read site behaves + * identically — users routinely add `// ...` comments to their config and we + * must not crash on them (see issue #1543). + */ +export function parseOpenClawConfig(raw: string): unknown { + return JSON5.parse(raw); +} + +/** Read + parse an openclaw.json file with JSON5 tolerance. */ +export function readOpenClawConfig(configPath: string): unknown { + return parseOpenClawConfig(fs.readFileSync(configPath, "utf-8")); +} diff --git a/apps/memos-local-openclaw/src/viewer/server.ts b/apps/memos-local-openclaw/src/viewer/server.ts index 852f3eda0..a8ae654b1 100644 --- a/apps/memos-local-openclaw/src/viewer/server.ts +++ b/apps/memos-local-openclaw/src/viewer/server.ts @@ -19,6 +19,7 @@ import { getHubStatus } from "../client/connector"; import { type ResolvedHubClient, hubGetMemoryDetail, hubListMemories, hubListTasks, hubListSkills, hubRequestJson, hubSearchMemories, hubSearchSkills, hubUpdateUsername, normalizeHubUrl, resolveHubClient } from "../client/hub"; import { buildSkillBundleForHub, fetchHubSkillBundle, restoreSkillBundleFromHub } from "../client/skill-sync"; import type { Logger, Chunk, PluginContext, MemosLocalConfig } from "../types"; +import { parseOpenClawConfig } from "../shared/openclaw-config"; import { viewerHTML } from "./html"; import { v4 as uuid } from "uuid"; @@ -649,7 +650,7 @@ export class ViewerServer { try { const cfgPath = this.getOpenClawConfigPath(); if (fs.existsSync(cfgPath)) { - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseOpenClawConfig(fs.readFileSync(cfgPath, "utf-8")) as any; const entries = raw?.plugins?.entries ?? {}; const pluginCfg = entries["memos-local-openclaw-plugin"]?.config ?? entries["memos-local"]?.config ?? {}; diff --git a/apps/memos-local-openclaw/tests/openclaw-json5.test.ts b/apps/memos-local-openclaw/tests/openclaw-json5.test.ts new file mode 100644 index 000000000..21b9ed911 --- /dev/null +++ b/apps/memos-local-openclaw/tests/openclaw-json5.test.ts @@ -0,0 +1,190 @@ +import { describe, it, expect, afterEach } from "vitest"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { parseOpenClawConfig, readOpenClawConfig } from "../src/shared/openclaw-config"; +import { loadOpenClawFallbackConfig } from "../src/shared/llm-call"; + +describe("openclaw.json JSON5 tolerance (issue #1543)", () => { + const noopLog = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + }; + let tmpDir: string | undefined; + let savedConfigPath: string | undefined; + let savedStateDir: string | undefined; + + afterEach(() => { + if (savedConfigPath !== undefined) process.env.OPENCLAW_CONFIG_PATH = savedConfigPath; + else delete process.env.OPENCLAW_CONFIG_PATH; + if (savedStateDir !== undefined) process.env.OPENCLAW_STATE_DIR = savedStateDir; + else delete process.env.OPENCLAW_STATE_DIR; + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + tmpDir = undefined; + savedConfigPath = undefined; + savedStateDir = undefined; + }); + + function writeConfig(raw: string): string { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-json5-")); + const cfgPath = path.join(tmpDir, "openclaw.json"); + fs.writeFileSync(cfgPath, raw, "utf-8"); + savedConfigPath = process.env.OPENCLAW_CONFIG_PATH; + savedStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_CONFIG_PATH = cfgPath; + return cfgPath; + } + + describe("parseOpenClawConfig", () => { + it("parses strict JSON identical to JSON.parse", () => { + const raw = '{"tools":{"allow":["core:read","core:write"]}}'; + expect(parseOpenClawConfig(raw)).toEqual({ + tools: { allow: ["core:read", "core:write"] }, + }); + }); + + it("accepts line comments (// ...) — the exact case from issue #1543", () => { + const raw = `{ + // this is my openclaw config + "tools": { + "allow": [ + "core:read", + "core:write" // last tool + ] + } + }`; + const cfg = parseOpenClawConfig(raw) as any; + expect(cfg.tools.allow).toEqual(["core:read", "core:write"]); + }); + + it("accepts block comments (/* ... */)", () => { + const raw = `{ + /* header */ + "tools": { "allow": ["core:read"] } + }`; + const cfg = parseOpenClawConfig(raw) as any; + expect(cfg.tools.allow).toEqual(["core:read"]); + }); + + it("accepts single-quoted strings", () => { + const raw = `{ 'tools': { 'allow': ['core:read', 'core:write'] } }`; + const cfg = parseOpenClawConfig(raw) as any; + expect(cfg.tools.allow).toEqual(["core:read", "core:write"]); + }); + + it("accepts trailing commas", () => { + const raw = `{ "tools": { "allow": ["core:read", "core:write",], }, }`; + const cfg = parseOpenClawConfig(raw) as any; + expect(cfg.tools.allow).toEqual(["core:read", "core:write"]); + }); + + it("accepts unquoted identifier keys", () => { + const raw = `{ tools: { allow: ["core:read"] } }`; + const cfg = parseOpenClawConfig(raw) as any; + expect(cfg.tools.allow).toEqual(["core:read"]); + }); + + it("still throws on malformed JSON5", () => { + expect(() => parseOpenClawConfig("{ this is not valid")).toThrow(); + }); + }); + + describe("readOpenClawConfig", () => { + it("reads a JSON5 file from disk", () => { + const cfgPath = writeConfig(`{ + // comment + "tools": { "allow": ["a", "b"] } + }`); + expect(readOpenClawConfig(cfgPath)).toEqual({ tools: { allow: ["a", "b"] } }); + }); + }); + + describe("loadOpenClawFallbackConfig", () => { + it("loads a JSON5 openclaw.json with comments and mixed quoting", () => { + const testKey = "sk-json5-test-" + Date.now(); + process.env.__MEMOS_TEST_JSON5_KEY = testKey; + try { + writeConfig(`{ + // top-level comment + agents: { + defaults: { + model: { + primary: 'anthropic/claude-3-haiku', // trailing comment + }, + }, + }, + models: { + providers: { + anthropic: { + baseUrl: "https://api.anthropic.com", + /* block comment */ + apiKey: { source: 'env', provider: 'anthropic', id: '__MEMOS_TEST_JSON5_KEY' }, + }, + }, + }, + }`); + const cfg = loadOpenClawFallbackConfig(noopLog); + expect(cfg).toBeDefined(); + expect(cfg!.apiKey).toBe(testKey); + expect(cfg!.provider).toBe("anthropic"); + expect(cfg!.model).toBe("claude-3-haiku"); + } finally { + delete process.env.__MEMOS_TEST_JSON5_KEY; + } + }); + }); + + describe("tools.allow patch (simulates index.ts:354-380)", () => { + /** + * Reproduce the exact flow from apps/memos-local-openclaw/index.ts that + * failed pre-fix with "SyntaxError: Expected double-quoted property name + * in JSON at position 2222" when the user's openclaw.json contained + * comments. We only assert the read step no longer throws. + */ + function readAllowFromRaw(raw: string): string[] | undefined { + const cfg = parseOpenClawConfig(raw) as any; + return cfg?.tools?.allow; + } + + it("does not crash on a JSON5 openclaw.json with '//' comments", () => { + const raw = `{ + // memory tools + "tools": { + "allow": [ + "core:read", + "core:write" // enable writes + ] + } + }`; + expect(() => readAllowFromRaw(raw)).not.toThrow(); + expect(readAllowFromRaw(raw)).toEqual(["core:read", "core:write"]); + }); + + it("patch regex still works when file uses double-quoted strings alongside comments", () => { + const raw = `{ + // top comment + "tools": { + "allow": [ + "core:read", + "core:write" + ] + } + }`; + const allow = readAllowFromRaw(raw)!; + const lastEntry = JSON.stringify(allow[allow.length - 1]); + const patched = raw.replace( + new RegExp(`(${lastEntry})(\\s*\\])`), + `$1,\n "group:plugins"$2`, + ); + expect(patched).not.toBe(raw); + expect(patched).toContain('"group:plugins"'); + // The comment must be preserved. + expect(patched).toContain("// top comment"); + // The patched file must still parse (as JSON5). + const reparsed = parseOpenClawConfig(patched) as any; + expect(reparsed.tools.allow).toContain("group:plugins"); + }); + }); +});