From 2f267af7a39ddd0af218339768b28e4bb1813d5f Mon Sep 17 00:00:00 2001 From: TJWater CI Date: Fri, 7 Aug 2026 18:06:08 +0800 Subject: [PATCH] revert: remove unused Agent PostgreSQL persistence --- .github/copilot-instructions.md | 124 ----- .opencode/skills/examples.md | 15 - .opencode/skills/runbook.md | 1 - .opencode/tools/dynamic_http_call.ts | 21 +- .opencode/tools/fetch_result_ref.ts | 10 +- .opencode/tools/memory_manager.ts | 12 +- .opencode/tools/session_search.ts | 10 +- .opencode/tools/skill_manager.ts | 8 +- .opencode/tools/store_render_ref.ts | 10 +- Dockerfile | 1 - README.md | 2 +- bun.lock | 30 - package.json | 3 - scripts/migrate-file-storage-to-postgres.ts | 578 -------------------- src/audit/llmRequestAudit.ts | 1 + src/chat/sessionBridge.ts | 122 +++-- src/config.ts | 26 - src/conversations/stateStore.ts | 90 +-- src/conversations/store.ts | 226 ++------ src/db/index.ts | 486 ---------------- src/history/store.ts | 378 ++++++------- src/learning/orchestrator.ts | 40 +- src/learning/stateStore.ts | 91 +-- src/memory/store.ts | 265 ++++----- src/results/resolver.ts | 12 +- src/results/store.ts | 421 +++++++------- src/routes/chat.ts | 86 ++- src/routes/chatStream.ts | 30 +- src/server.ts | 60 +- src/session/runtimeSessionStore.ts | 123 ----- src/session/toolContextStore.ts | 55 ++ src/tools/dynamicHttpExecutor.ts | 24 +- tests/conversations/store.test.ts | 56 +- tests/helpers/postgres.ts | 17 - tests/history/store.test.ts | 92 ++-- tests/results/store.test.ts | 351 ++++++++++-- tests/session/runtimeSessionStore.test.ts | 52 -- tests/session/toolContextStore.test.ts | 51 ++ tests/tools/dynamicHttpExecutor.test.ts | 110 ---- 39 files changed, 1330 insertions(+), 2760 deletions(-) delete mode 100644 .github/copilot-instructions.md delete mode 100644 scripts/migrate-file-storage-to-postgres.ts delete mode 100644 src/db/index.ts delete mode 100644 src/session/runtimeSessionStore.ts create mode 100644 src/session/toolContextStore.ts delete mode 100644 tests/helpers/postgres.ts delete mode 100644 tests/session/runtimeSessionStore.test.ts create mode 100644 tests/session/toolContextStore.test.ts delete mode 100644 tests/tools/dynamicHttpExecutor.test.ts diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index c531ee5..0000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1,124 +0,0 @@ -# Copilot Instructions for TJWaterAgent - -## Overview - -- TJWaterAgent is a Bun + TypeScript backend service. -- Main entrypoint: `src/server.ts` -- Chat API base path: `/api/v1/agent/chat` -- Persistent storage is **PostgreSQL-first**. Historical file storage only remains for one-time migration and external result payload files. - -## Core identity model - -- `sessionId` = durable chat thread identifier exposed to clients and used for persistence. -- `runtimeSessionId` = temporary opencode runtime execution identifier for a single active request. -- Do **not** reintroduce `conversationId` / `clientSessionId` naming for the durable thread id. Use `sessionId`. - -## Persistence design - -Canonical PostgreSQL tables: - -- `conversations` -- `conversation_states` -- `conversation_turns` -- `runtime_sessions` -- `learning_states` -- `result_refs` -- `memories` - -### conversations - -Stores durable thread metadata: - -- `session_id` -- actor / owner / project fields -- `parent_session_id` -- `title` -- `status` -- streaming lifecycle fields: - - `is_streaming` - - `active_runtime_session_id` - - `streaming_started_at` - -`is_streaming` is persisted so the frontend can recover streaming state after refresh. Stream cleanup must be guarded by `active_runtime_session_id` so an old stream cannot clear a newer one. - -### conversation_states - -Stores frontend-facing session UI state: - -- `messages` -- `branch_groups` -- `is_title_manually_edited` - -### conversation_turns - -Stores transcript turns as one row per turn: - -- keyed by `turn_id` -- linked by `session_id` -- ordered by deterministic `turn_index` - -Do not fall back to legacy transcript files in runtime code. - -### runtime_sessions - -Stores runtime binding context for internal tools and learning review/gate flows: - -- `runtime_session_id` -- `session_id` -- actor / project / trace fields -- `allow_learning_write` -- `learning_mode` -- `released_at` - -This replaces the old alias-based `tool_session_contexts` model. - -### result_refs - -`result_refs` is **metadata-only in PostgreSQL**: - -- metadata lives in PG -- payload body lives outside PG via `payload_path` or future `object_key` - -Most payloads are large, so do not inline them into PostgreSQL. - -### memories - -Durable memory is stored in PG, not markdown files. - -## Runtime flow - -1. Client sends/loads a `session_id`. -2. `ChatSessionBridge` binds that durable `sessionId` to a fresh `runtimeSessionId`. -3. Internal tools call back into the server using `runtimeSessionId`. -4. Transcript, memory, learning state, and result refs persist against `sessionId`. -5. Stream start sets `conversations.is_streaming = true`; stream completion clears it only if the runtime id still matches. - -## Important files - -- `src/server.ts` - bootstrap, internal tool endpoints, app wiring -- `src/chat/sessionBridge.ts` - `sessionId` vs `runtimeSessionId` binding -- `src/db/index.ts` - PostgreSQL bootstrap and schema migration/cleanup -- `src/conversations/store.ts` - durable session metadata + streaming state -- `src/conversations/stateStore.ts` - UI state persistence -- `src/history/store.ts` - transcript persistence -- `src/session/runtimeSessionStore.ts` - runtime session persistence -- `src/results/store.ts` - metadata-only result refs + external payload files -- `src/results/resolver.ts` - result ref normalization and retrieval -- `src/learning/orchestrator.ts` / `src/learning/stateStore.ts` - learning review pipeline -- `src/memory/store.ts` - persistent memory store -- `scripts/migrate-file-storage-to-postgres.ts` - one-time migration from legacy file storage - -## Compatibility rules - -- Runtime code should target the canonical schema only. -- Legacy column/table handling belongs only in `src/db/index.ts` bootstrap migration or `scripts/migrate-file-storage-to-postgres.ts`. -- Do not add back file-based runtime persistence or compatibility reads unless a migration explicitly requires it. - -## Validation - -Useful commands: - -```bash -bun run check -PGHOST=... PGPORT=... PGUSER=... PGPASSWORD=... PGDATABASE=agent PGSSLMODE=disable bun test tests/conversations/store.test.ts tests/history/store.test.ts tests/session/runtimeSessionStore.test.ts tests/results/store.test.ts tests/routes/chatSession.test.ts -``` diff --git a/.opencode/skills/examples.md b/.opencode/skills/examples.md index 1d6b5d7..6cf022d 100644 --- a/.opencode/skills/examples.md +++ b/.opencode/skills/examples.md @@ -44,21 +44,6 @@ - 第二步（可选）工具调用：查询补充数据接口。 - opencode agent 汇总工具结果，持续通过 SSE 输出 `progress` 与 `token`，最终返回 `done`。 -若接口要求 JSON 请求体，可这样调用： - -```json -{ - "path": "/api/v1/runproject/simulate", - "method": "POST", - "arguments": { - "network": "demo-network" - }, - "body": { - "duration_minutes": 15 - } -} -``` - `progress` 示例： ```json diff --git a/.opencode/skills/runbook.md b/.opencode/skills/runbook.md index 245640a..486b593 100644 --- a/.opencode/skills/runbook.md +++ b/.opencode/skills/runbook.md @@ -66,7 +66,6 @@ SSE 事件： - `path` 必须以 `/` 开头。 - `method` 支持：`GET/POST/PUT/PATCH/DELETE`。 - `arguments` 会编码为 query 参数（列表会转为逗号拼接）。 -- `body` 会作为 JSON request body 透传；`GET` 不支持传 `body`。 ## 3.1) 学习工具约定 diff --git a/.opencode/tools/dynamic_http_call.ts b/.opencode/tools/dynamic_http_call.ts index 18e16f0..80a5138 100644 --- a/.opencode/tools/dynamic_http_call.ts +++ b/.opencode/tools/dynamic_http_call.ts @@ -1,11 +1,14 @@ import { tool } from "@opencode-ai/plugin"; +import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; +const toolContextStore = new ToolSessionContextStore(); +const initializePromise = toolContextStore.initialize(); export default tool({ description: - "通过本地 Agent 桥接调用 TJWater 后端 API。支持 query 参数，且在 POST/PUT/PATCH/DELETE 时可选传递 JSON body。", + "通过本地 Agent 桥接调用 TJWater 后端 API。需提供 API 路径、可选的请求方法以及查询参数。", args: { reason: tool.schema .string() @@ -18,14 +21,15 @@ export default tool({ arguments: tool.schema .record(tool.schema.string(), tool.schema.unknown()) .optional() - .describe("Query arguments object. Encoded into URL query string."), - body: tool.schema - .unknown() - .optional() - .describe("Optional JSON body for POST/PUT/PATCH/DELETE requests."), + .describe("Query arguments object."), }, async execute(args, context) { - // 工具本身不直接持有用户 token；通过 runtimeSessionId 回调 Agent 服务，由服务侧补齐用户上下文。 + await initializePromise; + const sessionContext = await toolContextStore.read(context.sessionID); + if (!sessionContext) { + throw new Error(`session context not found for ${context.sessionID}`); + } + // 工具本身不直接持有用户 token；通过 sessionID 回调 Agent 服务，由服务侧补齐用户上下文。 const response = await fetch(`${internalBaseUrl}/internal/tools/dynamic-http-call`, { method: "POST", headers: { @@ -33,12 +37,11 @@ export default tool({ "x-agent-internal-token": internalToken, }, body: JSON.stringify({ - runtimeSessionId: context.sessionID, + sessionScopeKey: sessionContext.sessionScopeKey, reason: args.reason, path: args.path, method: args.method, arguments: args.arguments, - body: args.body, }), }); diff --git a/.opencode/tools/fetch_result_ref.ts b/.opencode/tools/fetch_result_ref.ts index 3060a0b..32a45c0 100644 --- a/.opencode/tools/fetch_result_ref.ts +++ b/.opencode/tools/fetch_result_ref.ts @@ -1,7 +1,10 @@ import { tool } from "@opencode-ai/plugin"; +import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; +const toolContextStore = new ToolSessionContextStore(); +const initializePromise = toolContextStore.initialize(); export default tool({ description: @@ -19,6 +22,11 @@ export default tool({ .describe("Optional maximum number of top-level items or fields to return."), }, async execute(args, context) { + await initializePromise; + const sessionContext = await toolContextStore.read(context.sessionID); + if (!sessionContext) { + throw new Error(`session context not found for ${context.sessionID}`); + } const response = await fetch(`${internalBaseUrl}/internal/tools/fetch-result-ref`, { method: "POST", headers: { @@ -26,7 +34,7 @@ export default tool({ "x-agent-internal-token": internalToken, }, body: JSON.stringify({ - runtimeSessionId: context.sessionID, + sessionScopeKey: sessionContext.sessionScopeKey, result_ref: args.result_ref, max_items: args.max_items, }), diff --git a/.opencode/tools/memory_manager.ts b/.opencode/tools/memory_manager.ts index b55c3da..22f7795 100644 --- a/.opencode/tools/memory_manager.ts +++ b/.opencode/tools/memory_manager.ts @@ -1,12 +1,12 @@ import { tool } from "@opencode-ai/plugin"; import { MemoryStore } from "../../src/memory/store.js"; -import { RuntimeSessionStore } from "../../src/session/runtimeSessionStore.js"; +import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const memoryStore = new MemoryStore(); -const runtimeSessionStore = new RuntimeSessionStore(); +const toolContextStore = new ToolSessionContextStore(); const initializePromise = Promise.all([ memoryStore.initialize(), - runtimeSessionStore.initialize(), + toolContextStore.initialize(), ]); export default tool({ @@ -37,7 +37,7 @@ export default tool({ }, async execute(args, context) { await initializePromise; - const sessionContext = await runtimeSessionStore.read(context.sessionID); + const sessionContext = await toolContextStore.read(context.sessionID); if (!sessionContext) { throw new Error(`session context not found for ${context.sessionID}`); } @@ -80,7 +80,7 @@ export default tool({ if (args.action === "add") { const result = await memoryStore.upsert(scope, scopeKey, { content: args.content ?? "", - sessionId: sessionContext.sessionId, + sessionId: sessionContext.clientSessionId, source: "tool", traceId: sessionContext.traceId, }); @@ -105,7 +105,7 @@ export default tool({ if (args.action === "replace") { const result = await memoryStore.replace(scope, scopeKey, args.target_id ?? "", { content: args.content ?? "", - sessionId: sessionContext.sessionId, + sessionId: sessionContext.clientSessionId, source: "tool", traceId: sessionContext.traceId, }); diff --git a/.opencode/tools/session_search.ts b/.opencode/tools/session_search.ts index f571df9..9631b3c 100644 --- a/.opencode/tools/session_search.ts +++ b/.opencode/tools/session_search.ts @@ -1,8 +1,11 @@ import { tool } from "@opencode-ai/plugin"; +import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; +const toolContextStore = new ToolSessionContextStore(); +const initializePromise = toolContextStore.initialize(); export default tool({ description: @@ -22,6 +25,11 @@ export default tool({ .describe("Optional maximum number of hits to return."), }, async execute(args, context) { + await initializePromise; + const sessionContext = await toolContextStore.read(context.sessionID); + if (!sessionContext) { + throw new Error(`session context not found for ${context.sessionID}`); + } const response = await fetch(`${internalBaseUrl}/internal/tools/session-search`, { method: "POST", headers: { @@ -31,7 +39,7 @@ export default tool({ body: JSON.stringify({ max_results: args.max_results, query: args.query, - runtimeSessionId: context.sessionID, + sessionScopeKey: sessionContext.sessionScopeKey, }), }); const text = await response.text(); diff --git a/.opencode/tools/skill_manager.ts b/.opencode/tools/skill_manager.ts index 9dabf62..da199b7 100644 --- a/.opencode/tools/skill_manager.ts +++ b/.opencode/tools/skill_manager.ts @@ -1,10 +1,10 @@ import { tool } from "@opencode-ai/plugin"; import { SkillStore } from "../../src/skills/store.js"; -import { RuntimeSessionStore } from "../../src/session/runtimeSessionStore.js"; +import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; -const runtimeSessionStore = new RuntimeSessionStore(); -const initializePromise = runtimeSessionStore.initialize(); +const toolContextStore = new ToolSessionContextStore(); +const initializePromise = toolContextStore.initialize(); const skillStore = new SkillStore(); export default tool({ @@ -51,7 +51,7 @@ export default tool({ }, async execute(args, context) { await initializePromise; - const sessionContext = await runtimeSessionStore.read(context.sessionID); + const sessionContext = await toolContextStore.read(context.sessionID); if (!sessionContext) { throw new Error(`session context not found for ${context.sessionID}`); } diff --git a/.opencode/tools/store_render_ref.ts b/.opencode/tools/store_render_ref.ts index 1b28213..20725f1 100644 --- a/.opencode/tools/store_render_ref.ts +++ b/.opencode/tools/store_render_ref.ts @@ -1,7 +1,10 @@ import { tool } from "@opencode-ai/plugin"; +import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; +const toolContextStore = new ToolSessionContextStore(); +const initializePromise = toolContextStore.initialize(); export default tool({ description: @@ -17,6 +20,11 @@ export default tool({ ), }, async execute(args, context) { + await initializePromise; + const sessionContext = await toolContextStore.read(context.sessionID); + if (!sessionContext) { + throw new Error(`session context not found for ${context.sessionID}`); + } const response = await fetch(`${internalBaseUrl}/internal/tools/store-render-ref`, { method: "POST", headers: { @@ -24,7 +32,7 @@ export default tool({ "x-agent-internal-token": internalToken, }, body: JSON.stringify({ - runtimeSessionId: context.sessionID, + sessionScopeKey: sessionContext.sessionScopeKey, file_path: args.file_path, }), }); diff --git a/Dockerfile b/Dockerfile index e3edae2..0ac0f9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,6 @@ COPY --from=deps /app/.opencode/node_modules ./.opencode/node_modules COPY package.json bun.lock ./ COPY tsconfig.json opencode.json ./ COPY src ./src -COPY scripts ./scripts COPY .opencode ./.opencode COPY entrypoint.sh /entrypoint.sh diff --git a/README.md b/README.md index 2be8d1f..c77e63e 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ src/ 这些是 opencode 可以调用的自定义工具。 -`dynamic_http_call.ts` 不直接保存用户 token，也不直接访问后端。它会回调 `TJWaterAgent` 的内部接口，由上级服务层根据当前 session 补上用户 token、项目 ID 和 trace ID，再调用 TJWater 后端。`arguments` 会编码为 query 参数；`body` 可用于 `POST/PUT/PATCH/DELETE` 的 JSON 请求体。 +`dynamic_http_call.ts` 不直接保存用户 token，也不直接访问后端。它会回调 `TJWaterAgent` 的内部接口，由上级服务层根据当前 session 补上用户 token、项目 ID 和 trace ID，再调用 TJWater 后端。 前端类工具如 `locate_features`、`view_history`、`view_scada`、`show_chart` 主要用于触发 UI 动作或可视化，不应被当作数据查询工具。 diff --git a/bun.lock b/bun.lock index 27636f8..90544b1 100644 --- a/bun.lock +++ b/bun.lock @@ -6,11 +6,9 @@ "name": "tjwater-agent", "dependencies": { "@opencode-ai/sdk": "^1.14.29", - "@types/pg": "^8.20.0", "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^4.21.2", - "pg": "^8.21.0", "pino": "^9.7.0", "pino-pretty": "^13.1.2", "zod": "^3.25.76", @@ -43,8 +41,6 @@ "@types/node": ["@types/node@24.12.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g=="], - "@types/pg": ["@types/pg@8.20.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow=="], - "@types/qs": ["@types/qs@6.15.0", "", {}, "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow=="], "@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="], @@ -185,22 +181,6 @@ "path-to-regexp": ["path-to-regexp@0.1.13", "", {}, "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA=="], - "pg": ["pg@8.21.0", "", { "dependencies": { "pg-connection-string": "^2.13.0", "pg-pool": "^3.14.0", "pg-protocol": "^1.14.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA=="], - - "pg-cloudflare": ["pg-cloudflare@1.4.0", "", {}, "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A=="], - - "pg-connection-string": ["pg-connection-string@2.13.0", "", {}, "sha512-EMnU9E2fSULdsbErBbMaXJvFeD9B4+nPcM3f+4lsiCR0BHLPrLVjv3DbyM2hgQQviKJaTWIRRTjKjWlHg3p2ig=="], - - "pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="], - - "pg-pool": ["pg-pool@3.14.0", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw=="], - - "pg-protocol": ["pg-protocol@1.14.0", "", {}, "sha512-n5taZ1kO3s9ngDTVxsEznOqCyToTgz0FLuPq0B33COy5pPpuWJpY3/2oRBVETuOgzdqRXfWpM9HIhp2LBBT1BA=="], - - "pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], - - "pgpass": ["pgpass@1.0.5", "", { "dependencies": { "split2": "^4.1.0" } }, "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug=="], - "pino": ["pino@9.14.0", "", { "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w=="], "pino-abstract-transport": ["pino-abstract-transport@2.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw=="], @@ -209,14 +189,6 @@ "pino-std-serializers": ["pino-std-serializers@7.1.0", "", {}, "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw=="], - "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], - - "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], - - "postgres-date": ["postgres-date@1.0.7", "", {}, "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="], - - "postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], - "process-warning": ["process-warning@5.0.0", "", {}, "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA=="], "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], @@ -287,8 +259,6 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], - "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], "body-parser/qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="], diff --git a/package.json b/package.json index 569961b..d606903 100644 --- a/package.json +++ b/package.json @@ -11,18 +11,15 @@ "dev": "bun --watch src/server.ts", "build": "bun run check", "check": "bun run typecheck && bun run typecheck:opencode", - "migrate:storage": "bun scripts/migrate-file-storage-to-postgres.ts", "pipeline:trigger": "bash scripts/trigger-gitea-pipeline.sh", "start": "bun src/server.ts", "start:prod": "bun run check && bun src/server.ts" }, "dependencies": { "@opencode-ai/sdk": "^1.14.29", - "@types/pg": "^8.20.0", "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^4.21.2", - "pg": "^8.21.0", "pino": "^9.7.0", "pino-pretty": "^13.1.2", "zod": "^3.25.76" diff --git a/scripts/migrate-file-storage-to-postgres.ts b/scripts/migrate-file-storage-to-postgres.ts deleted file mode 100644 index 10ebdbe..0000000 --- a/scripts/migrate-file-storage-to-postgres.ts +++ /dev/null @@ -1,578 +0,0 @@ -import { readdir, readFile } from "node:fs/promises"; -import { basename, extname, join } from "node:path"; - -import { config } from "../src/config.js"; -import { AgentDatabase } from "../src/db/index.js"; -import { - RESULT_REFERENCE_KIND, - RESULT_REFERENCE_SOURCE, - RESULT_REF_PATTERN, - type ResultPreview, -} from "../src/results/store.js"; -import { sanitizePersistentDocument, sanitizePersistentLine } from "../src/utils/persistencePolicy.js"; -import { toProjectKey, toStableId } from "../src/utils/fileStore.js"; - -type JsonRecord = Record; - -const db = new AgentDatabase(); -const runtimeConversationMap = new Map(); - -const counters = { - conversations: 0, - states: 0, - turns: 0, - runtimeSessions: 0, - learningStates: 0, - resultRefs: 0, - memories: 0, - skippedResultRefs: 0, -}; - -await db.initialize(); -await migrateConversations(); -await migrateConversationStates(); -await migrateConversationTurns(); -await migrateRuntimeSessions(); -await migrateLearningStates(); -await migrateResultRefs(); -await migrateMemories(); -await db.close(); - -console.log(JSON.stringify(counters, null, 2)); - -async function migrateConversations() { - for (const filePath of await listFiles(config.CONVERSATION_STORAGE_DIR, ".json")) { - const row = await readJson(filePath); - if (!row || typeof row.sessionId !== "string" || typeof row.actorKey !== "string") { - continue; - } - await upsertConversation({ - sessionId: row.sessionId, - actorKey: row.actorKey, - ownerUserId: stringOrUndefined(row.ownerUserId), - projectId: stringOrUndefined(row.projectId), - projectKey: - stringOrUndefined(row.projectKey) ?? - toProjectKey(stringOrUndefined(row.projectId)), - parentSessionId: stringOrUndefined(row.parentSessionId), - title: stringOrUndefined(row.title), - status: row.status === "archived" ? "archived" : "active", - createdAt: stringOrUndefined(row.createdAt), - updatedAt: stringOrUndefined(row.updatedAt), - }); - counters.conversations += 1; - } -} - -async function migrateConversationStates() { - for (const filePath of await listFiles(config.CONVERSATION_STATE_STORAGE_DIR, ".json")) { - const row = await readJson(filePath); - if (!row || typeof row.sessionId !== "string") { - continue; - } - await db.query( - ` - INSERT INTO ${db.table("conversation_states")} ( - session_id, - is_title_manually_edited, - messages, - branch_groups - ) - VALUES ($1, $2, $3::jsonb, $4::jsonb) - ON CONFLICT (session_id) - DO UPDATE SET - is_title_manually_edited = EXCLUDED.is_title_manually_edited, - messages = EXCLUDED.messages, - branch_groups = EXCLUDED.branch_groups - `, - [ - row.sessionId, - Boolean(row.isTitleManuallyEdited), - JSON.stringify(Array.isArray(row.messages) ? row.messages : []), - JSON.stringify(Array.isArray(row.branchGroups) ? row.branchGroups : []), - ], - ); - counters.states += 1; - } -} - -async function migrateConversationTurns() { - for (const filePath of await listFiles(config.SESSION_HISTORY_STORAGE_DIR, ".json")) { - const row = await readJson(filePath); - if (!row || typeof row.actorKey !== "string" || typeof row.projectKey !== "string") { - continue; - } - const sessionId = - stringOrUndefined(row.clientSessionId) ?? stringOrUndefined(row.sessionId); - if (!sessionId) { - continue; - } - const turns = Array.isArray(row.turns) ? row.turns : []; - const lastTimestamp = stringOrUndefined(row.updatedAt); - await upsertConversation({ - sessionId, - actorKey: row.actorKey, - projectKey: row.projectKey, - projectId: undefined, - createdAt: - turns.length > 0 && isRecord(turns[0]) ? stringOrUndefined(turns[0].timestamp) : undefined, - updatedAt: - lastTimestamp ?? - (turns.length > 0 && isRecord(turns[turns.length - 1]) - ? stringOrUndefined(turns[turns.length - 1].timestamp) - : undefined), - }); - for (const [turnIndex, turn] of turns.entries()) { - if (!isRecord(turn)) { - continue; - } - const userMessage = sanitizePersistentDocument(String(turn.userMessage ?? ""), 4000); - const assistantMessage = sanitizePersistentDocument( - String(turn.assistantMessage ?? ""), - 4000, - ); - if (!userMessage || !assistantMessage) { - continue; - } - const timestamp = stringOrUndefined(turn.timestamp) ?? new Date().toISOString(); - await db.query( - ` - INSERT INTO ${db.table("conversation_turns")} ( - turn_id, - session_id, - turn_index, - actor_key, - project_key, - user_message, - assistant_message, - tool_call_count, - turn_timestamp - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (turn_id) DO NOTHING - `, - [ - typeof turn.id === "string" - ? turn.id - : toStableId(sessionId, String(turnIndex), timestamp, userMessage, assistantMessage), - sessionId, - turnIndex, - row.actorKey, - row.projectKey, - userMessage, - assistantMessage, - Math.max(0, numberOrZero(turn.toolCallCount)), - timestamp, - ], - ); - counters.turns += 1; - } - } -} - -async function migrateRuntimeSessions() { - const seen = new Set(); - for (const filePath of await listFiles(config.SESSION_CONTEXT_STORAGE_DIR, ".json")) { - const row = await readJson(filePath); - if ( - !row || - typeof row.sessionId !== "string" || - typeof row.clientSessionId !== "string" || - typeof row.actorKey !== "string" || - typeof row.projectKey !== "string" - ) { - continue; - } - if (seen.has(row.sessionId)) { - continue; - } - seen.add(row.sessionId); - runtimeConversationMap.set(row.sessionId, row.clientSessionId); - await upsertConversation({ - sessionId: row.clientSessionId, - actorKey: row.actorKey, - projectId: stringOrUndefined(row.projectId), - projectKey: row.projectKey, - }); - await db.query( - ` - INSERT INTO ${db.table("runtime_sessions")} ( - runtime_session_id, - session_id, - actor_key, - allow_learning_write, - learning_mode, - project_id, - project_key, - trace_id, - released_at - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) - ON CONFLICT (runtime_session_id) - DO UPDATE SET - session_id = EXCLUDED.session_id, - actor_key = EXCLUDED.actor_key, - allow_learning_write = EXCLUDED.allow_learning_write, - learning_mode = EXCLUDED.learning_mode, - project_id = EXCLUDED.project_id, - project_key = EXCLUDED.project_key, - trace_id = EXCLUDED.trace_id, - released_at = NOW() - `, - [ - row.sessionId, - row.clientSessionId, - row.actorKey, - Boolean(row.allowLearningWrite), - row.learningMode === "review" ? "review" : row.learningMode === "interactive" ? "interactive" : null, - stringOrUndefined(row.projectId) ?? null, - row.projectKey, - typeof row.traceId === "string" ? row.traceId : `trace-${row.sessionId}`, - ], - ); - counters.runtimeSessions += 1; - } -} - -async function migrateLearningStates() { - for (const filePath of await listFiles(config.LEARNING_STATE_STORAGE_DIR, ".json")) { - const row = await readJson(filePath); - if (!row || typeof row.sessionId !== "string") { - continue; - } - const sessionId = runtimeConversationMap.get(row.sessionId) ?? row.sessionId; - const existingConversation = await db.query( - `SELECT 1 FROM ${db.table("conversations")} WHERE session_id = $1 LIMIT 1`, - [sessionId], - ); - if (existingConversation.rowCount === 0) { - continue; - } - await db.query( - ` - INSERT INTO ${db.table("learning_states")} ( - session_id, - last_gated_turn, - last_reviewed_turn, - pending_review - ) - VALUES ($1, $2, $3, $4) - ON CONFLICT (session_id) - DO UPDATE SET - last_gated_turn = EXCLUDED.last_gated_turn, - last_reviewed_turn = EXCLUDED.last_reviewed_turn, - pending_review = EXCLUDED.pending_review - `, - [ - sessionId, - numberOrZero(row.lastGatedTurn), - numberOrZero(row.lastReviewedTurn), - Boolean(row.pendingReview), - ], - ); - counters.learningStates += 1; - } -} - -async function migrateResultRefs() { - for (const filePath of await listFiles(config.RESULT_REF_STORAGE_DIR, ".json")) { - const row = await readJson(filePath); - if (!row) { - continue; - } - const resultRef = normalizeResultRef(stringOrUndefined(row.resultRef) ?? basename(filePath, extname(filePath))); - const actorKey = stringOrUndefined(row.actorKey); - const sessionId = - stringOrUndefined(row.clientSessionId) ?? stringOrUndefined(row.sessionId); - if (!resultRef || !actorKey || !sessionId) { - counters.skippedResultRefs += 1; - continue; - } - const projectId = stringOrUndefined(row.projectId); - const projectKey = stringOrUndefined(row.projectKey) ?? toProjectKey(projectId); - await upsertConversation({ - sessionId, - actorKey, - projectId, - projectKey, - }); - const payload = unwrapPayload(row); - await db.query( - ` - INSERT INTO ${db.table("result_refs")} ( - result_ref, - actor_key, - session_id, - created_at, - kind, - preview, - project_id, - project_key, - schema_version, - size_bytes, - source, - trace_id, - payload_path, - object_key - ) - VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11, $12, $13, NULL) - ON CONFLICT (result_ref) - DO UPDATE SET - actor_key = EXCLUDED.actor_key, - session_id = EXCLUDED.session_id, - created_at = EXCLUDED.created_at, - kind = EXCLUDED.kind, - preview = EXCLUDED.preview, - project_id = EXCLUDED.project_id, - project_key = EXCLUDED.project_key, - schema_version = EXCLUDED.schema_version, - size_bytes = EXCLUDED.size_bytes, - source = EXCLUDED.source, - trace_id = EXCLUDED.trace_id, - payload_path = EXCLUDED.payload_path - `, - [ - resultRef, - actorKey, - sessionId, - stringOrUndefined(row.createdAt) ?? new Date().toISOString(), - inferResultKind(row, payload), - JSON.stringify(isResultPreview(row.preview) ? row.preview : buildPreview(payload)), - projectId ?? null, - projectKey, - positiveIntegerOrOne(row.schemaVersion), - estimateBytes(payload), - inferResultSource(row.source), - stringOrUndefined(row.traceId) ?? `trace-${resultRef}`, - filePath, - ], - ); - counters.resultRefs += 1; - } -} - -async function migrateMemories() { - await migrateMemoryScope("user", join(config.MEMORY_STORAGE_DIR, "users")); - await migrateMemoryScope("workspace", join(config.MEMORY_STORAGE_DIR, "workspaces")); -} - -async function migrateMemoryScope(scope: "user" | "workspace", dir: string) { - for (const filePath of await listFiles(dir, ".md")) { - const markdown = await readText(filePath); - if (!markdown) { - continue; - } - const scopeKey = basename(filePath, ".md"); - for (const entry of parseMemoryMarkdown(markdown)) { - await db.query( - ` - INSERT INTO ${db.table("memories")} ( - memory_id, - scope, - scope_key, - content, - source - ) - VALUES ($1, $2, $3, $4, 'tool') - ON CONFLICT (memory_id) - DO UPDATE SET content = EXCLUDED.content - `, - [entry.id, scope, scopeKey, entry.content], - ); - counters.memories += 1; - } - } -} - -async function upsertConversation(input: { - sessionId: string; - actorKey: string; - projectKey: string; - projectId?: string; - ownerUserId?: string; - parentSessionId?: string; - title?: string; - status?: "active" | "archived"; - createdAt?: string; - updatedAt?: string; -}) { - await db.query( - ` - INSERT INTO ${db.table("conversations")} ( - session_id, - actor_key, - owner_user_id, - project_id, - project_key, - parent_session_id, - title, - status, - created_at, - updated_at - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, COALESCE($9, NOW()), COALESCE($10, COALESCE($9, NOW()))) - ON CONFLICT (session_id) - DO UPDATE SET - actor_key = EXCLUDED.actor_key, - owner_user_id = COALESCE(EXCLUDED.owner_user_id, ${db.table("conversations")}.owner_user_id), - project_id = COALESCE(EXCLUDED.project_id, ${db.table("conversations")}.project_id), - project_key = EXCLUDED.project_key, - parent_session_id = COALESCE(EXCLUDED.parent_session_id, ${db.table("conversations")}.parent_session_id), - title = COALESCE(EXCLUDED.title, ${db.table("conversations")}.title), - status = EXCLUDED.status, - created_at = LEAST(${db.table("conversations")}.created_at, EXCLUDED.created_at), - updated_at = GREATEST(${db.table("conversations")}.updated_at, EXCLUDED.updated_at) - `, - [ - input.sessionId, - input.actorKey, - input.ownerUserId ?? null, - input.projectId ?? null, - input.projectKey, - input.parentSessionId ?? null, - input.title ?? null, - input.status ?? "active", - input.createdAt ?? null, - input.updatedAt ?? null, - ], - ); -} - -async function listFiles(dir: string, extension: string) { - try { - const names = await readdir(dir); - return names - .filter((name) => name.endsWith(extension)) - .map((name) => join(dir, name)); - } catch { - return []; - } -} - -async function readJson(path: string): Promise { - try { - return JSON.parse(await readFile(path, "utf8")) as JsonRecord; - } catch { - return null; - } -} - -async function readText(path: string): Promise { - try { - return await readFile(path, "utf8"); - } catch { - return null; - } -} - -function parseMemoryMarkdown(content: string) { - return content - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.startsWith("- ")) - .map((line) => line.slice(2).trim()) - .map((line) => { - const match = line.match(/^\[([a-z0-9]{8,})\]\s+(.*)$/i); - const entryContent = sanitizePersistentLine(match?.[2] ?? line, 240); - return { - id: match?.[1] ?? toStableId("memory-entry", entryContent.toLowerCase()), - content: entryContent, - }; - }) - .filter((entry) => entry.content); -} - -function inferResultKind(row: JsonRecord, payload: unknown) { - if (row.kind === RESULT_REFERENCE_KIND.renderJunctionsPayload) { - return RESULT_REFERENCE_KIND.renderJunctionsPayload; - } - if (isRecord(payload) && isRecord(payload.node_area_map)) { - return RESULT_REFERENCE_KIND.renderJunctionsPayload; - } - return RESULT_REFERENCE_KIND.dynamicHttpResult; -} - -function inferResultSource(value: unknown) { - return value === RESULT_REFERENCE_SOURCE.dynamicHttp || - value === RESULT_REFERENCE_SOURCE.agentGenerated || - value === RESULT_REFERENCE_SOURCE.migration - ? value - : RESULT_REFERENCE_SOURCE.legacy; -} - -function unwrapPayload(row: JsonRecord) { - if ("data" in row) { - return row.data; - } - return row; -} - -function isRecord(value: unknown): value is JsonRecord { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function isResultPreview(value: unknown): value is ResultPreview { - return ( - isRecord(value) && - typeof value.count === "number" && - Array.isArray(value.fields) && - typeof value.summary === "string" && - "sample" in value - ); -} - -function buildPreview(data: unknown): ResultPreview { - if (Array.isArray(data)) { - const sample = data.slice(0, config.MAX_PREVIEW_SAMPLE_ITEMS); - const fields = - sample.length > 0 && isRecord(sample[0]) - ? Object.keys(sample[0]).slice(0, 30) - : []; - return { - count: data.length, - fields, - sample, - summary: `list[${data.length}]`, - }; - } - if (isRecord(data)) { - const fields = Object.keys(data).slice(0, 30); - const sample = Object.fromEntries( - fields - .slice(0, config.MAX_PREVIEW_SAMPLE_ITEMS) - .map((field) => [field, data[field]]), - ); - return { - count: fields.length, - fields, - sample, - summary: `object<${fields.length} fields>`, - }; - } - return { - count: 1, - fields: [], - sample: String(data).slice(0, 300), - summary: `scalar<${typeof data}>`, - }; -} - -function estimateBytes(data: unknown) { - return Buffer.byteLength(JSON.stringify(data)); -} - -function normalizeResultRef(value?: string) { - return value && RESULT_REF_PATTERN.test(value) ? value : null; -} - -function stringOrUndefined(value: unknown) { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; -} - -function numberOrZero(value: unknown) { - return typeof value === "number" && Number.isFinite(value) ? value : 0; -} - -function positiveIntegerOrOne(value: unknown) { - return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 1; -} diff --git a/src/audit/llmRequestAudit.ts b/src/audit/llmRequestAudit.ts index 49a1fdb..be9307a 100644 --- a/src/audit/llmRequestAudit.ts +++ b/src/audit/llmRequestAudit.ts @@ -6,6 +6,7 @@ import { config } from "../config.js"; export type LlmRequestAuditEntry = { kind: "tool" | "skill"; sessionId: string; + clientSessionId: string; traceId?: string; projectId?: string; target: string; diff --git a/src/chat/sessionBridge.ts b/src/chat/sessionBridge.ts index d75636e..b59d32b 100644 --- a/src/chat/sessionBridge.ts +++ b/src/chat/sessionBridge.ts @@ -2,17 +2,20 @@ import { randomUUID } from "node:crypto"; import { logger } from "../logger.js"; import { type OpencodeRuntimeAdapter } from "../runtime/opencode.js"; -import { RuntimeSessionStore } from "../session/runtimeSessionStore.js"; +import { + buildToolSessionScopeKey, + ToolSessionContextStore, +} from "../session/toolContextStore.js"; import { toActorKey, toProjectKey } from "../utils/fileStore.js"; export type SessionBinding = { + clientSessionId: string; sessionId: string; - runtimeSessionId: string; startedAt: number; }; export type SessionContext = { - sessionId: string; + clientSessionId: string; accessToken?: string; projectId?: string; userId?: string; @@ -25,15 +28,15 @@ export type ChatRequestContext = SessionContext & { }; export class ChatSessionBridge { - // runtime session 仅在单次请求生命周期内有效；线程连续性由 sessionId 对应的持久状态承担。 + // runtime session 仅在单次请求生命周期内有效；线程连续性由 clientSessionId 对应的持久状态承担。 private readonly activeRuntimeSessions = new Map(); private readonly activeSensitiveContexts = new Map(); - private readonly runtimeSessionStore = new RuntimeSessionStore(); + private readonly toolContextStore = new ToolSessionContextStore(); constructor(private readonly runtime: OpencodeRuntimeAdapter) {} async resolve(context: { - sessionId?: string; + clientSessionId?: string; accessToken?: string; projectId?: string; traceId?: string; @@ -44,24 +47,30 @@ export class ChatSessionBridge { created: boolean; }> { const requestContext = this.buildRequestContext(context); - await this.abortActiveRuntime(requestContext.sessionId); + await this.abortActiveRuntime(requestContext.clientSessionId); - const session = await this.runtime.createSession(requestContext.sessionId); + const session = await this.runtime.createSession(requestContext.clientSessionId); const binding: SessionBinding = { - sessionId: requestContext.sessionId, - runtimeSessionId: session.id, + clientSessionId: requestContext.clientSessionId, + sessionId: session.id, startedAt: Date.now(), }; - this.activeRuntimeSessions.set(requestContext.sessionId, session.id); - this.activeSensitiveContexts.set(session.id, requestContext); - await this.runtimeSessionStore.write({ - runtimeSessionId: session.id, + const sessionScopeKey = buildToolSessionScopeKey( + requestContext.actorKey, + requestContext.projectKey, + requestContext.clientSessionId, + ); + this.activeRuntimeSessions.set(requestContext.clientSessionId, session.id); + this.activeSensitiveContexts.set(sessionScopeKey, requestContext); + await this.toolContextStore.write({ actorKey: requestContext.actorKey, allowLearningWrite: true, - sessionId: requestContext.sessionId, + clientSessionId: requestContext.clientSessionId, learningMode: "interactive", projectId: requestContext.projectId, projectKey: requestContext.projectKey, + sessionId: session.id, + sessionScopeKey, traceId: requestContext.traceId, }); @@ -72,59 +81,58 @@ export class ChatSessionBridge { return this.activeRuntimeSessions.size; } - createSessionId() { + createClientSessionId() { return `agent-${randomUUID().slice(0, 12)}`; } - getActiveSensitiveContext(runtimeSessionId: string) { - return this.activeSensitiveContexts.get(runtimeSessionId) ?? null; + getActiveSensitiveContext(sessionScopeKey: string) { + return this.activeSensitiveContexts.get(sessionScopeKey) ?? null; } - async abort(context: { sessionId?: string }): Promise { - const sessionId = context.sessionId?.trim(); + async abort(context: { + clientSessionId?: string; + }): Promise { + const clientSessionId = context.clientSessionId?.trim(); + if (!clientSessionId) { + return null; + } + + const sessionId = this.activeRuntimeSessions.get(clientSessionId); if (!sessionId) { return null; } - const runtimeSessionId = this.activeRuntimeSessions.get(sessionId); - if (!runtimeSessionId) { - return null; - } - - await this.abortActiveRuntime(sessionId); + await this.abortActiveRuntime(clientSessionId); return { + clientSessionId, sessionId, - runtimeSessionId, startedAt: Date.now(), }; } - async releaseRuntimeSession(sessionId: string, runtimeSessionId: string) { - const activeSessionId = this.activeRuntimeSessions.get(sessionId); - if (activeSessionId === runtimeSessionId) { - this.activeRuntimeSessions.delete(sessionId); + async releaseRuntimeSession(clientSessionId: string, sessionId: string) { + const activeSessionId = this.activeRuntimeSessions.get(clientSessionId); + if (activeSessionId === sessionId) { + this.activeRuntimeSessions.delete(clientSessionId); } - this.activeSensitiveContexts.delete(runtimeSessionId); - await this.runtimeSessionStore.release(runtimeSessionId).catch((error) => { - logger.debug( - { runtimeSessionId, err: error }, - "failed to cleanup persisted runtime session", - ); + this.activeSensitiveContexts.delete(findScopeKey(this.activeSensitiveContexts, clientSessionId)); + await this.toolContextStore.remove(sessionId).catch((error) => { + logger.debug({ sessionId, err: error }, "failed to cleanup runtime tool context"); }); - await this.runtime.abortSession(runtimeSessionId).catch((error) => { - logger.debug({ runtimeSessionId, err: error }, "failed to cleanup runtime session"); + await this.runtime.abortSession(sessionId).catch((error) => { + logger.debug({ sessionId, err: error }, "failed to cleanup runtime session"); }); } private buildRequestContext(context: { - sessionId?: string; + clientSessionId?: string; accessToken?: string; projectId?: string; traceId?: string; userId?: string; }): ChatRequestContext { return { - sessionId: context.sessionId?.trim() || this.createSessionId(), + clientSessionId: context.clientSessionId?.trim() || this.createClientSessionId(), accessToken: context.accessToken, actorKey: toActorKey(context.userId), projectId: context.projectId, @@ -134,25 +142,37 @@ export class ChatSessionBridge { }; } - private async abortActiveRuntime(sessionId: string) { - const activeRuntimeSessionId = this.activeRuntimeSessions.get(sessionId); - if (!activeRuntimeSessionId) { + private async abortActiveRuntime(clientSessionId: string) { + const activeSessionId = this.activeRuntimeSessions.get(clientSessionId); + if (!activeSessionId) { return; } - this.activeRuntimeSessions.delete(sessionId); - this.activeSensitiveContexts.delete(activeRuntimeSessionId); - await this.runtimeSessionStore.release(activeRuntimeSessionId).catch(() => undefined); - await this.runtime.abortSession(activeRuntimeSessionId).catch((error) => { + this.activeRuntimeSessions.delete(clientSessionId); + this.activeSensitiveContexts.delete(findScopeKey(this.activeSensitiveContexts, clientSessionId)); + await this.toolContextStore.remove(activeSessionId).catch(() => undefined); + await this.runtime.abortSession(activeSessionId).catch((error) => { logger.warn( - { sessionId, runtimeSessionId: activeRuntimeSessionId, err: error }, + { clientSessionId, sessionId: activeSessionId, err: error }, "failed to abort previous active runtime session", ); }); - await this.runtime.waitForSessionIdle(activeRuntimeSessionId).catch((error) => { + await this.runtime.waitForSessionIdle(activeSessionId).catch((error) => { logger.warn( - { sessionId, runtimeSessionId: activeRuntimeSessionId, err: error }, + { clientSessionId, sessionId: activeSessionId, err: error }, "failed while waiting for previous runtime session to become idle", ); }); } } + +const findScopeKey = ( + contexts: Map, + clientSessionId: string, +) => { + for (const [scopeKey, context] of contexts.entries()) { + if (context.clientSessionId === clientSessionId) { + return scopeKey; + } + } + return clientSessionId; +}; diff --git a/src/config.ts b/src/config.ts index bbe3852..30a4022 100644 --- a/src/config.ts +++ b/src/config.ts @@ -25,22 +25,6 @@ const envSchema = z PORT: z.coerce.number().int().positive().default(8787), // HTTP 服务监听地址。 HOST: z.string().default("0.0.0.0"), - // PostgreSQL connection string; if omitted, PG* env vars are used. - DATABASE_URL: optionalString(), - // PostgreSQL host. - PGHOST: optionalString(), - // PostgreSQL port. - PGPORT: z.coerce.number().int().positive().optional(), - // PostgreSQL user. - PGUSER: optionalString(), - // PostgreSQL password. - PGPASSWORD: optionalString(), - // PostgreSQL database name. - PGDATABASE: optionalString(), - // PostgreSQL SSL mode. - PGSSLMODE: z.enum(["disable", "prefer", "require"]).default("disable"), - // PostgreSQL schema used by TJWaterAgent. - AGENT_DB_SCHEMA: z.string().default("public"), // Pino 日志级别。 LOG_LEVEL: z.string().default("info"), // LLM 工具/技能调用审计日志路径。 @@ -132,16 +116,6 @@ const envSchema = z message: "OPENCODE_CLIENT_BASE_URL is required when OPENCODE_MODE=client", }); } - if ( - !env.DATABASE_URL && - (!env.PGHOST || !env.PGUSER || !env.PGPASSWORD || !env.PGDATABASE) - ) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ["DATABASE_URL"], - message: "DATABASE_URL or PGHOST/PGUSER/PGPASSWORD/PGDATABASE must be set", - }); - } }); export type AppConfig = z.infer; diff --git a/src/conversations/stateStore.ts b/src/conversations/stateStore.ts index 898d524..fddaf83 100644 --- a/src/conversations/stateStore.ts +++ b/src/conversations/stateStore.ts @@ -1,6 +1,12 @@ -import { type QueryResultRow } from "pg"; +import { join } from "node:path"; -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; +import { config } from "../config.js"; +import { + atomicWriteJson, + ensureDirectory, + readJsonFile, + removeFileIfExists, +} from "../utils/fileStore.js"; export type ConversationStateRecord = { sessionId: string; @@ -9,81 +15,27 @@ export type ConversationStateRecord = { branchGroups: unknown[]; }; -type ConversationStateRow = QueryResultRow & { - session_id: string; - is_title_manually_edited: boolean; - messages: unknown[]; - branch_groups: unknown[]; -}; - export class ConversationStateStore { - constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} + constructor(private readonly baseDir = config.CONVERSATION_STATE_STORAGE_DIR) {} async initialize() { - await this.db.initialize(); + await ensureDirectory(this.baseDir); } - async read(sessionId: string) { - const result = await this.db.query( - ` - SELECT session_id, is_title_manually_edited, messages, branch_groups - FROM ${this.db.table("conversation_states")} - WHERE session_id = $1 - LIMIT 1 - `, - [sessionId], - ); - return mapConversationStateRow(result.rows[0]); + async read(sessionScopeKey: string) { + return await readJsonFile(this.filePath(sessionScopeKey)); } - async write(sessionId: string, state: ConversationStateRecord) { - const result = await this.db.query( - ` - INSERT INTO ${this.db.table("conversation_states")} ( - session_id, - is_title_manually_edited, - messages, - branch_groups - ) - VALUES ($1, $2, $3::jsonb, $4::jsonb) - ON CONFLICT (session_id) - DO UPDATE SET - is_title_manually_edited = EXCLUDED.is_title_manually_edited, - messages = EXCLUDED.messages, - branch_groups = EXCLUDED.branch_groups - RETURNING session_id, is_title_manually_edited, messages, branch_groups - `, - [ - sessionId, - state.isTitleManuallyEdited ?? false, - JSON.stringify(state.messages), - JSON.stringify(state.branchGroups), - ], - ); - return mapConversationStateRow(result.rows[0]) ?? state; + async write(sessionScopeKey: string, state: ConversationStateRecord) { + await atomicWriteJson(this.filePath(sessionScopeKey), state); + return state; } - async remove(sessionId: string) { - await this.db.query( - ` - DELETE FROM ${this.db.table("conversation_states")} - WHERE session_id = $1 - `, - [sessionId], - ); + async remove(sessionScopeKey: string) { + await removeFileIfExists(this.filePath(sessionScopeKey)); + } + + private filePath(sessionScopeKey: string) { + return join(this.baseDir, `${sessionScopeKey}.json`); } } - -const mapConversationStateRow = ( - row?: ConversationStateRow | null, -): ConversationStateRecord | null => { - if (!row) { - return null; - } - return { - sessionId: row.session_id, - isTitleManuallyEdited: row.is_title_manually_edited, - messages: Array.isArray(row.messages) ? row.messages : [], - branchGroups: Array.isArray(row.branch_groups) ? row.branch_groups : [], - }; -}; diff --git a/src/conversations/store.ts b/src/conversations/store.ts index 89674a7..2e5f3c8 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -1,13 +1,21 @@ import { randomUUID } from "node:crypto"; +import { join } from "node:path"; -import { type QueryResultRow } from "pg"; - -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; +import { config } from "../config.js"; +import { + atomicWriteJson, + ensureDirectory, + listJsonFiles, + readJsonFile, + removeFileIfExists, +} from "../utils/fileStore.js"; +import { toConversationScopeKey } from "../utils/fileStore.js"; export type ConversationStatus = "active" | "archived"; export type ConversationRecord = { sessionId: string; + sessionScopeKey: string; actorKey: string; ownerUserId?: string; projectId?: string; @@ -15,8 +23,6 @@ export type ConversationRecord = { parentSessionId?: string; createdAt: string; updatedAt: string; - isStreaming: boolean; - streamingStartedAt?: string; status: ConversationStatus; title?: string; }; @@ -33,59 +39,40 @@ type EnsureConversationInput = ConversationContext & { parentSessionId?: string; }; -type ConversationRow = QueryResultRow & { - session_id: string; - actor_key: string; - owner_user_id: string | null; - project_id: string | null; - project_key: string; - parent_session_id: string | null; - created_at: Date | string; - updated_at: Date | string; - is_streaming: boolean; - streaming_started_at: Date | string | null; - status: ConversationStatus; - title: string | null; -}; - export class ConversationStore { - constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} + constructor(private readonly baseDir = config.CONVERSATION_STORAGE_DIR) {} async initialize() { - await this.db.initialize(); + await ensureDirectory(this.baseDir); } async ensure(input: EnsureConversationInput) { const sessionId = normalizeSessionId(input.sessionId) ?? createConversationSessionId(); - const existing = await this.get(input, sessionId); + const sessionScopeKey = toConversationScopeKey( + input.actorKey, + input.projectKey, + sessionId, + ); + const existing = await readJsonFile(this.filePath(sessionScopeKey)); if (existing) { return { created: false, record: existing }; } - const inserted = await this.db.query( - ` - INSERT INTO ${this.db.table("conversations")} ( - session_id, - actor_key, - owner_user_id, - project_id, - project_key, - parent_session_id, - status - ) - VALUES ($1, $2, $3, $4, $5, $6, 'active') - RETURNING * - `, - [ - sessionId, - input.actorKey, - input.userId?.trim() || null, - input.projectId ?? null, - input.projectKey, - normalizeSessionId(input.parentSessionId) ?? null, - ], - ); - return { created: true, record: requireConversationRow(inserted.rows[0]) }; + const now = new Date().toISOString(); + const record: ConversationRecord = { + sessionId, + sessionScopeKey, + actorKey: input.actorKey, + ownerUserId: input.userId?.trim(), + projectId: input.projectId, + projectKey: input.projectKey, + parentSessionId: normalizeSessionId(input.parentSessionId), + createdAt: now, + updatedAt: now, + status: "active", + }; + await atomicWriteJson(this.filePath(sessionScopeKey), record); + return { created: true, record }; } async get(context: ConversationContext, sessionId: string) { @@ -93,111 +80,47 @@ export class ConversationStore { if (!normalizedSessionId) { return null; } - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("conversations")} - WHERE session_id = $1 - AND actor_key = $2 - AND project_key = $3 - LIMIT 1 - `, - [normalizedSessionId, context.actorKey, context.projectKey], + return await readJsonFile( + this.filePath( + toConversationScopeKey(context.actorKey, context.projectKey, normalizedSessionId), + ), ); - return mapConversationRow(result.rows[0]); } async touch( record: ConversationRecord, updates: Partial> = {}, ) { - const normalized = normalizeConversationUpdates(updates); - const result = await this.db.query( - ` - UPDATE ${this.db.table("conversations")} - SET - title = COALESCE($2, title), - status = COALESCE($3, status) - WHERE session_id = $1 - RETURNING * - `, - [record.sessionId, normalized.title ?? null, normalized.status ?? null], - ); - return requireConversationRow(result.rows[0]); + const next: ConversationRecord = { + ...record, + ...normalizeConversationUpdates(updates), + updatedAt: new Date().toISOString(), + }; + await atomicWriteJson(this.filePath(record.sessionScopeKey), next); + return next; } async list(context: ConversationContext) { - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("conversations")} - WHERE actor_key = $1 - AND project_key = $2 - ORDER BY updated_at DESC - `, - [context.actorKey, context.projectKey], + const files = await listJsonFiles(this.baseDir); + const records = await Promise.all( + files.map((file) => readJsonFile(file)), ); - return result.rows - .map(mapConversationRow) - .filter((record): record is ConversationRecord => Boolean(record)); + return records + .filter((record): record is ConversationRecord => Boolean(record)) + .filter( + (record) => + record.actorKey === context.actorKey && + record.projectKey === context.projectKey, + ) + .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); } async remove(record: ConversationRecord) { - await this.db.query( - ` - DELETE FROM ${this.db.table("conversations")} - WHERE session_id = $1 - `, - [record.sessionId], - ); + await removeFileIfExists(this.filePath(record.sessionScopeKey)); } - async markStreaming(record: ConversationRecord, runtimeSessionId: string) { - const result = await this.db.query( - ` - UPDATE ${this.db.table("conversations")} - SET - is_streaming = TRUE, - active_runtime_session_id = $2, - streaming_started_at = NOW() - WHERE session_id = $1 - RETURNING * - `, - [record.sessionId, runtimeSessionId], - ); - return requireConversationRow(result.rows[0]); - } - - async clearStreaming(sessionId: string, runtimeSessionId: string) { - const result = await this.db.query( - ` - UPDATE ${this.db.table("conversations")} - SET - is_streaming = FALSE, - active_runtime_session_id = NULL, - streaming_started_at = NULL - WHERE session_id = $1 - AND active_runtime_session_id = $2 - RETURNING * - `, - [sessionId, runtimeSessionId], - ); - return mapConversationRow(result.rows[0]); - } - - async resetStreamingSessions() { - await this.db.query( - ` - UPDATE ${this.db.table("conversations")} - SET - is_streaming = FALSE, - active_runtime_session_id = NULL, - streaming_started_at = NULL - WHERE is_streaming = TRUE - OR active_runtime_session_id IS NOT NULL - OR streaming_started_at IS NOT NULL - `, - ); + private filePath(sessionScopeKey: string) { + return join(this.baseDir, `${sessionScopeKey}.json`); } } @@ -223,34 +146,3 @@ const normalizeConversationUpdates = ( } return normalized; }; - -const mapConversationRow = (row?: ConversationRow | null): ConversationRecord | null => { - if (!row) { - return null; - } - return { - sessionId: row.session_id, - actorKey: row.actor_key, - ownerUserId: row.owner_user_id ?? undefined, - projectId: row.project_id ?? undefined, - projectKey: row.project_key, - parentSessionId: row.parent_session_id ?? undefined, - createdAt: toIsoString(row.created_at), - updatedAt: toIsoString(row.updated_at), - isStreaming: row.is_streaming, - streamingStartedAt: row.streaming_started_at ? toIsoString(row.streaming_started_at) : undefined, - status: row.status, - title: row.title ?? undefined, - }; -}; - -const requireConversationRow = (row?: ConversationRow | null) => { - const record = mapConversationRow(row); - if (!record) { - throw new Error("conversation row not found"); - } - return record; -}; - -const toIsoString = (value: Date | string) => - value instanceof Date ? value.toISOString() : new Date(value).toISOString(); diff --git a/src/db/index.ts b/src/db/index.ts deleted file mode 100644 index 3721922..0000000 --- a/src/db/index.ts +++ /dev/null @@ -1,486 +0,0 @@ -import { Pool, type PoolClient, type QueryResultRow } from "pg"; - -import { config } from "../config.js"; - -const IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; - -export class AgentDatabase { - private readonly pool: Pool; - private readonly schemaName: string; - private initialized = false; - private initializePromise: Promise | null = null; - - constructor(options?: { - connectionString?: string; - host?: string; - port?: number; - user?: string; - password?: string; - database?: string; - sslmode?: "disable" | "prefer" | "require"; - schema?: string; - }) { - const schema = options?.schema ?? config.AGENT_DB_SCHEMA; - if (!IDENTIFIER_PATTERN.test(schema)) { - throw new Error(`invalid PostgreSQL schema name: ${schema}`); - } - this.schemaName = schema; - this.pool = new Pool({ - ...(options?.connectionString ?? config.DATABASE_URL - ? { - connectionString: options?.connectionString ?? config.DATABASE_URL, - } - : { - host: options?.host ?? config.PGHOST, - port: options?.port ?? config.PGPORT ?? 5432, - user: options?.user ?? config.PGUSER, - password: options?.password ?? config.PGPASSWORD, - database: options?.database ?? config.PGDATABASE, - }), - ssl: - (options?.sslmode ?? config.PGSSLMODE) === "require" - ? { rejectUnauthorized: false } - : undefined, - }); - } - - get schema() { - return this.schemaName; - } - - table(name: string) { - if (!IDENTIFIER_PATTERN.test(name)) { - throw new Error(`invalid PostgreSQL table name: ${name}`); - } - return `"${this.schemaName}"."${name}"`; - } - - async initialize() { - if (this.initialized) { - return; - } - if (!this.initializePromise) { - this.initializePromise = (async () => { - await this.query(`CREATE SCHEMA IF NOT EXISTS "${this.schemaName}"`); - await this.query(buildSchemaSql(this.schemaName)); - this.initialized = true; - })().catch((error) => { - this.initializePromise = null; - throw error; - }); - } - await this.initializePromise; - } - - async query( - text: string, - values?: unknown[], - client?: PoolClient, - ) { - const executor = client ?? this.pool; - return await executor.query(text, values); - } - - async withTransaction(task: (client: PoolClient) => Promise) { - const client = await this.pool.connect(); - try { - await client.query("BEGIN"); - const result = await task(client); - await client.query("COMMIT"); - return result; - } catch (error) { - await client.query("ROLLBACK"); - throw error; - } finally { - client.release(); - } - } - - async close() { - await this.pool.end(); - } -} - -let defaultDatabase: AgentDatabase | undefined; - -export const getAgentDatabase = () => { - if (!defaultDatabase) { - defaultDatabase = new AgentDatabase(); - } - return defaultDatabase; -}; - -const buildSchemaSql = (schema: string) => ` -CREATE OR REPLACE FUNCTION "${schema}".set_updated_at() -RETURNS trigger AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TABLE IF NOT EXISTS "${schema}"."conversations" ( - session_id TEXT PRIMARY KEY, - actor_key TEXT NOT NULL, - owner_user_id TEXT, - project_id TEXT, - project_key TEXT NOT NULL, - parent_session_id TEXT REFERENCES "${schema}"."conversations"(session_id) ON DELETE SET NULL, - title VARCHAR(120), - is_streaming BOOLEAN NOT NULL DEFAULT FALSE, - active_runtime_session_id TEXT, - streaming_started_at TIMESTAMPTZ, - status TEXT NOT NULL CHECK (status IN ('active', 'archived')), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -ALTER TABLE "${schema}"."conversations" - ADD COLUMN IF NOT EXISTS is_streaming BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS active_runtime_session_id TEXT, - ADD COLUMN IF NOT EXISTS streaming_started_at TIMESTAMPTZ; - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'conversations' - AND column_name = 'session_scope_key' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."conversations" DROP COLUMN session_scope_key'; - END IF; -END $$; - -CREATE INDEX IF NOT EXISTS idx_conversations_actor_project_updated - ON "${schema}"."conversations" (actor_key, project_key, updated_at DESC); -CREATE INDEX IF NOT EXISTS idx_conversations_parent_session - ON "${schema}"."conversations" (parent_session_id); - -CREATE TABLE IF NOT EXISTS "${schema}"."conversation_states" ( - session_id TEXT PRIMARY KEY REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, - is_title_manually_edited BOOLEAN NOT NULL DEFAULT FALSE, - messages JSONB NOT NULL DEFAULT '[]'::jsonb, - branch_groups JSONB NOT NULL DEFAULT '[]'::jsonb, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -ALTER TABLE "${schema}"."conversation_states" - ADD COLUMN IF NOT EXISTS is_title_manually_edited BOOLEAN NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS messages JSONB NOT NULL DEFAULT '[]'::jsonb, - ADD COLUMN IF NOT EXISTS branch_groups JSONB NOT NULL DEFAULT '[]'::jsonb, - ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'conversation_states' - AND column_name = 'session_scope_key' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."conversation_states" DROP COLUMN session_scope_key'; - END IF; -END $$; - -CREATE TABLE IF NOT EXISTS "${schema}"."conversation_turns" ( - turn_id TEXT PRIMARY KEY, - session_id TEXT NOT NULL REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, - turn_index INTEGER NOT NULL CHECK (turn_index >= 0), - actor_key TEXT NOT NULL, - project_key TEXT NOT NULL, - user_message TEXT NOT NULL, - assistant_message TEXT NOT NULL, - tool_call_count INTEGER NOT NULL DEFAULT 0 CHECK (tool_call_count >= 0), - turn_timestamp TIMESTAMPTZ NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - UNIQUE (session_id, turn_index) -); - -ALTER TABLE "${schema}"."conversation_turns" - ADD COLUMN IF NOT EXISTS turn_index INTEGER; - -WITH ranked_turns AS ( - SELECT - turn_id, - ROW_NUMBER() OVER ( - PARTITION BY session_id - ORDER BY turn_timestamp ASC, turn_id ASC - ) - 1 AS computed_turn_index - FROM "${schema}"."conversation_turns" - WHERE turn_index IS NULL -) -UPDATE "${schema}"."conversation_turns" turns -SET turn_index = ranked_turns.computed_turn_index -FROM ranked_turns -WHERE turns.turn_id = ranked_turns.turn_id; - -ALTER TABLE "${schema}"."conversation_turns" - ALTER COLUMN turn_index SET NOT NULL; - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'conversation_turns' - AND column_name = 'client_session_id' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."conversation_turns" DROP COLUMN client_session_id'; - END IF; -END $$; - -CREATE INDEX IF NOT EXISTS idx_conversation_turns_session_time - ON "${schema}"."conversation_turns" (session_id, turn_index DESC); -CREATE INDEX IF NOT EXISTS idx_conversation_turns_actor_project_time - ON "${schema}"."conversation_turns" (actor_key, project_key, turn_timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_conversation_turns_search - ON "${schema}"."conversation_turns" - USING GIN (to_tsvector('simple', coalesce(user_message, '') || ' ' || coalesce(assistant_message, ''))); - -CREATE TABLE IF NOT EXISTS "${schema}"."runtime_sessions" ( - runtime_session_id TEXT PRIMARY KEY, - session_id TEXT NOT NULL REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, - actor_key TEXT NOT NULL, - allow_learning_write BOOLEAN NOT NULL DEFAULT FALSE, - learning_mode TEXT CHECK (learning_mode IN ('interactive', 'review') OR learning_mode IS NULL), - project_id TEXT, - project_key TEXT NOT NULL, - trace_id TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - released_at TIMESTAMPTZ -); - -ALTER TABLE "${schema}"."runtime_sessions" - ADD COLUMN IF NOT EXISTS session_id TEXT; - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'runtime_sessions' - AND column_name = 'conversation_id' - ) THEN - EXECUTE 'UPDATE "${schema}"."runtime_sessions" SET session_id = COALESCE(session_id, conversation_id) WHERE session_id IS NULL'; - END IF; -END $$; - -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_constraint - WHERE conname = 'runtime_sessions_session_id_fkey' - AND connamespace = to_regnamespace('${schema}') - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."runtime_sessions" ADD CONSTRAINT runtime_sessions_session_id_fkey FOREIGN KEY (session_id) REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE'; - END IF; -END $$; - -ALTER TABLE "${schema}"."runtime_sessions" - ALTER COLUMN session_id SET NOT NULL; - -DO $$ -BEGIN - IF to_regclass('"${schema}"."tool_session_contexts"') IS NOT NULL THEN - INSERT INTO "${schema}"."runtime_sessions" ( - runtime_session_id, - session_id, - actor_key, - allow_learning_write, - learning_mode, - project_id, - project_key, - trace_id, - released_at - ) - SELECT - runtime_session_id, - client_session_id, - actor_key, - allow_learning_write, - learning_mode, - project_id, - project_key, - trace_id, - NOW() - FROM "${schema}"."tool_session_contexts" - WHERE client_session_id IS NOT NULL - ON CONFLICT (runtime_session_id) DO NOTHING; - END IF; -END $$; - -DROP INDEX IF EXISTS "${schema}"."idx_runtime_sessions_conversation"; -CREATE INDEX IF NOT EXISTS idx_runtime_sessions_session - ON "${schema}"."runtime_sessions" (session_id); -CREATE INDEX IF NOT EXISTS idx_runtime_sessions_trace_id - ON "${schema}"."runtime_sessions" (trace_id); - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'runtime_sessions' - AND column_name = 'conversation_id' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."runtime_sessions" DROP COLUMN conversation_id'; - END IF; -END $$; - -DROP TABLE IF EXISTS "${schema}"."tool_session_contexts"; - -CREATE TABLE IF NOT EXISTS "${schema}"."learning_states" ( - session_id TEXT PRIMARY KEY REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, - last_gated_turn INTEGER NOT NULL DEFAULT 0 CHECK (last_gated_turn >= 0), - last_reviewed_turn INTEGER NOT NULL DEFAULT 0 CHECK (last_reviewed_turn >= 0), - pending_review BOOLEAN NOT NULL DEFAULT FALSE, - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE IF NOT EXISTS "${schema}"."result_refs" ( - result_ref TEXT PRIMARY KEY, - actor_key TEXT NOT NULL, - session_id TEXT NOT NULL REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, - created_at TIMESTAMPTZ NOT NULL, - kind TEXT NOT NULL CHECK (kind IN ('dynamic-http-result', 'render-junctions-payload')), - preview JSONB NOT NULL, - project_id TEXT, - project_key TEXT NOT NULL, - schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version > 0), - size_bytes INTEGER NOT NULL CHECK (size_bytes >= 0), - source TEXT NOT NULL CHECK (source IN ('dynamic_http', 'agent_generated', 'legacy', 'migration')), - trace_id TEXT NOT NULL, - payload_path TEXT, - object_key TEXT, - CONSTRAINT chk_result_refs_payload_locator CHECK ( - num_nonnulls(payload_path, object_key) = 1 - ) -); - -ALTER TABLE "${schema}"."result_refs" - ADD COLUMN IF NOT EXISTS session_id TEXT, - ADD COLUMN IF NOT EXISTS payload_path TEXT, - ADD COLUMN IF NOT EXISTS object_key TEXT; - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'result_refs' - AND column_name = 'conversation_id' - ) THEN - EXECUTE 'UPDATE "${schema}"."result_refs" SET session_id = COALESCE(session_id, conversation_id) WHERE session_id IS NULL'; - END IF; - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'result_refs' - AND column_name = 'client_session_id' - ) THEN - EXECUTE 'UPDATE "${schema}"."result_refs" SET session_id = COALESCE(session_id, client_session_id) WHERE session_id IS NULL'; - END IF; -END $$; - -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_constraint - WHERE conname = 'result_refs_session_id_fkey' - AND connamespace = to_regnamespace('${schema}') - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."result_refs" ADD CONSTRAINT result_refs_session_id_fkey FOREIGN KEY (session_id) REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE'; - END IF; - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'result_refs' - AND column_name = 'client_session_id' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."result_refs" DROP COLUMN client_session_id'; - END IF; - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'result_refs' - AND column_name = 'data' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."result_refs" ALTER COLUMN data DROP NOT NULL'; - END IF; - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = '${schema}' - AND table_name = 'result_refs' - AND column_name = 'conversation_id' - ) THEN - EXECUTE 'ALTER TABLE "${schema}"."result_refs" DROP COLUMN conversation_id'; - END IF; -END $$; - -ALTER TABLE "${schema}"."result_refs" - ALTER COLUMN session_id SET NOT NULL; - -CREATE INDEX IF NOT EXISTS idx_result_refs_session_created - ON "${schema}"."result_refs" (session_id, created_at DESC); -CREATE INDEX IF NOT EXISTS idx_result_refs_actor_project_created - ON "${schema}"."result_refs" (actor_key, project_key, created_at DESC); -CREATE INDEX IF NOT EXISTS idx_result_refs_trace_id - ON "${schema}"."result_refs" (trace_id); - -CREATE TABLE IF NOT EXISTS "${schema}"."memories" ( - memory_id TEXT PRIMARY KEY, - scope TEXT NOT NULL CHECK (scope IN ('user', 'workspace')), - scope_key TEXT NOT NULL, - content TEXT NOT NULL, - source TEXT NOT NULL CHECK (source IN ('review', 'tool')), - session_id TEXT REFERENCES "${schema}"."conversations"(session_id) ON DELETE SET NULL, - trace_id TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - UNIQUE (scope, scope_key, content) -); - -CREATE INDEX IF NOT EXISTS idx_memories_scope_key - ON "${schema}"."memories" (scope, scope_key, updated_at DESC); - -DROP TRIGGER IF EXISTS trg_conversations_set_updated_at ON "${schema}"."conversations"; -CREATE TRIGGER trg_conversations_set_updated_at -BEFORE UPDATE ON "${schema}"."conversations" -FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); - -DROP TRIGGER IF EXISTS trg_conversation_states_set_updated_at ON "${schema}"."conversation_states"; -CREATE TRIGGER trg_conversation_states_set_updated_at -BEFORE UPDATE ON "${schema}"."conversation_states" -FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); - -DROP TRIGGER IF EXISTS trg_runtime_sessions_set_updated_at ON "${schema}"."runtime_sessions"; -CREATE TRIGGER trg_runtime_sessions_set_updated_at -BEFORE UPDATE ON "${schema}"."runtime_sessions" -FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); - -DROP TRIGGER IF EXISTS trg_learning_states_set_updated_at ON "${schema}"."learning_states"; -CREATE TRIGGER trg_learning_states_set_updated_at -BEFORE UPDATE ON "${schema}"."learning_states" -FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); - -DROP TRIGGER IF EXISTS trg_memories_set_updated_at ON "${schema}"."memories"; -CREATE TRIGGER trg_memories_set_updated_at -BEFORE UPDATE ON "${schema}"."memories" -FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); -`; diff --git a/src/history/store.ts b/src/history/store.ts index 180cf5e..11244b0 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -1,9 +1,14 @@ -import { type QueryResultRow } from "pg"; +import { join } from "node:path"; import { config } from "../config.js"; -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; +import { + atomicWriteJson, + ensureDirectory, + listJsonFiles, + readJsonFile, + toStableId, +} from "../utils/fileStore.js"; import { sanitizePersistentDocument } from "../utils/persistencePolicy.js"; -import { toStableId } from "../utils/fileStore.js"; export type SessionTurnRecord = { id: string; @@ -15,6 +20,7 @@ export type SessionTurnRecord = { type SessionTranscriptRecord = { actorKey: string; + clientSessionId?: string; projectKey: string; sessionId: string; turns: SessionTurnRecord[]; @@ -32,28 +38,18 @@ export type SessionSearchHit = { type SessionHistoryContext = { actorKey: string; + clientSessionId?: string; projectKey: string; sessionId: string; }; -type ConversationTurnRow = QueryResultRow & { - turn_id: string; - session_id: string; - turn_index: number; - actor_key: string; - project_key: string; - user_message: string; - assistant_message: string; - tool_call_count: number; - turn_timestamp: Date | string; - created_at: Date | string; -}; - export class SessionHistoryStore { - constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} + private readonly writeQueues = new Map>(); + + constructor(private readonly baseDir = config.SESSION_HISTORY_STORAGE_DIR) {} async initialize() { - await this.db.initialize(); + await ensureDirectory(this.baseDir); } async appendTurn( @@ -63,146 +59,73 @@ export class SessionHistoryStore { toolCallCount: number; userMessage: string; }, - ): Promise { - const userMessage = sanitizePersistentDocument(turn.userMessage, 4000); - const assistantMessage = sanitizePersistentDocument(turn.assistantMessage, 4000); - if (!userMessage || !assistantMessage) { - return (await this.readTranscript(context)) ?? emptyTranscript(context); - } + ) { + const key = this.filePath(context); + return this.serializeWrite(key, async () => { + const transcript = (await this.readTranscript(context)) ?? { + actorKey: context.actorKey, + clientSessionId: context.clientSessionId, + projectKey: context.projectKey, + sessionId: context.sessionId, + turns: [], + updatedAt: new Date().toISOString(), + }; + const userMessage = sanitizePersistentDocument(turn.userMessage, 4000); + const assistantMessage = sanitizePersistentDocument(turn.assistantMessage, 4000); + if (!userMessage || !assistantMessage) { + return transcript; + } - const timestamp = new Date().toISOString(); - await this.db.withTransaction(async (client) => { - await this.db.query("SELECT pg_advisory_xact_lock(hashtext($1))", [context.sessionId], client); - const nextIndexResult = await this.db.query<{ next_index: number }>( - ` - SELECT COALESCE(MAX(turn_index), -1) + 1 AS next_index - FROM ${this.db.table("conversation_turns")} - WHERE session_id = $1 - `, - [context.sessionId], - client, - ); - const nextIndex = nextIndexResult.rows[0]?.next_index ?? 0; - await this.db.query( - ` - INSERT INTO ${this.db.table("conversation_turns")} ( - turn_id, - session_id, - turn_index, - actor_key, - project_key, - user_message, - assistant_message, - tool_call_count, - turn_timestamp - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - `, - [ - toStableId(context.sessionId, String(nextIndex), timestamp, userMessage, assistantMessage), - context.sessionId, - nextIndex, - context.actorKey, - context.projectKey, - userMessage, - assistantMessage, - Math.max(0, turn.toolCallCount), - timestamp, - ], - client, - ); + const timestamp = new Date().toISOString(); + const record: SessionTurnRecord = { + id: toStableId(context.sessionId, timestamp, userMessage, assistantMessage), + assistantMessage, + timestamp, + toolCallCount: Math.max(0, turn.toolCallCount), + userMessage, + }; + transcript.clientSessionId = context.clientSessionId ?? transcript.clientSessionId; + transcript.sessionId = context.sessionId; + transcript.turns.push(record); + if (transcript.turns.length > config.SESSION_HISTORY_MAX_TURNS_PER_SESSION) { + transcript.turns = transcript.turns.slice( + transcript.turns.length - config.SESSION_HISTORY_MAX_TURNS_PER_SESSION, + ); + } + transcript.updatedAt = timestamp; + await atomicWriteJson(key, transcript); + return transcript; }); - return (await this.readTranscript(context)) ?? emptyTranscript(context); } async getRecentTurns( context: SessionHistoryContext, limit: number, ): Promise { - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("conversation_turns")} - WHERE session_id = $1 - AND actor_key = $2 - AND project_key = $3 - ORDER BY turn_index DESC - LIMIT $4 - `, - [context.sessionId, context.actorKey, context.projectKey, Math.max(1, limit)], - ); - return result.rows - .slice() - .reverse() - .map(mapTurnRow); + const transcript = await this.readTranscript(context); + if (!transcript) { + return []; + } + return transcript.turns.slice(-Math.max(1, limit)); } async cloneThread( sourceContext: SessionHistoryContext, targetContext: SessionHistoryContext, keepMessageCount: number, - ): Promise { - const keepTurnCount = Math.floor(keepMessageCount / 2); - const sourceTurns = keepTurnCount - ? await this.db.query( - ` - SELECT * - FROM ${this.db.table("conversation_turns")} - WHERE session_id = $1 - AND actor_key = $2 - AND project_key = $3 - ORDER BY turn_index ASC - LIMIT $4 - `, - [ - sourceContext.sessionId, - sourceContext.actorKey, - sourceContext.projectKey, - keepTurnCount, - ], - ) - : { rows: [] as ConversationTurnRow[] }; - - await this.db.withTransaction(async (client) => { - for (const [turnIndex, row] of sourceTurns.rows.entries()) { - await this.db.query( - ` - INSERT INTO ${this.db.table("conversation_turns")} ( - turn_id, - session_id, - turn_index, - actor_key, - project_key, - user_message, - assistant_message, - tool_call_count, - turn_timestamp - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - `, - [ - toStableId( - targetContext.sessionId, - String(turnIndex), - toIsoString(row.turn_timestamp), - row.user_message, - row.assistant_message, - ), - targetContext.sessionId, - turnIndex, - targetContext.actorKey, - targetContext.projectKey, - row.user_message, - row.assistant_message, - row.tool_call_count, - row.turn_timestamp, - ], - client, - ); - } - }); - - return (await this.readTranscript(targetContext)) ?? emptyTranscript(targetContext); + ) { + const sourceTranscript = await this.readTranscript(sourceContext); + const timestamp = new Date().toISOString(); + const nextTranscript: SessionTranscriptRecord = { + actorKey: targetContext.actorKey, + clientSessionId: targetContext.clientSessionId, + projectKey: targetContext.projectKey, + sessionId: targetContext.sessionId, + turns: projectTurnsForFork(sourceTranscript?.turns ?? [], keepMessageCount), + updatedAt: timestamp, + }; + await atomicWriteJson(this.filePath(targetContext), nextTranscript); + return nextTranscript; } async search( @@ -214,86 +137,104 @@ export class SessionHistoryStore { if (!normalizedQuery) { return []; } - const rows = await this.db.query( - ` - SELECT * - FROM ${this.db.table("conversation_turns")} - WHERE actor_key = $1 - AND project_key = $2 - AND ( - LOWER(user_message) LIKE $3 - OR LOWER(assistant_message) LIKE $3 - ) - ORDER BY turn_timestamp DESC - `, - [context.actorKey, context.projectKey, `%${normalizedQuery}%`], - ); const queryTokens = normalizedQuery.split(/\s+/).filter(Boolean); const hits: SessionSearchHit[] = []; - for (const row of rows.rows) { - const candidates: Array<["user" | "assistant", string]> = [ - ["user", row.user_message], - ["assistant", row.assistant_message], - ]; - for (const [matchedField, text] of candidates) { - const score = scoreText(text, normalizedQuery, queryTokens); - if (score <= 0) { - continue; + const files = await listJsonFiles(this.baseDir); + for (const file of files) { + const transcript = await readJsonFile(file); + if (!transcript) { + continue; + } + if ( + transcript.actorKey !== context.actorKey || + transcript.projectKey !== context.projectKey + ) { + continue; + } + for (const turn of transcript.turns) { + const candidates: Array<["user" | "assistant", string]> = [ + ["user", turn.userMessage], + ["assistant", turn.assistantMessage], + ]; + for (const [matchedField, text] of candidates) { + const score = scoreText(text, normalizedQuery, queryTokens); + if (score <= 0) { + continue; + } + hits.push({ + matchedField, + score, + sessionId: transcript.sessionId, + snippet: buildSnippet(text, normalizedQuery), + timestamp: turn.timestamp, + turnId: turn.id, + }); } - hits.push({ - matchedField, - score, - sessionId: row.session_id, - snippet: buildSnippet(text, normalizedQuery), - timestamp: toIsoString(row.turn_timestamp), - turnId: row.turn_id, - }); } } return hits.sort((a, b) => b.score - a.score).slice(0, Math.max(1, maxResults)); } private async readTranscript(context: SessionHistoryContext) { - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("conversation_turns")} - WHERE session_id = $1 - AND actor_key = $2 - AND project_key = $3 - ORDER BY turn_index ASC - `, - [context.sessionId, context.actorKey, context.projectKey], - ); - if (result.rows.length === 0) { + const direct = await readJsonFile(this.filePath(context)); + if (direct) { + return direct; + } + + const clientSessionId = context.clientSessionId?.trim(); + if (!clientSessionId) { return null; } - return { - actorKey: context.actorKey, - projectKey: context.projectKey, - sessionId: context.sessionId, - turns: result.rows.map(mapTurnRow), - updatedAt: toIsoString(result.rows[result.rows.length - 1]?.turn_timestamp ?? new Date()), - }; + + const files = await listJsonFiles(this.baseDir); + const matches: SessionTranscriptRecord[] = []; + for (const file of files) { + const transcript = await readJsonFile(file); + if (!transcript) { + continue; + } + if ( + transcript.actorKey !== context.actorKey || + transcript.projectKey !== context.projectKey || + transcript.clientSessionId !== clientSessionId + ) { + continue; + } + matches.push(transcript); + } + + if (matches.length === 0) { + return null; + } + + return matches.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt))[0] ?? null; + } + + private filePath(context: SessionHistoryContext) { + return join( + this.baseDir, + `${context.actorKey}__${context.projectKey}__${context.sessionId}.json`, + ); + } + + private async serializeWrite(key: string, task: () => Promise) { + const previous = this.writeQueues.get(key) ?? Promise.resolve(); + const run = previous.catch(() => undefined).then(task); + const next = run.then( + () => undefined, + () => undefined, + ); + this.writeQueues.set(key, next); + try { + return await run; + } finally { + if (this.writeQueues.get(key) === next) { + this.writeQueues.delete(key); + } + } } } -const emptyTranscript = (context: SessionHistoryContext): SessionTranscriptRecord => ({ - actorKey: context.actorKey, - projectKey: context.projectKey, - sessionId: context.sessionId, - turns: [], - updatedAt: new Date().toISOString(), -}); - -const mapTurnRow = (row: ConversationTurnRow): SessionTurnRecord => ({ - id: row.turn_id, - assistantMessage: row.assistant_message, - timestamp: toIsoString(row.turn_timestamp), - toolCallCount: row.tool_call_count, - userMessage: row.user_message, -}); - const scoreText = (text: string, query: string, queryTokens: string[]) => { const normalized = text.toLowerCase(); let score = 0; @@ -322,5 +263,18 @@ const buildSnippet = (text: string, query: string) => { return `${prefix}${snippet}${suffix}`; }; -const toIsoString = (value: Date | string) => - value instanceof Date ? value.toISOString() : new Date(value).toISOString(); +const projectTurnsForFork = ( + turns: SessionTurnRecord[], + keepMessageCount: number, +): SessionTurnRecord[] => { + if (keepMessageCount <= 0) { + return []; + } + + const keepTurnCount = Math.floor(keepMessageCount / 2); + if (keepTurnCount <= 0) { + return []; + } + + return turns.slice(0, keepTurnCount); +}; diff --git a/src/learning/orchestrator.ts b/src/learning/orchestrator.ts index c393957..7fae655 100644 --- a/src/learning/orchestrator.ts +++ b/src/learning/orchestrator.ts @@ -9,7 +9,10 @@ import { LearningStateStore } from "./stateStore.js"; import { MemoryStore, type MemoryScope } from "../memory/store.js"; import { type OpencodeRuntimeAdapter } from "../runtime/opencode.js"; import { SkillStore } from "../skills/store.js"; -import { RuntimeSessionStore } from "../session/runtimeSessionStore.js"; +import { + buildToolSessionScopeKey, + ToolSessionContextStore, +} from "../session/toolContextStore.js"; import { sanitizePersistentDocument, sanitizePersistentLine, @@ -70,7 +73,7 @@ export class LearningOrchestrator { private readonly activeReviews = new Set(); private readonly learningStateStore = new LearningStateStore(); private readonly skillStore = new SkillStore(); - private readonly runtimeSessionStore = new RuntimeSessionStore(); + private readonly toolContextStore = new ToolSessionContextStore(); constructor( private readonly runtime: OpencodeRuntimeAdapter, @@ -81,7 +84,7 @@ export class LearningOrchestrator { async initialize() { await Promise.all([ this.learningStateStore.initialize(), - this.runtimeSessionStore.initialize(), + this.toolContextStore.initialize(), ]); } @@ -89,6 +92,7 @@ export class LearningOrchestrator { const transcript = await this.historyStore.appendTurn( { actorKey: input.requestContext.actorKey, + clientSessionId: input.requestContext.clientSessionId, projectKey: input.requestContext.projectKey, sessionId: input.sessionId, }, @@ -138,17 +142,22 @@ export class LearningOrchestrator { let gateSessionId: string | null = null; try { const gateSession = await this.runtime.createSession( - `learning-gate-${input.requestContext.sessionId}`, + `learning-gate-${input.requestContext.clientSessionId}`, ); gateSessionId = gateSession.id; - await this.runtimeSessionStore.write({ - runtimeSessionId: gateSession.id, + await this.toolContextStore.write({ actorKey: input.requestContext.actorKey, allowLearningWrite: false, - sessionId: input.requestContext.sessionId, + clientSessionId: `gate-${input.requestContext.clientSessionId}`, learningMode: "review", projectId: input.requestContext.projectId, projectKey: input.requestContext.projectKey, + sessionId: gateSession.id, + sessionScopeKey: buildToolSessionScopeKey( + input.requestContext.actorKey, + input.requestContext.projectKey, + input.requestContext.clientSessionId, + ), traceId: input.requestContext.traceId, }); await this.runtime.prompt( @@ -210,7 +219,7 @@ export class LearningOrchestrator { }); } finally { if (gateSessionId) { - await this.runtimeSessionStore.release(gateSessionId).catch(() => undefined); + await this.toolContextStore.remove(gateSessionId).catch(() => undefined); await this.runtime.abortSession(gateSessionId).catch(() => undefined); } } @@ -228,16 +237,21 @@ export class LearningOrchestrator { turnCount: number; }) { const reviewSession = await this.runtime.createSession( - `learning-review-${input.requestContext.sessionId}`, + `learning-review-${input.requestContext.clientSessionId}`, ); - await this.runtimeSessionStore.write({ - runtimeSessionId: reviewSession.id, + await this.toolContextStore.write({ actorKey: input.requestContext.actorKey, allowLearningWrite: false, - sessionId: input.requestContext.sessionId, + clientSessionId: `review-${input.requestContext.clientSessionId}`, learningMode: "review", projectId: input.requestContext.projectId, projectKey: input.requestContext.projectKey, + sessionId: reviewSession.id, + sessionScopeKey: buildToolSessionScopeKey( + input.requestContext.actorKey, + input.requestContext.projectKey, + input.requestContext.clientSessionId, + ), traceId: input.requestContext.traceId, }); try { @@ -278,7 +292,7 @@ export class LearningOrchestrator { traceId: input.requestContext.traceId, }); } finally { - await this.runtimeSessionStore.release(reviewSession.id).catch(() => undefined); + await this.toolContextStore.remove(reviewSession.id).catch(() => undefined); await this.runtime.abortSession(reviewSession.id).catch(() => undefined); } } diff --git a/src/learning/stateStore.ts b/src/learning/stateStore.ts index 16e1e91..33e032b 100644 --- a/src/learning/stateStore.ts +++ b/src/learning/stateStore.ts @@ -1,6 +1,11 @@ -import { type QueryResultRow } from "pg"; +import { join } from "node:path"; -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; +import { config } from "../config.js"; +import { + atomicWriteJson, + ensureDirectory, + readJsonFile, +} from "../utils/fileStore.js"; export type LearningSessionState = { lastGatedTurn: number; @@ -10,65 +15,32 @@ export type LearningSessionState = { updatedAt: string; }; -type LearningStateRow = QueryResultRow & { - session_id: string; - last_gated_turn: number; - last_reviewed_turn: number; - pending_review: boolean; - updated_at: Date | string; -}; - export class LearningStateStore { - constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} + constructor(private readonly baseDir = config.LEARNING_STATE_STORAGE_DIR) {} async initialize() { - await this.db.initialize(); + await ensureDirectory(this.baseDir); } async read(sessionId: string): Promise { - const result = await this.db.query( - ` - SELECT session_id, last_gated_turn, last_reviewed_turn, pending_review, updated_at - FROM ${this.db.table("learning_states")} - WHERE session_id = $1 - LIMIT 1 - `, - [sessionId], - ); - return ( - mapLearningStateRow(result.rows[0]) ?? { - lastGatedTurn: 0, - lastReviewedTurn: 0, - pendingReview: false, - sessionId, - updatedAt: new Date(0).toISOString(), - } - ); + const existing = await readJsonFile(this.filePath(sessionId)); + if (existing) { + return existing; + } + return { + lastGatedTurn: 0, + lastReviewedTurn: 0, + pendingReview: false, + sessionId, + updatedAt: new Date(0).toISOString(), + }; } async write(state: LearningSessionState) { - await this.db.query( - ` - INSERT INTO ${this.db.table("learning_states")} ( - session_id, - last_gated_turn, - last_reviewed_turn, - pending_review - ) - VALUES ($1, $2, $3, $4) - ON CONFLICT (session_id) - DO UPDATE SET - last_gated_turn = EXCLUDED.last_gated_turn, - last_reviewed_turn = EXCLUDED.last_reviewed_turn, - pending_review = EXCLUDED.pending_review - `, - [ - state.sessionId, - state.lastGatedTurn, - state.lastReviewedTurn, - state.pendingReview, - ], - ); + await atomicWriteJson(this.filePath(state.sessionId), { + ...state, + updatedAt: new Date().toISOString(), + }); } async markPending(sessionId: string, pendingReview: boolean) { @@ -97,17 +69,8 @@ export class LearningStateStore { pendingReview: false, }); } -} -const mapLearningStateRow = (row?: LearningStateRow | null): LearningSessionState | null => { - if (!row) { - return null; + private filePath(sessionId: string) { + return join(this.baseDir, `${sessionId}.json`); } - return { - sessionId: row.session_id, - lastGatedTurn: row.last_gated_turn, - lastReviewedTurn: row.last_reviewed_turn, - pendingReview: row.pending_review, - updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at, - }; -}; +} diff --git a/src/memory/store.ts b/src/memory/store.ts index 025b13f..c8771fe 100644 --- a/src/memory/store.ts +++ b/src/memory/store.ts @@ -1,9 +1,13 @@ -import { type QueryResultRow } from "pg"; +import { join } from "node:path"; import { config } from "../config.js"; -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; import { sanitizePersistentLine } from "../utils/persistencePolicy.js"; -import { toStableId } from "../utils/fileStore.js"; +import { + atomicWriteFileWithHistory, + ensureDirectory, + readTextFile, + toStableId, +} from "../utils/fileStore.js"; export type MemoryScope = "user" | "workspace"; export type MemoryEntrySource = "review" | "tool"; @@ -25,17 +29,6 @@ type MemoryContext = { projectKey: string; }; -type MemoryRow = QueryResultRow & { - memory_id: string; - scope: MemoryScope; - scope_key: string; - content: string; - source: MemoryEntrySource; - session_id: string | null; - trace_id: string | null; - updated_at: Date | string; -}; - const SUSPICIOUS_MEMORY_PATTERNS = [ /ignore\s+(all|previous|prior|above)\s+instructions/i, /system\s+prompt/i, @@ -44,118 +37,113 @@ const SUSPICIOUS_MEMORY_PATTERNS = [ ]; export class MemoryStore { - constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} + // Memory 文件可能被多次连续追加，串行化可避免并发覆盖掉刚写入的条目。 + private writeQueue: Promise = Promise.resolve(); + + constructor( + private readonly baseDir = config.MEMORY_STORAGE_DIR, + private readonly historyDir = join(config.PERSISTENCE_HISTORY_DIR, "memory"), + ) {} async initialize() { - await this.db.initialize(); + await ensureDirectory(this.baseDir); + await ensureDirectory(join(this.baseDir, "users")); + await ensureDirectory(join(this.baseDir, "workspaces")); + // 历史备份与正式数据分目录存放，便于排查和手工恢复。 + await ensureDirectory(this.historyDir); } async upsert(scope: MemoryScope, key: string, draft: MemoryDraft) { - const content = normalizeMemoryContent(draft.content); - if (!content) { - return { changed: false, entry: null as MemoryEntry | null }; - } - const existing = await this.findByContent(scope, key, content); - if (existing) { - return { changed: false, entry: existing }; - } - const id = toStableId(scope, key, content.toLowerCase()); - await this.db.query( - ` - INSERT INTO ${this.db.table("memories")} ( - memory_id, - scope, - scope_key, - content, - source, - session_id, - trace_id - ) - VALUES ($1, $2, $3, $4, $5, $6, $7) - `, - [id, scope, key, content, draft.source, draft.sessionId ?? null, draft.traceId ?? null], - ); - return { - changed: true, - entry: { - id, + return this.serializeWrite(async () => { + const content = normalizeMemoryContent(draft.content); + if (!content) { + return { changed: false, entry: null as MemoryEntry | null }; + } + + const entries = await this.readEntries(scope, key); + const existing = entries.find((entry) => entry.content === content); + if (existing) { + return { changed: false, entry: existing }; + } + + const entry: MemoryEntry = { content, - }, - }; + id: toStableId(scope, key, content.toLowerCase()), + }; + entries.unshift(entry); + // 每次覆盖 memory 文件前先保留上一版，写入失败时由底层工具恢复。 + await atomicWriteFileWithHistory( + this.filePath(scope, key), + renderMemoryMarkdown(scope, entries), + { + historyDir: this.historyDir, + rootDir: this.baseDir, + }, + ); + return { changed: true, entry }; + }); } async list(scope: MemoryScope, key: string) { - const rows = await this.db.query( - ` - SELECT memory_id, content - FROM ${this.db.table("memories")} - WHERE scope = $1 - AND scope_key = $2 - ORDER BY updated_at DESC, created_at DESC - `, - [scope, key], - ); - return rows.rows.map((row) => ({ - id: row.memory_id, - content: row.content, - })); + return await this.readEntries(scope, key); } async replace(scope: MemoryScope, key: string, targetId: string, draft: MemoryDraft) { - const content = normalizeMemoryContent(draft.content); - if (!content) { - return { changed: false, detail: "content rejected by persistence policy" }; - } - const duplicate = await this.findByContent(scope, key, content); - if (duplicate && duplicate.id !== targetId.trim()) { - return { changed: false, detail: "replacement would duplicate an existing memory" }; - } - const result = await this.db.query( - ` - UPDATE ${this.db.table("memories")} - SET - content = $4, - source = $5, - session_id = $6, - trace_id = $7 - WHERE memory_id = $1 - AND scope = $2 - AND scope_key = $3 - `, - [ - targetId.trim(), - scope, - key, + return this.serializeWrite(async () => { + const content = normalizeMemoryContent(draft.content); + if (!content) { + return { changed: false, detail: "content rejected by persistence policy" }; + } + const entries = await this.readEntries(scope, key); + const index = entries.findIndex((entry) => entry.id === targetId.trim()); + if (index === -1) { + return { changed: false, detail: "memory entry not found" }; + } + const duplicate = entries.find( + (entry, currentIndex) => currentIndex !== index && entry.content === content, + ); + if (duplicate) { + return { changed: false, detail: "replacement would duplicate an existing memory" }; + } + entries[index] = { content, - draft.source, - draft.sessionId ?? null, - draft.traceId ?? null, - ], - ); - return result.rowCount === 0 - ? { changed: false, detail: "memory entry not found" } - : { changed: true, detail: "memory replaced" }; + id: entries[index]?.id ?? toStableId(scope, key, content.toLowerCase()), + }; + await atomicWriteFileWithHistory( + this.filePath(scope, key), + renderMemoryMarkdown(scope, entries), + { + historyDir: this.historyDir, + rootDir: this.baseDir, + }, + ); + return { changed: true, detail: "memory replaced" }; + }); } async remove(scope: MemoryScope, key: string, targetId: string) { - const result = await this.db.query( - ` - DELETE FROM ${this.db.table("memories")} - WHERE memory_id = $1 - AND scope = $2 - AND scope_key = $3 - `, - [targetId.trim(), scope, key], - ); - return result.rowCount === 0 - ? { changed: false, detail: "memory entry not found" } - : { changed: true, detail: "memory removed" }; + return this.serializeWrite(async () => { + const entries = await this.readEntries(scope, key); + const next = entries.filter((entry) => entry.id !== targetId.trim()); + if (next.length === entries.length) { + return { changed: false, detail: "memory entry not found" }; + } + await atomicWriteFileWithHistory( + this.filePath(scope, key), + renderMemoryMarkdown(scope, next), + { + historyDir: this.historyDir, + rootDir: this.baseDir, + }, + ); + return { changed: true, detail: "memory removed" }; + }); } async buildPromptSnapshot(context: MemoryContext) { const [userMemory, workspaceMemory] = await Promise.all([ - this.list("user", context.actorKey), - this.list("workspace", context.projectKey), + this.readEntries("user", context.actorKey), + this.readEntries("workspace", context.projectKey), ]); const sections: string[] = []; @@ -192,25 +180,26 @@ export class MemoryStore { : block; } - private async findByContent(scope: MemoryScope, key: string, content: string) { - const result = await this.db.query( - ` - SELECT memory_id, content - FROM ${this.db.table("memories")} - WHERE scope = $1 - AND scope_key = $2 - AND content = $3 - LIMIT 1 - `, - [scope, key, content], + private async readEntries(scope: MemoryScope, key: string) { + const markdown = await readTextFile(this.filePath(scope, key)); + if (!markdown) { + return []; + } + return parseMemoryMarkdown(markdown); + } + + private filePath(scope: MemoryScope, key: string) { + const dir = scope === "user" ? "users" : "workspaces"; + return join(this.baseDir, dir, `${key}.md`); + } + + private async serializeWrite(task: () => Promise) { + const run = this.writeQueue.catch(() => undefined).then(task); + this.writeQueue = run.then( + () => undefined, + () => undefined, ); - const row = result.rows[0]; - return row - ? { - id: row.memory_id, - content: row.content, - } - : null; + return run; } } @@ -224,3 +213,31 @@ const normalizeMemoryContent = (content: string) => { } return normalized; }; + +const parseMemoryMarkdown = (content: string): MemoryEntry[] => + content + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.startsWith("- ")) + .map((line) => line.slice(2).trim()) + .map((line) => { + const match = line.match(/^\[([a-z0-9]{8,})\]\s+(.*)$/i); + if (match) { + return { + content: normalizeMemoryContent(match[2]), + id: match[1], + }; + } + const normalized = normalizeMemoryContent(line); + return { + content: normalized, + id: normalized ? toStableId("memory-entry", normalized.toLowerCase()) : "", + }; + }) + .filter((entry) => entry.content); + +const renderMemoryMarkdown = (scope: MemoryScope, entries: MemoryEntry[]) => { + const title = scope === "user" ? "# User Memory" : "# Workspace Memory"; + const bullets = entries.map((entry) => `- [${entry.id}] ${entry.content}`); + return [title, "", ...bullets, ""].join("\n"); +}; diff --git a/src/results/resolver.ts b/src/results/resolver.ts index f9b1615..95986a2 100644 --- a/src/results/resolver.ts +++ b/src/results/resolver.ts @@ -16,9 +16,9 @@ type ResolveOptions = { type RegisterResultReferenceInput = { actorKey: string; + clientSessionId: string; data: unknown; kind: ResultReferenceKind; - payloadPath?: string; projectId?: string; projectKey: string; schemaVersion: number; @@ -47,9 +47,9 @@ export class ResultReferenceResolver { } return this.store.store({ actorKey: input.actorKey, + clientSessionId: input.clientSessionId, data: normalizedData, kind: input.kind, - payloadPath: input.payloadPath, projectId: input.projectId, projectKey: input.projectKey, schemaVersion: input.schemaVersion, @@ -82,16 +82,10 @@ export class ResultReferenceResolver { } return this.register({ - actorKey: input.actorKey, + ...input, data: payload, kind: RESULT_REFERENCE_KIND.renderJunctionsPayload, - payloadPath: filePath, - projectId: input.projectId, - projectKey: input.projectKey, schemaVersion: 1, - sessionId: input.sessionId, - source: input.source, - traceId: input.traceId, }); } diff --git a/src/results/store.ts b/src/results/store.ts index 23161ca..81530f5 100644 --- a/src/results/store.ts +++ b/src/results/store.ts @@ -1,19 +1,20 @@ import { randomUUID } from "node:crypto"; -import { join, resolve } from "node:path"; - -import { type QueryResultRow } from "pg"; +import { join } from "node:path"; import { config } from "../config.js"; -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; import { logger } from "../logger.js"; import { atomicWriteJson, ensureDirectory, + getFileStat, + listJsonFiles, readJsonFile, removeFileIfExists, + toProjectKey, } from "../utils/fileStore.js"; export const RESULT_REF_PATTERN = /^res-[a-f0-9-]{8,64}$/; +const RESULT_REF_FILE_PATTERN = /^(res-[a-f0-9-]{8,64})(?:\.json)?$/; export const RESULT_REFERENCE_KIND = { dynamicHttpResult: "dynamic-http-result", @@ -43,6 +44,7 @@ export type ResultPreview = { export type ResultReferenceRecord = { resultRef: string; actorKey: string; + clientSessionId: string; createdAt: string; data: unknown; kind: ResultReferenceKind; @@ -54,12 +56,11 @@ export type ResultReferenceRecord = { sizeBytes: number; source: ResultReferenceSource; traceId: string; - payloadPath?: string; - objectKey?: string; }; export type StoreResultInput = { actorKey: string; + clientSessionId: string; data: unknown; kind: ResultReferenceKind; projectId?: string; @@ -68,13 +69,11 @@ export type StoreResultInput = { sessionId: string; source: ResultReferenceSource; traceId: string; - payloadPath?: string; - objectKey?: string; }; export type RetrievalContext = { actorKey: string; - sessionId?: string; + clientSessionId?: string; projectId?: string; }; @@ -85,37 +84,18 @@ export type ResultReferencePeek = { storedAt: string; }; -type ResultReferenceRow = QueryResultRow & { - result_ref: string; - actor_key: string; - session_id: string; - created_at: Date | string; - kind: ResultReferenceKind; - preview: ResultPreview; - project_id: string | null; - project_key: string; - schema_version: number; - size_bytes: number; - source: ResultReferenceSource; - trace_id: string; - payload_path: string | null; - object_key: string | null; -}; +type PartialRecord = Partial & { data?: unknown }; export class ResultReferenceStore { private cleanupTimer: NodeJS.Timeout | null = null; - private readonly managedPayloadDir: string; constructor( - private readonly db: AgentDatabase = getAgentDatabase(), private readonly baseDir = config.RESULT_REF_STORAGE_DIR, private readonly ttlMs = config.RESULT_REF_TTL_HOURS * 60 * 60 * 1000, - ) { - this.managedPayloadDir = join(this.baseDir, "payloads"); - } + ) {} async initialize() { - await Promise.all([this.db.initialize(), ensureDirectory(this.managedPayloadDir)]); + await ensureDirectory(this.baseDir); } startCleanupLoop() { @@ -139,77 +119,24 @@ export class ResultReferenceStore { async store(input: StoreResultInput) { const resultRef = `res-${randomUUID().slice(0, 16)}`; - const createdAt = new Date().toISOString(); - const payloadPath = - input.payloadPath ?? - (input.objectKey - ? undefined - : this.managedPayloadPath(resultRef)); - - if (!payloadPath && !input.objectKey) { - throw new Error("result ref requires payloadPath or objectKey"); - } - if (!input.payloadPath && payloadPath) { - await atomicWriteJson(payloadPath, wrapPayload(input.data, createdAt, input.projectId)); - } - - const preview = buildPreview(input.data); - const sizeBytes = estimateBytes(input.data); - await this.db.query( - ` - INSERT INTO ${this.db.table("result_refs")} ( - result_ref, - actor_key, - session_id, - created_at, - kind, - preview, - project_id, - project_key, - schema_version, - size_bytes, - source, - trace_id, - payload_path, - object_key - ) - VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11, $12, $13, $14) - `, - [ - resultRef, - input.actorKey, - input.sessionId, - createdAt, - input.kind, - JSON.stringify(preview), - input.projectId ?? null, - input.projectKey, - input.schemaVersion, - sizeBytes, - input.source, - input.traceId, - payloadPath ?? null, - input.objectKey ?? null, - ], - ); - - return { + const record: ResultReferenceRecord = { resultRef, actorKey: input.actorKey, - createdAt, + clientSessionId: input.clientSessionId, + createdAt: new Date().toISOString(), data: input.data, kind: input.kind, - preview, + preview: buildPreview(input.data), projectId: input.projectId, projectKey: input.projectKey, schemaVersion: input.schemaVersion, sessionId: input.sessionId, - sizeBytes, + sizeBytes: estimateBytes(input.data), source: input.source, traceId: input.traceId, - payloadPath, - objectKey: input.objectKey, - } satisfies ResultReferenceRecord; + }; + await atomicWriteJson(this.filePath(resultRef), record); + return record; } async getAuthorizedRecord(resultRef: string, context: RetrievalContext) { @@ -218,49 +145,26 @@ export class ResultReferenceStore { return null; } - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("result_refs")} - WHERE result_ref = $1 - LIMIT 1 - `, - [normalizedResultRef], - ); - const row = result.rows[0]; - if (!row) { + const rawRecord = await readJsonFile(this.filePath(normalizedResultRef)); + const record = + normalizeResultReferenceRecord(rawRecord) ?? + normalizeLegacyRenderReferenceRecord(rawRecord, normalizedResultRef, context); + if (!record) { return null; } - if (row.actor_key !== context.actorKey) { + if (record.actorKey !== context.actorKey) { return null; } - if ((row.project_id ?? "") !== (context.projectId ?? "")) { + if ((record.projectId ?? "") !== (context.projectId ?? "")) { return null; } - if (context.sessionId && row.session_id !== context.sessionId) { + if ( + context.clientSessionId && + record.clientSessionId !== context.clientSessionId + ) { return null; } - const data = await this.readPayload(row); - if (data === null) { - return null; - } - return { - resultRef: row.result_ref, - actorKey: row.actor_key, - createdAt: toIsoString(row.created_at), - data, - kind: row.kind, - preview: row.preview, - projectId: row.project_id ?? undefined, - projectKey: row.project_key, - schemaVersion: row.schema_version, - sessionId: row.session_id, - sizeBytes: row.size_bytes, - source: row.source, - traceId: row.trace_id, - payloadPath: row.payload_path ?? undefined, - objectKey: row.object_key ?? undefined, - } satisfies ResultReferenceRecord; + return record; } async peekAuthorized( @@ -280,82 +184,196 @@ export class ResultReferenceStore { } async listBySession(sessionId: string) { - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("result_refs")} - WHERE session_id = $1 - ORDER BY created_at DESC - `, - [sessionId], - ); + const files = await listJsonFiles(this.baseDir); const records = await Promise.all( - result.rows.map(async (row) => { - const data = await this.readPayload(row); - if (data === null) { - return null; - } - return { - resultRef: row.result_ref, - actorKey: row.actor_key, - createdAt: toIsoString(row.created_at), - data, - kind: row.kind, - preview: row.preview, - projectId: row.project_id ?? undefined, - projectKey: row.project_key, - schemaVersion: row.schema_version, - sessionId: row.session_id, - sizeBytes: row.size_bytes, - source: row.source, - traceId: row.trace_id, - payloadPath: row.payload_path ?? undefined, - objectKey: row.object_key ?? undefined, - } satisfies ResultReferenceRecord; - }), + files.map(async (filePath) => + normalizeResultReferenceRecord(await readJsonFile(filePath)), + ), ); - return records.filter(Boolean) as ResultReferenceRecord[]; + return records + .filter((record): record is ResultReferenceRecord => Boolean(record)) + .filter((record) => record.sessionId === sessionId) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)); } async cleanupExpired() { - const result = await this.db.query( - ` - DELETE FROM ${this.db.table("result_refs")} - WHERE created_at < NOW() - ($1 * INTERVAL '1 millisecond') - RETURNING * - `, - [this.ttlMs], - ); - await Promise.all( - result.rows.map(async (row) => { - if (row.payload_path && isManagedPayloadPath(row.payload_path, this.managedPayloadDir)) { - await removeFileIfExists(row.payload_path); - } - }), - ); + const files = await listJsonFiles(this.baseDir); + const now = Date.now(); + for (const filePath of files) { + const stats = await getFileStat(filePath); + if (!stats) { + continue; + } + if (now - stats.mtimeMs > this.ttlMs) { + await removeFileIfExists(filePath); + } + } } - private managedPayloadPath(resultRef: string) { - return join(this.managedPayloadDir, `${resultRef}.json`); - } - - private async readPayload(row: ResultReferenceRow) { - if (row.payload_path) { - return unwrapStoredPayload(await readJsonFile(row.payload_path)); - } - if (row.object_key) { - logger.warn({ resultRef: row.result_ref, objectKey: row.object_key }, "object storage payloads are not implemented"); - return null; - } - return null; + private filePath(resultRef: string) { + return join(this.baseDir, `${resultRef}.json`); } } -const normalizeResultRef = (value: string) => { - const normalized = value.trim(); - return RESULT_REF_PATTERN.test(normalized) ? normalized : null; +export const normalizeResultReferenceRecord = ( + value: unknown, +): ResultReferenceRecord | null => { + if (!isRecord(value)) { + return null; + } + + const partial = value as PartialRecord; + if ( + !isValidResultRef(partial.resultRef) || + typeof partial.actorKey !== "string" || + typeof partial.clientSessionId !== "string" || + typeof partial.createdAt !== "string" || + !("data" in partial) || + !isResultPreview(partial.preview) || + typeof partial.projectKey !== "string" || + typeof partial.sessionId !== "string" || + typeof partial.sizeBytes !== "number" || + !Number.isFinite(partial.sizeBytes) || + typeof partial.traceId !== "string" + ) { + return null; + } + + const kind = normalizeResultReferenceKind(partial.kind); + const source = normalizeResultReferenceSource(partial.source); + const schemaVersion = + typeof partial.schemaVersion === "number" && + Number.isInteger(partial.schemaVersion) && + partial.schemaVersion > 0 + ? partial.schemaVersion + : 1; + + if (!kind || !source) { + return null; + } + + if ( + partial.projectId !== undefined && + typeof partial.projectId !== "string" + ) { + return null; + } + + return { + resultRef: partial.resultRef, + actorKey: partial.actorKey, + clientSessionId: partial.clientSessionId, + createdAt: partial.createdAt, + data: partial.data, + kind, + preview: partial.preview, + projectId: partial.projectId, + projectKey: partial.projectKey, + schemaVersion, + sessionId: partial.sessionId, + sizeBytes: partial.sizeBytes, + source, + traceId: partial.traceId, + }; }; +const normalizeResultReferenceKind = ( + value: unknown, +): ResultReferenceKind | null => { + if (value === undefined) { + return RESULT_REFERENCE_KIND.dynamicHttpResult; + } + return Object.values(RESULT_REFERENCE_KIND).includes( + value as ResultReferenceKind, + ) + ? (value as ResultReferenceKind) + : null; +}; + +const normalizeResultReferenceSource = ( + value: unknown, +): ResultReferenceSource | null => { + if (value === undefined) { + return RESULT_REFERENCE_SOURCE.legacy; + } + return Object.values(RESULT_REFERENCE_SOURCE).includes( + value as ResultReferenceSource, + ) + ? (value as ResultReferenceSource) + : null; +}; + +const isValidResultRef = (value: unknown): value is string => + typeof value === "string" && RESULT_REF_PATTERN.test(value); + +const normalizeResultRef = (value: string) => { + const match = value.trim().match(RESULT_REF_FILE_PATTERN); + return match?.[1] ?? null; +}; + +const normalizeLegacyRenderReferenceRecord = ( + value: unknown, + resultRef: string, + context: RetrievalContext, +): ResultReferenceRecord | null => { + const data = extractLegacyRenderPayload(value); + if (!data) { + return null; + } + + const root = isRecord(value) ? value : {}; + const metadata = isRecord(root.metadata) ? root.metadata : {}; + const projectId = firstNonEmptyString(root.projectId, metadata.projectId); + const createdAt = + firstNonEmptyString(root.createdAt, metadata.createdAt) ?? new Date().toISOString(); + + return { + resultRef, + actorKey: context.actorKey, + clientSessionId: context.clientSessionId ?? "", + createdAt, + data, + kind: RESULT_REFERENCE_KIND.renderJunctionsPayload, + preview: buildPreview(data), + projectId, + projectKey: toProjectKey(projectId), + schemaVersion: 1, + sessionId: context.clientSessionId ?? resultRef, + sizeBytes: estimateBytes(data), + source: RESULT_REFERENCE_SOURCE.legacy, + traceId: "legacy-render-ref", + }; +}; + +const extractLegacyRenderPayload = (value: unknown) => { + if (!isRecord(value)) { + return null; + } + const candidate = isRecord(value.data) ? value.data : value; + if (!isRecord(candidate.node_area_map)) { + return null; + } + return candidate; +}; + +const firstNonEmptyString = (...values: unknown[]) => { + for (const value of values) { + if (typeof value === "string" && value.trim().length > 0) { + return value.trim(); + } + } + return undefined; +}; + +const isResultPreview = (value: unknown): value is ResultPreview => + isRecord(value) && + typeof value.count === "number" && + Number.isFinite(value.count) && + Array.isArray(value.fields) && + value.fields.every((field) => typeof field === "string") && + typeof value.summary === "string" && + "sample" in value; + const estimateBytes = (data: unknown) => Buffer.byteLength(JSON.stringify(data)); const buildPreview = (data: unknown): ResultPreview => { @@ -396,26 +414,5 @@ const buildPreview = (data: unknown): ResultPreview => { }; }; -const wrapPayload = (data: unknown, createdAt: string, projectId?: string) => ({ - metadata: { - createdAt, - ...(projectId ? { projectId } : {}), - }, - data, -}); - const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); - -const unwrapStoredPayload = (value: unknown) => { - if (!isRecord(value)) { - return value; - } - return "data" in value ? value.data : value; -}; - -const toIsoString = (value: Date | string) => - value instanceof Date ? value.toISOString() : new Date(value).toISOString(); - -const isManagedPayloadPath = (payloadPath: string, managedPayloadDir: string) => - resolve(payloadPath).startsWith(resolve(managedPayloadDir)); diff --git a/src/routes/chat.ts b/src/routes/chat.ts index b6acfc9..25e153a 100644 --- a/src/routes/chat.ts +++ b/src/routes/chat.ts @@ -91,8 +91,6 @@ export const buildChatRouter = ( session_id: record.sessionId, created_at: record.createdAt, updated_at: record.updatedAt, - is_streaming: record.isStreaming, - streaming_started_at: record.streamingStartedAt, status: record.status, title: record.title, parent_session_id: record.parentSessionId, @@ -116,8 +114,6 @@ export const buildChatRouter = ( title: record.title ?? "新对话", created_at: record.createdAt, updated_at: record.updatedAt, - is_streaming: record.isStreaming, - streaming_started_at: record.streamingStartedAt, status: record.status, parent_session_id: record.parentSessionId, })), @@ -149,15 +145,13 @@ export const buildChatRouter = ( return; } - const state = await conversationStateStore.read(conversation.sessionId); + const state = await conversationStateStore.read(conversation.sessionScopeKey); res.json({ id: conversation.sessionId, title: conversation.title ?? "新对话", is_title_manually_edited: state?.isTitleManuallyEdited ?? false, created_at: conversation.createdAt, updated_at: conversation.updatedAt, - is_streaming: conversation.isStreaming, - streaming_started_at: conversation.streamingStartedAt, status: conversation.status, session_id: conversation.sessionId, messages: state?.messages ?? [], @@ -196,7 +190,7 @@ export const buildChatRouter = ( const nextRecord = await conversationStore.touch(record, { ...(parsed.data.title ? { title: parsed.data.title } : {}), }); - await conversationStateStore.write(nextRecord.sessionId, { + await conversationStateStore.write(nextRecord.sessionScopeKey, { sessionId: nextRecord.sessionId, isTitleManuallyEdited: parsed.data.is_title_manually_edited, messages: parsed.data.messages, @@ -207,8 +201,6 @@ export const buildChatRouter = ( title: nextRecord.title ?? "新对话", created_at: nextRecord.createdAt, updated_at: nextRecord.updatedAt, - is_streaming: nextRecord.isStreaming, - streaming_started_at: nextRecord.streamingStartedAt, status: nextRecord.status, session_id: nextRecord.sessionId, }); @@ -239,9 +231,9 @@ export const buildChatRouter = ( return; } const nextConversation = await conversationStore.touch(conversation, { title }); - const state = await conversationStateStore.read(nextConversation.sessionId); + const state = await conversationStateStore.read(nextConversation.sessionScopeKey); if (state) { - await conversationStateStore.write(nextConversation.sessionId, { + await conversationStateStore.write(nextConversation.sessionScopeKey, { ...state, isTitleManuallyEdited: isTitleManuallyEdited ?? state.isTitleManuallyEdited, @@ -272,7 +264,7 @@ export const buildChatRouter = ( res.status(204).end(); return; } - await conversationStateStore.remove(conversation.sessionId); + await conversationStateStore.remove(conversation.sessionScopeKey); await conversationStore.remove(conversation); res.status(204).end(); }); @@ -281,7 +273,7 @@ export const buildChatRouter = ( const renderRef = req.params.renderRef?.trim(); const userId = req.header("x-user-id")?.trim(); const projectId = req.header("x-project-id") ?? undefined; - const sessionId = + const clientSessionId = typeof req.query.session_id === "string" ? req.query.session_id.trim() : undefined; @@ -304,7 +296,7 @@ export const buildChatRouter = ( renderRef, { actorKey: toActorKey(userId), - sessionId, + clientSessionId, projectId, }, { @@ -332,7 +324,7 @@ export const buildChatRouter = ( try { const binding = await sessionBridge.abort({ - sessionId: parsed.data.session_id, + clientSessionId: parsed.data.session_id, }); if (!binding) { @@ -342,8 +334,8 @@ export const buildChatRouter = ( logger.info( { - sessionId: parsed.data.session_id, - runtimeSessionId: binding.runtimeSessionId, + clientSessionId: parsed.data.session_id, + sessionId: binding.sessionId, }, "aborted chat session by client request", ); @@ -378,8 +370,8 @@ export const buildChatRouter = ( const actorKey = toActorKey(userId); const projectKey = toProjectKey(projectId); - const sourceSessionId = parsed.data.session_id?.trim(); - const sourceConversation = sourceSessionId + const sourceClientSessionId = parsed.data.session_id?.trim(); + const sourceConversation = sourceClientSessionId ? await conversationStore.get( { actorKey, @@ -387,29 +379,31 @@ export const buildChatRouter = ( projectKey, userId, }, - sourceSessionId, + sourceClientSessionId, ) : null; const { record: targetConversation } = await conversationStore.ensure({ actorKey, - parentSessionId: sourceSessionId, + parentSessionId: sourceClientSessionId, projectId, projectKey, userId, }); - const nextSessionId = targetConversation.sessionId; + const nextClientSessionId = targetConversation.sessionId; - if (sourceSessionId && parsed.data.keep_message_count > 0) { + if (sourceClientSessionId && parsed.data.keep_message_count > 0) { await sessionHistoryStore.cloneThread( { actorKey, + clientSessionId: sourceClientSessionId, projectKey, - sessionId: sourceSessionId, + sessionId: sourceClientSessionId, }, { actorKey, + clientSessionId: nextClientSessionId, projectKey, - sessionId: nextSessionId, + sessionId: nextClientSessionId, }, parsed.data.keep_message_count, ); @@ -422,8 +416,8 @@ export const buildChatRouter = ( logger.info( { - sourceSessionId: parsed.data.session_id, - sessionId: nextSessionId, + sourceClientSessionId: parsed.data.session_id, + clientSessionId: nextClientSessionId, traceId, projectId, keepMessageCount: parsed.data.keep_message_count, @@ -432,7 +426,7 @@ export const buildChatRouter = ( ); res.status(200).json({ - session_id: nextSessionId, + session_id: nextClientSessionId, }); } catch (error) { const detail = error instanceof Error ? error.message : String(error); @@ -475,7 +469,7 @@ export const buildChatRouter = ( const activeConversation = await conversationStore.touch(conversation); const { binding, requestContext, created } = await sessionBridge.resolve({ - sessionId: activeConversation.sessionId, + clientSessionId: activeConversation.sessionId, accessToken, projectId, traceId, @@ -483,16 +477,16 @@ export const buildChatRouter = ( }); const historyContext = { actorKey: requestContext.actorKey, + clientSessionId: requestContext.clientSessionId, projectKey: requestContext.projectKey, - sessionId: requestContext.sessionId, + sessionId: requestContext.clientSessionId, }; - await conversationStore.markStreaming(activeConversation, binding.runtimeSessionId); const recentTurns = await sessionHistoryStore.getRecentTurns(historyContext, 8); logger.info( { - sessionId: requestContext.sessionId, - runtimeSessionId: binding.runtimeSessionId, + clientSessionId: requestContext.clientSessionId, + sessionId: binding.sessionId, created: created || conversationCreated, model: parsed.data.model, traceId: requestContext.traceId, @@ -508,7 +502,7 @@ export const buildChatRouter = ( res.setHeader("X-Accel-Buffering", "no"); res.flushHeaders?.(); - const sessionId = requestContext.sessionId; + const clientSessionId = requestContext.clientSessionId; let streamClosed = false; const abortController = new AbortController(); const handleClientClose = () => { @@ -531,8 +525,8 @@ export const buildChatRouter = ( ); const streamResult = await streamPromptResponse({ runtime, - opencodeSessionId: binding.runtimeSessionId, - sessionId, + opencodeSessionId: binding.sessionId, + clientSessionId, message: preparedMessage, model: parsed.data.model, traceId: requestContext.traceId, @@ -547,7 +541,7 @@ export const buildChatRouter = ( }); if (!streamResult.aborted && !streamResult.failed) { - const messages = await runtime.messages(binding.runtimeSessionId, 60); + const messages = await runtime.messages(binding.sessionId, 60); const assistantMessage = [...messages] .reverse() .find((message) => message.info.role === "assistant"); @@ -558,7 +552,7 @@ export const buildChatRouter = ( activeConversation.sessionId, )) ?? activeConversation; const latestConversationState = await conversationStateStore.read( - latestConversation.sessionId, + latestConversation.sessionScopeKey, ); const existingSessionTitle = latestConversation.title; let sessionTitle = existingSessionTitle; @@ -569,7 +563,7 @@ export const buildChatRouter = ( }); if (shouldGenerateTitle) { sessionTitle = await generateSessionTitle(runtime, { - sessionId: binding.runtimeSessionId, + sessionId: binding.sessionId, latestAssistantMessage: assistantText, latestUserMessage: parsed.data.message, fallbackTitle: existingSessionTitle, @@ -588,7 +582,7 @@ export const buildChatRouter = ( ) { res.write( toSse("session_title", { - session_id: sessionId, + session_id: clientSessionId, title: sessionTitle, }), ); @@ -599,23 +593,19 @@ export const buildChatRouter = ( assistantMessage: assistantText, model: parsed.data.model, requestContext, - sessionId, + sessionId: clientSessionId, toolCallCount: streamResult.toolCallCount, userMessage: parsed.data.message, }).catch((error) => { logger.warn( - { err: error, sessionId }, + { err: error, sessionId: clientSessionId }, "post-turn learning failed", ); }); } } } finally { - await sessionBridge.releaseRuntimeSession( - sessionId, - binding.runtimeSessionId, - ); - await conversationStore.clearStreaming(sessionId, binding.runtimeSessionId); + await sessionBridge.releaseRuntimeSession(clientSessionId, binding.sessionId); streamClosed = true; req.off("close", handleClientClose); res.off("close", handleClientClose); diff --git a/src/routes/chatStream.ts b/src/routes/chatStream.ts index 5453886..60adf6c 100644 --- a/src/routes/chatStream.ts +++ b/src/routes/chatStream.ts @@ -14,7 +14,7 @@ export type SupportedModel = (typeof supportedModels)[number]; type StreamPromptOptions = { runtime: OpencodeRuntimeAdapter; opencodeSessionId: string; - sessionId: string; + clientSessionId: string; message: string; model?: SupportedModel; traceId?: string; @@ -169,7 +169,7 @@ export const collectTextContent = (parts: Part[]) => const emitFallbackMessage = async ( runtime: OpencodeRuntimeAdapter, opencodeSessionId: string, - sessionId: string, + clientSessionId: string, write: (event: string, data: Record) => void, ) => { const messages = await runtime.messages(opencodeSessionId); @@ -180,7 +180,7 @@ const emitFallbackMessage = async ( const text = collectTextContent(parts); if (text) { write("token", { - session_id: sessionId, + session_id: clientSessionId, content: text, }); } @@ -294,7 +294,7 @@ const getToolProgressTitle = (tool: string, status: string) => { export const streamPromptResponse = async ({ runtime, opencodeSessionId, - sessionId, + clientSessionId, message, model, traceId, @@ -333,7 +333,7 @@ export const streamPromptResponse = async ({ let failed = false; const debugContext = { opencodeSessionId, - sessionId, + clientSessionId, traceId, projectId, model: model ?? null, @@ -369,7 +369,7 @@ export const streamPromptResponse = async ({ if (status === "running") { write("progress", { - session_id: sessionId, + session_id: clientSessionId, id, phase, status, @@ -385,7 +385,7 @@ export const streamPromptResponse = async ({ finalizedProgressIds.add(id); progressStartedAtMap.delete(id); write("progress", { - session_id: sessionId, + session_id: clientSessionId, id, phase, status, @@ -542,6 +542,7 @@ export const streamPromptResponse = async ({ void writeLlmRequestAuditLog({ kind: "skill", sessionId: opencodeSessionId, + clientSessionId, traceId, projectId, target: name, @@ -567,7 +568,7 @@ export const streamPromptResponse = async ({ } emittedText = true; write("token", { - session_id: sessionId, + session_id: clientSessionId, content: event.properties.delta, }); } else if (partType === "reasoning") { @@ -600,7 +601,7 @@ export const streamPromptResponse = async ({ for (const content of pending) { emittedText = true; write("token", { - session_id: sessionId, + session_id: clientSessionId, content, }); } @@ -691,7 +692,7 @@ export const streamPromptResponse = async ({ { tool: part.tool, sessionId: opencodeSessionId, - requestSessionId: sessionId, + clientSessionId, }, "llm tool request missing reason", ); @@ -699,6 +700,7 @@ export const streamPromptResponse = async ({ void writeLlmRequestAuditLog({ kind: "tool", sessionId: opencodeSessionId, + clientSessionId, traceId, projectId, target: part.tool, @@ -709,7 +711,7 @@ export const streamPromptResponse = async ({ logger.warn({ err: error }, "failed to write tool audit log"); }); write("tool_call", { - session_id: sessionId, + session_id: clientSessionId, tool: part.tool, params: toolParams, reason, @@ -744,7 +746,7 @@ export const streamPromptResponse = async ({ : "opencode session error", }); write("error", { - session_id: sessionId, + session_id: clientSessionId, message: event.properties.error ? getErrorMessage(event.properties.error) : "opencode session error", @@ -801,7 +803,7 @@ export const streamPromptResponse = async ({ ...debugContext, elapsedMs: Math.max(0, Date.now() - requestStartedAt), }); - await emitFallbackMessage(runtime, opencodeSessionId, sessionId, write); + await emitFallbackMessage(runtime, opencodeSessionId, clientSessionId, write); } emitProgress({ id: "request-received", @@ -820,7 +822,7 @@ export const streamPromptResponse = async ({ : "已完成分析，并通过兜底消息补发最终回答内容。", }); write("done", { - session_id: sessionId, + session_id: clientSessionId, total_duration_ms: Math.max(0, Date.now() - requestStartedAt), }); logDevelopmentDebug("chat stream completed", { diff --git a/src/server.ts b/src/server.ts index 81b8141..24bcc46 100644 --- a/src/server.ts +++ b/src/server.ts @@ -14,7 +14,7 @@ import { ResultReferenceResolver } from "./results/resolver.js"; import { ResultReferenceStore } from "./results/store.js"; import { buildChatRouter } from "./routes/chat.js"; import { opencodeRuntime } from "./runtime/opencode.js"; -import { RuntimeSessionStore } from "./session/runtimeSessionStore.js"; +import { ToolSessionContextStore } from "./session/toolContextStore.js"; import { DynamicHttpExecutor } from "./tools/dynamicHttpExecutor.js"; const app = express(); @@ -23,7 +23,7 @@ const conversationStore = new ConversationStore(); const conversationStateStore = new ConversationStateStore(); const memoryStore = new MemoryStore(); const sessionHistoryStore = new SessionHistoryStore(); -const runtimeSessionStore = new RuntimeSessionStore(); +const toolContextStore = new ToolSessionContextStore(); const learningOrchestrator = new LearningOrchestrator( opencodeRuntime, memoryStore, @@ -65,26 +65,22 @@ app.post("/internal/tools/dynamic-http-call", async (req, res) => { return; } - const runtimeSessionId = - typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; - const persistedContext = runtimeSessionId - ? await runtimeSessionStore.read(runtimeSessionId) - : null; - const runtimeContext = runtimeSessionId - ? sessionBridge.getActiveSensitiveContext(runtimeSessionId) - : null; - if (!persistedContext && !runtimeContext) { + const sessionScopeKey = + typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; + const threadContext = await toolContextStore.read(sessionScopeKey); + const runtimeContext = sessionBridge.getActiveSensitiveContext(sessionScopeKey); + if (!threadContext && !runtimeContext) { res.status(404).json({ message: "runtime or session context not found", - detail: runtimeSessionId, + detail: sessionScopeKey, }); return; } - const context = persistedContext; + const context = runtimeContext ?? threadContext; if (!context) { res.status(404).json({ message: "runtime or session context not found", - detail: runtimeSessionId, + detail: sessionScopeKey, }); return; } @@ -97,14 +93,14 @@ app.post("/internal/tools/dynamic-http-call", async (req, res) => { path: req.body?.path, method: req.body?.method, arguments: req.body?.arguments, - body: req.body?.body, }, { accessToken: runtimeContext?.accessToken, actorKey: context.actorKey, - sessionId: context.sessionId, + clientSessionId: context.clientSessionId, projectId: context.projectId, projectKey: context.projectKey, + sessionId: context.clientSessionId, traceId: context.traceId, }, ); @@ -124,14 +120,14 @@ app.post("/internal/tools/fetch-result-ref", async (req, res) => { return; } - const runtimeSessionId = - typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; + const sessionScopeKey = + typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; const resultRef = typeof req.body?.result_ref === "string" ? req.body.result_ref : ""; - const context = runtimeSessionId ? await runtimeSessionStore.read(runtimeSessionId) : null; + const context = await toolContextStore.read(sessionScopeKey); if (!context) { res.status(404).json({ message: "session context not found", - detail: runtimeSessionId, + detail: sessionScopeKey, }); return; } @@ -144,7 +140,7 @@ app.post("/internal/tools/fetch-result-ref", async (req, res) => { resultRef, { actorKey: context.actorKey, - sessionId: context.sessionId, + clientSessionId: context.clientSessionId, projectId: context.projectId, }, { @@ -167,14 +163,14 @@ app.post("/internal/tools/store-render-ref", async (req, res) => { return; } - const runtimeSessionId = - typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; + const sessionScopeKey = + typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; const filePath = typeof req.body?.file_path === "string" ? req.body.file_path.trim() : ""; - const context = runtimeSessionId ? await runtimeSessionStore.read(runtimeSessionId) : null; + const context = await toolContextStore.read(sessionScopeKey); if (!context) { res.status(404).json({ message: "session context not found", - detail: runtimeSessionId, + detail: sessionScopeKey, }); return; } @@ -186,9 +182,10 @@ app.post("/internal/tools/store-render-ref", async (req, res) => { try { const record = await resultReferenceResolver.registerRenderPayloadFile(filePath, { actorKey: context.actorKey, + clientSessionId: context.clientSessionId, projectId: context.projectId, projectKey: context.projectKey, - sessionId: context.sessionId, + sessionId: context.clientSessionId, source: "migration", traceId: context.traceId, }); @@ -216,14 +213,14 @@ app.post("/internal/tools/session-search", async (req, res) => { return; } - const runtimeSessionId = - typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; + const sessionScopeKey = + typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; const query = typeof req.body?.query === "string" ? req.body.query : ""; - const context = runtimeSessionId ? await runtimeSessionStore.read(runtimeSessionId) : null; + const context = await toolContextStore.read(sessionScopeKey); if (!context) { res.status(404).json({ message: "session context not found", - detail: runtimeSessionId, + detail: sessionScopeKey, }); return; } @@ -267,9 +264,8 @@ const bootstrap = async () => { memoryStore.initialize(), resultReferenceStore.initialize(), sessionHistoryStore.initialize(), - runtimeSessionStore.initialize(), + toolContextStore.initialize(), ]); - await conversationStore.resetStreamingSessions(); resultReferenceStore.startCleanupLoop(); }; diff --git a/src/session/runtimeSessionStore.ts b/src/session/runtimeSessionStore.ts deleted file mode 100644 index b2a627b..0000000 --- a/src/session/runtimeSessionStore.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { type QueryResultRow } from "pg"; - -import { AgentDatabase, getAgentDatabase } from "../db/index.js"; - -export type RuntimeSessionContext = { - runtimeSessionId: string; - actorKey: string; - allowLearningWrite?: boolean; - sessionId: string; - learningMode?: "interactive" | "review"; - projectId?: string; - projectKey: string; - traceId: string; - releasedAt?: string; -}; - -type RuntimeSessionRow = QueryResultRow & { - runtime_session_id: string; - actor_key: string; - allow_learning_write: boolean; - session_id: string; - learning_mode: "interactive" | "review" | null; - project_id: string | null; - project_key: string; - trace_id: string; - released_at: Date | string | null; -}; - -export class RuntimeSessionStore { - constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} - - async initialize() { - await this.db.initialize(); - } - - async write(context: RuntimeSessionContext) { - const result = await this.db.query( - ` - INSERT INTO ${this.db.table("runtime_sessions")} ( - runtime_session_id, - actor_key, - allow_learning_write, - session_id, - learning_mode, - project_id, - project_key, - trace_id, - released_at - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NULL) - ON CONFLICT (runtime_session_id) - DO UPDATE SET - actor_key = EXCLUDED.actor_key, - allow_learning_write = EXCLUDED.allow_learning_write, - session_id = EXCLUDED.session_id, - learning_mode = EXCLUDED.learning_mode, - project_id = EXCLUDED.project_id, - project_key = EXCLUDED.project_key, - trace_id = EXCLUDED.trace_id, - released_at = NULL - RETURNING * - `, - [ - context.runtimeSessionId, - context.actorKey, - context.allowLearningWrite ?? false, - context.sessionId, - context.learningMode ?? null, - context.projectId ?? null, - context.projectKey, - context.traceId, - ], - ); - return mapRuntimeSessionRow(result.rows[0]); - } - - async read(runtimeSessionId: string, options: { includeReleased?: boolean } = {}) { - const result = await this.db.query( - ` - SELECT * - FROM ${this.db.table("runtime_sessions")} - WHERE runtime_session_id = $1 - AND ($2::boolean OR released_at IS NULL) - LIMIT 1 - `, - [runtimeSessionId, options.includeReleased ?? false], - ); - return mapRuntimeSessionRow(result.rows[0]); - } - - async release(runtimeSessionId: string) { - await this.db.query( - ` - UPDATE ${this.db.table("runtime_sessions")} - SET released_at = NOW() - WHERE runtime_session_id = $1 - `, - [runtimeSessionId], - ); - } -} - -const mapRuntimeSessionRow = ( - row?: RuntimeSessionRow | null, -): RuntimeSessionContext | null => { - if (!row) { - return null; - } - return { - runtimeSessionId: row.runtime_session_id, - actorKey: row.actor_key, - allowLearningWrite: row.allow_learning_write, - sessionId: row.session_id, - learningMode: row.learning_mode ?? undefined, - projectId: row.project_id ?? undefined, - projectKey: row.project_key, - traceId: row.trace_id, - releasedAt: - row.released_at instanceof Date - ? row.released_at.toISOString() - : row.released_at ?? undefined, - }; -}; diff --git a/src/session/toolContextStore.ts b/src/session/toolContextStore.ts new file mode 100644 index 0000000..bc8fa5f --- /dev/null +++ b/src/session/toolContextStore.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; + +import { config } from "../config.js"; +import { + atomicWriteJson, + ensureDirectory, + readJsonFile, + removeFileIfExists, +} from "../utils/fileStore.js"; +import { toConversationScopeKey } from "../utils/fileStore.js"; + +export type ToolSessionContext = { + actorKey: string; + allowLearningWrite?: boolean; + clientSessionId: string; + learningMode?: "interactive" | "review"; + projectId?: string; + projectKey: string; + sessionId: string; + sessionScopeKey: string; + traceId: string; +}; + +export class ToolSessionContextStore { + constructor(private readonly baseDir = config.SESSION_CONTEXT_STORAGE_DIR) {} + + async initialize() { + await ensureDirectory(this.baseDir); + } + + async write(context: ToolSessionContext) { + await atomicWriteJson(this.filePath(context.sessionId), context); + if (context.learningMode === "interactive" && context.sessionScopeKey) { + await atomicWriteJson(this.filePath(context.sessionScopeKey), context); + } + } + + async read(sessionId: string) { + return await readJsonFile(this.filePath(sessionId)); + } + + async remove(sessionId: string) { + await removeFileIfExists(this.filePath(sessionId)); + } + + private filePath(sessionId: string) { + return join(this.baseDir, `${sessionId}.json`); + } +} + +export const buildToolSessionScopeKey = ( + actorKey: string, + projectKey: string, + clientSessionId: string, +) => toConversationScopeKey(actorKey, projectKey, clientSessionId); diff --git a/src/tools/dynamicHttpExecutor.ts b/src/tools/dynamicHttpExecutor.ts index 71a3a2a..d002395 100644 --- a/src/tools/dynamicHttpExecutor.ts +++ b/src/tools/dynamicHttpExecutor.ts @@ -8,12 +8,12 @@ export type DynamicHttpInput = { path: string; method?: string; arguments?: Record; - body?: unknown; }; export type SessionToolContext = { accessToken?: string; actorKey: string; + clientSessionId: string; projectKey: string; sessionId: string; projectId?: string; @@ -53,13 +53,11 @@ export class DynamicHttpExecutor { if (context.projectId) { headers.set("x-project-id", context.projectId); } - const body = buildRequestBody(method, input.body, headers); const startedAt = Date.now(); const response = await fetch(url, { method, headers, - body, signal: AbortSignal.timeout(config.TJWATER_API_TIMEOUT_MS), }); const durationMs = Date.now() - startedAt; @@ -130,25 +128,6 @@ const buildQuery = (argumentsObject: Record) => { return pairs; }; -const buildRequestBody = ( - method: string, - body: unknown, - headers: Headers, -) => { - if (body === undefined) { - return undefined; - } - if (method === "GET") { - throw new Error("GET requests do not support body"); - } - const serialized = JSON.stringify(body); - if (serialized === undefined) { - throw new Error("body must be JSON-serializable"); - } - headers.set("Content-Type", "application/json"); - return serialized; -}; - const normalizeSuccessResult = async ( data: unknown, context: SessionToolContext, @@ -166,6 +145,7 @@ const normalizeSuccessResult = async ( // 大结果转成持久化引用，支持 review 和跨重启回读。 const record = await resultStore.store({ actorKey: context.actorKey, + clientSessionId: context.clientSessionId, data, kind: RESULT_REFERENCE_KIND.dynamicHttpResult, projectId: context.projectId, diff --git a/tests/conversations/store.test.ts b/tests/conversations/store.test.ts index 375a61d..f7dc550 100644 --- a/tests/conversations/store.test.ts +++ b/tests/conversations/store.test.ts @@ -1,21 +1,22 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { ConversationStore } from "../../src/conversations/store.js"; -import { createTestDatabase } from "../helpers/postgres.js"; describe("ConversationStore", () => { + let tempDir: string; let store: ConversationStore; - let dispose: (() => Promise) | undefined; beforeEach(async () => { - const testDb = await createTestDatabase(); - store = new ConversationStore(testDb.db); - dispose = testDb.dispose; + tempDir = await mkdtemp(join(tmpdir(), "tjwater-conversation-")); + store = new ConversationStore(tempDir); await store.initialize(); }); afterEach(async () => { - await dispose?.(); + await rm(tempDir, { force: true, recursive: true }); }); it("issues backend-managed session ids when absent", async () => { @@ -57,48 +58,7 @@ describe("ConversationStore", () => { }, "existing-session", ); - expect(fetched?.sessionId).toBe(record.sessionId); + expect(fetched?.sessionScopeKey).toBe(record.sessionScopeKey); expect(fetched?.title).toBe("新标题"); }); - - it("tracks streaming state and clears it with runtime-session guarding", async () => { - const { record } = await store.ensure({ - actorKey: "actor-3", - projectId: "project-3", - projectKey: "project-key-3", - sessionId: "streaming-session", - userId: "user-3", - }); - - const streaming = await store.markStreaming(record, "runtime-1"); - expect(streaming.isStreaming).toBe(true); - expect(streaming.streamingStartedAt).toBeDefined(); - - const skipped = await store.clearStreaming(record.sessionId, "runtime-2"); - expect(skipped).toBeNull(); - - const stillStreaming = await store.get( - { - actorKey: "actor-3", - projectId: "project-3", - projectKey: "project-key-3", - userId: "user-3", - }, - record.sessionId, - ); - expect(stillStreaming?.isStreaming).toBe(true); - - await store.clearStreaming(record.sessionId, "runtime-1"); - const cleared = await store.get( - { - actorKey: "actor-3", - projectId: "project-3", - projectKey: "project-key-3", - userId: "user-3", - }, - record.sessionId, - ); - expect(cleared?.isStreaming).toBe(false); - expect(cleared?.streamingStartedAt).toBeUndefined(); - }); }); diff --git a/tests/helpers/postgres.ts b/tests/helpers/postgres.ts deleted file mode 100644 index a57b275..0000000 --- a/tests/helpers/postgres.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { randomUUID } from "node:crypto"; - -import { AgentDatabase } from "../../src/db/index.js"; - -export const createTestDatabase = async () => { - const schema = `test_${randomUUID().replace(/-/g, "")}`; - const db = new AgentDatabase({ schema }); - await db.initialize(); - return { - db, - schema, - async dispose() { - await db.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`); - await db.close(); - }, - }; -}; diff --git a/tests/history/store.test.ts b/tests/history/store.test.ts index ea77025..4667765 100644 --- a/tests/history/store.test.ts +++ b/tests/history/store.test.ts @@ -1,53 +1,67 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -import { ConversationStore } from "../../src/conversations/store.js"; import { SessionHistoryStore } from "../../src/history/store.js"; -import { createTestDatabase } from "../helpers/postgres.js"; describe("SessionHistoryStore", () => { - let conversationStore: ConversationStore; + let tempDir: string; let store: SessionHistoryStore; - let dispose: (() => Promise) | undefined; beforeEach(async () => { - const testDb = await createTestDatabase(); - conversationStore = new ConversationStore(testDb.db); - store = new SessionHistoryStore(testDb.db); - dispose = testDb.dispose; - await Promise.all([conversationStore.initialize(), store.initialize()]); + tempDir = await mkdtemp(join(tmpdir(), "tjwater-history-")); + store = new SessionHistoryStore(tempDir); + await store.initialize(); }); afterEach(async () => { - await dispose?.(); + await rm(tempDir, { force: true, recursive: true }); }); - it("stores ordered turns for the durable conversation id", async () => { - await conversationStore.ensure({ - actorKey: "actor-1", - projectId: "project-1", - projectKey: "project-1", - sessionId: "thread-1", - userId: "user-1", - }); + it("falls back to legacy runtime-session transcripts by client session id and migrates on append", async () => { + await writeFile( + join(tempDir, "actor-1__project-1__runtime-session-1.json"), + JSON.stringify( + { + actorKey: "actor-1", + clientSessionId: "thread-1", + projectKey: "project-1", + sessionId: "runtime-session-1", + turns: [ + { + id: "turn-1", + assistantMessage: "先检查泵站流量。", + timestamp: "2026-05-21T00:00:00.000Z", + toolCallCount: 1, + userMessage: "帮我看一下当前异常。", + }, + ], + updatedAt: "2026-05-21T00:00:00.000Z", + }, + null, + 2, + ), + "utf8", + ); - const first = await store.appendTurn( + const recentTurns = await store.getRecentTurns( { actorKey: "actor-1", + clientSessionId: "thread-1", projectKey: "project-1", sessionId: "thread-1", }, - { - assistantMessage: "先检查泵站流量。", - toolCallCount: 1, - userMessage: "帮我看一下当前异常。", - }, + 5, ); - expect(first.turns).toHaveLength(1); + expect(recentTurns).toHaveLength(1); + expect(recentTurns[0]?.userMessage).toBe("帮我看一下当前异常。"); - const second = await store.appendTurn( + const transcript = await store.appendTurn( { actorKey: "actor-1", + clientSessionId: "thread-1", projectKey: "project-1", sessionId: "thread-1", }, @@ -58,31 +72,15 @@ describe("SessionHistoryStore", () => { }, ); - expect(second.sessionId).toBe("thread-1"); - expect(second.turns).toHaveLength(2); - expect(second.turns[0]?.userMessage).toBe("帮我看一下当前异常。"); - expect(second.turns[1]?.assistantMessage).toBe("已经定位到 3 条疑似异常支路。"); + expect(transcript.sessionId).toBe("thread-1"); + expect(transcript.turns).toHaveLength(2); }); it("clones only the kept prefix when forking a thread", async () => { - await conversationStore.ensure({ - actorKey: "actor-2", - projectId: "project-2", - projectKey: "project-2", - sessionId: "thread-source", - userId: "user-2", - }); - await conversationStore.ensure({ - actorKey: "actor-2", - projectId: "project-2", - projectKey: "project-2", - sessionId: "thread-fork", - userId: "user-2", - }); - await store.appendTurn( { actorKey: "actor-2", + clientSessionId: "thread-source", projectKey: "project-2", sessionId: "thread-source", }, @@ -95,6 +93,7 @@ describe("SessionHistoryStore", () => { await store.appendTurn( { actorKey: "actor-2", + clientSessionId: "thread-source", projectKey: "project-2", sessionId: "thread-source", }, @@ -108,11 +107,13 @@ describe("SessionHistoryStore", () => { const cloned = await store.cloneThread( { actorKey: "actor-2", + clientSessionId: "thread-source", projectKey: "project-2", sessionId: "thread-source", }, { actorKey: "actor-2", + clientSessionId: "thread-fork", projectKey: "project-2", sessionId: "thread-fork", }, @@ -125,6 +126,7 @@ describe("SessionHistoryStore", () => { const forkRecentTurns = await store.getRecentTurns( { actorKey: "actor-2", + clientSessionId: "thread-fork", projectKey: "project-2", sessionId: "thread-fork", }, diff --git a/tests/results/store.test.ts b/tests/results/store.test.ts index a086246..a86424d 100644 --- a/tests/results/store.test.ts +++ b/tests/results/store.test.ts @@ -3,65 +3,46 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { ConversationStore } from "../../src/conversations/store.js"; import { ResultReferenceResolver } from "../../src/results/resolver.js"; import { RESULT_REFERENCE_KIND, RESULT_REFERENCE_SOURCE, ResultReferenceStore, } from "../../src/results/store.js"; -import { createTestDatabase } from "../helpers/postgres.js"; describe("ResultReferenceResolver", () => { - let payloadDir: string; - let conversationStore: ConversationStore; + let tempDir: string; let store: ResultReferenceStore; let resolver: ResultReferenceResolver; - let dispose: (() => Promise) | undefined; beforeEach(async () => { - payloadDir = await mkdtemp(join(tmpdir(), "tjwater-result-ref-")); - const testDb = await createTestDatabase(); - conversationStore = new ConversationStore(testDb.db); - store = new ResultReferenceStore(testDb.db, payloadDir, 60_000); + tempDir = await mkdtemp(join(tmpdir(), "tjwater-result-ref-")); + store = new ResultReferenceStore(tempDir, 60_000); resolver = new ResultReferenceResolver(store); - dispose = async () => { - await testDb.dispose(); - await rm(payloadDir, { force: true, recursive: true }); - }; - await Promise.all([conversationStore.initialize(), store.initialize()]); + await store.initialize(); }); afterEach(async () => { - await dispose?.(); + await rm(tempDir, { force: true, recursive: true }); }); - it("stores referenced results as metadata plus an external payload file", async () => { - await conversationStore.ensure({ - actorKey: "actor-1", - projectId: "project-1", - projectKey: "project-key-1", - sessionId: "client-1", - userId: "user-1", - }); - + it("stores metadata for new referenced results and resolves them", async () => { const record = await resolver.register({ actorKey: "actor-1", + clientSessionId: "client-1", data: [{ id: "J1" }, { id: "J2" }], kind: RESULT_REFERENCE_KIND.dynamicHttpResult, projectId: "project-1", projectKey: "project-key-1", schemaVersion: 1, - sessionId: "client-1", + sessionId: "session-1", source: RESULT_REFERENCE_SOURCE.dynamicHttp, traceId: "trace-1", }); - expect(record.payloadPath).toContain("/payloads/"); - const payload = JSON.parse(await readFile(record.payloadPath!, "utf8")) as { - data?: unknown; - }; - expect(payload.data).toEqual([{ id: "J1" }, { id: "J2" }]); + expect(record.kind).toBe(RESULT_REFERENCE_KIND.dynamicHttpResult); + expect(record.schemaVersion).toBe(1); + expect(record.source).toBe(RESULT_REFERENCE_SOURCE.dynamicHttp); const result = await resolver.getAuthorized( record.resultRef, @@ -74,21 +55,124 @@ describe("ResultReferenceResolver", () => { }, ); + expect(result).not.toBeNull(); expect(result?.kind).toBe(RESULT_REFERENCE_KIND.dynamicHttpResult); + expect(result?.schema_version).toBe(1); expect(result?.source).toBe(RESULT_REFERENCE_SOURCE.dynamicHttp); expect(result?.data).toEqual([{ id: "J1" }]); }); - it("reuses an existing external render payload file as the payload locator", async () => { - await conversationStore.ensure({ - actorKey: "actor-3", - projectId: "project-3", - projectKey: "project-key-3", - sessionId: "client-3", - userId: "user-3", + it("keeps legacy result refs readable while defaulting metadata", async () => { + const legacyRef = "res-aaaaaaaaaaaaaaaa"; + await writeFile( + join(tempDir, `${legacyRef}.json`), + JSON.stringify( + { + resultRef: legacyRef, + actorKey: "actor-legacy", + clientSessionId: "client-legacy", + createdAt: "2026-05-21T00:00:00.000Z", + data: { nodes: ["J1"] }, + preview: { + count: 1, + fields: ["nodes"], + sample: { nodes: ["J1"] }, + summary: "object<1 fields>", + }, + projectId: "project-legacy", + projectKey: "project-key-legacy", + sessionId: "session-legacy", + sizeBytes: 16, + traceId: "trace-legacy", + }, + null, + 2, + ), + "utf8", + ); + + const record = await store.getAuthorizedRecord(legacyRef, { + actorKey: "actor-legacy", + projectId: "project-legacy", }); - const filePath = join(payloadDir, "render-wrapper.json"); + expect(record).not.toBeNull(); + expect(record?.kind).toBe(RESULT_REFERENCE_KIND.dynamicHttpResult); + expect(record?.schemaVersion).toBe(1); + expect(record?.source).toBe(RESULT_REFERENCE_SOURCE.legacy); + }); + + it("rejects malformed refs, mismatched kinds, and auth mismatches", async () => { + const malformedRef = "res-bbbbbbbbbbbbbbbb"; + await writeFile( + join(tempDir, `${malformedRef}.json`), + JSON.stringify( + { + resultRef: malformedRef, + createdAt: "2026-05-21T00:00:00.000Z", + data: { value: 1 }, + preview: { + count: 1, + fields: ["value"], + sample: { value: 1 }, + summary: "object<1 fields>", + }, + projectId: "project-1", + projectKey: "project-key-1", + sessionId: "session-1", + sizeBytes: 10, + traceId: "trace-1", + }, + null, + 2, + ), + "utf8", + ); + + const malformed = await store.getAuthorizedRecord(malformedRef, { + actorKey: "actor-1", + projectId: "project-1", + }); + expect(malformed).toBeNull(); + + const renderRecord = await resolver.register({ + actorKey: "actor-2", + clientSessionId: "client-2", + data: { + node_area_map: { + J1: "DMA-1", + }, + }, + kind: RESULT_REFERENCE_KIND.renderJunctionsPayload, + projectId: "project-2", + projectKey: "project-key-2", + schemaVersion: 1, + sessionId: "session-2", + source: RESULT_REFERENCE_SOURCE.agentGenerated, + traceId: "trace-2", + }); + + const wrongKind = await resolver.getFullAuthorized( + renderRecord.resultRef, + { + actorKey: "actor-2", + projectId: "project-2", + }, + { + expectedKind: RESULT_REFERENCE_KIND.dynamicHttpResult, + }, + ); + expect(wrongKind).toBeNull(); + + const wrongActor = await resolver.getFullAuthorized(renderRecord.resultRef, { + actorKey: "actor-other", + projectId: "project-2", + }); + expect(wrongActor).toBeNull(); + }); + + it("registers render refs from local wrapper files and normalizes payloads", async () => { + const filePath = join(tempDir, "render-wrapper.json"); await writeFile( filePath, JSON.stringify( @@ -106,7 +190,12 @@ describe("ResultReferenceResolver", () => { J2: 2, }, area_ids: ["DMA-1", " DMA-2 "], + area_colors: { + "DMA-1": "#ff0000", + "DMA-2": "#00ff00", + }, }, + createdAt: "2026-05-21T00:00:00.000Z", }, null, 2, @@ -116,21 +205,22 @@ describe("ResultReferenceResolver", () => { const record = await resolver.registerRenderPayloadFile(filePath, { actorKey: "actor-3", + clientSessionId: "client-3", projectId: "project-3", projectKey: "project-key-3", - sessionId: "client-3", + sessionId: "session-3", source: RESULT_REFERENCE_SOURCE.migration, traceId: "trace-3", }); - expect(record.payloadPath).toBe(filePath); + expect(record.kind).toBe(RESULT_REFERENCE_KIND.renderJunctionsPayload); + expect(record.source).toBe(RESULT_REFERENCE_SOURCE.migration); const result = await resolver.getFullAuthorized( record.resultRef, { actorKey: "actor-3", projectId: "project-3", - sessionId: "client-3", }, { expectedKind: RESULT_REFERENCE_KIND.renderJunctionsPayload, @@ -143,6 +233,185 @@ describe("ResultReferenceResolver", () => { J2: "2", }, area_ids: ["DMA-1", "DMA-2"], + area_colors: { + "DMA-1": "#ff0000", + "DMA-2": "#00ff00", + }, }); }); + + it("repairs wrapper files that omit metadata and location", async () => { + const filePath = join(tempDir, "render-wrapper-missing-fields.json"); + await writeFile( + filePath, + JSON.stringify( + { + data: { + node_area_map: { + J1: "DMA-1", + }, + }, + createdAt: "2026-05-21T00:00:00.000Z", + }, + null, + 2, + ), + "utf8", + ); + + const record = await resolver.registerRenderPayloadFile(filePath, { + actorKey: "actor-4", + clientSessionId: "client-4", + projectId: "project-4", + projectKey: "project-key-4", + sessionId: "session-4", + source: RESULT_REFERENCE_SOURCE.migration, + traceId: "trace-4", + }); + + expect(record.kind).toBe(RESULT_REFERENCE_KIND.renderJunctionsPayload); + + const repaired = JSON.parse(await readFile(filePath, "utf8")) as { + metadata?: Record; + location?: Record; + }; + + expect(repaired.metadata).toEqual({ + createdAt: "2026-05-21T00:00:00.000Z", + projectId: "project-4", + }); + expect(repaired.location).toEqual({ + file_path: filePath, + }); + }); + + it("repairs wrapper files whose location points elsewhere", async () => { + const filePath = join(tempDir, "render-wrapper-wrong-location.json"); + await writeFile( + filePath, + JSON.stringify( + { + metadata: { + createdAt: "2026-05-21T00:00:00.000Z", + }, + location: { + file_path: "/tmp/elsewhere.json", + source: "legacy", + }, + data: { + node_area_map: { + J1: "DMA-1", + }, + }, + }, + null, + 2, + ), + "utf8", + ); + + await resolver.registerRenderPayloadFile(filePath, { + actorKey: "actor-4", + clientSessionId: "client-4", + projectId: "project-4", + projectKey: "project-key-4", + sessionId: "session-4", + source: RESULT_REFERENCE_SOURCE.migration, + traceId: "trace-4", + }); + + const repaired = JSON.parse(await readFile(filePath, "utf8")) as { + metadata?: Record; + location?: Record; + }; + + expect(repaired.metadata).toEqual({ + createdAt: "2026-05-21T00:00:00.000Z", + projectId: "project-4", + }); + expect(repaired.location).toEqual({ + file_path: filePath, + source: "legacy", + }); + }); + + it("resolves legacy render payload files when callers include the json suffix", async () => { + const legacyRef = "res-c2fcee33-577e"; + await writeFile( + join(tempDir, `${legacyRef}.json`), + JSON.stringify( + { + data: { + node_area_map: { + J1: "DMA-1", + J2: 2, + }, + area_ids: ["DMA-1"], + }, + createdAt: "2026-05-21T00:00:00.000Z", + projectId: "project-legacy-render", + }, + null, + 2, + ), + "utf8", + ); + + const result = await resolver.getFullAuthorized( + `${legacyRef}.json`, + { + actorKey: "actor-legacy-render", + clientSessionId: "chat-legacy-render", + projectId: "project-legacy-render", + }, + { + expectedKind: RESULT_REFERENCE_KIND.renderJunctionsPayload, + }, + ); + + expect(result?.result_ref).toBe(legacyRef); + expect(result?.kind).toBe(RESULT_REFERENCE_KIND.renderJunctionsPayload); + expect(result?.source).toBe(RESULT_REFERENCE_SOURCE.legacy); + expect(result?.data).toEqual({ + node_area_map: { + J1: "DMA-1", + J2: "2", + }, + area_ids: ["DMA-1"], + }); + }); + + it("keeps legacy render payload files scoped to their project", async () => { + const legacyRef = "res-dddddddddddddddd"; + await writeFile( + join(tempDir, `${legacyRef}.json`), + JSON.stringify( + { + data: { + node_area_map: { + J1: "DMA-1", + }, + }, + projectId: "project-allowed", + }, + null, + 2, + ), + "utf8", + ); + + const result = await resolver.getFullAuthorized( + legacyRef, + { + actorKey: "actor-legacy-render", + clientSessionId: "chat-legacy-render", + projectId: "project-denied", + }, + { + expectedKind: RESULT_REFERENCE_KIND.renderJunctionsPayload, + }, + ); + + expect(result).toBeNull(); + }); }); diff --git a/tests/session/runtimeSessionStore.test.ts b/tests/session/runtimeSessionStore.test.ts deleted file mode 100644 index 4e26417..0000000 --- a/tests/session/runtimeSessionStore.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; - -import { ConversationStore } from "../../src/conversations/store.js"; -import { RuntimeSessionStore } from "../../src/session/runtimeSessionStore.js"; -import { createTestDatabase } from "../helpers/postgres.js"; - -describe("RuntimeSessionStore", () => { - let conversationStore: ConversationStore; - let store: RuntimeSessionStore; - let dispose: (() => Promise) | undefined; - - beforeEach(async () => { - const testDb = await createTestDatabase(); - conversationStore = new ConversationStore(testDb.db); - store = new RuntimeSessionStore(testDb.db); - dispose = testDb.dispose; - await Promise.all([conversationStore.initialize(), store.initialize()]); - }); - - afterEach(async () => { - await dispose?.(); - }); - - it("stores and releases runtime sessions by runtime session id", async () => { - await conversationStore.ensure({ - actorKey: "actor-1", - projectId: "project-1", - projectKey: "project-1", - sessionId: "chat-session-1", - userId: "user-1", - }); - - await store.write({ - runtimeSessionId: "runtime-session-1", - actorKey: "actor-1", - allowLearningWrite: true, - sessionId: "chat-session-1", - learningMode: "interactive", - projectId: "project-id-1", - projectKey: "project-1", - traceId: "trace-1", - }); - - const runtimeContext = await store.read("runtime-session-1"); - expect(runtimeContext?.sessionId).toBe("chat-session-1"); - expect(runtimeContext?.runtimeSessionId).toBe("runtime-session-1"); - - await store.release("runtime-session-1"); - expect(await store.read("runtime-session-1")).toBeNull(); - expect((await store.read("runtime-session-1", { includeReleased: true }))?.releasedAt).toBeDefined(); - }); -}); diff --git a/tests/session/toolContextStore.test.ts b/tests/session/toolContextStore.test.ts new file mode 100644 index 0000000..7ff6b3d --- /dev/null +++ b/tests/session/toolContextStore.test.ts @@ -0,0 +1,51 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + buildToolSessionScopeKey, + ToolSessionContextStore, +} from "../../src/session/toolContextStore.js"; + +describe("ToolSessionContextStore", () => { + let tempDir: string; + let store: ToolSessionContextStore; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), "tjwater-tool-context-")); + store = new ToolSessionContextStore(tempDir); + await store.initialize(); + }); + + afterEach(async () => { + await rm(tempDir, { force: true, recursive: true }); + }); + + it("writes interactive aliases under scoped session keys", async () => { + const sessionScopeKey = buildToolSessionScopeKey( + "actor-1", + "project-1", + "chat-session-1", + ); + + await store.write({ + actorKey: "actor-1", + allowLearningWrite: true, + clientSessionId: "chat-session-1", + learningMode: "interactive", + projectId: "project-id-1", + projectKey: "project-1", + sessionId: "runtime-session-1", + sessionScopeKey, + traceId: "trace-1", + }); + + const runtimeContext = await store.read("runtime-session-1"); + const scopedContext = await store.read(sessionScopeKey); + + expect(runtimeContext?.clientSessionId).toBe("chat-session-1"); + expect(scopedContext?.sessionScopeKey).toBe(sessionScopeKey); + expect(scopedContext?.sessionId).toBe("runtime-session-1"); + }); +}); diff --git a/tests/tools/dynamicHttpExecutor.test.ts b/tests/tools/dynamicHttpExecutor.test.ts deleted file mode 100644 index 1f055c6..0000000 --- a/tests/tools/dynamicHttpExecutor.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { ResultReferenceStore } from "../../src/results/store.js"; -import { DynamicHttpExecutor } from "../../src/tools/dynamicHttpExecutor.js"; -import { createTestDatabase } from "../helpers/postgres.js"; - -describe("DynamicHttpExecutor", () => { - const originalFetch = globalThis.fetch; - let payloadDir: string; - let store: ResultReferenceStore; - let executor: DynamicHttpExecutor; - let dispose: (() => Promise) | undefined; - - beforeEach(async () => { - payloadDir = await mkdtemp(join(tmpdir(), "tjwater-dynamic-http-")); - const testDb = await createTestDatabase(); - store = new ResultReferenceStore(testDb.db, payloadDir, 60_000); - executor = new DynamicHttpExecutor(store); - dispose = async () => { - globalThis.fetch = originalFetch; - await testDb.dispose(); - await rm(payloadDir, { force: true, recursive: true }); - }; - await store.initialize(); - }); - - afterEach(async () => { - await dispose?.(); - }); - - it("sends JSON body for POST requests while preserving query arguments", async () => { - let capturedUrl = ""; - let capturedInit: RequestInit | undefined; - globalThis.fetch = Object.assign( - async (input: URL | RequestInfo, init?: RequestInit | BunFetchRequestInit) => { - capturedUrl = String(input); - capturedInit = init; - return new Response(JSON.stringify({ status: "ok" }), { - status: 200, - headers: { "content-type": "application/json" }, - }); - }, - originalFetch, - ); - - const result = await executor.execute( - { - path: "/api/v1/runproject/simulate", - method: "POST", - arguments: { network: "demo-network" }, - body: { duration_minutes: 15 }, - }, - { - accessToken: "token-1", - actorKey: "actor-1", - projectId: "project-1", - projectKey: "project-key-1", - sessionId: "session-1", - traceId: "trace-1", - }, - ); - - const url = new URL(capturedUrl); - const headers = new Headers(capturedInit?.headers); - - expect(url.pathname).toBe("/api/v1/runproject/simulate"); - expect(url.searchParams.get("network")).toBe("demo-network"); - expect(capturedInit?.method).toBe("POST"); - expect(headers.get("content-type")).toBe("application/json"); - expect(headers.get("authorization")).toBe("Bearer token-1"); - expect(headers.get("x-project-id")).toBe("project-1"); - expect(headers.get("x-trace-id")).toBe("trace-1"); - expect(capturedInit?.body).toBe(JSON.stringify({ duration_minutes: 15 })); - expect(result).toEqual({ - ok: true, - trace_id: "trace-1", - upstream: { - method: "POST", - path: "/api/v1/runproject/simulate", - status_code: 200, - }, - result_mode: "inline", - result_size_bytes: expect.any(Number), - data: { status: "ok" }, - }); - }); - - it("rejects GET requests that include a body", async () => { - await expect( - executor.execute( - { - path: "/api/v1/runproject/", - method: "GET", - arguments: { network: "demo-network" }, - body: { duration_minutes: 15 }, - }, - { - actorKey: "actor-1", - projectId: "project-1", - projectKey: "project-key-1", - sessionId: "session-1", - traceId: "trace-1", - }, - ), - ).rejects.toThrow("GET requests do not support body"); - }); -});