From 5ac50bfeaac03a7e28caf4e2334060a5ee446738 Mon Sep 17 00:00:00 2001 From: Huarch Date: Thu, 28 May 2026 18:22:39 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=87=E6=8D=A2=E5=88=B0=E4=BD=BF=E7=94=A8pg?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/copilot-instructions.md | 124 +++++ .opencode/skills/examples.md | 15 + .opencode/skills/runbook.md | 1 + .opencode/tools/dynamic_http_call.ts | 21 +- .opencode/tools/fetch_result_ref.ts | 10 +- .opencode/tools/memory_manager.ts | 12 +- .opencode/tools/session_search.ts | 10 +- .opencode/tools/skill_manager.ts | 8 +- .opencode/tools/store_render_ref.ts | 10 +- README.md | 2 +- bun.lock | 30 + package.json | 3 + scripts/migrate-file-storage-to-postgres.ts | 578 ++++++++++++++++++++ src/audit/llmRequestAudit.ts | 1 - src/chat/sessionBridge.ts | 122 ++--- src/config.ts | 26 + src/conversations/stateStore.ts | 90 ++- src/conversations/store.ts | 226 ++++++-- src/db/index.ts | 486 ++++++++++++++++ src/history/store.ts | 378 +++++++------ src/learning/orchestrator.ts | 40 +- src/learning/stateStore.ts | 93 +++- src/memory/store.ts | 265 +++++---- src/results/resolver.ts | 12 +- src/results/store.ts | 421 +++++++------- src/routes/chat.ts | 86 +-- src/routes/chatStream.ts | 30 +- src/server.ts | 60 +- src/session/runtimeSessionStore.ts | 123 +++++ src/session/toolContextStore.ts | 55 -- src/tools/dynamicHttpExecutor.ts | 24 +- tests/conversations/store.test.ts | 56 +- tests/helpers/postgres.ts | 17 + tests/history/store.test.ts | 92 ++-- tests/results/store.test.ts | 351 ++---------- tests/session/runtimeSessionStore.test.ts | 52 ++ tests/session/toolContextStore.test.ts | 51 -- tests/tools/dynamicHttpExecutor.test.ts | 110 ++++ 38 files changed, 2760 insertions(+), 1331 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 scripts/migrate-file-storage-to-postgres.ts create mode 100644 src/db/index.ts create mode 100644 src/session/runtimeSessionStore.ts delete mode 100644 src/session/toolContextStore.ts create mode 100644 tests/helpers/postgres.ts create mode 100644 tests/session/runtimeSessionStore.test.ts delete mode 100644 tests/session/toolContextStore.test.ts create mode 100644 tests/tools/dynamicHttpExecutor.test.ts diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..c531ee5 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,124 @@ +# Copilot Instructions for TJWaterAgent + +## Overview + +- TJWaterAgent is a Bun + TypeScript backend service. +- Main entrypoint: `src/server.ts` +- Chat API base path: `/api/v1/agent/chat` +- Persistent storage is **PostgreSQL-first**. Historical file storage only remains for one-time migration and external result payload files. + +## Core identity model + +- `sessionId` = durable chat thread identifier exposed to clients and used for persistence. +- `runtimeSessionId` = temporary opencode runtime execution identifier for a single active request. +- Do **not** reintroduce `conversationId` / `clientSessionId` naming for the durable thread id. Use `sessionId`. + +## Persistence design + +Canonical PostgreSQL tables: + +- `conversations` +- `conversation_states` +- `conversation_turns` +- `runtime_sessions` +- `learning_states` +- `result_refs` +- `memories` + +### conversations + +Stores durable thread metadata: + +- `session_id` +- actor / owner / project fields +- `parent_session_id` +- `title` +- `status` +- streaming lifecycle fields: + - `is_streaming` + - `active_runtime_session_id` + - `streaming_started_at` + +`is_streaming` is persisted so the frontend can recover streaming state after refresh. Stream cleanup must be guarded by `active_runtime_session_id` so an old stream cannot clear a newer one. + +### conversation_states + +Stores frontend-facing session UI state: + +- `messages` +- `branch_groups` +- `is_title_manually_edited` + +### conversation_turns + +Stores transcript turns as one row per turn: + +- keyed by `turn_id` +- linked by `session_id` +- ordered by deterministic `turn_index` + +Do not fall back to legacy transcript files in runtime code. + +### runtime_sessions + +Stores runtime binding context for internal tools and learning review/gate flows: + +- `runtime_session_id` +- `session_id` +- actor / project / trace fields +- `allow_learning_write` +- `learning_mode` +- `released_at` + +This replaces the old alias-based `tool_session_contexts` model. + +### result_refs + +`result_refs` is **metadata-only in PostgreSQL**: + +- metadata lives in PG +- payload body lives outside PG via `payload_path` or future `object_key` + +Most payloads are large, so do not inline them into PostgreSQL. + +### memories + +Durable memory is stored in PG, not markdown files. + +## Runtime flow + +1. Client sends/loads a `session_id`. +2. `ChatSessionBridge` binds that durable `sessionId` to a fresh `runtimeSessionId`. +3. Internal tools call back into the server using `runtimeSessionId`. +4. Transcript, memory, learning state, and result refs persist against `sessionId`. +5. Stream start sets `conversations.is_streaming = true`; stream completion clears it only if the runtime id still matches. + +## Important files + +- `src/server.ts` - bootstrap, internal tool endpoints, app wiring +- `src/chat/sessionBridge.ts` - `sessionId` vs `runtimeSessionId` binding +- `src/db/index.ts` - PostgreSQL bootstrap and schema migration/cleanup +- `src/conversations/store.ts` - durable session metadata + streaming state +- `src/conversations/stateStore.ts` - UI state persistence +- `src/history/store.ts` - transcript persistence +- `src/session/runtimeSessionStore.ts` - runtime session persistence +- `src/results/store.ts` - metadata-only result refs + external payload files +- `src/results/resolver.ts` - result ref normalization and retrieval +- `src/learning/orchestrator.ts` / `src/learning/stateStore.ts` - learning review pipeline +- `src/memory/store.ts` - persistent memory store +- `scripts/migrate-file-storage-to-postgres.ts` - one-time migration from legacy file storage + +## Compatibility rules + +- Runtime code should target the canonical schema only. +- Legacy column/table handling belongs only in `src/db/index.ts` bootstrap migration or `scripts/migrate-file-storage-to-postgres.ts`. +- Do not add back file-based runtime persistence or compatibility reads unless a migration explicitly requires it. + +## Validation + +Useful commands: + +```bash +bun run check +PGHOST=... PGPORT=... PGUSER=... PGPASSWORD=... PGDATABASE=agent PGSSLMODE=disable bun test tests/conversations/store.test.ts tests/history/store.test.ts tests/session/runtimeSessionStore.test.ts tests/results/store.test.ts tests/routes/chatSession.test.ts +``` diff --git a/.opencode/skills/examples.md b/.opencode/skills/examples.md index 6cf022d..1d6b5d7 100644 --- a/.opencode/skills/examples.md +++ b/.opencode/skills/examples.md @@ -44,6 +44,21 @@ - 第二步（可选）工具调用：查询补充数据接口。 - opencode agent 汇总工具结果，持续通过 SSE 输出 `progress` 与 `token`，最终返回 `done`。 +若接口要求 JSON 请求体，可这样调用： + +```json +{ + "path": "/api/v1/runproject/simulate", + "method": "POST", + "arguments": { + "network": "demo-network" + }, + "body": { + "duration_minutes": 15 + } +} +``` + `progress` 示例： ```json diff --git a/.opencode/skills/runbook.md b/.opencode/skills/runbook.md index 486b593..245640a 100644 --- a/.opencode/skills/runbook.md +++ b/.opencode/skills/runbook.md @@ -66,6 +66,7 @@ SSE 事件： - `path` 必须以 `/` 开头。 - `method` 支持：`GET/POST/PUT/PATCH/DELETE`。 - `arguments` 会编码为 query 参数（列表会转为逗号拼接）。 +- `body` 会作为 JSON request body 透传；`GET` 不支持传 `body`。 ## 3.1) 学习工具约定 diff --git a/.opencode/tools/dynamic_http_call.ts b/.opencode/tools/dynamic_http_call.ts index 80a5138..18e16f0 100644 --- a/.opencode/tools/dynamic_http_call.ts +++ b/.opencode/tools/dynamic_http_call.ts @@ -1,14 +1,11 @@ import { tool } from "@opencode-ai/plugin"; -import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; -const toolContextStore = new ToolSessionContextStore(); -const initializePromise = toolContextStore.initialize(); export default tool({ description: - "通过本地 Agent 桥接调用 TJWater 后端 API。需提供 API 路径、可选的请求方法以及查询参数。", + "通过本地 Agent 桥接调用 TJWater 后端 API。支持 query 参数，且在 POST/PUT/PATCH/DELETE 时可选传递 JSON body。", args: { reason: tool.schema .string() @@ -21,15 +18,14 @@ export default tool({ arguments: tool.schema .record(tool.schema.string(), tool.schema.unknown()) .optional() - .describe("Query arguments object."), + .describe("Query arguments object. Encoded into URL query string."), + body: tool.schema + .unknown() + .optional() + .describe("Optional JSON body for POST/PUT/PATCH/DELETE requests."), }, async execute(args, context) { - await initializePromise; - const sessionContext = await toolContextStore.read(context.sessionID); - if (!sessionContext) { - throw new Error(`session context not found for ${context.sessionID}`); - } - // 工具本身不直接持有用户 token；通过 sessionID 回调 Agent 服务，由服务侧补齐用户上下文。 + // 工具本身不直接持有用户 token；通过 runtimeSessionId 回调 Agent 服务，由服务侧补齐用户上下文。 const response = await fetch(`${internalBaseUrl}/internal/tools/dynamic-http-call`, { method: "POST", headers: { @@ -37,11 +33,12 @@ export default tool({ "x-agent-internal-token": internalToken, }, body: JSON.stringify({ - sessionScopeKey: sessionContext.sessionScopeKey, + runtimeSessionId: context.sessionID, reason: args.reason, path: args.path, method: args.method, arguments: args.arguments, + body: args.body, }), }); diff --git a/.opencode/tools/fetch_result_ref.ts b/.opencode/tools/fetch_result_ref.ts index 32a45c0..3060a0b 100644 --- a/.opencode/tools/fetch_result_ref.ts +++ b/.opencode/tools/fetch_result_ref.ts @@ -1,10 +1,7 @@ import { tool } from "@opencode-ai/plugin"; -import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; -const toolContextStore = new ToolSessionContextStore(); -const initializePromise = toolContextStore.initialize(); export default tool({ description: @@ -22,11 +19,6 @@ export default tool({ .describe("Optional maximum number of top-level items or fields to return."), }, async execute(args, context) { - await initializePromise; - const sessionContext = await toolContextStore.read(context.sessionID); - if (!sessionContext) { - throw new Error(`session context not found for ${context.sessionID}`); - } const response = await fetch(`${internalBaseUrl}/internal/tools/fetch-result-ref`, { method: "POST", headers: { @@ -34,7 +26,7 @@ export default tool({ "x-agent-internal-token": internalToken, }, body: JSON.stringify({ - sessionScopeKey: sessionContext.sessionScopeKey, + runtimeSessionId: context.sessionID, result_ref: args.result_ref, max_items: args.max_items, }), diff --git a/.opencode/tools/memory_manager.ts b/.opencode/tools/memory_manager.ts index 22f7795..b55c3da 100644 --- a/.opencode/tools/memory_manager.ts +++ b/.opencode/tools/memory_manager.ts @@ -1,12 +1,12 @@ import { tool } from "@opencode-ai/plugin"; import { MemoryStore } from "../../src/memory/store.js"; -import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; +import { RuntimeSessionStore } from "../../src/session/runtimeSessionStore.js"; const memoryStore = new MemoryStore(); -const toolContextStore = new ToolSessionContextStore(); +const runtimeSessionStore = new RuntimeSessionStore(); const initializePromise = Promise.all([ memoryStore.initialize(), - toolContextStore.initialize(), + runtimeSessionStore.initialize(), ]); export default tool({ @@ -37,7 +37,7 @@ export default tool({ }, async execute(args, context) { await initializePromise; - const sessionContext = await toolContextStore.read(context.sessionID); + const sessionContext = await runtimeSessionStore.read(context.sessionID); if (!sessionContext) { throw new Error(`session context not found for ${context.sessionID}`); } @@ -80,7 +80,7 @@ export default tool({ if (args.action === "add") { const result = await memoryStore.upsert(scope, scopeKey, { content: args.content ?? "", - sessionId: sessionContext.clientSessionId, + sessionId: sessionContext.sessionId, source: "tool", traceId: sessionContext.traceId, }); @@ -105,7 +105,7 @@ export default tool({ if (args.action === "replace") { const result = await memoryStore.replace(scope, scopeKey, args.target_id ?? "", { content: args.content ?? "", - sessionId: sessionContext.clientSessionId, + sessionId: sessionContext.sessionId, source: "tool", traceId: sessionContext.traceId, }); diff --git a/.opencode/tools/session_search.ts b/.opencode/tools/session_search.ts index 9631b3c..f571df9 100644 --- a/.opencode/tools/session_search.ts +++ b/.opencode/tools/session_search.ts @@ -1,11 +1,8 @@ import { tool } from "@opencode-ai/plugin"; -import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; -const toolContextStore = new ToolSessionContextStore(); -const initializePromise = toolContextStore.initialize(); export default tool({ description: @@ -25,11 +22,6 @@ export default tool({ .describe("Optional maximum number of hits to return."), }, async execute(args, context) { - await initializePromise; - const sessionContext = await toolContextStore.read(context.sessionID); - if (!sessionContext) { - throw new Error(`session context not found for ${context.sessionID}`); - } const response = await fetch(`${internalBaseUrl}/internal/tools/session-search`, { method: "POST", headers: { @@ -39,7 +31,7 @@ export default tool({ body: JSON.stringify({ max_results: args.max_results, query: args.query, - sessionScopeKey: sessionContext.sessionScopeKey, + runtimeSessionId: context.sessionID, }), }); const text = await response.text(); diff --git a/.opencode/tools/skill_manager.ts b/.opencode/tools/skill_manager.ts index da199b7..9dabf62 100644 --- a/.opencode/tools/skill_manager.ts +++ b/.opencode/tools/skill_manager.ts @@ -1,10 +1,10 @@ import { tool } from "@opencode-ai/plugin"; import { SkillStore } from "../../src/skills/store.js"; -import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; +import { RuntimeSessionStore } from "../../src/session/runtimeSessionStore.js"; -const toolContextStore = new ToolSessionContextStore(); -const initializePromise = toolContextStore.initialize(); +const runtimeSessionStore = new RuntimeSessionStore(); +const initializePromise = runtimeSessionStore.initialize(); const skillStore = new SkillStore(); export default tool({ @@ -51,7 +51,7 @@ export default tool({ }, async execute(args, context) { await initializePromise; - const sessionContext = await toolContextStore.read(context.sessionID); + const sessionContext = await runtimeSessionStore.read(context.sessionID); if (!sessionContext) { throw new Error(`session context not found for ${context.sessionID}`); } diff --git a/.opencode/tools/store_render_ref.ts b/.opencode/tools/store_render_ref.ts index 20725f1..1b28213 100644 --- a/.opencode/tools/store_render_ref.ts +++ b/.opencode/tools/store_render_ref.ts @@ -1,10 +1,7 @@ import { tool } from "@opencode-ai/plugin"; -import { ToolSessionContextStore } from "../../src/session/toolContextStore.js"; const internalBaseUrl = process.env.TJWATER_AGENT_INTERNAL_BASE_URL ?? "http://127.0.0.1:8787"; const internalToken = process.env.TJWATER_AGENT_INTERNAL_TOKEN ?? ""; -const toolContextStore = new ToolSessionContextStore(); -const initializePromise = toolContextStore.initialize(); export default tool({ description: @@ -20,11 +17,6 @@ export default tool({ ), }, async execute(args, context) { - await initializePromise; - const sessionContext = await toolContextStore.read(context.sessionID); - if (!sessionContext) { - throw new Error(`session context not found for ${context.sessionID}`); - } const response = await fetch(`${internalBaseUrl}/internal/tools/store-render-ref`, { method: "POST", headers: { @@ -32,7 +24,7 @@ export default tool({ "x-agent-internal-token": internalToken, }, body: JSON.stringify({ - sessionScopeKey: sessionContext.sessionScopeKey, + runtimeSessionId: context.sessionID, file_path: args.file_path, }), }); diff --git a/README.md b/README.md index c77e63e..2be8d1f 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ src/ 这些是 opencode 可以调用的自定义工具。 -`dynamic_http_call.ts` 不直接保存用户 token，也不直接访问后端。它会回调 `TJWaterAgent` 的内部接口，由上级服务层根据当前 session 补上用户 token、项目 ID 和 trace ID，再调用 TJWater 后端。 +`dynamic_http_call.ts` 不直接保存用户 token，也不直接访问后端。它会回调 `TJWaterAgent` 的内部接口，由上级服务层根据当前 session 补上用户 token、项目 ID 和 trace ID，再调用 TJWater 后端。`arguments` 会编码为 query 参数；`body` 可用于 `POST/PUT/PATCH/DELETE` 的 JSON 请求体。 前端类工具如 `locate_features`、`view_history`、`view_scada`、`show_chart` 主要用于触发 UI 动作或可视化，不应被当作数据查询工具。 diff --git a/bun.lock b/bun.lock index 90544b1..27636f8 100644 --- a/bun.lock +++ b/bun.lock @@ -6,9 +6,11 @@ "name": "tjwater-agent", "dependencies": { "@opencode-ai/sdk": "^1.14.29", + "@types/pg": "^8.20.0", "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^4.21.2", + "pg": "^8.21.0", "pino": "^9.7.0", "pino-pretty": "^13.1.2", "zod": "^3.25.76", @@ -41,6 +43,8 @@ "@types/node": ["@types/node@24.12.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g=="], + "@types/pg": ["@types/pg@8.20.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow=="], + "@types/qs": ["@types/qs@6.15.0", "", {}, "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow=="], "@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="], @@ -181,6 +185,22 @@ "path-to-regexp": ["path-to-regexp@0.1.13", "", {}, "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA=="], + "pg": ["pg@8.21.0", "", { "dependencies": { "pg-connection-string": "^2.13.0", "pg-pool": "^3.14.0", "pg-protocol": "^1.14.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA=="], + + "pg-cloudflare": ["pg-cloudflare@1.4.0", "", {}, "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A=="], + + "pg-connection-string": ["pg-connection-string@2.13.0", "", {}, "sha512-EMnU9E2fSULdsbErBbMaXJvFeD9B4+nPcM3f+4lsiCR0BHLPrLVjv3DbyM2hgQQviKJaTWIRRTjKjWlHg3p2ig=="], + + "pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="], + + "pg-pool": ["pg-pool@3.14.0", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw=="], + + "pg-protocol": ["pg-protocol@1.14.0", "", {}, "sha512-n5taZ1kO3s9ngDTVxsEznOqCyToTgz0FLuPq0B33COy5pPpuWJpY3/2oRBVETuOgzdqRXfWpM9HIhp2LBBT1BA=="], + + "pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], + + "pgpass": ["pgpass@1.0.5", "", { "dependencies": { "split2": "^4.1.0" } }, "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug=="], + "pino": ["pino@9.14.0", "", { "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w=="], "pino-abstract-transport": ["pino-abstract-transport@2.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw=="], @@ -189,6 +209,14 @@ "pino-std-serializers": ["pino-std-serializers@7.1.0", "", {}, "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw=="], + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], + + "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], + + "postgres-date": ["postgres-date@1.0.7", "", {}, "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="], + + "postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], + "process-warning": ["process-warning@5.0.0", "", {}, "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA=="], "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], @@ -259,6 +287,8 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], "body-parser/qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="], diff --git a/package.json b/package.json index d606903..569961b 100644 --- a/package.json +++ b/package.json @@ -11,15 +11,18 @@ "dev": "bun --watch src/server.ts", "build": "bun run check", "check": "bun run typecheck && bun run typecheck:opencode", + "migrate:storage": "bun scripts/migrate-file-storage-to-postgres.ts", "pipeline:trigger": "bash scripts/trigger-gitea-pipeline.sh", "start": "bun src/server.ts", "start:prod": "bun run check && bun src/server.ts" }, "dependencies": { "@opencode-ai/sdk": "^1.14.29", + "@types/pg": "^8.20.0", "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^4.21.2", + "pg": "^8.21.0", "pino": "^9.7.0", "pino-pretty": "^13.1.2", "zod": "^3.25.76" diff --git a/scripts/migrate-file-storage-to-postgres.ts b/scripts/migrate-file-storage-to-postgres.ts new file mode 100644 index 0000000..10ebdbe --- /dev/null +++ b/scripts/migrate-file-storage-to-postgres.ts @@ -0,0 +1,578 @@ +import { readdir, readFile } from "node:fs/promises"; +import { basename, extname, join } from "node:path"; + +import { config } from "../src/config.js"; +import { AgentDatabase } from "../src/db/index.js"; +import { + RESULT_REFERENCE_KIND, + RESULT_REFERENCE_SOURCE, + RESULT_REF_PATTERN, + type ResultPreview, +} from "../src/results/store.js"; +import { sanitizePersistentDocument, sanitizePersistentLine } from "../src/utils/persistencePolicy.js"; +import { toProjectKey, toStableId } from "../src/utils/fileStore.js"; + +type JsonRecord = Record; + +const db = new AgentDatabase(); +const runtimeConversationMap = new Map(); + +const counters = { + conversations: 0, + states: 0, + turns: 0, + runtimeSessions: 0, + learningStates: 0, + resultRefs: 0, + memories: 0, + skippedResultRefs: 0, +}; + +await db.initialize(); +await migrateConversations(); +await migrateConversationStates(); +await migrateConversationTurns(); +await migrateRuntimeSessions(); +await migrateLearningStates(); +await migrateResultRefs(); +await migrateMemories(); +await db.close(); + +console.log(JSON.stringify(counters, null, 2)); + +async function migrateConversations() { + for (const filePath of await listFiles(config.CONVERSATION_STORAGE_DIR, ".json")) { + const row = await readJson(filePath); + if (!row || typeof row.sessionId !== "string" || typeof row.actorKey !== "string") { + continue; + } + await upsertConversation({ + sessionId: row.sessionId, + actorKey: row.actorKey, + ownerUserId: stringOrUndefined(row.ownerUserId), + projectId: stringOrUndefined(row.projectId), + projectKey: + stringOrUndefined(row.projectKey) ?? + toProjectKey(stringOrUndefined(row.projectId)), + parentSessionId: stringOrUndefined(row.parentSessionId), + title: stringOrUndefined(row.title), + status: row.status === "archived" ? "archived" : "active", + createdAt: stringOrUndefined(row.createdAt), + updatedAt: stringOrUndefined(row.updatedAt), + }); + counters.conversations += 1; + } +} + +async function migrateConversationStates() { + for (const filePath of await listFiles(config.CONVERSATION_STATE_STORAGE_DIR, ".json")) { + const row = await readJson(filePath); + if (!row || typeof row.sessionId !== "string") { + continue; + } + await db.query( + ` + INSERT INTO ${db.table("conversation_states")} ( + session_id, + is_title_manually_edited, + messages, + branch_groups + ) + VALUES ($1, $2, $3::jsonb, $4::jsonb) + ON CONFLICT (session_id) + DO UPDATE SET + is_title_manually_edited = EXCLUDED.is_title_manually_edited, + messages = EXCLUDED.messages, + branch_groups = EXCLUDED.branch_groups + `, + [ + row.sessionId, + Boolean(row.isTitleManuallyEdited), + JSON.stringify(Array.isArray(row.messages) ? row.messages : []), + JSON.stringify(Array.isArray(row.branchGroups) ? row.branchGroups : []), + ], + ); + counters.states += 1; + } +} + +async function migrateConversationTurns() { + for (const filePath of await listFiles(config.SESSION_HISTORY_STORAGE_DIR, ".json")) { + const row = await readJson(filePath); + if (!row || typeof row.actorKey !== "string" || typeof row.projectKey !== "string") { + continue; + } + const sessionId = + stringOrUndefined(row.clientSessionId) ?? stringOrUndefined(row.sessionId); + if (!sessionId) { + continue; + } + const turns = Array.isArray(row.turns) ? row.turns : []; + const lastTimestamp = stringOrUndefined(row.updatedAt); + await upsertConversation({ + sessionId, + actorKey: row.actorKey, + projectKey: row.projectKey, + projectId: undefined, + createdAt: + turns.length > 0 && isRecord(turns[0]) ? stringOrUndefined(turns[0].timestamp) : undefined, + updatedAt: + lastTimestamp ?? + (turns.length > 0 && isRecord(turns[turns.length - 1]) + ? stringOrUndefined(turns[turns.length - 1].timestamp) + : undefined), + }); + for (const [turnIndex, turn] of turns.entries()) { + if (!isRecord(turn)) { + continue; + } + const userMessage = sanitizePersistentDocument(String(turn.userMessage ?? ""), 4000); + const assistantMessage = sanitizePersistentDocument( + String(turn.assistantMessage ?? ""), + 4000, + ); + if (!userMessage || !assistantMessage) { + continue; + } + const timestamp = stringOrUndefined(turn.timestamp) ?? new Date().toISOString(); + await db.query( + ` + INSERT INTO ${db.table("conversation_turns")} ( + turn_id, + session_id, + turn_index, + actor_key, + project_key, + user_message, + assistant_message, + tool_call_count, + turn_timestamp + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (turn_id) DO NOTHING + `, + [ + typeof turn.id === "string" + ? turn.id + : toStableId(sessionId, String(turnIndex), timestamp, userMessage, assistantMessage), + sessionId, + turnIndex, + row.actorKey, + row.projectKey, + userMessage, + assistantMessage, + Math.max(0, numberOrZero(turn.toolCallCount)), + timestamp, + ], + ); + counters.turns += 1; + } + } +} + +async function migrateRuntimeSessions() { + const seen = new Set(); + for (const filePath of await listFiles(config.SESSION_CONTEXT_STORAGE_DIR, ".json")) { + const row = await readJson(filePath); + if ( + !row || + typeof row.sessionId !== "string" || + typeof row.clientSessionId !== "string" || + typeof row.actorKey !== "string" || + typeof row.projectKey !== "string" + ) { + continue; + } + if (seen.has(row.sessionId)) { + continue; + } + seen.add(row.sessionId); + runtimeConversationMap.set(row.sessionId, row.clientSessionId); + await upsertConversation({ + sessionId: row.clientSessionId, + actorKey: row.actorKey, + projectId: stringOrUndefined(row.projectId), + projectKey: row.projectKey, + }); + await db.query( + ` + INSERT INTO ${db.table("runtime_sessions")} ( + runtime_session_id, + session_id, + actor_key, + allow_learning_write, + learning_mode, + project_id, + project_key, + trace_id, + released_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) + ON CONFLICT (runtime_session_id) + DO UPDATE SET + session_id = EXCLUDED.session_id, + actor_key = EXCLUDED.actor_key, + allow_learning_write = EXCLUDED.allow_learning_write, + learning_mode = EXCLUDED.learning_mode, + project_id = EXCLUDED.project_id, + project_key = EXCLUDED.project_key, + trace_id = EXCLUDED.trace_id, + released_at = NOW() + `, + [ + row.sessionId, + row.clientSessionId, + row.actorKey, + Boolean(row.allowLearningWrite), + row.learningMode === "review" ? "review" : row.learningMode === "interactive" ? "interactive" : null, + stringOrUndefined(row.projectId) ?? null, + row.projectKey, + typeof row.traceId === "string" ? row.traceId : `trace-${row.sessionId}`, + ], + ); + counters.runtimeSessions += 1; + } +} + +async function migrateLearningStates() { + for (const filePath of await listFiles(config.LEARNING_STATE_STORAGE_DIR, ".json")) { + const row = await readJson(filePath); + if (!row || typeof row.sessionId !== "string") { + continue; + } + const sessionId = runtimeConversationMap.get(row.sessionId) ?? row.sessionId; + const existingConversation = await db.query( + `SELECT 1 FROM ${db.table("conversations")} WHERE session_id = $1 LIMIT 1`, + [sessionId], + ); + if (existingConversation.rowCount === 0) { + continue; + } + await db.query( + ` + INSERT INTO ${db.table("learning_states")} ( + session_id, + last_gated_turn, + last_reviewed_turn, + pending_review + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (session_id) + DO UPDATE SET + last_gated_turn = EXCLUDED.last_gated_turn, + last_reviewed_turn = EXCLUDED.last_reviewed_turn, + pending_review = EXCLUDED.pending_review + `, + [ + sessionId, + numberOrZero(row.lastGatedTurn), + numberOrZero(row.lastReviewedTurn), + Boolean(row.pendingReview), + ], + ); + counters.learningStates += 1; + } +} + +async function migrateResultRefs() { + for (const filePath of await listFiles(config.RESULT_REF_STORAGE_DIR, ".json")) { + const row = await readJson(filePath); + if (!row) { + continue; + } + const resultRef = normalizeResultRef(stringOrUndefined(row.resultRef) ?? basename(filePath, extname(filePath))); + const actorKey = stringOrUndefined(row.actorKey); + const sessionId = + stringOrUndefined(row.clientSessionId) ?? stringOrUndefined(row.sessionId); + if (!resultRef || !actorKey || !sessionId) { + counters.skippedResultRefs += 1; + continue; + } + const projectId = stringOrUndefined(row.projectId); + const projectKey = stringOrUndefined(row.projectKey) ?? toProjectKey(projectId); + await upsertConversation({ + sessionId, + actorKey, + projectId, + projectKey, + }); + const payload = unwrapPayload(row); + await db.query( + ` + INSERT INTO ${db.table("result_refs")} ( + result_ref, + actor_key, + session_id, + created_at, + kind, + preview, + project_id, + project_key, + schema_version, + size_bytes, + source, + trace_id, + payload_path, + object_key + ) + VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11, $12, $13, NULL) + ON CONFLICT (result_ref) + DO UPDATE SET + actor_key = EXCLUDED.actor_key, + session_id = EXCLUDED.session_id, + created_at = EXCLUDED.created_at, + kind = EXCLUDED.kind, + preview = EXCLUDED.preview, + project_id = EXCLUDED.project_id, + project_key = EXCLUDED.project_key, + schema_version = EXCLUDED.schema_version, + size_bytes = EXCLUDED.size_bytes, + source = EXCLUDED.source, + trace_id = EXCLUDED.trace_id, + payload_path = EXCLUDED.payload_path + `, + [ + resultRef, + actorKey, + sessionId, + stringOrUndefined(row.createdAt) ?? new Date().toISOString(), + inferResultKind(row, payload), + JSON.stringify(isResultPreview(row.preview) ? row.preview : buildPreview(payload)), + projectId ?? null, + projectKey, + positiveIntegerOrOne(row.schemaVersion), + estimateBytes(payload), + inferResultSource(row.source), + stringOrUndefined(row.traceId) ?? `trace-${resultRef}`, + filePath, + ], + ); + counters.resultRefs += 1; + } +} + +async function migrateMemories() { + await migrateMemoryScope("user", join(config.MEMORY_STORAGE_DIR, "users")); + await migrateMemoryScope("workspace", join(config.MEMORY_STORAGE_DIR, "workspaces")); +} + +async function migrateMemoryScope(scope: "user" | "workspace", dir: string) { + for (const filePath of await listFiles(dir, ".md")) { + const markdown = await readText(filePath); + if (!markdown) { + continue; + } + const scopeKey = basename(filePath, ".md"); + for (const entry of parseMemoryMarkdown(markdown)) { + await db.query( + ` + INSERT INTO ${db.table("memories")} ( + memory_id, + scope, + scope_key, + content, + source + ) + VALUES ($1, $2, $3, $4, 'tool') + ON CONFLICT (memory_id) + DO UPDATE SET content = EXCLUDED.content + `, + [entry.id, scope, scopeKey, entry.content], + ); + counters.memories += 1; + } + } +} + +async function upsertConversation(input: { + sessionId: string; + actorKey: string; + projectKey: string; + projectId?: string; + ownerUserId?: string; + parentSessionId?: string; + title?: string; + status?: "active" | "archived"; + createdAt?: string; + updatedAt?: string; +}) { + await db.query( + ` + INSERT INTO ${db.table("conversations")} ( + session_id, + actor_key, + owner_user_id, + project_id, + project_key, + parent_session_id, + title, + status, + created_at, + updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, COALESCE($9, NOW()), COALESCE($10, COALESCE($9, NOW()))) + ON CONFLICT (session_id) + DO UPDATE SET + actor_key = EXCLUDED.actor_key, + owner_user_id = COALESCE(EXCLUDED.owner_user_id, ${db.table("conversations")}.owner_user_id), + project_id = COALESCE(EXCLUDED.project_id, ${db.table("conversations")}.project_id), + project_key = EXCLUDED.project_key, + parent_session_id = COALESCE(EXCLUDED.parent_session_id, ${db.table("conversations")}.parent_session_id), + title = COALESCE(EXCLUDED.title, ${db.table("conversations")}.title), + status = EXCLUDED.status, + created_at = LEAST(${db.table("conversations")}.created_at, EXCLUDED.created_at), + updated_at = GREATEST(${db.table("conversations")}.updated_at, EXCLUDED.updated_at) + `, + [ + input.sessionId, + input.actorKey, + input.ownerUserId ?? null, + input.projectId ?? null, + input.projectKey, + input.parentSessionId ?? null, + input.title ?? null, + input.status ?? "active", + input.createdAt ?? null, + input.updatedAt ?? null, + ], + ); +} + +async function listFiles(dir: string, extension: string) { + try { + const names = await readdir(dir); + return names + .filter((name) => name.endsWith(extension)) + .map((name) => join(dir, name)); + } catch { + return []; + } +} + +async function readJson(path: string): Promise { + try { + return JSON.parse(await readFile(path, "utf8")) as JsonRecord; + } catch { + return null; + } +} + +async function readText(path: string): Promise { + try { + return await readFile(path, "utf8"); + } catch { + return null; + } +} + +function parseMemoryMarkdown(content: string) { + return content + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.startsWith("- ")) + .map((line) => line.slice(2).trim()) + .map((line) => { + const match = line.match(/^\[([a-z0-9]{8,})\]\s+(.*)$/i); + const entryContent = sanitizePersistentLine(match?.[2] ?? line, 240); + return { + id: match?.[1] ?? toStableId("memory-entry", entryContent.toLowerCase()), + content: entryContent, + }; + }) + .filter((entry) => entry.content); +} + +function inferResultKind(row: JsonRecord, payload: unknown) { + if (row.kind === RESULT_REFERENCE_KIND.renderJunctionsPayload) { + return RESULT_REFERENCE_KIND.renderJunctionsPayload; + } + if (isRecord(payload) && isRecord(payload.node_area_map)) { + return RESULT_REFERENCE_KIND.renderJunctionsPayload; + } + return RESULT_REFERENCE_KIND.dynamicHttpResult; +} + +function inferResultSource(value: unknown) { + return value === RESULT_REFERENCE_SOURCE.dynamicHttp || + value === RESULT_REFERENCE_SOURCE.agentGenerated || + value === RESULT_REFERENCE_SOURCE.migration + ? value + : RESULT_REFERENCE_SOURCE.legacy; +} + +function unwrapPayload(row: JsonRecord) { + if ("data" in row) { + return row.data; + } + return row; +} + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isResultPreview(value: unknown): value is ResultPreview { + return ( + isRecord(value) && + typeof value.count === "number" && + Array.isArray(value.fields) && + typeof value.summary === "string" && + "sample" in value + ); +} + +function buildPreview(data: unknown): ResultPreview { + if (Array.isArray(data)) { + const sample = data.slice(0, config.MAX_PREVIEW_SAMPLE_ITEMS); + const fields = + sample.length > 0 && isRecord(sample[0]) + ? Object.keys(sample[0]).slice(0, 30) + : []; + return { + count: data.length, + fields, + sample, + summary: `list[${data.length}]`, + }; + } + if (isRecord(data)) { + const fields = Object.keys(data).slice(0, 30); + const sample = Object.fromEntries( + fields + .slice(0, config.MAX_PREVIEW_SAMPLE_ITEMS) + .map((field) => [field, data[field]]), + ); + return { + count: fields.length, + fields, + sample, + summary: `object<${fields.length} fields>`, + }; + } + return { + count: 1, + fields: [], + sample: String(data).slice(0, 300), + summary: `scalar<${typeof data}>`, + }; +} + +function estimateBytes(data: unknown) { + return Buffer.byteLength(JSON.stringify(data)); +} + +function normalizeResultRef(value?: string) { + return value && RESULT_REF_PATTERN.test(value) ? value : null; +} + +function stringOrUndefined(value: unknown) { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +function numberOrZero(value: unknown) { + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function positiveIntegerOrOne(value: unknown) { + return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 1; +} diff --git a/src/audit/llmRequestAudit.ts b/src/audit/llmRequestAudit.ts index be9307a..49a1fdb 100644 --- a/src/audit/llmRequestAudit.ts +++ b/src/audit/llmRequestAudit.ts @@ -6,7 +6,6 @@ import { config } from "../config.js"; export type LlmRequestAuditEntry = { kind: "tool" | "skill"; sessionId: string; - clientSessionId: string; traceId?: string; projectId?: string; target: string; diff --git a/src/chat/sessionBridge.ts b/src/chat/sessionBridge.ts index b59d32b..d75636e 100644 --- a/src/chat/sessionBridge.ts +++ b/src/chat/sessionBridge.ts @@ -2,20 +2,17 @@ import { randomUUID } from "node:crypto"; import { logger } from "../logger.js"; import { type OpencodeRuntimeAdapter } from "../runtime/opencode.js"; -import { - buildToolSessionScopeKey, - ToolSessionContextStore, -} from "../session/toolContextStore.js"; +import { RuntimeSessionStore } from "../session/runtimeSessionStore.js"; import { toActorKey, toProjectKey } from "../utils/fileStore.js"; export type SessionBinding = { - clientSessionId: string; sessionId: string; + runtimeSessionId: string; startedAt: number; }; export type SessionContext = { - clientSessionId: string; + sessionId: string; accessToken?: string; projectId?: string; userId?: string; @@ -28,15 +25,15 @@ export type ChatRequestContext = SessionContext & { }; export class ChatSessionBridge { - // runtime session 仅在单次请求生命周期内有效；线程连续性由 clientSessionId 对应的持久状态承担。 + // runtime session 仅在单次请求生命周期内有效；线程连续性由 sessionId 对应的持久状态承担。 private readonly activeRuntimeSessions = new Map(); private readonly activeSensitiveContexts = new Map(); - private readonly toolContextStore = new ToolSessionContextStore(); + private readonly runtimeSessionStore = new RuntimeSessionStore(); constructor(private readonly runtime: OpencodeRuntimeAdapter) {} async resolve(context: { - clientSessionId?: string; + sessionId?: string; accessToken?: string; projectId?: string; traceId?: string; @@ -47,30 +44,24 @@ export class ChatSessionBridge { created: boolean; }> { const requestContext = this.buildRequestContext(context); - await this.abortActiveRuntime(requestContext.clientSessionId); + await this.abortActiveRuntime(requestContext.sessionId); - const session = await this.runtime.createSession(requestContext.clientSessionId); + const session = await this.runtime.createSession(requestContext.sessionId); const binding: SessionBinding = { - clientSessionId: requestContext.clientSessionId, - sessionId: session.id, + sessionId: requestContext.sessionId, + runtimeSessionId: session.id, startedAt: Date.now(), }; - const sessionScopeKey = buildToolSessionScopeKey( - requestContext.actorKey, - requestContext.projectKey, - requestContext.clientSessionId, - ); - this.activeRuntimeSessions.set(requestContext.clientSessionId, session.id); - this.activeSensitiveContexts.set(sessionScopeKey, requestContext); - await this.toolContextStore.write({ + this.activeRuntimeSessions.set(requestContext.sessionId, session.id); + this.activeSensitiveContexts.set(session.id, requestContext); + await this.runtimeSessionStore.write({ + runtimeSessionId: session.id, actorKey: requestContext.actorKey, allowLearningWrite: true, - clientSessionId: requestContext.clientSessionId, + sessionId: requestContext.sessionId, learningMode: "interactive", projectId: requestContext.projectId, projectKey: requestContext.projectKey, - sessionId: session.id, - sessionScopeKey, traceId: requestContext.traceId, }); @@ -81,58 +72,59 @@ export class ChatSessionBridge { return this.activeRuntimeSessions.size; } - createClientSessionId() { + createSessionId() { return `agent-${randomUUID().slice(0, 12)}`; } - getActiveSensitiveContext(sessionScopeKey: string) { - return this.activeSensitiveContexts.get(sessionScopeKey) ?? null; + getActiveSensitiveContext(runtimeSessionId: string) { + return this.activeSensitiveContexts.get(runtimeSessionId) ?? null; } - async abort(context: { - clientSessionId?: string; - }): Promise { - const clientSessionId = context.clientSessionId?.trim(); - if (!clientSessionId) { - return null; - } - - const sessionId = this.activeRuntimeSessions.get(clientSessionId); + async abort(context: { sessionId?: string }): Promise { + const sessionId = context.sessionId?.trim(); if (!sessionId) { return null; } - await this.abortActiveRuntime(clientSessionId); + const runtimeSessionId = this.activeRuntimeSessions.get(sessionId); + if (!runtimeSessionId) { + return null; + } + + await this.abortActiveRuntime(sessionId); return { - clientSessionId, sessionId, + runtimeSessionId, startedAt: Date.now(), }; } - async releaseRuntimeSession(clientSessionId: string, sessionId: string) { - const activeSessionId = this.activeRuntimeSessions.get(clientSessionId); - if (activeSessionId === sessionId) { - this.activeRuntimeSessions.delete(clientSessionId); + async releaseRuntimeSession(sessionId: string, runtimeSessionId: string) { + const activeSessionId = this.activeRuntimeSessions.get(sessionId); + if (activeSessionId === runtimeSessionId) { + this.activeRuntimeSessions.delete(sessionId); } - this.activeSensitiveContexts.delete(findScopeKey(this.activeSensitiveContexts, clientSessionId)); - await this.toolContextStore.remove(sessionId).catch((error) => { - logger.debug({ sessionId, err: error }, "failed to cleanup runtime tool context"); + this.activeSensitiveContexts.delete(runtimeSessionId); + await this.runtimeSessionStore.release(runtimeSessionId).catch((error) => { + logger.debug( + { runtimeSessionId, err: error }, + "failed to cleanup persisted runtime session", + ); }); - await this.runtime.abortSession(sessionId).catch((error) => { - logger.debug({ sessionId, err: error }, "failed to cleanup runtime session"); + await this.runtime.abortSession(runtimeSessionId).catch((error) => { + logger.debug({ runtimeSessionId, err: error }, "failed to cleanup runtime session"); }); } private buildRequestContext(context: { - clientSessionId?: string; + sessionId?: string; accessToken?: string; projectId?: string; traceId?: string; userId?: string; }): ChatRequestContext { return { - clientSessionId: context.clientSessionId?.trim() || this.createClientSessionId(), + sessionId: context.sessionId?.trim() || this.createSessionId(), accessToken: context.accessToken, actorKey: toActorKey(context.userId), projectId: context.projectId, @@ -142,37 +134,25 @@ export class ChatSessionBridge { }; } - private async abortActiveRuntime(clientSessionId: string) { - const activeSessionId = this.activeRuntimeSessions.get(clientSessionId); - if (!activeSessionId) { + private async abortActiveRuntime(sessionId: string) { + const activeRuntimeSessionId = this.activeRuntimeSessions.get(sessionId); + if (!activeRuntimeSessionId) { return; } - this.activeRuntimeSessions.delete(clientSessionId); - this.activeSensitiveContexts.delete(findScopeKey(this.activeSensitiveContexts, clientSessionId)); - await this.toolContextStore.remove(activeSessionId).catch(() => undefined); - await this.runtime.abortSession(activeSessionId).catch((error) => { + this.activeRuntimeSessions.delete(sessionId); + this.activeSensitiveContexts.delete(activeRuntimeSessionId); + await this.runtimeSessionStore.release(activeRuntimeSessionId).catch(() => undefined); + await this.runtime.abortSession(activeRuntimeSessionId).catch((error) => { logger.warn( - { clientSessionId, sessionId: activeSessionId, err: error }, + { sessionId, runtimeSessionId: activeRuntimeSessionId, err: error }, "failed to abort previous active runtime session", ); }); - await this.runtime.waitForSessionIdle(activeSessionId).catch((error) => { + await this.runtime.waitForSessionIdle(activeRuntimeSessionId).catch((error) => { logger.warn( - { clientSessionId, sessionId: activeSessionId, err: error }, + { sessionId, runtimeSessionId: activeRuntimeSessionId, err: error }, "failed while waiting for previous runtime session to become idle", ); }); } } - -const findScopeKey = ( - contexts: Map, - clientSessionId: string, -) => { - for (const [scopeKey, context] of contexts.entries()) { - if (context.clientSessionId === clientSessionId) { - return scopeKey; - } - } - return clientSessionId; -}; diff --git a/src/config.ts b/src/config.ts index 30a4022..bbe3852 100644 --- a/src/config.ts +++ b/src/config.ts @@ -25,6 +25,22 @@ const envSchema = z PORT: z.coerce.number().int().positive().default(8787), // HTTP 服务监听地址。 HOST: z.string().default("0.0.0.0"), + // PostgreSQL connection string; if omitted, PG* env vars are used. + DATABASE_URL: optionalString(), + // PostgreSQL host. + PGHOST: optionalString(), + // PostgreSQL port. + PGPORT: z.coerce.number().int().positive().optional(), + // PostgreSQL user. + PGUSER: optionalString(), + // PostgreSQL password. + PGPASSWORD: optionalString(), + // PostgreSQL database name. + PGDATABASE: optionalString(), + // PostgreSQL SSL mode. + PGSSLMODE: z.enum(["disable", "prefer", "require"]).default("disable"), + // PostgreSQL schema used by TJWaterAgent. + AGENT_DB_SCHEMA: z.string().default("public"), // Pino 日志级别。 LOG_LEVEL: z.string().default("info"), // LLM 工具/技能调用审计日志路径。 @@ -116,6 +132,16 @@ const envSchema = z message: "OPENCODE_CLIENT_BASE_URL is required when OPENCODE_MODE=client", }); } + if ( + !env.DATABASE_URL && + (!env.PGHOST || !env.PGUSER || !env.PGPASSWORD || !env.PGDATABASE) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["DATABASE_URL"], + message: "DATABASE_URL or PGHOST/PGUSER/PGPASSWORD/PGDATABASE must be set", + }); + } }); export type AppConfig = z.infer; diff --git a/src/conversations/stateStore.ts b/src/conversations/stateStore.ts index fddaf83..898d524 100644 --- a/src/conversations/stateStore.ts +++ b/src/conversations/stateStore.ts @@ -1,12 +1,6 @@ -import { join } from "node:path"; +import { type QueryResultRow } from "pg"; -import { config } from "../config.js"; -import { - atomicWriteJson, - ensureDirectory, - readJsonFile, - removeFileIfExists, -} from "../utils/fileStore.js"; +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; export type ConversationStateRecord = { sessionId: string; @@ -15,27 +9,81 @@ export type ConversationStateRecord = { branchGroups: unknown[]; }; +type ConversationStateRow = QueryResultRow & { + session_id: string; + is_title_manually_edited: boolean; + messages: unknown[]; + branch_groups: unknown[]; +}; + export class ConversationStateStore { - constructor(private readonly baseDir = config.CONVERSATION_STATE_STORAGE_DIR) {} + constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} async initialize() { - await ensureDirectory(this.baseDir); + await this.db.initialize(); } - async read(sessionScopeKey: string) { - return await readJsonFile(this.filePath(sessionScopeKey)); + async read(sessionId: string) { + const result = await this.db.query( + ` + SELECT session_id, is_title_manually_edited, messages, branch_groups + FROM ${this.db.table("conversation_states")} + WHERE session_id = $1 + LIMIT 1 + `, + [sessionId], + ); + return mapConversationStateRow(result.rows[0]); } - async write(sessionScopeKey: string, state: ConversationStateRecord) { - await atomicWriteJson(this.filePath(sessionScopeKey), state); - return state; + async write(sessionId: string, state: ConversationStateRecord) { + const result = await this.db.query( + ` + INSERT INTO ${this.db.table("conversation_states")} ( + session_id, + is_title_manually_edited, + messages, + branch_groups + ) + VALUES ($1, $2, $3::jsonb, $4::jsonb) + ON CONFLICT (session_id) + DO UPDATE SET + is_title_manually_edited = EXCLUDED.is_title_manually_edited, + messages = EXCLUDED.messages, + branch_groups = EXCLUDED.branch_groups + RETURNING session_id, is_title_manually_edited, messages, branch_groups + `, + [ + sessionId, + state.isTitleManuallyEdited ?? false, + JSON.stringify(state.messages), + JSON.stringify(state.branchGroups), + ], + ); + return mapConversationStateRow(result.rows[0]) ?? state; } - async remove(sessionScopeKey: string) { - await removeFileIfExists(this.filePath(sessionScopeKey)); - } - - private filePath(sessionScopeKey: string) { - return join(this.baseDir, `${sessionScopeKey}.json`); + async remove(sessionId: string) { + await this.db.query( + ` + DELETE FROM ${this.db.table("conversation_states")} + WHERE session_id = $1 + `, + [sessionId], + ); } } + +const mapConversationStateRow = ( + row?: ConversationStateRow | null, +): ConversationStateRecord | null => { + if (!row) { + return null; + } + return { + sessionId: row.session_id, + isTitleManuallyEdited: row.is_title_manually_edited, + messages: Array.isArray(row.messages) ? row.messages : [], + branchGroups: Array.isArray(row.branch_groups) ? row.branch_groups : [], + }; +}; diff --git a/src/conversations/store.ts b/src/conversations/store.ts index 2e5f3c8..89674a7 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -1,21 +1,13 @@ import { randomUUID } from "node:crypto"; -import { join } from "node:path"; -import { config } from "../config.js"; -import { - atomicWriteJson, - ensureDirectory, - listJsonFiles, - readJsonFile, - removeFileIfExists, -} from "../utils/fileStore.js"; -import { toConversationScopeKey } from "../utils/fileStore.js"; +import { type QueryResultRow } from "pg"; + +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; export type ConversationStatus = "active" | "archived"; export type ConversationRecord = { sessionId: string; - sessionScopeKey: string; actorKey: string; ownerUserId?: string; projectId?: string; @@ -23,6 +15,8 @@ export type ConversationRecord = { parentSessionId?: string; createdAt: string; updatedAt: string; + isStreaming: boolean; + streamingStartedAt?: string; status: ConversationStatus; title?: string; }; @@ -39,40 +33,59 @@ type EnsureConversationInput = ConversationContext & { parentSessionId?: string; }; +type ConversationRow = QueryResultRow & { + session_id: string; + actor_key: string; + owner_user_id: string | null; + project_id: string | null; + project_key: string; + parent_session_id: string | null; + created_at: Date | string; + updated_at: Date | string; + is_streaming: boolean; + streaming_started_at: Date | string | null; + status: ConversationStatus; + title: string | null; +}; + export class ConversationStore { - constructor(private readonly baseDir = config.CONVERSATION_STORAGE_DIR) {} + constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} async initialize() { - await ensureDirectory(this.baseDir); + await this.db.initialize(); } async ensure(input: EnsureConversationInput) { const sessionId = normalizeSessionId(input.sessionId) ?? createConversationSessionId(); - const sessionScopeKey = toConversationScopeKey( - input.actorKey, - input.projectKey, - sessionId, - ); - const existing = await readJsonFile(this.filePath(sessionScopeKey)); + const existing = await this.get(input, sessionId); if (existing) { return { created: false, record: existing }; } - const now = new Date().toISOString(); - const record: ConversationRecord = { - sessionId, - sessionScopeKey, - actorKey: input.actorKey, - ownerUserId: input.userId?.trim(), - projectId: input.projectId, - projectKey: input.projectKey, - parentSessionId: normalizeSessionId(input.parentSessionId), - createdAt: now, - updatedAt: now, - status: "active", - }; - await atomicWriteJson(this.filePath(sessionScopeKey), record); - return { created: true, record }; + const inserted = await this.db.query( + ` + INSERT INTO ${this.db.table("conversations")} ( + session_id, + actor_key, + owner_user_id, + project_id, + project_key, + parent_session_id, + status + ) + VALUES ($1, $2, $3, $4, $5, $6, 'active') + RETURNING * + `, + [ + sessionId, + input.actorKey, + input.userId?.trim() || null, + input.projectId ?? null, + input.projectKey, + normalizeSessionId(input.parentSessionId) ?? null, + ], + ); + return { created: true, record: requireConversationRow(inserted.rows[0]) }; } async get(context: ConversationContext, sessionId: string) { @@ -80,47 +93,111 @@ export class ConversationStore { if (!normalizedSessionId) { return null; } - return await readJsonFile( - this.filePath( - toConversationScopeKey(context.actorKey, context.projectKey, normalizedSessionId), - ), + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("conversations")} + WHERE session_id = $1 + AND actor_key = $2 + AND project_key = $3 + LIMIT 1 + `, + [normalizedSessionId, context.actorKey, context.projectKey], ); + return mapConversationRow(result.rows[0]); } async touch( record: ConversationRecord, updates: Partial> = {}, ) { - const next: ConversationRecord = { - ...record, - ...normalizeConversationUpdates(updates), - updatedAt: new Date().toISOString(), - }; - await atomicWriteJson(this.filePath(record.sessionScopeKey), next); - return next; + const normalized = normalizeConversationUpdates(updates); + const result = await this.db.query( + ` + UPDATE ${this.db.table("conversations")} + SET + title = COALESCE($2, title), + status = COALESCE($3, status) + WHERE session_id = $1 + RETURNING * + `, + [record.sessionId, normalized.title ?? null, normalized.status ?? null], + ); + return requireConversationRow(result.rows[0]); } async list(context: ConversationContext) { - const files = await listJsonFiles(this.baseDir); - const records = await Promise.all( - files.map((file) => readJsonFile(file)), + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("conversations")} + WHERE actor_key = $1 + AND project_key = $2 + ORDER BY updated_at DESC + `, + [context.actorKey, context.projectKey], ); - return records - .filter((record): record is ConversationRecord => Boolean(record)) - .filter( - (record) => - record.actorKey === context.actorKey && - record.projectKey === context.projectKey, - ) - .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); + return result.rows + .map(mapConversationRow) + .filter((record): record is ConversationRecord => Boolean(record)); } async remove(record: ConversationRecord) { - await removeFileIfExists(this.filePath(record.sessionScopeKey)); + await this.db.query( + ` + DELETE FROM ${this.db.table("conversations")} + WHERE session_id = $1 + `, + [record.sessionId], + ); } - private filePath(sessionScopeKey: string) { - return join(this.baseDir, `${sessionScopeKey}.json`); + async markStreaming(record: ConversationRecord, runtimeSessionId: string) { + const result = await this.db.query( + ` + UPDATE ${this.db.table("conversations")} + SET + is_streaming = TRUE, + active_runtime_session_id = $2, + streaming_started_at = NOW() + WHERE session_id = $1 + RETURNING * + `, + [record.sessionId, runtimeSessionId], + ); + return requireConversationRow(result.rows[0]); + } + + async clearStreaming(sessionId: string, runtimeSessionId: string) { + const result = await this.db.query( + ` + UPDATE ${this.db.table("conversations")} + SET + is_streaming = FALSE, + active_runtime_session_id = NULL, + streaming_started_at = NULL + WHERE session_id = $1 + AND active_runtime_session_id = $2 + RETURNING * + `, + [sessionId, runtimeSessionId], + ); + return mapConversationRow(result.rows[0]); + } + + async resetStreamingSessions() { + await this.db.query( + ` + UPDATE ${this.db.table("conversations")} + SET + is_streaming = FALSE, + active_runtime_session_id = NULL, + streaming_started_at = NULL + WHERE is_streaming = TRUE + OR active_runtime_session_id IS NOT NULL + OR streaming_started_at IS NOT NULL + `, + ); } } @@ -146,3 +223,34 @@ const normalizeConversationUpdates = ( } return normalized; }; + +const mapConversationRow = (row?: ConversationRow | null): ConversationRecord | null => { + if (!row) { + return null; + } + return { + sessionId: row.session_id, + actorKey: row.actor_key, + ownerUserId: row.owner_user_id ?? undefined, + projectId: row.project_id ?? undefined, + projectKey: row.project_key, + parentSessionId: row.parent_session_id ?? undefined, + createdAt: toIsoString(row.created_at), + updatedAt: toIsoString(row.updated_at), + isStreaming: row.is_streaming, + streamingStartedAt: row.streaming_started_at ? toIsoString(row.streaming_started_at) : undefined, + status: row.status, + title: row.title ?? undefined, + }; +}; + +const requireConversationRow = (row?: ConversationRow | null) => { + const record = mapConversationRow(row); + if (!record) { + throw new Error("conversation row not found"); + } + return record; +}; + +const toIsoString = (value: Date | string) => + value instanceof Date ? value.toISOString() : new Date(value).toISOString(); diff --git a/src/db/index.ts b/src/db/index.ts new file mode 100644 index 0000000..3721922 --- /dev/null +++ b/src/db/index.ts @@ -0,0 +1,486 @@ +import { Pool, type PoolClient, type QueryResultRow } from "pg"; + +import { config } from "../config.js"; + +const IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +export class AgentDatabase { + private readonly pool: Pool; + private readonly schemaName: string; + private initialized = false; + private initializePromise: Promise | null = null; + + constructor(options?: { + connectionString?: string; + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + sslmode?: "disable" | "prefer" | "require"; + schema?: string; + }) { + const schema = options?.schema ?? config.AGENT_DB_SCHEMA; + if (!IDENTIFIER_PATTERN.test(schema)) { + throw new Error(`invalid PostgreSQL schema name: ${schema}`); + } + this.schemaName = schema; + this.pool = new Pool({ + ...(options?.connectionString ?? config.DATABASE_URL + ? { + connectionString: options?.connectionString ?? config.DATABASE_URL, + } + : { + host: options?.host ?? config.PGHOST, + port: options?.port ?? config.PGPORT ?? 5432, + user: options?.user ?? config.PGUSER, + password: options?.password ?? config.PGPASSWORD, + database: options?.database ?? config.PGDATABASE, + }), + ssl: + (options?.sslmode ?? config.PGSSLMODE) === "require" + ? { rejectUnauthorized: false } + : undefined, + }); + } + + get schema() { + return this.schemaName; + } + + table(name: string) { + if (!IDENTIFIER_PATTERN.test(name)) { + throw new Error(`invalid PostgreSQL table name: ${name}`); + } + return `"${this.schemaName}"."${name}"`; + } + + async initialize() { + if (this.initialized) { + return; + } + if (!this.initializePromise) { + this.initializePromise = (async () => { + await this.query(`CREATE SCHEMA IF NOT EXISTS "${this.schemaName}"`); + await this.query(buildSchemaSql(this.schemaName)); + this.initialized = true; + })().catch((error) => { + this.initializePromise = null; + throw error; + }); + } + await this.initializePromise; + } + + async query( + text: string, + values?: unknown[], + client?: PoolClient, + ) { + const executor = client ?? this.pool; + return await executor.query(text, values); + } + + async withTransaction(task: (client: PoolClient) => Promise) { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + const result = await task(client); + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + } + } + + async close() { + await this.pool.end(); + } +} + +let defaultDatabase: AgentDatabase | undefined; + +export const getAgentDatabase = () => { + if (!defaultDatabase) { + defaultDatabase = new AgentDatabase(); + } + return defaultDatabase; +}; + +const buildSchemaSql = (schema: string) => ` +CREATE OR REPLACE FUNCTION "${schema}".set_updated_at() +RETURNS trigger AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TABLE IF NOT EXISTS "${schema}"."conversations" ( + session_id TEXT PRIMARY KEY, + actor_key TEXT NOT NULL, + owner_user_id TEXT, + project_id TEXT, + project_key TEXT NOT NULL, + parent_session_id TEXT REFERENCES "${schema}"."conversations"(session_id) ON DELETE SET NULL, + title VARCHAR(120), + is_streaming BOOLEAN NOT NULL DEFAULT FALSE, + active_runtime_session_id TEXT, + streaming_started_at TIMESTAMPTZ, + status TEXT NOT NULL CHECK (status IN ('active', 'archived')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +ALTER TABLE "${schema}"."conversations" + ADD COLUMN IF NOT EXISTS is_streaming BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS active_runtime_session_id TEXT, + ADD COLUMN IF NOT EXISTS streaming_started_at TIMESTAMPTZ; + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'conversations' + AND column_name = 'session_scope_key' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."conversations" DROP COLUMN session_scope_key'; + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_conversations_actor_project_updated + ON "${schema}"."conversations" (actor_key, project_key, updated_at DESC); +CREATE INDEX IF NOT EXISTS idx_conversations_parent_session + ON "${schema}"."conversations" (parent_session_id); + +CREATE TABLE IF NOT EXISTS "${schema}"."conversation_states" ( + session_id TEXT PRIMARY KEY REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, + is_title_manually_edited BOOLEAN NOT NULL DEFAULT FALSE, + messages JSONB NOT NULL DEFAULT '[]'::jsonb, + branch_groups JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +ALTER TABLE "${schema}"."conversation_states" + ADD COLUMN IF NOT EXISTS is_title_manually_edited BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS messages JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS branch_groups JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'conversation_states' + AND column_name = 'session_scope_key' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."conversation_states" DROP COLUMN session_scope_key'; + END IF; +END $$; + +CREATE TABLE IF NOT EXISTS "${schema}"."conversation_turns" ( + turn_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, + turn_index INTEGER NOT NULL CHECK (turn_index >= 0), + actor_key TEXT NOT NULL, + project_key TEXT NOT NULL, + user_message TEXT NOT NULL, + assistant_message TEXT NOT NULL, + tool_call_count INTEGER NOT NULL DEFAULT 0 CHECK (tool_call_count >= 0), + turn_timestamp TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (session_id, turn_index) +); + +ALTER TABLE "${schema}"."conversation_turns" + ADD COLUMN IF NOT EXISTS turn_index INTEGER; + +WITH ranked_turns AS ( + SELECT + turn_id, + ROW_NUMBER() OVER ( + PARTITION BY session_id + ORDER BY turn_timestamp ASC, turn_id ASC + ) - 1 AS computed_turn_index + FROM "${schema}"."conversation_turns" + WHERE turn_index IS NULL +) +UPDATE "${schema}"."conversation_turns" turns +SET turn_index = ranked_turns.computed_turn_index +FROM ranked_turns +WHERE turns.turn_id = ranked_turns.turn_id; + +ALTER TABLE "${schema}"."conversation_turns" + ALTER COLUMN turn_index SET NOT NULL; + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'conversation_turns' + AND column_name = 'client_session_id' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."conversation_turns" DROP COLUMN client_session_id'; + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_conversation_turns_session_time + ON "${schema}"."conversation_turns" (session_id, turn_index DESC); +CREATE INDEX IF NOT EXISTS idx_conversation_turns_actor_project_time + ON "${schema}"."conversation_turns" (actor_key, project_key, turn_timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_conversation_turns_search + ON "${schema}"."conversation_turns" + USING GIN (to_tsvector('simple', coalesce(user_message, '') || ' ' || coalesce(assistant_message, ''))); + +CREATE TABLE IF NOT EXISTS "${schema}"."runtime_sessions" ( + runtime_session_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, + actor_key TEXT NOT NULL, + allow_learning_write BOOLEAN NOT NULL DEFAULT FALSE, + learning_mode TEXT CHECK (learning_mode IN ('interactive', 'review') OR learning_mode IS NULL), + project_id TEXT, + project_key TEXT NOT NULL, + trace_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + released_at TIMESTAMPTZ +); + +ALTER TABLE "${schema}"."runtime_sessions" + ADD COLUMN IF NOT EXISTS session_id TEXT; + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'runtime_sessions' + AND column_name = 'conversation_id' + ) THEN + EXECUTE 'UPDATE "${schema}"."runtime_sessions" SET session_id = COALESCE(session_id, conversation_id) WHERE session_id IS NULL'; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'runtime_sessions_session_id_fkey' + AND connamespace = to_regnamespace('${schema}') + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."runtime_sessions" ADD CONSTRAINT runtime_sessions_session_id_fkey FOREIGN KEY (session_id) REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE'; + END IF; +END $$; + +ALTER TABLE "${schema}"."runtime_sessions" + ALTER COLUMN session_id SET NOT NULL; + +DO $$ +BEGIN + IF to_regclass('"${schema}"."tool_session_contexts"') IS NOT NULL THEN + INSERT INTO "${schema}"."runtime_sessions" ( + runtime_session_id, + session_id, + actor_key, + allow_learning_write, + learning_mode, + project_id, + project_key, + trace_id, + released_at + ) + SELECT + runtime_session_id, + client_session_id, + actor_key, + allow_learning_write, + learning_mode, + project_id, + project_key, + trace_id, + NOW() + FROM "${schema}"."tool_session_contexts" + WHERE client_session_id IS NOT NULL + ON CONFLICT (runtime_session_id) DO NOTHING; + END IF; +END $$; + +DROP INDEX IF EXISTS "${schema}"."idx_runtime_sessions_conversation"; +CREATE INDEX IF NOT EXISTS idx_runtime_sessions_session + ON "${schema}"."runtime_sessions" (session_id); +CREATE INDEX IF NOT EXISTS idx_runtime_sessions_trace_id + ON "${schema}"."runtime_sessions" (trace_id); + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'runtime_sessions' + AND column_name = 'conversation_id' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."runtime_sessions" DROP COLUMN conversation_id'; + END IF; +END $$; + +DROP TABLE IF EXISTS "${schema}"."tool_session_contexts"; + +CREATE TABLE IF NOT EXISTS "${schema}"."learning_states" ( + session_id TEXT PRIMARY KEY REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, + last_gated_turn INTEGER NOT NULL DEFAULT 0 CHECK (last_gated_turn >= 0), + last_reviewed_turn INTEGER NOT NULL DEFAULT 0 CHECK (last_reviewed_turn >= 0), + pending_review BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS "${schema}"."result_refs" ( + result_ref TEXT PRIMARY KEY, + actor_key TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('dynamic-http-result', 'render-junctions-payload')), + preview JSONB NOT NULL, + project_id TEXT, + project_key TEXT NOT NULL, + schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version > 0), + size_bytes INTEGER NOT NULL CHECK (size_bytes >= 0), + source TEXT NOT NULL CHECK (source IN ('dynamic_http', 'agent_generated', 'legacy', 'migration')), + trace_id TEXT NOT NULL, + payload_path TEXT, + object_key TEXT, + CONSTRAINT chk_result_refs_payload_locator CHECK ( + num_nonnulls(payload_path, object_key) = 1 + ) +); + +ALTER TABLE "${schema}"."result_refs" + ADD COLUMN IF NOT EXISTS session_id TEXT, + ADD COLUMN IF NOT EXISTS payload_path TEXT, + ADD COLUMN IF NOT EXISTS object_key TEXT; + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'result_refs' + AND column_name = 'conversation_id' + ) THEN + EXECUTE 'UPDATE "${schema}"."result_refs" SET session_id = COALESCE(session_id, conversation_id) WHERE session_id IS NULL'; + END IF; + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'result_refs' + AND column_name = 'client_session_id' + ) THEN + EXECUTE 'UPDATE "${schema}"."result_refs" SET session_id = COALESCE(session_id, client_session_id) WHERE session_id IS NULL'; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'result_refs_session_id_fkey' + AND connamespace = to_regnamespace('${schema}') + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."result_refs" ADD CONSTRAINT result_refs_session_id_fkey FOREIGN KEY (session_id) REFERENCES "${schema}"."conversations"(session_id) ON DELETE CASCADE'; + END IF; + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'result_refs' + AND column_name = 'client_session_id' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."result_refs" DROP COLUMN client_session_id'; + END IF; + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'result_refs' + AND column_name = 'data' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."result_refs" ALTER COLUMN data DROP NOT NULL'; + END IF; + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = '${schema}' + AND table_name = 'result_refs' + AND column_name = 'conversation_id' + ) THEN + EXECUTE 'ALTER TABLE "${schema}"."result_refs" DROP COLUMN conversation_id'; + END IF; +END $$; + +ALTER TABLE "${schema}"."result_refs" + ALTER COLUMN session_id SET NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_result_refs_session_created + ON "${schema}"."result_refs" (session_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_result_refs_actor_project_created + ON "${schema}"."result_refs" (actor_key, project_key, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_result_refs_trace_id + ON "${schema}"."result_refs" (trace_id); + +CREATE TABLE IF NOT EXISTS "${schema}"."memories" ( + memory_id TEXT PRIMARY KEY, + scope TEXT NOT NULL CHECK (scope IN ('user', 'workspace')), + scope_key TEXT NOT NULL, + content TEXT NOT NULL, + source TEXT NOT NULL CHECK (source IN ('review', 'tool')), + session_id TEXT REFERENCES "${schema}"."conversations"(session_id) ON DELETE SET NULL, + trace_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (scope, scope_key, content) +); + +CREATE INDEX IF NOT EXISTS idx_memories_scope_key + ON "${schema}"."memories" (scope, scope_key, updated_at DESC); + +DROP TRIGGER IF EXISTS trg_conversations_set_updated_at ON "${schema}"."conversations"; +CREATE TRIGGER trg_conversations_set_updated_at +BEFORE UPDATE ON "${schema}"."conversations" +FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); + +DROP TRIGGER IF EXISTS trg_conversation_states_set_updated_at ON "${schema}"."conversation_states"; +CREATE TRIGGER trg_conversation_states_set_updated_at +BEFORE UPDATE ON "${schema}"."conversation_states" +FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); + +DROP TRIGGER IF EXISTS trg_runtime_sessions_set_updated_at ON "${schema}"."runtime_sessions"; +CREATE TRIGGER trg_runtime_sessions_set_updated_at +BEFORE UPDATE ON "${schema}"."runtime_sessions" +FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); + +DROP TRIGGER IF EXISTS trg_learning_states_set_updated_at ON "${schema}"."learning_states"; +CREATE TRIGGER trg_learning_states_set_updated_at +BEFORE UPDATE ON "${schema}"."learning_states" +FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); + +DROP TRIGGER IF EXISTS trg_memories_set_updated_at ON "${schema}"."memories"; +CREATE TRIGGER trg_memories_set_updated_at +BEFORE UPDATE ON "${schema}"."memories" +FOR EACH ROW EXECUTE FUNCTION "${schema}".set_updated_at(); +`; diff --git a/src/history/store.ts b/src/history/store.ts index 11244b0..180cf5e 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -1,14 +1,9 @@ -import { join } from "node:path"; +import { type QueryResultRow } from "pg"; import { config } from "../config.js"; -import { - atomicWriteJson, - ensureDirectory, - listJsonFiles, - readJsonFile, - toStableId, -} from "../utils/fileStore.js"; +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; import { sanitizePersistentDocument } from "../utils/persistencePolicy.js"; +import { toStableId } from "../utils/fileStore.js"; export type SessionTurnRecord = { id: string; @@ -20,7 +15,6 @@ export type SessionTurnRecord = { type SessionTranscriptRecord = { actorKey: string; - clientSessionId?: string; projectKey: string; sessionId: string; turns: SessionTurnRecord[]; @@ -38,18 +32,28 @@ export type SessionSearchHit = { type SessionHistoryContext = { actorKey: string; - clientSessionId?: string; projectKey: string; sessionId: string; }; -export class SessionHistoryStore { - private readonly writeQueues = new Map>(); +type ConversationTurnRow = QueryResultRow & { + turn_id: string; + session_id: string; + turn_index: number; + actor_key: string; + project_key: string; + user_message: string; + assistant_message: string; + tool_call_count: number; + turn_timestamp: Date | string; + created_at: Date | string; +}; - constructor(private readonly baseDir = config.SESSION_HISTORY_STORAGE_DIR) {} +export class SessionHistoryStore { + constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} async initialize() { - await ensureDirectory(this.baseDir); + await this.db.initialize(); } async appendTurn( @@ -59,73 +63,146 @@ export class SessionHistoryStore { toolCallCount: number; userMessage: string; }, - ) { - const key = this.filePath(context); - return this.serializeWrite(key, async () => { - const transcript = (await this.readTranscript(context)) ?? { - actorKey: context.actorKey, - clientSessionId: context.clientSessionId, - projectKey: context.projectKey, - sessionId: context.sessionId, - turns: [], - updatedAt: new Date().toISOString(), - }; - const userMessage = sanitizePersistentDocument(turn.userMessage, 4000); - const assistantMessage = sanitizePersistentDocument(turn.assistantMessage, 4000); - if (!userMessage || !assistantMessage) { - return transcript; - } + ): Promise { + const userMessage = sanitizePersistentDocument(turn.userMessage, 4000); + const assistantMessage = sanitizePersistentDocument(turn.assistantMessage, 4000); + if (!userMessage || !assistantMessage) { + return (await this.readTranscript(context)) ?? emptyTranscript(context); + } - const timestamp = new Date().toISOString(); - const record: SessionTurnRecord = { - id: toStableId(context.sessionId, timestamp, userMessage, assistantMessage), - assistantMessage, - timestamp, - toolCallCount: Math.max(0, turn.toolCallCount), - userMessage, - }; - transcript.clientSessionId = context.clientSessionId ?? transcript.clientSessionId; - transcript.sessionId = context.sessionId; - transcript.turns.push(record); - if (transcript.turns.length > config.SESSION_HISTORY_MAX_TURNS_PER_SESSION) { - transcript.turns = transcript.turns.slice( - transcript.turns.length - config.SESSION_HISTORY_MAX_TURNS_PER_SESSION, - ); - } - transcript.updatedAt = timestamp; - await atomicWriteJson(key, transcript); - return transcript; + const timestamp = new Date().toISOString(); + await this.db.withTransaction(async (client) => { + await this.db.query("SELECT pg_advisory_xact_lock(hashtext($1))", [context.sessionId], client); + const nextIndexResult = await this.db.query<{ next_index: number }>( + ` + SELECT COALESCE(MAX(turn_index), -1) + 1 AS next_index + FROM ${this.db.table("conversation_turns")} + WHERE session_id = $1 + `, + [context.sessionId], + client, + ); + const nextIndex = nextIndexResult.rows[0]?.next_index ?? 0; + await this.db.query( + ` + INSERT INTO ${this.db.table("conversation_turns")} ( + turn_id, + session_id, + turn_index, + actor_key, + project_key, + user_message, + assistant_message, + tool_call_count, + turn_timestamp + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, + [ + toStableId(context.sessionId, String(nextIndex), timestamp, userMessage, assistantMessage), + context.sessionId, + nextIndex, + context.actorKey, + context.projectKey, + userMessage, + assistantMessage, + Math.max(0, turn.toolCallCount), + timestamp, + ], + client, + ); }); + return (await this.readTranscript(context)) ?? emptyTranscript(context); } async getRecentTurns( context: SessionHistoryContext, limit: number, ): Promise { - const transcript = await this.readTranscript(context); - if (!transcript) { - return []; - } - return transcript.turns.slice(-Math.max(1, limit)); + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("conversation_turns")} + WHERE session_id = $1 + AND actor_key = $2 + AND project_key = $3 + ORDER BY turn_index DESC + LIMIT $4 + `, + [context.sessionId, context.actorKey, context.projectKey, Math.max(1, limit)], + ); + return result.rows + .slice() + .reverse() + .map(mapTurnRow); } async cloneThread( sourceContext: SessionHistoryContext, targetContext: SessionHistoryContext, keepMessageCount: number, - ) { - const sourceTranscript = await this.readTranscript(sourceContext); - const timestamp = new Date().toISOString(); - const nextTranscript: SessionTranscriptRecord = { - actorKey: targetContext.actorKey, - clientSessionId: targetContext.clientSessionId, - projectKey: targetContext.projectKey, - sessionId: targetContext.sessionId, - turns: projectTurnsForFork(sourceTranscript?.turns ?? [], keepMessageCount), - updatedAt: timestamp, - }; - await atomicWriteJson(this.filePath(targetContext), nextTranscript); - return nextTranscript; + ): Promise { + const keepTurnCount = Math.floor(keepMessageCount / 2); + const sourceTurns = keepTurnCount + ? await this.db.query( + ` + SELECT * + FROM ${this.db.table("conversation_turns")} + WHERE session_id = $1 + AND actor_key = $2 + AND project_key = $3 + ORDER BY turn_index ASC + LIMIT $4 + `, + [ + sourceContext.sessionId, + sourceContext.actorKey, + sourceContext.projectKey, + keepTurnCount, + ], + ) + : { rows: [] as ConversationTurnRow[] }; + + await this.db.withTransaction(async (client) => { + for (const [turnIndex, row] of sourceTurns.rows.entries()) { + await this.db.query( + ` + INSERT INTO ${this.db.table("conversation_turns")} ( + turn_id, + session_id, + turn_index, + actor_key, + project_key, + user_message, + assistant_message, + tool_call_count, + turn_timestamp + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, + [ + toStableId( + targetContext.sessionId, + String(turnIndex), + toIsoString(row.turn_timestamp), + row.user_message, + row.assistant_message, + ), + targetContext.sessionId, + turnIndex, + targetContext.actorKey, + targetContext.projectKey, + row.user_message, + row.assistant_message, + row.tool_call_count, + row.turn_timestamp, + ], + client, + ); + } + }); + + return (await this.readTranscript(targetContext)) ?? emptyTranscript(targetContext); } async search( @@ -137,104 +214,86 @@ export class SessionHistoryStore { if (!normalizedQuery) { return []; } + const rows = await this.db.query( + ` + SELECT * + FROM ${this.db.table("conversation_turns")} + WHERE actor_key = $1 + AND project_key = $2 + AND ( + LOWER(user_message) LIKE $3 + OR LOWER(assistant_message) LIKE $3 + ) + ORDER BY turn_timestamp DESC + `, + [context.actorKey, context.projectKey, `%${normalizedQuery}%`], + ); const queryTokens = normalizedQuery.split(/\s+/).filter(Boolean); const hits: SessionSearchHit[] = []; - const files = await listJsonFiles(this.baseDir); - for (const file of files) { - const transcript = await readJsonFile(file); - if (!transcript) { - continue; - } - if ( - transcript.actorKey !== context.actorKey || - transcript.projectKey !== context.projectKey - ) { - continue; - } - for (const turn of transcript.turns) { - const candidates: Array<["user" | "assistant", string]> = [ - ["user", turn.userMessage], - ["assistant", turn.assistantMessage], - ]; - for (const [matchedField, text] of candidates) { - const score = scoreText(text, normalizedQuery, queryTokens); - if (score <= 0) { - continue; - } - hits.push({ - matchedField, - score, - sessionId: transcript.sessionId, - snippet: buildSnippet(text, normalizedQuery), - timestamp: turn.timestamp, - turnId: turn.id, - }); + for (const row of rows.rows) { + const candidates: Array<["user" | "assistant", string]> = [ + ["user", row.user_message], + ["assistant", row.assistant_message], + ]; + for (const [matchedField, text] of candidates) { + const score = scoreText(text, normalizedQuery, queryTokens); + if (score <= 0) { + continue; } + hits.push({ + matchedField, + score, + sessionId: row.session_id, + snippet: buildSnippet(text, normalizedQuery), + timestamp: toIsoString(row.turn_timestamp), + turnId: row.turn_id, + }); } } return hits.sort((a, b) => b.score - a.score).slice(0, Math.max(1, maxResults)); } private async readTranscript(context: SessionHistoryContext) { - const direct = await readJsonFile(this.filePath(context)); - if (direct) { - return direct; - } - - const clientSessionId = context.clientSessionId?.trim(); - if (!clientSessionId) { + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("conversation_turns")} + WHERE session_id = $1 + AND actor_key = $2 + AND project_key = $3 + ORDER BY turn_index ASC + `, + [context.sessionId, context.actorKey, context.projectKey], + ); + if (result.rows.length === 0) { return null; } - - const files = await listJsonFiles(this.baseDir); - const matches: SessionTranscriptRecord[] = []; - for (const file of files) { - const transcript = await readJsonFile(file); - if (!transcript) { - continue; - } - if ( - transcript.actorKey !== context.actorKey || - transcript.projectKey !== context.projectKey || - transcript.clientSessionId !== clientSessionId - ) { - continue; - } - matches.push(transcript); - } - - if (matches.length === 0) { - return null; - } - - return matches.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt))[0] ?? null; - } - - private filePath(context: SessionHistoryContext) { - return join( - this.baseDir, - `${context.actorKey}__${context.projectKey}__${context.sessionId}.json`, - ); - } - - private async serializeWrite(key: string, task: () => Promise) { - const previous = this.writeQueues.get(key) ?? Promise.resolve(); - const run = previous.catch(() => undefined).then(task); - const next = run.then( - () => undefined, - () => undefined, - ); - this.writeQueues.set(key, next); - try { - return await run; - } finally { - if (this.writeQueues.get(key) === next) { - this.writeQueues.delete(key); - } - } + return { + actorKey: context.actorKey, + projectKey: context.projectKey, + sessionId: context.sessionId, + turns: result.rows.map(mapTurnRow), + updatedAt: toIsoString(result.rows[result.rows.length - 1]?.turn_timestamp ?? new Date()), + }; } } +const emptyTranscript = (context: SessionHistoryContext): SessionTranscriptRecord => ({ + actorKey: context.actorKey, + projectKey: context.projectKey, + sessionId: context.sessionId, + turns: [], + updatedAt: new Date().toISOString(), +}); + +const mapTurnRow = (row: ConversationTurnRow): SessionTurnRecord => ({ + id: row.turn_id, + assistantMessage: row.assistant_message, + timestamp: toIsoString(row.turn_timestamp), + toolCallCount: row.tool_call_count, + userMessage: row.user_message, +}); + const scoreText = (text: string, query: string, queryTokens: string[]) => { const normalized = text.toLowerCase(); let score = 0; @@ -263,18 +322,5 @@ const buildSnippet = (text: string, query: string) => { return `${prefix}${snippet}${suffix}`; }; -const projectTurnsForFork = ( - turns: SessionTurnRecord[], - keepMessageCount: number, -): SessionTurnRecord[] => { - if (keepMessageCount <= 0) { - return []; - } - - const keepTurnCount = Math.floor(keepMessageCount / 2); - if (keepTurnCount <= 0) { - return []; - } - - return turns.slice(0, keepTurnCount); -}; +const toIsoString = (value: Date | string) => + value instanceof Date ? value.toISOString() : new Date(value).toISOString(); diff --git a/src/learning/orchestrator.ts b/src/learning/orchestrator.ts index 7fae655..c393957 100644 --- a/src/learning/orchestrator.ts +++ b/src/learning/orchestrator.ts @@ -9,10 +9,7 @@ import { LearningStateStore } from "./stateStore.js"; import { MemoryStore, type MemoryScope } from "../memory/store.js"; import { type OpencodeRuntimeAdapter } from "../runtime/opencode.js"; import { SkillStore } from "../skills/store.js"; -import { - buildToolSessionScopeKey, - ToolSessionContextStore, -} from "../session/toolContextStore.js"; +import { RuntimeSessionStore } from "../session/runtimeSessionStore.js"; import { sanitizePersistentDocument, sanitizePersistentLine, @@ -73,7 +70,7 @@ export class LearningOrchestrator { private readonly activeReviews = new Set(); private readonly learningStateStore = new LearningStateStore(); private readonly skillStore = new SkillStore(); - private readonly toolContextStore = new ToolSessionContextStore(); + private readonly runtimeSessionStore = new RuntimeSessionStore(); constructor( private readonly runtime: OpencodeRuntimeAdapter, @@ -84,7 +81,7 @@ export class LearningOrchestrator { async initialize() { await Promise.all([ this.learningStateStore.initialize(), - this.toolContextStore.initialize(), + this.runtimeSessionStore.initialize(), ]); } @@ -92,7 +89,6 @@ export class LearningOrchestrator { const transcript = await this.historyStore.appendTurn( { actorKey: input.requestContext.actorKey, - clientSessionId: input.requestContext.clientSessionId, projectKey: input.requestContext.projectKey, sessionId: input.sessionId, }, @@ -142,22 +138,17 @@ export class LearningOrchestrator { let gateSessionId: string | null = null; try { const gateSession = await this.runtime.createSession( - `learning-gate-${input.requestContext.clientSessionId}`, + `learning-gate-${input.requestContext.sessionId}`, ); gateSessionId = gateSession.id; - await this.toolContextStore.write({ + await this.runtimeSessionStore.write({ + runtimeSessionId: gateSession.id, actorKey: input.requestContext.actorKey, allowLearningWrite: false, - clientSessionId: `gate-${input.requestContext.clientSessionId}`, + sessionId: input.requestContext.sessionId, learningMode: "review", projectId: input.requestContext.projectId, projectKey: input.requestContext.projectKey, - sessionId: gateSession.id, - sessionScopeKey: buildToolSessionScopeKey( - input.requestContext.actorKey, - input.requestContext.projectKey, - input.requestContext.clientSessionId, - ), traceId: input.requestContext.traceId, }); await this.runtime.prompt( @@ -219,7 +210,7 @@ export class LearningOrchestrator { }); } finally { if (gateSessionId) { - await this.toolContextStore.remove(gateSessionId).catch(() => undefined); + await this.runtimeSessionStore.release(gateSessionId).catch(() => undefined); await this.runtime.abortSession(gateSessionId).catch(() => undefined); } } @@ -237,21 +228,16 @@ export class LearningOrchestrator { turnCount: number; }) { const reviewSession = await this.runtime.createSession( - `learning-review-${input.requestContext.clientSessionId}`, + `learning-review-${input.requestContext.sessionId}`, ); - await this.toolContextStore.write({ + await this.runtimeSessionStore.write({ + runtimeSessionId: reviewSession.id, actorKey: input.requestContext.actorKey, allowLearningWrite: false, - clientSessionId: `review-${input.requestContext.clientSessionId}`, + sessionId: input.requestContext.sessionId, learningMode: "review", projectId: input.requestContext.projectId, projectKey: input.requestContext.projectKey, - sessionId: reviewSession.id, - sessionScopeKey: buildToolSessionScopeKey( - input.requestContext.actorKey, - input.requestContext.projectKey, - input.requestContext.clientSessionId, - ), traceId: input.requestContext.traceId, }); try { @@ -292,7 +278,7 @@ export class LearningOrchestrator { traceId: input.requestContext.traceId, }); } finally { - await this.toolContextStore.remove(reviewSession.id).catch(() => undefined); + await this.runtimeSessionStore.release(reviewSession.id).catch(() => undefined); await this.runtime.abortSession(reviewSession.id).catch(() => undefined); } } diff --git a/src/learning/stateStore.ts b/src/learning/stateStore.ts index 33e032b..16e1e91 100644 --- a/src/learning/stateStore.ts +++ b/src/learning/stateStore.ts @@ -1,11 +1,6 @@ -import { join } from "node:path"; +import { type QueryResultRow } from "pg"; -import { config } from "../config.js"; -import { - atomicWriteJson, - ensureDirectory, - readJsonFile, -} from "../utils/fileStore.js"; +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; export type LearningSessionState = { lastGatedTurn: number; @@ -15,32 +10,65 @@ export type LearningSessionState = { updatedAt: string; }; +type LearningStateRow = QueryResultRow & { + session_id: string; + last_gated_turn: number; + last_reviewed_turn: number; + pending_review: boolean; + updated_at: Date | string; +}; + export class LearningStateStore { - constructor(private readonly baseDir = config.LEARNING_STATE_STORAGE_DIR) {} + constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} async initialize() { - await ensureDirectory(this.baseDir); + await this.db.initialize(); } async read(sessionId: string): Promise { - const existing = await readJsonFile(this.filePath(sessionId)); - if (existing) { - return existing; - } - return { - lastGatedTurn: 0, - lastReviewedTurn: 0, - pendingReview: false, - sessionId, - updatedAt: new Date(0).toISOString(), - }; + const result = await this.db.query( + ` + SELECT session_id, last_gated_turn, last_reviewed_turn, pending_review, updated_at + FROM ${this.db.table("learning_states")} + WHERE session_id = $1 + LIMIT 1 + `, + [sessionId], + ); + return ( + mapLearningStateRow(result.rows[0]) ?? { + lastGatedTurn: 0, + lastReviewedTurn: 0, + pendingReview: false, + sessionId, + updatedAt: new Date(0).toISOString(), + } + ); } async write(state: LearningSessionState) { - await atomicWriteJson(this.filePath(state.sessionId), { - ...state, - updatedAt: new Date().toISOString(), - }); + await this.db.query( + ` + INSERT INTO ${this.db.table("learning_states")} ( + session_id, + last_gated_turn, + last_reviewed_turn, + pending_review + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (session_id) + DO UPDATE SET + last_gated_turn = EXCLUDED.last_gated_turn, + last_reviewed_turn = EXCLUDED.last_reviewed_turn, + pending_review = EXCLUDED.pending_review + `, + [ + state.sessionId, + state.lastGatedTurn, + state.lastReviewedTurn, + state.pendingReview, + ], + ); } async markPending(sessionId: string, pendingReview: boolean) { @@ -69,8 +97,17 @@ export class LearningStateStore { pendingReview: false, }); } - - private filePath(sessionId: string) { - return join(this.baseDir, `${sessionId}.json`); - } } + +const mapLearningStateRow = (row?: LearningStateRow | null): LearningSessionState | null => { + if (!row) { + return null; + } + return { + sessionId: row.session_id, + lastGatedTurn: row.last_gated_turn, + lastReviewedTurn: row.last_reviewed_turn, + pendingReview: row.pending_review, + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at, + }; +}; diff --git a/src/memory/store.ts b/src/memory/store.ts index c8771fe..025b13f 100644 --- a/src/memory/store.ts +++ b/src/memory/store.ts @@ -1,13 +1,9 @@ -import { join } from "node:path"; +import { type QueryResultRow } from "pg"; import { config } from "../config.js"; +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; import { sanitizePersistentLine } from "../utils/persistencePolicy.js"; -import { - atomicWriteFileWithHistory, - ensureDirectory, - readTextFile, - toStableId, -} from "../utils/fileStore.js"; +import { toStableId } from "../utils/fileStore.js"; export type MemoryScope = "user" | "workspace"; export type MemoryEntrySource = "review" | "tool"; @@ -29,6 +25,17 @@ type MemoryContext = { projectKey: string; }; +type MemoryRow = QueryResultRow & { + memory_id: string; + scope: MemoryScope; + scope_key: string; + content: string; + source: MemoryEntrySource; + session_id: string | null; + trace_id: string | null; + updated_at: Date | string; +}; + const SUSPICIOUS_MEMORY_PATTERNS = [ /ignore\s+(all|previous|prior|above)\s+instructions/i, /system\s+prompt/i, @@ -37,113 +44,118 @@ const SUSPICIOUS_MEMORY_PATTERNS = [ ]; export class MemoryStore { - // Memory 文件可能被多次连续追加，串行化可避免并发覆盖掉刚写入的条目。 - private writeQueue: Promise = Promise.resolve(); - - constructor( - private readonly baseDir = config.MEMORY_STORAGE_DIR, - private readonly historyDir = join(config.PERSISTENCE_HISTORY_DIR, "memory"), - ) {} + constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} async initialize() { - await ensureDirectory(this.baseDir); - await ensureDirectory(join(this.baseDir, "users")); - await ensureDirectory(join(this.baseDir, "workspaces")); - // 历史备份与正式数据分目录存放，便于排查和手工恢复。 - await ensureDirectory(this.historyDir); + await this.db.initialize(); } async upsert(scope: MemoryScope, key: string, draft: MemoryDraft) { - return this.serializeWrite(async () => { - const content = normalizeMemoryContent(draft.content); - if (!content) { - return { changed: false, entry: null as MemoryEntry | null }; - } - - const entries = await this.readEntries(scope, key); - const existing = entries.find((entry) => entry.content === content); - if (existing) { - return { changed: false, entry: existing }; - } - - const entry: MemoryEntry = { + const content = normalizeMemoryContent(draft.content); + if (!content) { + return { changed: false, entry: null as MemoryEntry | null }; + } + const existing = await this.findByContent(scope, key, content); + if (existing) { + return { changed: false, entry: existing }; + } + const id = toStableId(scope, key, content.toLowerCase()); + await this.db.query( + ` + INSERT INTO ${this.db.table("memories")} ( + memory_id, + scope, + scope_key, + content, + source, + session_id, + trace_id + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, + [id, scope, key, content, draft.source, draft.sessionId ?? null, draft.traceId ?? null], + ); + return { + changed: true, + entry: { + id, content, - id: toStableId(scope, key, content.toLowerCase()), - }; - entries.unshift(entry); - // 每次覆盖 memory 文件前先保留上一版，写入失败时由底层工具恢复。 - await atomicWriteFileWithHistory( - this.filePath(scope, key), - renderMemoryMarkdown(scope, entries), - { - historyDir: this.historyDir, - rootDir: this.baseDir, - }, - ); - return { changed: true, entry }; - }); + }, + }; } async list(scope: MemoryScope, key: string) { - return await this.readEntries(scope, key); + const rows = await this.db.query( + ` + SELECT memory_id, content + FROM ${this.db.table("memories")} + WHERE scope = $1 + AND scope_key = $2 + ORDER BY updated_at DESC, created_at DESC + `, + [scope, key], + ); + return rows.rows.map((row) => ({ + id: row.memory_id, + content: row.content, + })); } async replace(scope: MemoryScope, key: string, targetId: string, draft: MemoryDraft) { - return this.serializeWrite(async () => { - const content = normalizeMemoryContent(draft.content); - if (!content) { - return { changed: false, detail: "content rejected by persistence policy" }; - } - const entries = await this.readEntries(scope, key); - const index = entries.findIndex((entry) => entry.id === targetId.trim()); - if (index === -1) { - return { changed: false, detail: "memory entry not found" }; - } - const duplicate = entries.find( - (entry, currentIndex) => currentIndex !== index && entry.content === content, - ); - if (duplicate) { - return { changed: false, detail: "replacement would duplicate an existing memory" }; - } - entries[index] = { + const content = normalizeMemoryContent(draft.content); + if (!content) { + return { changed: false, detail: "content rejected by persistence policy" }; + } + const duplicate = await this.findByContent(scope, key, content); + if (duplicate && duplicate.id !== targetId.trim()) { + return { changed: false, detail: "replacement would duplicate an existing memory" }; + } + const result = await this.db.query( + ` + UPDATE ${this.db.table("memories")} + SET + content = $4, + source = $5, + session_id = $6, + trace_id = $7 + WHERE memory_id = $1 + AND scope = $2 + AND scope_key = $3 + `, + [ + targetId.trim(), + scope, + key, content, - id: entries[index]?.id ?? toStableId(scope, key, content.toLowerCase()), - }; - await atomicWriteFileWithHistory( - this.filePath(scope, key), - renderMemoryMarkdown(scope, entries), - { - historyDir: this.historyDir, - rootDir: this.baseDir, - }, - ); - return { changed: true, detail: "memory replaced" }; - }); + draft.source, + draft.sessionId ?? null, + draft.traceId ?? null, + ], + ); + return result.rowCount === 0 + ? { changed: false, detail: "memory entry not found" } + : { changed: true, detail: "memory replaced" }; } async remove(scope: MemoryScope, key: string, targetId: string) { - return this.serializeWrite(async () => { - const entries = await this.readEntries(scope, key); - const next = entries.filter((entry) => entry.id !== targetId.trim()); - if (next.length === entries.length) { - return { changed: false, detail: "memory entry not found" }; - } - await atomicWriteFileWithHistory( - this.filePath(scope, key), - renderMemoryMarkdown(scope, next), - { - historyDir: this.historyDir, - rootDir: this.baseDir, - }, - ); - return { changed: true, detail: "memory removed" }; - }); + const result = await this.db.query( + ` + DELETE FROM ${this.db.table("memories")} + WHERE memory_id = $1 + AND scope = $2 + AND scope_key = $3 + `, + [targetId.trim(), scope, key], + ); + return result.rowCount === 0 + ? { changed: false, detail: "memory entry not found" } + : { changed: true, detail: "memory removed" }; } async buildPromptSnapshot(context: MemoryContext) { const [userMemory, workspaceMemory] = await Promise.all([ - this.readEntries("user", context.actorKey), - this.readEntries("workspace", context.projectKey), + this.list("user", context.actorKey), + this.list("workspace", context.projectKey), ]); const sections: string[] = []; @@ -180,26 +192,25 @@ export class MemoryStore { : block; } - private async readEntries(scope: MemoryScope, key: string) { - const markdown = await readTextFile(this.filePath(scope, key)); - if (!markdown) { - return []; - } - return parseMemoryMarkdown(markdown); - } - - private filePath(scope: MemoryScope, key: string) { - const dir = scope === "user" ? "users" : "workspaces"; - return join(this.baseDir, dir, `${key}.md`); - } - - private async serializeWrite(task: () => Promise) { - const run = this.writeQueue.catch(() => undefined).then(task); - this.writeQueue = run.then( - () => undefined, - () => undefined, + private async findByContent(scope: MemoryScope, key: string, content: string) { + const result = await this.db.query( + ` + SELECT memory_id, content + FROM ${this.db.table("memories")} + WHERE scope = $1 + AND scope_key = $2 + AND content = $3 + LIMIT 1 + `, + [scope, key, content], ); - return run; + const row = result.rows[0]; + return row + ? { + id: row.memory_id, + content: row.content, + } + : null; } } @@ -213,31 +224,3 @@ const normalizeMemoryContent = (content: string) => { } return normalized; }; - -const parseMemoryMarkdown = (content: string): MemoryEntry[] => - content - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.startsWith("- ")) - .map((line) => line.slice(2).trim()) - .map((line) => { - const match = line.match(/^\[([a-z0-9]{8,})\]\s+(.*)$/i); - if (match) { - return { - content: normalizeMemoryContent(match[2]), - id: match[1], - }; - } - const normalized = normalizeMemoryContent(line); - return { - content: normalized, - id: normalized ? toStableId("memory-entry", normalized.toLowerCase()) : "", - }; - }) - .filter((entry) => entry.content); - -const renderMemoryMarkdown = (scope: MemoryScope, entries: MemoryEntry[]) => { - const title = scope === "user" ? "# User Memory" : "# Workspace Memory"; - const bullets = entries.map((entry) => `- [${entry.id}] ${entry.content}`); - return [title, "", ...bullets, ""].join("\n"); -}; diff --git a/src/results/resolver.ts b/src/results/resolver.ts index 95986a2..f9b1615 100644 --- a/src/results/resolver.ts +++ b/src/results/resolver.ts @@ -16,9 +16,9 @@ type ResolveOptions = { type RegisterResultReferenceInput = { actorKey: string; - clientSessionId: string; data: unknown; kind: ResultReferenceKind; + payloadPath?: string; projectId?: string; projectKey: string; schemaVersion: number; @@ -47,9 +47,9 @@ export class ResultReferenceResolver { } return this.store.store({ actorKey: input.actorKey, - clientSessionId: input.clientSessionId, data: normalizedData, kind: input.kind, + payloadPath: input.payloadPath, projectId: input.projectId, projectKey: input.projectKey, schemaVersion: input.schemaVersion, @@ -82,10 +82,16 @@ export class ResultReferenceResolver { } return this.register({ - ...input, + actorKey: input.actorKey, data: payload, kind: RESULT_REFERENCE_KIND.renderJunctionsPayload, + payloadPath: filePath, + projectId: input.projectId, + projectKey: input.projectKey, schemaVersion: 1, + sessionId: input.sessionId, + source: input.source, + traceId: input.traceId, }); } diff --git a/src/results/store.ts b/src/results/store.ts index 81530f5..23161ca 100644 --- a/src/results/store.ts +++ b/src/results/store.ts @@ -1,20 +1,19 @@ import { randomUUID } from "node:crypto"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; + +import { type QueryResultRow } from "pg"; import { config } from "../config.js"; +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; import { logger } from "../logger.js"; import { atomicWriteJson, ensureDirectory, - getFileStat, - listJsonFiles, readJsonFile, removeFileIfExists, - toProjectKey, } from "../utils/fileStore.js"; export const RESULT_REF_PATTERN = /^res-[a-f0-9-]{8,64}$/; -const RESULT_REF_FILE_PATTERN = /^(res-[a-f0-9-]{8,64})(?:\.json)?$/; export const RESULT_REFERENCE_KIND = { dynamicHttpResult: "dynamic-http-result", @@ -44,7 +43,6 @@ export type ResultPreview = { export type ResultReferenceRecord = { resultRef: string; actorKey: string; - clientSessionId: string; createdAt: string; data: unknown; kind: ResultReferenceKind; @@ -56,11 +54,12 @@ export type ResultReferenceRecord = { sizeBytes: number; source: ResultReferenceSource; traceId: string; + payloadPath?: string; + objectKey?: string; }; export type StoreResultInput = { actorKey: string; - clientSessionId: string; data: unknown; kind: ResultReferenceKind; projectId?: string; @@ -69,11 +68,13 @@ export type StoreResultInput = { sessionId: string; source: ResultReferenceSource; traceId: string; + payloadPath?: string; + objectKey?: string; }; export type RetrievalContext = { actorKey: string; - clientSessionId?: string; + sessionId?: string; projectId?: string; }; @@ -84,18 +85,37 @@ export type ResultReferencePeek = { storedAt: string; }; -type PartialRecord = Partial & { data?: unknown }; +type ResultReferenceRow = QueryResultRow & { + result_ref: string; + actor_key: string; + session_id: string; + created_at: Date | string; + kind: ResultReferenceKind; + preview: ResultPreview; + project_id: string | null; + project_key: string; + schema_version: number; + size_bytes: number; + source: ResultReferenceSource; + trace_id: string; + payload_path: string | null; + object_key: string | null; +}; export class ResultReferenceStore { private cleanupTimer: NodeJS.Timeout | null = null; + private readonly managedPayloadDir: string; constructor( + private readonly db: AgentDatabase = getAgentDatabase(), private readonly baseDir = config.RESULT_REF_STORAGE_DIR, private readonly ttlMs = config.RESULT_REF_TTL_HOURS * 60 * 60 * 1000, - ) {} + ) { + this.managedPayloadDir = join(this.baseDir, "payloads"); + } async initialize() { - await ensureDirectory(this.baseDir); + await Promise.all([this.db.initialize(), ensureDirectory(this.managedPayloadDir)]); } startCleanupLoop() { @@ -119,24 +139,77 @@ export class ResultReferenceStore { async store(input: StoreResultInput) { const resultRef = `res-${randomUUID().slice(0, 16)}`; - const record: ResultReferenceRecord = { + const createdAt = new Date().toISOString(); + const payloadPath = + input.payloadPath ?? + (input.objectKey + ? undefined + : this.managedPayloadPath(resultRef)); + + if (!payloadPath && !input.objectKey) { + throw new Error("result ref requires payloadPath or objectKey"); + } + if (!input.payloadPath && payloadPath) { + await atomicWriteJson(payloadPath, wrapPayload(input.data, createdAt, input.projectId)); + } + + const preview = buildPreview(input.data); + const sizeBytes = estimateBytes(input.data); + await this.db.query( + ` + INSERT INTO ${this.db.table("result_refs")} ( + result_ref, + actor_key, + session_id, + created_at, + kind, + preview, + project_id, + project_key, + schema_version, + size_bytes, + source, + trace_id, + payload_path, + object_key + ) + VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11, $12, $13, $14) + `, + [ + resultRef, + input.actorKey, + input.sessionId, + createdAt, + input.kind, + JSON.stringify(preview), + input.projectId ?? null, + input.projectKey, + input.schemaVersion, + sizeBytes, + input.source, + input.traceId, + payloadPath ?? null, + input.objectKey ?? null, + ], + ); + + return { resultRef, actorKey: input.actorKey, - clientSessionId: input.clientSessionId, - createdAt: new Date().toISOString(), + createdAt, data: input.data, kind: input.kind, - preview: buildPreview(input.data), + preview, projectId: input.projectId, projectKey: input.projectKey, schemaVersion: input.schemaVersion, sessionId: input.sessionId, - sizeBytes: estimateBytes(input.data), + sizeBytes, source: input.source, traceId: input.traceId, - }; - await atomicWriteJson(this.filePath(resultRef), record); - return record; + payloadPath, + objectKey: input.objectKey, + } satisfies ResultReferenceRecord; } async getAuthorizedRecord(resultRef: string, context: RetrievalContext) { @@ -145,26 +218,49 @@ export class ResultReferenceStore { return null; } - const rawRecord = await readJsonFile(this.filePath(normalizedResultRef)); - const record = - normalizeResultReferenceRecord(rawRecord) ?? - normalizeLegacyRenderReferenceRecord(rawRecord, normalizedResultRef, context); - if (!record) { + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("result_refs")} + WHERE result_ref = $1 + LIMIT 1 + `, + [normalizedResultRef], + ); + const row = result.rows[0]; + if (!row) { return null; } - if (record.actorKey !== context.actorKey) { + if (row.actor_key !== context.actorKey) { return null; } - if ((record.projectId ?? "") !== (context.projectId ?? "")) { + if ((row.project_id ?? "") !== (context.projectId ?? "")) { return null; } - if ( - context.clientSessionId && - record.clientSessionId !== context.clientSessionId - ) { + if (context.sessionId && row.session_id !== context.sessionId) { return null; } - return record; + const data = await this.readPayload(row); + if (data === null) { + return null; + } + return { + resultRef: row.result_ref, + actorKey: row.actor_key, + createdAt: toIsoString(row.created_at), + data, + kind: row.kind, + preview: row.preview, + projectId: row.project_id ?? undefined, + projectKey: row.project_key, + schemaVersion: row.schema_version, + sessionId: row.session_id, + sizeBytes: row.size_bytes, + source: row.source, + traceId: row.trace_id, + payloadPath: row.payload_path ?? undefined, + objectKey: row.object_key ?? undefined, + } satisfies ResultReferenceRecord; } async peekAuthorized( @@ -184,196 +280,82 @@ export class ResultReferenceStore { } async listBySession(sessionId: string) { - const files = await listJsonFiles(this.baseDir); - const records = await Promise.all( - files.map(async (filePath) => - normalizeResultReferenceRecord(await readJsonFile(filePath)), - ), + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("result_refs")} + WHERE session_id = $1 + ORDER BY created_at DESC + `, + [sessionId], ); - return records - .filter((record): record is ResultReferenceRecord => Boolean(record)) - .filter((record) => record.sessionId === sessionId) - .sort((left, right) => right.createdAt.localeCompare(left.createdAt)); + const records = await Promise.all( + result.rows.map(async (row) => { + const data = await this.readPayload(row); + if (data === null) { + return null; + } + return { + resultRef: row.result_ref, + actorKey: row.actor_key, + createdAt: toIsoString(row.created_at), + data, + kind: row.kind, + preview: row.preview, + projectId: row.project_id ?? undefined, + projectKey: row.project_key, + schemaVersion: row.schema_version, + sessionId: row.session_id, + sizeBytes: row.size_bytes, + source: row.source, + traceId: row.trace_id, + payloadPath: row.payload_path ?? undefined, + objectKey: row.object_key ?? undefined, + } satisfies ResultReferenceRecord; + }), + ); + return records.filter(Boolean) as ResultReferenceRecord[]; } async cleanupExpired() { - const files = await listJsonFiles(this.baseDir); - const now = Date.now(); - for (const filePath of files) { - const stats = await getFileStat(filePath); - if (!stats) { - continue; - } - if (now - stats.mtimeMs > this.ttlMs) { - await removeFileIfExists(filePath); - } - } + const result = await this.db.query( + ` + DELETE FROM ${this.db.table("result_refs")} + WHERE created_at < NOW() - ($1 * INTERVAL '1 millisecond') + RETURNING * + `, + [this.ttlMs], + ); + await Promise.all( + result.rows.map(async (row) => { + if (row.payload_path && isManagedPayloadPath(row.payload_path, this.managedPayloadDir)) { + await removeFileIfExists(row.payload_path); + } + }), + ); } - private filePath(resultRef: string) { - return join(this.baseDir, `${resultRef}.json`); + private managedPayloadPath(resultRef: string) { + return join(this.managedPayloadDir, `${resultRef}.json`); + } + + private async readPayload(row: ResultReferenceRow) { + if (row.payload_path) { + return unwrapStoredPayload(await readJsonFile(row.payload_path)); + } + if (row.object_key) { + logger.warn({ resultRef: row.result_ref, objectKey: row.object_key }, "object storage payloads are not implemented"); + return null; + } + return null; } } -export const normalizeResultReferenceRecord = ( - value: unknown, -): ResultReferenceRecord | null => { - if (!isRecord(value)) { - return null; - } - - const partial = value as PartialRecord; - if ( - !isValidResultRef(partial.resultRef) || - typeof partial.actorKey !== "string" || - typeof partial.clientSessionId !== "string" || - typeof partial.createdAt !== "string" || - !("data" in partial) || - !isResultPreview(partial.preview) || - typeof partial.projectKey !== "string" || - typeof partial.sessionId !== "string" || - typeof partial.sizeBytes !== "number" || - !Number.isFinite(partial.sizeBytes) || - typeof partial.traceId !== "string" - ) { - return null; - } - - const kind = normalizeResultReferenceKind(partial.kind); - const source = normalizeResultReferenceSource(partial.source); - const schemaVersion = - typeof partial.schemaVersion === "number" && - Number.isInteger(partial.schemaVersion) && - partial.schemaVersion > 0 - ? partial.schemaVersion - : 1; - - if (!kind || !source) { - return null; - } - - if ( - partial.projectId !== undefined && - typeof partial.projectId !== "string" - ) { - return null; - } - - return { - resultRef: partial.resultRef, - actorKey: partial.actorKey, - clientSessionId: partial.clientSessionId, - createdAt: partial.createdAt, - data: partial.data, - kind, - preview: partial.preview, - projectId: partial.projectId, - projectKey: partial.projectKey, - schemaVersion, - sessionId: partial.sessionId, - sizeBytes: partial.sizeBytes, - source, - traceId: partial.traceId, - }; -}; - -const normalizeResultReferenceKind = ( - value: unknown, -): ResultReferenceKind | null => { - if (value === undefined) { - return RESULT_REFERENCE_KIND.dynamicHttpResult; - } - return Object.values(RESULT_REFERENCE_KIND).includes( - value as ResultReferenceKind, - ) - ? (value as ResultReferenceKind) - : null; -}; - -const normalizeResultReferenceSource = ( - value: unknown, -): ResultReferenceSource | null => { - if (value === undefined) { - return RESULT_REFERENCE_SOURCE.legacy; - } - return Object.values(RESULT_REFERENCE_SOURCE).includes( - value as ResultReferenceSource, - ) - ? (value as ResultReferenceSource) - : null; -}; - -const isValidResultRef = (value: unknown): value is string => - typeof value === "string" && RESULT_REF_PATTERN.test(value); - const normalizeResultRef = (value: string) => { - const match = value.trim().match(RESULT_REF_FILE_PATTERN); - return match?.[1] ?? null; + const normalized = value.trim(); + return RESULT_REF_PATTERN.test(normalized) ? normalized : null; }; -const normalizeLegacyRenderReferenceRecord = ( - value: unknown, - resultRef: string, - context: RetrievalContext, -): ResultReferenceRecord | null => { - const data = extractLegacyRenderPayload(value); - if (!data) { - return null; - } - - const root = isRecord(value) ? value : {}; - const metadata = isRecord(root.metadata) ? root.metadata : {}; - const projectId = firstNonEmptyString(root.projectId, metadata.projectId); - const createdAt = - firstNonEmptyString(root.createdAt, metadata.createdAt) ?? new Date().toISOString(); - - return { - resultRef, - actorKey: context.actorKey, - clientSessionId: context.clientSessionId ?? "", - createdAt, - data, - kind: RESULT_REFERENCE_KIND.renderJunctionsPayload, - preview: buildPreview(data), - projectId, - projectKey: toProjectKey(projectId), - schemaVersion: 1, - sessionId: context.clientSessionId ?? resultRef, - sizeBytes: estimateBytes(data), - source: RESULT_REFERENCE_SOURCE.legacy, - traceId: "legacy-render-ref", - }; -}; - -const extractLegacyRenderPayload = (value: unknown) => { - if (!isRecord(value)) { - return null; - } - const candidate = isRecord(value.data) ? value.data : value; - if (!isRecord(candidate.node_area_map)) { - return null; - } - return candidate; -}; - -const firstNonEmptyString = (...values: unknown[]) => { - for (const value of values) { - if (typeof value === "string" && value.trim().length > 0) { - return value.trim(); - } - } - return undefined; -}; - -const isResultPreview = (value: unknown): value is ResultPreview => - isRecord(value) && - typeof value.count === "number" && - Number.isFinite(value.count) && - Array.isArray(value.fields) && - value.fields.every((field) => typeof field === "string") && - typeof value.summary === "string" && - "sample" in value; - const estimateBytes = (data: unknown) => Buffer.byteLength(JSON.stringify(data)); const buildPreview = (data: unknown): ResultPreview => { @@ -414,5 +396,26 @@ const buildPreview = (data: unknown): ResultPreview => { }; }; +const wrapPayload = (data: unknown, createdAt: string, projectId?: string) => ({ + metadata: { + createdAt, + ...(projectId ? { projectId } : {}), + }, + data, +}); + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); + +const unwrapStoredPayload = (value: unknown) => { + if (!isRecord(value)) { + return value; + } + return "data" in value ? value.data : value; +}; + +const toIsoString = (value: Date | string) => + value instanceof Date ? value.toISOString() : new Date(value).toISOString(); + +const isManagedPayloadPath = (payloadPath: string, managedPayloadDir: string) => + resolve(payloadPath).startsWith(resolve(managedPayloadDir)); diff --git a/src/routes/chat.ts b/src/routes/chat.ts index 25e153a..b6acfc9 100644 --- a/src/routes/chat.ts +++ b/src/routes/chat.ts @@ -91,6 +91,8 @@ export const buildChatRouter = ( session_id: record.sessionId, created_at: record.createdAt, updated_at: record.updatedAt, + is_streaming: record.isStreaming, + streaming_started_at: record.streamingStartedAt, status: record.status, title: record.title, parent_session_id: record.parentSessionId, @@ -114,6 +116,8 @@ export const buildChatRouter = ( title: record.title ?? "新对话", created_at: record.createdAt, updated_at: record.updatedAt, + is_streaming: record.isStreaming, + streaming_started_at: record.streamingStartedAt, status: record.status, parent_session_id: record.parentSessionId, })), @@ -145,13 +149,15 @@ export const buildChatRouter = ( return; } - const state = await conversationStateStore.read(conversation.sessionScopeKey); + const state = await conversationStateStore.read(conversation.sessionId); res.json({ id: conversation.sessionId, title: conversation.title ?? "新对话", is_title_manually_edited: state?.isTitleManuallyEdited ?? false, created_at: conversation.createdAt, updated_at: conversation.updatedAt, + is_streaming: conversation.isStreaming, + streaming_started_at: conversation.streamingStartedAt, status: conversation.status, session_id: conversation.sessionId, messages: state?.messages ?? [], @@ -190,7 +196,7 @@ export const buildChatRouter = ( const nextRecord = await conversationStore.touch(record, { ...(parsed.data.title ? { title: parsed.data.title } : {}), }); - await conversationStateStore.write(nextRecord.sessionScopeKey, { + await conversationStateStore.write(nextRecord.sessionId, { sessionId: nextRecord.sessionId, isTitleManuallyEdited: parsed.data.is_title_manually_edited, messages: parsed.data.messages, @@ -201,6 +207,8 @@ export const buildChatRouter = ( title: nextRecord.title ?? "新对话", created_at: nextRecord.createdAt, updated_at: nextRecord.updatedAt, + is_streaming: nextRecord.isStreaming, + streaming_started_at: nextRecord.streamingStartedAt, status: nextRecord.status, session_id: nextRecord.sessionId, }); @@ -231,9 +239,9 @@ export const buildChatRouter = ( return; } const nextConversation = await conversationStore.touch(conversation, { title }); - const state = await conversationStateStore.read(nextConversation.sessionScopeKey); + const state = await conversationStateStore.read(nextConversation.sessionId); if (state) { - await conversationStateStore.write(nextConversation.sessionScopeKey, { + await conversationStateStore.write(nextConversation.sessionId, { ...state, isTitleManuallyEdited: isTitleManuallyEdited ?? state.isTitleManuallyEdited, @@ -264,7 +272,7 @@ export const buildChatRouter = ( res.status(204).end(); return; } - await conversationStateStore.remove(conversation.sessionScopeKey); + await conversationStateStore.remove(conversation.sessionId); await conversationStore.remove(conversation); res.status(204).end(); }); @@ -273,7 +281,7 @@ export const buildChatRouter = ( const renderRef = req.params.renderRef?.trim(); const userId = req.header("x-user-id")?.trim(); const projectId = req.header("x-project-id") ?? undefined; - const clientSessionId = + const sessionId = typeof req.query.session_id === "string" ? req.query.session_id.trim() : undefined; @@ -296,7 +304,7 @@ export const buildChatRouter = ( renderRef, { actorKey: toActorKey(userId), - clientSessionId, + sessionId, projectId, }, { @@ -324,7 +332,7 @@ export const buildChatRouter = ( try { const binding = await sessionBridge.abort({ - clientSessionId: parsed.data.session_id, + sessionId: parsed.data.session_id, }); if (!binding) { @@ -334,8 +342,8 @@ export const buildChatRouter = ( logger.info( { - clientSessionId: parsed.data.session_id, - sessionId: binding.sessionId, + sessionId: parsed.data.session_id, + runtimeSessionId: binding.runtimeSessionId, }, "aborted chat session by client request", ); @@ -370,8 +378,8 @@ export const buildChatRouter = ( const actorKey = toActorKey(userId); const projectKey = toProjectKey(projectId); - const sourceClientSessionId = parsed.data.session_id?.trim(); - const sourceConversation = sourceClientSessionId + const sourceSessionId = parsed.data.session_id?.trim(); + const sourceConversation = sourceSessionId ? await conversationStore.get( { actorKey, @@ -379,31 +387,29 @@ export const buildChatRouter = ( projectKey, userId, }, - sourceClientSessionId, + sourceSessionId, ) : null; const { record: targetConversation } = await conversationStore.ensure({ actorKey, - parentSessionId: sourceClientSessionId, + parentSessionId: sourceSessionId, projectId, projectKey, userId, }); - const nextClientSessionId = targetConversation.sessionId; + const nextSessionId = targetConversation.sessionId; - if (sourceClientSessionId && parsed.data.keep_message_count > 0) { + if (sourceSessionId && parsed.data.keep_message_count > 0) { await sessionHistoryStore.cloneThread( { actorKey, - clientSessionId: sourceClientSessionId, projectKey, - sessionId: sourceClientSessionId, + sessionId: sourceSessionId, }, { actorKey, - clientSessionId: nextClientSessionId, projectKey, - sessionId: nextClientSessionId, + sessionId: nextSessionId, }, parsed.data.keep_message_count, ); @@ -416,8 +422,8 @@ export const buildChatRouter = ( logger.info( { - sourceClientSessionId: parsed.data.session_id, - clientSessionId: nextClientSessionId, + sourceSessionId: parsed.data.session_id, + sessionId: nextSessionId, traceId, projectId, keepMessageCount: parsed.data.keep_message_count, @@ -426,7 +432,7 @@ export const buildChatRouter = ( ); res.status(200).json({ - session_id: nextClientSessionId, + session_id: nextSessionId, }); } catch (error) { const detail = error instanceof Error ? error.message : String(error); @@ -469,7 +475,7 @@ export const buildChatRouter = ( const activeConversation = await conversationStore.touch(conversation); const { binding, requestContext, created } = await sessionBridge.resolve({ - clientSessionId: activeConversation.sessionId, + sessionId: activeConversation.sessionId, accessToken, projectId, traceId, @@ -477,16 +483,16 @@ export const buildChatRouter = ( }); const historyContext = { actorKey: requestContext.actorKey, - clientSessionId: requestContext.clientSessionId, projectKey: requestContext.projectKey, - sessionId: requestContext.clientSessionId, + sessionId: requestContext.sessionId, }; + await conversationStore.markStreaming(activeConversation, binding.runtimeSessionId); const recentTurns = await sessionHistoryStore.getRecentTurns(historyContext, 8); logger.info( { - clientSessionId: requestContext.clientSessionId, - sessionId: binding.sessionId, + sessionId: requestContext.sessionId, + runtimeSessionId: binding.runtimeSessionId, created: created || conversationCreated, model: parsed.data.model, traceId: requestContext.traceId, @@ -502,7 +508,7 @@ export const buildChatRouter = ( res.setHeader("X-Accel-Buffering", "no"); res.flushHeaders?.(); - const clientSessionId = requestContext.clientSessionId; + const sessionId = requestContext.sessionId; let streamClosed = false; const abortController = new AbortController(); const handleClientClose = () => { @@ -525,8 +531,8 @@ export const buildChatRouter = ( ); const streamResult = await streamPromptResponse({ runtime, - opencodeSessionId: binding.sessionId, - clientSessionId, + opencodeSessionId: binding.runtimeSessionId, + sessionId, message: preparedMessage, model: parsed.data.model, traceId: requestContext.traceId, @@ -541,7 +547,7 @@ export const buildChatRouter = ( }); if (!streamResult.aborted && !streamResult.failed) { - const messages = await runtime.messages(binding.sessionId, 60); + const messages = await runtime.messages(binding.runtimeSessionId, 60); const assistantMessage = [...messages] .reverse() .find((message) => message.info.role === "assistant"); @@ -552,7 +558,7 @@ export const buildChatRouter = ( activeConversation.sessionId, )) ?? activeConversation; const latestConversationState = await conversationStateStore.read( - latestConversation.sessionScopeKey, + latestConversation.sessionId, ); const existingSessionTitle = latestConversation.title; let sessionTitle = existingSessionTitle; @@ -563,7 +569,7 @@ export const buildChatRouter = ( }); if (shouldGenerateTitle) { sessionTitle = await generateSessionTitle(runtime, { - sessionId: binding.sessionId, + sessionId: binding.runtimeSessionId, latestAssistantMessage: assistantText, latestUserMessage: parsed.data.message, fallbackTitle: existingSessionTitle, @@ -582,7 +588,7 @@ export const buildChatRouter = ( ) { res.write( toSse("session_title", { - session_id: clientSessionId, + session_id: sessionId, title: sessionTitle, }), ); @@ -593,19 +599,23 @@ export const buildChatRouter = ( assistantMessage: assistantText, model: parsed.data.model, requestContext, - sessionId: clientSessionId, + sessionId, toolCallCount: streamResult.toolCallCount, userMessage: parsed.data.message, }).catch((error) => { logger.warn( - { err: error, sessionId: clientSessionId }, + { err: error, sessionId }, "post-turn learning failed", ); }); } } } finally { - await sessionBridge.releaseRuntimeSession(clientSessionId, binding.sessionId); + await sessionBridge.releaseRuntimeSession( + sessionId, + binding.runtimeSessionId, + ); + await conversationStore.clearStreaming(sessionId, binding.runtimeSessionId); streamClosed = true; req.off("close", handleClientClose); res.off("close", handleClientClose); diff --git a/src/routes/chatStream.ts b/src/routes/chatStream.ts index 60adf6c..5453886 100644 --- a/src/routes/chatStream.ts +++ b/src/routes/chatStream.ts @@ -14,7 +14,7 @@ export type SupportedModel = (typeof supportedModels)[number]; type StreamPromptOptions = { runtime: OpencodeRuntimeAdapter; opencodeSessionId: string; - clientSessionId: string; + sessionId: string; message: string; model?: SupportedModel; traceId?: string; @@ -169,7 +169,7 @@ export const collectTextContent = (parts: Part[]) => const emitFallbackMessage = async ( runtime: OpencodeRuntimeAdapter, opencodeSessionId: string, - clientSessionId: string, + sessionId: string, write: (event: string, data: Record) => void, ) => { const messages = await runtime.messages(opencodeSessionId); @@ -180,7 +180,7 @@ const emitFallbackMessage = async ( const text = collectTextContent(parts); if (text) { write("token", { - session_id: clientSessionId, + session_id: sessionId, content: text, }); } @@ -294,7 +294,7 @@ const getToolProgressTitle = (tool: string, status: string) => { export const streamPromptResponse = async ({ runtime, opencodeSessionId, - clientSessionId, + sessionId, message, model, traceId, @@ -333,7 +333,7 @@ export const streamPromptResponse = async ({ let failed = false; const debugContext = { opencodeSessionId, - clientSessionId, + sessionId, traceId, projectId, model: model ?? null, @@ -369,7 +369,7 @@ export const streamPromptResponse = async ({ if (status === "running") { write("progress", { - session_id: clientSessionId, + session_id: sessionId, id, phase, status, @@ -385,7 +385,7 @@ export const streamPromptResponse = async ({ finalizedProgressIds.add(id); progressStartedAtMap.delete(id); write("progress", { - session_id: clientSessionId, + session_id: sessionId, id, phase, status, @@ -542,7 +542,6 @@ export const streamPromptResponse = async ({ void writeLlmRequestAuditLog({ kind: "skill", sessionId: opencodeSessionId, - clientSessionId, traceId, projectId, target: name, @@ -568,7 +567,7 @@ export const streamPromptResponse = async ({ } emittedText = true; write("token", { - session_id: clientSessionId, + session_id: sessionId, content: event.properties.delta, }); } else if (partType === "reasoning") { @@ -601,7 +600,7 @@ export const streamPromptResponse = async ({ for (const content of pending) { emittedText = true; write("token", { - session_id: clientSessionId, + session_id: sessionId, content, }); } @@ -692,7 +691,7 @@ export const streamPromptResponse = async ({ { tool: part.tool, sessionId: opencodeSessionId, - clientSessionId, + requestSessionId: sessionId, }, "llm tool request missing reason", ); @@ -700,7 +699,6 @@ export const streamPromptResponse = async ({ void writeLlmRequestAuditLog({ kind: "tool", sessionId: opencodeSessionId, - clientSessionId, traceId, projectId, target: part.tool, @@ -711,7 +709,7 @@ export const streamPromptResponse = async ({ logger.warn({ err: error }, "failed to write tool audit log"); }); write("tool_call", { - session_id: clientSessionId, + session_id: sessionId, tool: part.tool, params: toolParams, reason, @@ -746,7 +744,7 @@ export const streamPromptResponse = async ({ : "opencode session error", }); write("error", { - session_id: clientSessionId, + session_id: sessionId, message: event.properties.error ? getErrorMessage(event.properties.error) : "opencode session error", @@ -803,7 +801,7 @@ export const streamPromptResponse = async ({ ...debugContext, elapsedMs: Math.max(0, Date.now() - requestStartedAt), }); - await emitFallbackMessage(runtime, opencodeSessionId, clientSessionId, write); + await emitFallbackMessage(runtime, opencodeSessionId, sessionId, write); } emitProgress({ id: "request-received", @@ -822,7 +820,7 @@ export const streamPromptResponse = async ({ : "已完成分析，并通过兜底消息补发最终回答内容。", }); write("done", { - session_id: clientSessionId, + session_id: sessionId, total_duration_ms: Math.max(0, Date.now() - requestStartedAt), }); logDevelopmentDebug("chat stream completed", { diff --git a/src/server.ts b/src/server.ts index 24bcc46..81b8141 100644 --- a/src/server.ts +++ b/src/server.ts @@ -14,7 +14,7 @@ import { ResultReferenceResolver } from "./results/resolver.js"; import { ResultReferenceStore } from "./results/store.js"; import { buildChatRouter } from "./routes/chat.js"; import { opencodeRuntime } from "./runtime/opencode.js"; -import { ToolSessionContextStore } from "./session/toolContextStore.js"; +import { RuntimeSessionStore } from "./session/runtimeSessionStore.js"; import { DynamicHttpExecutor } from "./tools/dynamicHttpExecutor.js"; const app = express(); @@ -23,7 +23,7 @@ const conversationStore = new ConversationStore(); const conversationStateStore = new ConversationStateStore(); const memoryStore = new MemoryStore(); const sessionHistoryStore = new SessionHistoryStore(); -const toolContextStore = new ToolSessionContextStore(); +const runtimeSessionStore = new RuntimeSessionStore(); const learningOrchestrator = new LearningOrchestrator( opencodeRuntime, memoryStore, @@ -65,22 +65,26 @@ app.post("/internal/tools/dynamic-http-call", async (req, res) => { return; } - const sessionScopeKey = - typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; - const threadContext = await toolContextStore.read(sessionScopeKey); - const runtimeContext = sessionBridge.getActiveSensitiveContext(sessionScopeKey); - if (!threadContext && !runtimeContext) { + const runtimeSessionId = + typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; + const persistedContext = runtimeSessionId + ? await runtimeSessionStore.read(runtimeSessionId) + : null; + const runtimeContext = runtimeSessionId + ? sessionBridge.getActiveSensitiveContext(runtimeSessionId) + : null; + if (!persistedContext && !runtimeContext) { res.status(404).json({ message: "runtime or session context not found", - detail: sessionScopeKey, + detail: runtimeSessionId, }); return; } - const context = runtimeContext ?? threadContext; + const context = persistedContext; if (!context) { res.status(404).json({ message: "runtime or session context not found", - detail: sessionScopeKey, + detail: runtimeSessionId, }); return; } @@ -93,14 +97,14 @@ app.post("/internal/tools/dynamic-http-call", async (req, res) => { path: req.body?.path, method: req.body?.method, arguments: req.body?.arguments, + body: req.body?.body, }, { accessToken: runtimeContext?.accessToken, actorKey: context.actorKey, - clientSessionId: context.clientSessionId, + sessionId: context.sessionId, projectId: context.projectId, projectKey: context.projectKey, - sessionId: context.clientSessionId, traceId: context.traceId, }, ); @@ -120,14 +124,14 @@ app.post("/internal/tools/fetch-result-ref", async (req, res) => { return; } - const sessionScopeKey = - typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; + const runtimeSessionId = + typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; const resultRef = typeof req.body?.result_ref === "string" ? req.body.result_ref : ""; - const context = await toolContextStore.read(sessionScopeKey); + const context = runtimeSessionId ? await runtimeSessionStore.read(runtimeSessionId) : null; if (!context) { res.status(404).json({ message: "session context not found", - detail: sessionScopeKey, + detail: runtimeSessionId, }); return; } @@ -140,7 +144,7 @@ app.post("/internal/tools/fetch-result-ref", async (req, res) => { resultRef, { actorKey: context.actorKey, - clientSessionId: context.clientSessionId, + sessionId: context.sessionId, projectId: context.projectId, }, { @@ -163,14 +167,14 @@ app.post("/internal/tools/store-render-ref", async (req, res) => { return; } - const sessionScopeKey = - typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; + const runtimeSessionId = + typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; const filePath = typeof req.body?.file_path === "string" ? req.body.file_path.trim() : ""; - const context = await toolContextStore.read(sessionScopeKey); + const context = runtimeSessionId ? await runtimeSessionStore.read(runtimeSessionId) : null; if (!context) { res.status(404).json({ message: "session context not found", - detail: sessionScopeKey, + detail: runtimeSessionId, }); return; } @@ -182,10 +186,9 @@ app.post("/internal/tools/store-render-ref", async (req, res) => { try { const record = await resultReferenceResolver.registerRenderPayloadFile(filePath, { actorKey: context.actorKey, - clientSessionId: context.clientSessionId, projectId: context.projectId, projectKey: context.projectKey, - sessionId: context.clientSessionId, + sessionId: context.sessionId, source: "migration", traceId: context.traceId, }); @@ -213,14 +216,14 @@ app.post("/internal/tools/session-search", async (req, res) => { return; } - const sessionScopeKey = - typeof req.body?.sessionScopeKey === "string" ? req.body.sessionScopeKey : ""; + const runtimeSessionId = + typeof req.body?.runtimeSessionId === "string" ? req.body.runtimeSessionId.trim() : ""; const query = typeof req.body?.query === "string" ? req.body.query : ""; - const context = await toolContextStore.read(sessionScopeKey); + const context = runtimeSessionId ? await runtimeSessionStore.read(runtimeSessionId) : null; if (!context) { res.status(404).json({ message: "session context not found", - detail: sessionScopeKey, + detail: runtimeSessionId, }); return; } @@ -264,8 +267,9 @@ const bootstrap = async () => { memoryStore.initialize(), resultReferenceStore.initialize(), sessionHistoryStore.initialize(), - toolContextStore.initialize(), + runtimeSessionStore.initialize(), ]); + await conversationStore.resetStreamingSessions(); resultReferenceStore.startCleanupLoop(); }; diff --git a/src/session/runtimeSessionStore.ts b/src/session/runtimeSessionStore.ts new file mode 100644 index 0000000..b2a627b --- /dev/null +++ b/src/session/runtimeSessionStore.ts @@ -0,0 +1,123 @@ +import { type QueryResultRow } from "pg"; + +import { AgentDatabase, getAgentDatabase } from "../db/index.js"; + +export type RuntimeSessionContext = { + runtimeSessionId: string; + actorKey: string; + allowLearningWrite?: boolean; + sessionId: string; + learningMode?: "interactive" | "review"; + projectId?: string; + projectKey: string; + traceId: string; + releasedAt?: string; +}; + +type RuntimeSessionRow = QueryResultRow & { + runtime_session_id: string; + actor_key: string; + allow_learning_write: boolean; + session_id: string; + learning_mode: "interactive" | "review" | null; + project_id: string | null; + project_key: string; + trace_id: string; + released_at: Date | string | null; +}; + +export class RuntimeSessionStore { + constructor(private readonly db: AgentDatabase = getAgentDatabase()) {} + + async initialize() { + await this.db.initialize(); + } + + async write(context: RuntimeSessionContext) { + const result = await this.db.query( + ` + INSERT INTO ${this.db.table("runtime_sessions")} ( + runtime_session_id, + actor_key, + allow_learning_write, + session_id, + learning_mode, + project_id, + project_key, + trace_id, + released_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NULL) + ON CONFLICT (runtime_session_id) + DO UPDATE SET + actor_key = EXCLUDED.actor_key, + allow_learning_write = EXCLUDED.allow_learning_write, + session_id = EXCLUDED.session_id, + learning_mode = EXCLUDED.learning_mode, + project_id = EXCLUDED.project_id, + project_key = EXCLUDED.project_key, + trace_id = EXCLUDED.trace_id, + released_at = NULL + RETURNING * + `, + [ + context.runtimeSessionId, + context.actorKey, + context.allowLearningWrite ?? false, + context.sessionId, + context.learningMode ?? null, + context.projectId ?? null, + context.projectKey, + context.traceId, + ], + ); + return mapRuntimeSessionRow(result.rows[0]); + } + + async read(runtimeSessionId: string, options: { includeReleased?: boolean } = {}) { + const result = await this.db.query( + ` + SELECT * + FROM ${this.db.table("runtime_sessions")} + WHERE runtime_session_id = $1 + AND ($2::boolean OR released_at IS NULL) + LIMIT 1 + `, + [runtimeSessionId, options.includeReleased ?? false], + ); + return mapRuntimeSessionRow(result.rows[0]); + } + + async release(runtimeSessionId: string) { + await this.db.query( + ` + UPDATE ${this.db.table("runtime_sessions")} + SET released_at = NOW() + WHERE runtime_session_id = $1 + `, + [runtimeSessionId], + ); + } +} + +const mapRuntimeSessionRow = ( + row?: RuntimeSessionRow | null, +): RuntimeSessionContext | null => { + if (!row) { + return null; + } + return { + runtimeSessionId: row.runtime_session_id, + actorKey: row.actor_key, + allowLearningWrite: row.allow_learning_write, + sessionId: row.session_id, + learningMode: row.learning_mode ?? undefined, + projectId: row.project_id ?? undefined, + projectKey: row.project_key, + traceId: row.trace_id, + releasedAt: + row.released_at instanceof Date + ? row.released_at.toISOString() + : row.released_at ?? undefined, + }; +}; diff --git a/src/session/toolContextStore.ts b/src/session/toolContextStore.ts deleted file mode 100644 index bc8fa5f..0000000 --- a/src/session/toolContextStore.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { join } from "node:path"; - -import { config } from "../config.js"; -import { - atomicWriteJson, - ensureDirectory, - readJsonFile, - removeFileIfExists, -} from "../utils/fileStore.js"; -import { toConversationScopeKey } from "../utils/fileStore.js"; - -export type ToolSessionContext = { - actorKey: string; - allowLearningWrite?: boolean; - clientSessionId: string; - learningMode?: "interactive" | "review"; - projectId?: string; - projectKey: string; - sessionId: string; - sessionScopeKey: string; - traceId: string; -}; - -export class ToolSessionContextStore { - constructor(private readonly baseDir = config.SESSION_CONTEXT_STORAGE_DIR) {} - - async initialize() { - await ensureDirectory(this.baseDir); - } - - async write(context: ToolSessionContext) { - await atomicWriteJson(this.filePath(context.sessionId), context); - if (context.learningMode === "interactive" && context.sessionScopeKey) { - await atomicWriteJson(this.filePath(context.sessionScopeKey), context); - } - } - - async read(sessionId: string) { - return await readJsonFile(this.filePath(sessionId)); - } - - async remove(sessionId: string) { - await removeFileIfExists(this.filePath(sessionId)); - } - - private filePath(sessionId: string) { - return join(this.baseDir, `${sessionId}.json`); - } -} - -export const buildToolSessionScopeKey = ( - actorKey: string, - projectKey: string, - clientSessionId: string, -) => toConversationScopeKey(actorKey, projectKey, clientSessionId); diff --git a/src/tools/dynamicHttpExecutor.ts b/src/tools/dynamicHttpExecutor.ts index d002395..71a3a2a 100644 --- a/src/tools/dynamicHttpExecutor.ts +++ b/src/tools/dynamicHttpExecutor.ts @@ -8,12 +8,12 @@ export type DynamicHttpInput = { path: string; method?: string; arguments?: Record; + body?: unknown; }; export type SessionToolContext = { accessToken?: string; actorKey: string; - clientSessionId: string; projectKey: string; sessionId: string; projectId?: string; @@ -53,11 +53,13 @@ export class DynamicHttpExecutor { if (context.projectId) { headers.set("x-project-id", context.projectId); } + const body = buildRequestBody(method, input.body, headers); const startedAt = Date.now(); const response = await fetch(url, { method, headers, + body, signal: AbortSignal.timeout(config.TJWATER_API_TIMEOUT_MS), }); const durationMs = Date.now() - startedAt; @@ -128,6 +130,25 @@ const buildQuery = (argumentsObject: Record) => { return pairs; }; +const buildRequestBody = ( + method: string, + body: unknown, + headers: Headers, +) => { + if (body === undefined) { + return undefined; + } + if (method === "GET") { + throw new Error("GET requests do not support body"); + } + const serialized = JSON.stringify(body); + if (serialized === undefined) { + throw new Error("body must be JSON-serializable"); + } + headers.set("Content-Type", "application/json"); + return serialized; +}; + const normalizeSuccessResult = async ( data: unknown, context: SessionToolContext, @@ -145,7 +166,6 @@ const normalizeSuccessResult = async ( // 大结果转成持久化引用，支持 review 和跨重启回读。 const record = await resultStore.store({ actorKey: context.actorKey, - clientSessionId: context.clientSessionId, data, kind: RESULT_REFERENCE_KIND.dynamicHttpResult, projectId: context.projectId, diff --git a/tests/conversations/store.test.ts b/tests/conversations/store.test.ts index f7dc550..375a61d 100644 --- a/tests/conversations/store.test.ts +++ b/tests/conversations/store.test.ts @@ -1,22 +1,21 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { ConversationStore } from "../../src/conversations/store.js"; +import { createTestDatabase } from "../helpers/postgres.js"; describe("ConversationStore", () => { - let tempDir: string; let store: ConversationStore; + let dispose: (() => Promise) | undefined; beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), "tjwater-conversation-")); - store = new ConversationStore(tempDir); + const testDb = await createTestDatabase(); + store = new ConversationStore(testDb.db); + dispose = testDb.dispose; await store.initialize(); }); afterEach(async () => { - await rm(tempDir, { force: true, recursive: true }); + await dispose?.(); }); it("issues backend-managed session ids when absent", async () => { @@ -58,7 +57,48 @@ describe("ConversationStore", () => { }, "existing-session", ); - expect(fetched?.sessionScopeKey).toBe(record.sessionScopeKey); + expect(fetched?.sessionId).toBe(record.sessionId); expect(fetched?.title).toBe("新标题"); }); + + it("tracks streaming state and clears it with runtime-session guarding", async () => { + const { record } = await store.ensure({ + actorKey: "actor-3", + projectId: "project-3", + projectKey: "project-key-3", + sessionId: "streaming-session", + userId: "user-3", + }); + + const streaming = await store.markStreaming(record, "runtime-1"); + expect(streaming.isStreaming).toBe(true); + expect(streaming.streamingStartedAt).toBeDefined(); + + const skipped = await store.clearStreaming(record.sessionId, "runtime-2"); + expect(skipped).toBeNull(); + + const stillStreaming = await store.get( + { + actorKey: "actor-3", + projectId: "project-3", + projectKey: "project-key-3", + userId: "user-3", + }, + record.sessionId, + ); + expect(stillStreaming?.isStreaming).toBe(true); + + await store.clearStreaming(record.sessionId, "runtime-1"); + const cleared = await store.get( + { + actorKey: "actor-3", + projectId: "project-3", + projectKey: "project-key-3", + userId: "user-3", + }, + record.sessionId, + ); + expect(cleared?.isStreaming).toBe(false); + expect(cleared?.streamingStartedAt).toBeUndefined(); + }); }); diff --git a/tests/helpers/postgres.ts b/tests/helpers/postgres.ts new file mode 100644 index 0000000..a57b275 --- /dev/null +++ b/tests/helpers/postgres.ts @@ -0,0 +1,17 @@ +import { randomUUID } from "node:crypto"; + +import { AgentDatabase } from "../../src/db/index.js"; + +export const createTestDatabase = async () => { + const schema = `test_${randomUUID().replace(/-/g, "")}`; + const db = new AgentDatabase({ schema }); + await db.initialize(); + return { + db, + schema, + async dispose() { + await db.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`); + await db.close(); + }, + }; +}; diff --git a/tests/history/store.test.ts b/tests/history/store.test.ts index 4667765..ea77025 100644 --- a/tests/history/store.test.ts +++ b/tests/history/store.test.ts @@ -1,67 +1,53 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { ConversationStore } from "../../src/conversations/store.js"; import { SessionHistoryStore } from "../../src/history/store.js"; +import { createTestDatabase } from "../helpers/postgres.js"; describe("SessionHistoryStore", () => { - let tempDir: string; + let conversationStore: ConversationStore; let store: SessionHistoryStore; + let dispose: (() => Promise) | undefined; beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), "tjwater-history-")); - store = new SessionHistoryStore(tempDir); - await store.initialize(); + const testDb = await createTestDatabase(); + conversationStore = new ConversationStore(testDb.db); + store = new SessionHistoryStore(testDb.db); + dispose = testDb.dispose; + await Promise.all([conversationStore.initialize(), store.initialize()]); }); afterEach(async () => { - await rm(tempDir, { force: true, recursive: true }); + await dispose?.(); }); - it("falls back to legacy runtime-session transcripts by client session id and migrates on append", async () => { - await writeFile( - join(tempDir, "actor-1__project-1__runtime-session-1.json"), - JSON.stringify( - { - actorKey: "actor-1", - clientSessionId: "thread-1", - projectKey: "project-1", - sessionId: "runtime-session-1", - turns: [ - { - id: "turn-1", - assistantMessage: "先检查泵站流量。", - timestamp: "2026-05-21T00:00:00.000Z", - toolCallCount: 1, - userMessage: "帮我看一下当前异常。", - }, - ], - updatedAt: "2026-05-21T00:00:00.000Z", - }, - null, - 2, - ), - "utf8", - ); + it("stores ordered turns for the durable conversation id", async () => { + await conversationStore.ensure({ + actorKey: "actor-1", + projectId: "project-1", + projectKey: "project-1", + sessionId: "thread-1", + userId: "user-1", + }); - const recentTurns = await store.getRecentTurns( + const first = await store.appendTurn( { actorKey: "actor-1", - clientSessionId: "thread-1", projectKey: "project-1", sessionId: "thread-1", }, - 5, + { + assistantMessage: "先检查泵站流量。", + toolCallCount: 1, + userMessage: "帮我看一下当前异常。", + }, ); - expect(recentTurns).toHaveLength(1); - expect(recentTurns[0]?.userMessage).toBe("帮我看一下当前异常。"); + expect(first.turns).toHaveLength(1); - const transcript = await store.appendTurn( + const second = await store.appendTurn( { actorKey: "actor-1", - clientSessionId: "thread-1", projectKey: "project-1", sessionId: "thread-1", }, @@ -72,15 +58,31 @@ describe("SessionHistoryStore", () => { }, ); - expect(transcript.sessionId).toBe("thread-1"); - expect(transcript.turns).toHaveLength(2); + expect(second.sessionId).toBe("thread-1"); + expect(second.turns).toHaveLength(2); + expect(second.turns[0]?.userMessage).toBe("帮我看一下当前异常。"); + expect(second.turns[1]?.assistantMessage).toBe("已经定位到 3 条疑似异常支路。"); }); it("clones only the kept prefix when forking a thread", async () => { + await conversationStore.ensure({ + actorKey: "actor-2", + projectId: "project-2", + projectKey: "project-2", + sessionId: "thread-source", + userId: "user-2", + }); + await conversationStore.ensure({ + actorKey: "actor-2", + projectId: "project-2", + projectKey: "project-2", + sessionId: "thread-fork", + userId: "user-2", + }); + await store.appendTurn( { actorKey: "actor-2", - clientSessionId: "thread-source", projectKey: "project-2", sessionId: "thread-source", }, @@ -93,7 +95,6 @@ describe("SessionHistoryStore", () => { await store.appendTurn( { actorKey: "actor-2", - clientSessionId: "thread-source", projectKey: "project-2", sessionId: "thread-source", }, @@ -107,13 +108,11 @@ describe("SessionHistoryStore", () => { const cloned = await store.cloneThread( { actorKey: "actor-2", - clientSessionId: "thread-source", projectKey: "project-2", sessionId: "thread-source", }, { actorKey: "actor-2", - clientSessionId: "thread-fork", projectKey: "project-2", sessionId: "thread-fork", }, @@ -126,7 +125,6 @@ describe("SessionHistoryStore", () => { const forkRecentTurns = await store.getRecentTurns( { actorKey: "actor-2", - clientSessionId: "thread-fork", projectKey: "project-2", sessionId: "thread-fork", }, diff --git a/tests/results/store.test.ts b/tests/results/store.test.ts index a86424d..a086246 100644 --- a/tests/results/store.test.ts +++ b/tests/results/store.test.ts @@ -3,46 +3,65 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { ConversationStore } from "../../src/conversations/store.js"; import { ResultReferenceResolver } from "../../src/results/resolver.js"; import { RESULT_REFERENCE_KIND, RESULT_REFERENCE_SOURCE, ResultReferenceStore, } from "../../src/results/store.js"; +import { createTestDatabase } from "../helpers/postgres.js"; describe("ResultReferenceResolver", () => { - let tempDir: string; + let payloadDir: string; + let conversationStore: ConversationStore; let store: ResultReferenceStore; let resolver: ResultReferenceResolver; + let dispose: (() => Promise) | undefined; beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), "tjwater-result-ref-")); - store = new ResultReferenceStore(tempDir, 60_000); + payloadDir = await mkdtemp(join(tmpdir(), "tjwater-result-ref-")); + const testDb = await createTestDatabase(); + conversationStore = new ConversationStore(testDb.db); + store = new ResultReferenceStore(testDb.db, payloadDir, 60_000); resolver = new ResultReferenceResolver(store); - await store.initialize(); + dispose = async () => { + await testDb.dispose(); + await rm(payloadDir, { force: true, recursive: true }); + }; + await Promise.all([conversationStore.initialize(), store.initialize()]); }); afterEach(async () => { - await rm(tempDir, { force: true, recursive: true }); + await dispose?.(); }); - it("stores metadata for new referenced results and resolves them", async () => { + it("stores referenced results as metadata plus an external payload file", async () => { + await conversationStore.ensure({ + actorKey: "actor-1", + projectId: "project-1", + projectKey: "project-key-1", + sessionId: "client-1", + userId: "user-1", + }); + const record = await resolver.register({ actorKey: "actor-1", - clientSessionId: "client-1", data: [{ id: "J1" }, { id: "J2" }], kind: RESULT_REFERENCE_KIND.dynamicHttpResult, projectId: "project-1", projectKey: "project-key-1", schemaVersion: 1, - sessionId: "session-1", + sessionId: "client-1", source: RESULT_REFERENCE_SOURCE.dynamicHttp, traceId: "trace-1", }); - expect(record.kind).toBe(RESULT_REFERENCE_KIND.dynamicHttpResult); - expect(record.schemaVersion).toBe(1); - expect(record.source).toBe(RESULT_REFERENCE_SOURCE.dynamicHttp); + expect(record.payloadPath).toContain("/payloads/"); + const payload = JSON.parse(await readFile(record.payloadPath!, "utf8")) as { + data?: unknown; + }; + expect(payload.data).toEqual([{ id: "J1" }, { id: "J2" }]); const result = await resolver.getAuthorized( record.resultRef, @@ -55,124 +74,21 @@ describe("ResultReferenceResolver", () => { }, ); - expect(result).not.toBeNull(); expect(result?.kind).toBe(RESULT_REFERENCE_KIND.dynamicHttpResult); - expect(result?.schema_version).toBe(1); expect(result?.source).toBe(RESULT_REFERENCE_SOURCE.dynamicHttp); expect(result?.data).toEqual([{ id: "J1" }]); }); - it("keeps legacy result refs readable while defaulting metadata", async () => { - const legacyRef = "res-aaaaaaaaaaaaaaaa"; - await writeFile( - join(tempDir, `${legacyRef}.json`), - JSON.stringify( - { - resultRef: legacyRef, - actorKey: "actor-legacy", - clientSessionId: "client-legacy", - createdAt: "2026-05-21T00:00:00.000Z", - data: { nodes: ["J1"] }, - preview: { - count: 1, - fields: ["nodes"], - sample: { nodes: ["J1"] }, - summary: "object<1 fields>", - }, - projectId: "project-legacy", - projectKey: "project-key-legacy", - sessionId: "session-legacy", - sizeBytes: 16, - traceId: "trace-legacy", - }, - null, - 2, - ), - "utf8", - ); - - const record = await store.getAuthorizedRecord(legacyRef, { - actorKey: "actor-legacy", - projectId: "project-legacy", + it("reuses an existing external render payload file as the payload locator", async () => { + await conversationStore.ensure({ + actorKey: "actor-3", + projectId: "project-3", + projectKey: "project-key-3", + sessionId: "client-3", + userId: "user-3", }); - expect(record).not.toBeNull(); - expect(record?.kind).toBe(RESULT_REFERENCE_KIND.dynamicHttpResult); - expect(record?.schemaVersion).toBe(1); - expect(record?.source).toBe(RESULT_REFERENCE_SOURCE.legacy); - }); - - it("rejects malformed refs, mismatched kinds, and auth mismatches", async () => { - const malformedRef = "res-bbbbbbbbbbbbbbbb"; - await writeFile( - join(tempDir, `${malformedRef}.json`), - JSON.stringify( - { - resultRef: malformedRef, - createdAt: "2026-05-21T00:00:00.000Z", - data: { value: 1 }, - preview: { - count: 1, - fields: ["value"], - sample: { value: 1 }, - summary: "object<1 fields>", - }, - projectId: "project-1", - projectKey: "project-key-1", - sessionId: "session-1", - sizeBytes: 10, - traceId: "trace-1", - }, - null, - 2, - ), - "utf8", - ); - - const malformed = await store.getAuthorizedRecord(malformedRef, { - actorKey: "actor-1", - projectId: "project-1", - }); - expect(malformed).toBeNull(); - - const renderRecord = await resolver.register({ - actorKey: "actor-2", - clientSessionId: "client-2", - data: { - node_area_map: { - J1: "DMA-1", - }, - }, - kind: RESULT_REFERENCE_KIND.renderJunctionsPayload, - projectId: "project-2", - projectKey: "project-key-2", - schemaVersion: 1, - sessionId: "session-2", - source: RESULT_REFERENCE_SOURCE.agentGenerated, - traceId: "trace-2", - }); - - const wrongKind = await resolver.getFullAuthorized( - renderRecord.resultRef, - { - actorKey: "actor-2", - projectId: "project-2", - }, - { - expectedKind: RESULT_REFERENCE_KIND.dynamicHttpResult, - }, - ); - expect(wrongKind).toBeNull(); - - const wrongActor = await resolver.getFullAuthorized(renderRecord.resultRef, { - actorKey: "actor-other", - projectId: "project-2", - }); - expect(wrongActor).toBeNull(); - }); - - it("registers render refs from local wrapper files and normalizes payloads", async () => { - const filePath = join(tempDir, "render-wrapper.json"); + const filePath = join(payloadDir, "render-wrapper.json"); await writeFile( filePath, JSON.stringify( @@ -190,12 +106,7 @@ describe("ResultReferenceResolver", () => { J2: 2, }, area_ids: ["DMA-1", " DMA-2 "], - area_colors: { - "DMA-1": "#ff0000", - "DMA-2": "#00ff00", - }, }, - createdAt: "2026-05-21T00:00:00.000Z", }, null, 2, @@ -205,22 +116,21 @@ describe("ResultReferenceResolver", () => { const record = await resolver.registerRenderPayloadFile(filePath, { actorKey: "actor-3", - clientSessionId: "client-3", projectId: "project-3", projectKey: "project-key-3", - sessionId: "session-3", + sessionId: "client-3", source: RESULT_REFERENCE_SOURCE.migration, traceId: "trace-3", }); - expect(record.kind).toBe(RESULT_REFERENCE_KIND.renderJunctionsPayload); - expect(record.source).toBe(RESULT_REFERENCE_SOURCE.migration); + expect(record.payloadPath).toBe(filePath); const result = await resolver.getFullAuthorized( record.resultRef, { actorKey: "actor-3", projectId: "project-3", + sessionId: "client-3", }, { expectedKind: RESULT_REFERENCE_KIND.renderJunctionsPayload, @@ -233,185 +143,6 @@ describe("ResultReferenceResolver", () => { J2: "2", }, area_ids: ["DMA-1", "DMA-2"], - area_colors: { - "DMA-1": "#ff0000", - "DMA-2": "#00ff00", - }, }); }); - - it("repairs wrapper files that omit metadata and location", async () => { - const filePath = join(tempDir, "render-wrapper-missing-fields.json"); - await writeFile( - filePath, - JSON.stringify( - { - data: { - node_area_map: { - J1: "DMA-1", - }, - }, - createdAt: "2026-05-21T00:00:00.000Z", - }, - null, - 2, - ), - "utf8", - ); - - const record = await resolver.registerRenderPayloadFile(filePath, { - actorKey: "actor-4", - clientSessionId: "client-4", - projectId: "project-4", - projectKey: "project-key-4", - sessionId: "session-4", - source: RESULT_REFERENCE_SOURCE.migration, - traceId: "trace-4", - }); - - expect(record.kind).toBe(RESULT_REFERENCE_KIND.renderJunctionsPayload); - - const repaired = JSON.parse(await readFile(filePath, "utf8")) as { - metadata?: Record; - location?: Record; - }; - - expect(repaired.metadata).toEqual({ - createdAt: "2026-05-21T00:00:00.000Z", - projectId: "project-4", - }); - expect(repaired.location).toEqual({ - file_path: filePath, - }); - }); - - it("repairs wrapper files whose location points elsewhere", async () => { - const filePath = join(tempDir, "render-wrapper-wrong-location.json"); - await writeFile( - filePath, - JSON.stringify( - { - metadata: { - createdAt: "2026-05-21T00:00:00.000Z", - }, - location: { - file_path: "/tmp/elsewhere.json", - source: "legacy", - }, - data: { - node_area_map: { - J1: "DMA-1", - }, - }, - }, - null, - 2, - ), - "utf8", - ); - - await resolver.registerRenderPayloadFile(filePath, { - actorKey: "actor-4", - clientSessionId: "client-4", - projectId: "project-4", - projectKey: "project-key-4", - sessionId: "session-4", - source: RESULT_REFERENCE_SOURCE.migration, - traceId: "trace-4", - }); - - const repaired = JSON.parse(await readFile(filePath, "utf8")) as { - metadata?: Record; - location?: Record; - }; - - expect(repaired.metadata).toEqual({ - createdAt: "2026-05-21T00:00:00.000Z", - projectId: "project-4", - }); - expect(repaired.location).toEqual({ - file_path: filePath, - source: "legacy", - }); - }); - - it("resolves legacy render payload files when callers include the json suffix", async () => { - const legacyRef = "res-c2fcee33-577e"; - await writeFile( - join(tempDir, `${legacyRef}.json`), - JSON.stringify( - { - data: { - node_area_map: { - J1: "DMA-1", - J2: 2, - }, - area_ids: ["DMA-1"], - }, - createdAt: "2026-05-21T00:00:00.000Z", - projectId: "project-legacy-render", - }, - null, - 2, - ), - "utf8", - ); - - const result = await resolver.getFullAuthorized( - `${legacyRef}.json`, - { - actorKey: "actor-legacy-render", - clientSessionId: "chat-legacy-render", - projectId: "project-legacy-render", - }, - { - expectedKind: RESULT_REFERENCE_KIND.renderJunctionsPayload, - }, - ); - - expect(result?.result_ref).toBe(legacyRef); - expect(result?.kind).toBe(RESULT_REFERENCE_KIND.renderJunctionsPayload); - expect(result?.source).toBe(RESULT_REFERENCE_SOURCE.legacy); - expect(result?.data).toEqual({ - node_area_map: { - J1: "DMA-1", - J2: "2", - }, - area_ids: ["DMA-1"], - }); - }); - - it("keeps legacy render payload files scoped to their project", async () => { - const legacyRef = "res-dddddddddddddddd"; - await writeFile( - join(tempDir, `${legacyRef}.json`), - JSON.stringify( - { - data: { - node_area_map: { - J1: "DMA-1", - }, - }, - projectId: "project-allowed", - }, - null, - 2, - ), - "utf8", - ); - - const result = await resolver.getFullAuthorized( - legacyRef, - { - actorKey: "actor-legacy-render", - clientSessionId: "chat-legacy-render", - projectId: "project-denied", - }, - { - expectedKind: RESULT_REFERENCE_KIND.renderJunctionsPayload, - }, - ); - - expect(result).toBeNull(); - }); }); diff --git a/tests/session/runtimeSessionStore.test.ts b/tests/session/runtimeSessionStore.test.ts new file mode 100644 index 0000000..4e26417 --- /dev/null +++ b/tests/session/runtimeSessionStore.test.ts @@ -0,0 +1,52 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; + +import { ConversationStore } from "../../src/conversations/store.js"; +import { RuntimeSessionStore } from "../../src/session/runtimeSessionStore.js"; +import { createTestDatabase } from "../helpers/postgres.js"; + +describe("RuntimeSessionStore", () => { + let conversationStore: ConversationStore; + let store: RuntimeSessionStore; + let dispose: (() => Promise) | undefined; + + beforeEach(async () => { + const testDb = await createTestDatabase(); + conversationStore = new ConversationStore(testDb.db); + store = new RuntimeSessionStore(testDb.db); + dispose = testDb.dispose; + await Promise.all([conversationStore.initialize(), store.initialize()]); + }); + + afterEach(async () => { + await dispose?.(); + }); + + it("stores and releases runtime sessions by runtime session id", async () => { + await conversationStore.ensure({ + actorKey: "actor-1", + projectId: "project-1", + projectKey: "project-1", + sessionId: "chat-session-1", + userId: "user-1", + }); + + await store.write({ + runtimeSessionId: "runtime-session-1", + actorKey: "actor-1", + allowLearningWrite: true, + sessionId: "chat-session-1", + learningMode: "interactive", + projectId: "project-id-1", + projectKey: "project-1", + traceId: "trace-1", + }); + + const runtimeContext = await store.read("runtime-session-1"); + expect(runtimeContext?.sessionId).toBe("chat-session-1"); + expect(runtimeContext?.runtimeSessionId).toBe("runtime-session-1"); + + await store.release("runtime-session-1"); + expect(await store.read("runtime-session-1")).toBeNull(); + expect((await store.read("runtime-session-1", { includeReleased: true }))?.releasedAt).toBeDefined(); + }); +}); diff --git a/tests/session/toolContextStore.test.ts b/tests/session/toolContextStore.test.ts deleted file mode 100644 index 7ff6b3d..0000000 --- a/tests/session/toolContextStore.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { - buildToolSessionScopeKey, - ToolSessionContextStore, -} from "../../src/session/toolContextStore.js"; - -describe("ToolSessionContextStore", () => { - let tempDir: string; - let store: ToolSessionContextStore; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), "tjwater-tool-context-")); - store = new ToolSessionContextStore(tempDir); - await store.initialize(); - }); - - afterEach(async () => { - await rm(tempDir, { force: true, recursive: true }); - }); - - it("writes interactive aliases under scoped session keys", async () => { - const sessionScopeKey = buildToolSessionScopeKey( - "actor-1", - "project-1", - "chat-session-1", - ); - - await store.write({ - actorKey: "actor-1", - allowLearningWrite: true, - clientSessionId: "chat-session-1", - learningMode: "interactive", - projectId: "project-id-1", - projectKey: "project-1", - sessionId: "runtime-session-1", - sessionScopeKey, - traceId: "trace-1", - }); - - const runtimeContext = await store.read("runtime-session-1"); - const scopedContext = await store.read(sessionScopeKey); - - expect(runtimeContext?.clientSessionId).toBe("chat-session-1"); - expect(scopedContext?.sessionScopeKey).toBe(sessionScopeKey); - expect(scopedContext?.sessionId).toBe("runtime-session-1"); - }); -}); diff --git a/tests/tools/dynamicHttpExecutor.test.ts b/tests/tools/dynamicHttpExecutor.test.ts new file mode 100644 index 0000000..1f055c6 --- /dev/null +++ b/tests/tools/dynamicHttpExecutor.test.ts @@ -0,0 +1,110 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { ResultReferenceStore } from "../../src/results/store.js"; +import { DynamicHttpExecutor } from "../../src/tools/dynamicHttpExecutor.js"; +import { createTestDatabase } from "../helpers/postgres.js"; + +describe("DynamicHttpExecutor", () => { + const originalFetch = globalThis.fetch; + let payloadDir: string; + let store: ResultReferenceStore; + let executor: DynamicHttpExecutor; + let dispose: (() => Promise) | undefined; + + beforeEach(async () => { + payloadDir = await mkdtemp(join(tmpdir(), "tjwater-dynamic-http-")); + const testDb = await createTestDatabase(); + store = new ResultReferenceStore(testDb.db, payloadDir, 60_000); + executor = new DynamicHttpExecutor(store); + dispose = async () => { + globalThis.fetch = originalFetch; + await testDb.dispose(); + await rm(payloadDir, { force: true, recursive: true }); + }; + await store.initialize(); + }); + + afterEach(async () => { + await dispose?.(); + }); + + it("sends JSON body for POST requests while preserving query arguments", async () => { + let capturedUrl = ""; + let capturedInit: RequestInit | undefined; + globalThis.fetch = Object.assign( + async (input: URL | RequestInfo, init?: RequestInit | BunFetchRequestInit) => { + capturedUrl = String(input); + capturedInit = init; + return new Response(JSON.stringify({ status: "ok" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }, + originalFetch, + ); + + const result = await executor.execute( + { + path: "/api/v1/runproject/simulate", + method: "POST", + arguments: { network: "demo-network" }, + body: { duration_minutes: 15 }, + }, + { + accessToken: "token-1", + actorKey: "actor-1", + projectId: "project-1", + projectKey: "project-key-1", + sessionId: "session-1", + traceId: "trace-1", + }, + ); + + const url = new URL(capturedUrl); + const headers = new Headers(capturedInit?.headers); + + expect(url.pathname).toBe("/api/v1/runproject/simulate"); + expect(url.searchParams.get("network")).toBe("demo-network"); + expect(capturedInit?.method).toBe("POST"); + expect(headers.get("content-type")).toBe("application/json"); + expect(headers.get("authorization")).toBe("Bearer token-1"); + expect(headers.get("x-project-id")).toBe("project-1"); + expect(headers.get("x-trace-id")).toBe("trace-1"); + expect(capturedInit?.body).toBe(JSON.stringify({ duration_minutes: 15 })); + expect(result).toEqual({ + ok: true, + trace_id: "trace-1", + upstream: { + method: "POST", + path: "/api/v1/runproject/simulate", + status_code: 200, + }, + result_mode: "inline", + result_size_bytes: expect.any(Number), + data: { status: "ok" }, + }); + }); + + it("rejects GET requests that include a body", async () => { + await expect( + executor.execute( + { + path: "/api/v1/runproject/", + method: "GET", + arguments: { network: "demo-network" }, + body: { duration_minutes: 15 }, + }, + { + actorKey: "actor-1", + projectId: "project-1", + projectKey: "project-key-1", + sessionId: "session-1", + traceId: "trace-1", + }, + ), + ).rejects.toThrow("GET requests do not support body"); + }); +});