feat: extend operational investigation workflows

app/services/calculation_worker.py

@@ -46,22 +46,10 @@ class WorkerSettings:
 @asynccontextmanager
 async def queue_repository(project_id):
     """Resolve current routing from metadata, never from a job payload."""
-    from psycopg import AsyncConnection
-    from psycopg.rows import dict_row
-    from app.infra.db.metadb.database import SessionLocal
-    from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
-    from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
-    async with SessionLocal() as session:
-        metadata = MetadataRepository(session)
-        project = await metadata.get_project_by_id(UUID(str(project_id)))
-        if not project or project.status != 'active':
-            raise ValueError('Calculation project is not active')
-        routing = await metadata.get_project_db_routing(UUID(str(project_id)), 'biz_data')
-        if not routing or routing.db_type != 'postgresql':
-            raise ValueError('Project business PostgreSQL routing is unavailable')
-        await session.commit()
-        async with await AsyncConnection.connect(routing.dsn, row_factory=dict_row) as conn:
-            yield CalculationJobRepository(conn, project_id)
+    from app.infra.db.calculation_worker import queue_repository as open_repository
+
+    async with open_repository(project_id) as repository:
+        yield repository
 
 
 @asynccontextmanager
@@ -246,15 +234,11 @@ class CalculationWorker:
                     await cleanup_job_resources(row, directory)
 
     async def run(self):
-        from sqlalchemy import text
-        from app.infra.db.metadb.database import engine
+        from app.infra.db.calculation_worker import calculation_supervisor_lease
+
         # Dedicated session-level metadata lock: independent API workers never
         # participate; duplicate supervisors exit before claiming any jobs.
-        async with engine.connect() as lock:
-            acquired = await lock.scalar(text("SELECT pg_try_advisory_lock(1748531781, 2)"))
-            await lock.commit()
-            if not acquired:
-                raise RuntimeError('Another calculation supervisor is already active')
+        async with calculation_supervisor_lease() as lease:
             tasks = set()
             project_cursor = 0
             try:
@@ -264,8 +248,7 @@ class CalculationWorker:
                 await recover_job_resources(self.settings.project_ids)
                 while True:
                     async with asyncio.timeout(self.settings.heartbeat_seconds):
-                        await lock.execute(text('SELECT 1'))
-                        await lock.commit()
+                        await lease.ping()
                     for task in tuple(tasks):
                         if task.done():
                             tasks.remove(task)
@@ -286,8 +269,6 @@ class CalculationWorker:
                 for task in tasks:
                     task.cancel()
                 await asyncio.gather(*tasks, return_exceptions=True)
-                await lock.execute(text('SELECT pg_advisory_unlock(1748531781, 2)'))
-                await lock.commit()
 
 
 async def guard_child(input_path, output_path):

app/services/operational_presentation.py

@@ -88,7 +88,7 @@ def resolve_overlay(snapshot: TaskDataSnapshot, mode: str) -> dict:
         properties = feature.get("properties") or {}
         require(isinstance(properties, dict), "OVERLAY_GEOMETRY_INVALID", "Feature properties are invalid", status=422)
         normalized = {"type": "Feature", "geometry": geometry, "properties": dict(properties)}
-        if isinstance(feature.get("id"), (str, int)) and not isinstance(feature.get("id"), bool):
+        if isinstance(feature.get("id"), str) or _number(feature.get("id")):
             normalized["id"] = feature["id"]
         if mode == "heatmap":
             weight = properties.get("weight", 1)

app/services/operational_tasks.py

@@ -19,6 +19,7 @@ from app.auth.permissions import (
     SIMULATION_VIEW,
 )
 from app.domain.domain_errors import DomainError, require
+from app.services import task_questions
 from app.domain.schemas.operational_task import (
     ActionDecisionCommand,
     ActionExecuteCommand,
@@ -65,6 +66,8 @@ from app.domain.schemas.operational_task import (
     TaskPresentationEventCommand,
     TaskPresentationEventResult,
     TaskPresentationCursor,
+    TaskPresentationHistoryItem,
+    TaskPresentationHistoryPage,
 )
 from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
 from app.services.content_digest import digest
@@ -97,6 +100,15 @@ class OperationalTaskService:
     def _actor(self) -> str:
         return f"principal:{self.context.user_id}"
 
+    @staticmethod
+    def _require_run_owner(run: dict, command: AgentRunCommand) -> None:
+        require(run.get("agent_session_id") == command.source_session_id,
+                "SESSION_MISMATCH", "Agent session does not own this run", status=409)
+        owner = run.get("agent_instance_id")
+        if owner is not None:
+            require(owner == command.agent_instance_id,
+                    "AGENT_INSTANCE_MISMATCH", "Agent instance does not own this run", status=409)
+
     @property
     def _supports_runs(self) -> bool:
         return hasattr(self.repo, "get_active_run") and hasattr(self.repo, "update_run")
@@ -138,6 +150,7 @@ class OperationalTaskService:
         permissions = self.permissions
         return OperationalTaskPermissions(
             can_publish_artifacts=OPERATIONAL_TASK_EXECUTE in permissions,
+            can_answer_questions=OPERATIONAL_TASK_WRITE in permissions,
             can_propose_actions=OPERATIONAL_TASK_WRITE in permissions,
             can_sync_external_feedback=OPERATIONAL_TASK_EXTERNAL_SYNC in permissions,
             can_decide_work_orders=OPERATIONAL_ACTION_WORK_ORDER_APPROVE in permissions,
@@ -258,6 +271,7 @@ class OperationalTaskService:
             evidence=evidence,
             actions=actions,
             access_requests=access_requests,
+            questions=await task_questions.list_questions(self, task_id),
             permissions=self._permission_view(),
             work_order_snapshot=work_order_snapshot,
         )
@@ -483,6 +497,7 @@ class OperationalTaskService:
                 status="running",
                 revision=expected + 1,
                 agent_session_id=command.source_session_id,
+                agent_instance_id=command.agent_instance_id,
                 lease_expires_at=now + LEASE_DURATION,
                 heartbeat_at=now,
                 started_at=run.get("started_at") or now,
@@ -527,8 +542,7 @@ class OperationalTaskService:
                     "Run revision changed", status=409)
             require(run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
                     "Only an active run has a lease", status=409)
-            require(run.get("agent_session_id") == command.source_session_id,
-                    "SESSION_MISMATCH", "Agent session does not own this run", status=409)
+            self._require_run_owner(run, command)
             now = self._now()
             self._require_active_run_lease(run, now)
             expected = run["revision"]
@@ -580,6 +594,7 @@ class OperationalTaskService:
             replay = await self._request_replay(scope, command.request_id, body)
             if replay:
                 return await self.detail(task_id)
+            await self.repo.lock(f"task:{task_id}")
             await self.repo.lock(f"run:{run_id}")
             run = await self._run(task_id, run_id)
             require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
@@ -587,6 +602,7 @@ class OperationalTaskService:
             require(run["status"] in {"queued", "claimed", "running"},
                     "INVALID_TRANSITION", "Run is already terminal", status=409)
             now = self._now()
+            await task_questions.cancel_pending(self, task_id, now)
             expected = run["revision"]
             run.update(
                 status="cancelled", revision=expected + 1, finished_at=now,
@@ -624,6 +640,7 @@ class OperationalTaskService:
             require(await self.repo.get_active_run(str(task_id)) is None,
                     "ACTIVE_RUN_EXISTS", "Task already has an active run", status=409)
             now = self._now()
+            await task_questions.cancel_pending(self, task_id, now)
             generation = task["execution_generation"] + 1
             run = self._new_run(task_id, generation, now)
             await self.repo.insert_run(_json(run))
@@ -986,6 +1003,27 @@ class OperationalTaskService:
                 event_cursor=cursor,
             )
 
+    async def presentation_history(
+        self, task_id: UUID, *, after: int = 0, through: int | None = None, limit: int = 100,
+    ) -> TaskPresentationHistoryPage:
+        require(0 <= after and (through is None or through >= 0) and 1 <= limit <= 200,
+                "INVALID_ARGUMENT", "Invalid presentation history page", status=422)
+        async with self.repo.transaction():
+            await self._task(task_id)
+            cursor = await self.repo.current_event_cursor()
+            boundary = min(through, cursor) if through is not None else cursor
+            rows = await self.repo.list_presentation_events(str(task_id), after, boundary, limit + 1)
+            items = [TaskPresentationHistoryItem(
+                event_seq=row["event_seq"], occurred_at=row["occurred_at"],
+                run_id=row["metadata"]["run_id"],
+                execution_generation=row["metadata"]["execution_generation"],
+                command=row["metadata"]["command"],
+            ) for row in rows[:limit]]
+            return TaskPresentationHistoryPage(
+                task_id=task_id, event_cursor=boundary, items=items,
+                next_after=items[-1].event_seq if len(rows) > limit else None,
+            )
+
     async def publish_presentation_event(
         self, task_id: UUID, command: TaskPresentationEventCommand
     ) -> TaskPresentationEventResult:
@@ -1019,7 +1057,16 @@ class OperationalTaskService:
                 evidence = StoredEvidenceRecord.model_validate(stored)
                 require(evidence.data_snapshot is not None, "SNAPSHOT_NOT_FOUND",
                         "Overlay requires a Server-ingested immutable data snapshot", status=422)
-                payload["data"] = resolve_overlay(evidence.data_snapshot, command.command.mode)
+                resolved = resolve_overlay(evidence.data_snapshot, command.command.mode)
+                if command.command.data is not None:
+                    supplied = resolve_overlay(
+                        evidence.data_snapshot.model_copy(update={
+                            "data": command.command.data.model_dump(mode="json", exclude_unset=True),
+                        }), command.command.mode,
+                    )
+                    require(supplied == resolved, "OVERLAY_DATA_MISMATCH",
+                            "Overlay data must match the immutable evidence snapshot", status=422)
+                payload["data"] = resolved
             metadata = {"task_id": str(task_id), "run_id": str(run["run_id"]),
                         "execution_generation": run["generation"], "command": payload}
             try:
@@ -1890,18 +1937,21 @@ class OperationalTaskService:
             replay = await self._request_replay(scope, command.request_id, body)
             if replay:
                 return await self.detail(task_id)
+            await self.repo.lock(f"task:{task_id}")
             await self.repo.lock(f"run:{run_id}")
             run = await self._run(task_id, run_id)
             require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
                     "Run revision changed", status=409)
             require(run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
                     "Only an active run can finish", status=409)
-            require(run.get("agent_session_id") == command.source_session_id,
-                    "SESSION_MISMATCH", "Agent session does not own this run", status=409)
+            self._require_run_owner(run, command)
             now = self._now()
             self._require_active_run_lease(run, now)
             task = await self._task(task_id)
             final_outcome = outcome
+            pending_question = any(q["status"] == "pending" and str(q["run_id"]) == str(run_id)
+                for q in await task_questions.list_questions(self, task_id))
+            actions = []
             if outcome == "succeeded":
                 await self._require_no_pending_calculations(task_id, run)
                 if hasattr(self.repo, "pending_run_messages"):
@@ -1929,7 +1979,9 @@ class OperationalTaskService:
                         and plan.source_session_id == command.source_session_id
                     ))
                 )
-                if not same_generation or report.report.stage != "final":
+                if pending_question:
+                    lifecycle = "waiting_human"
+                elif not same_generation or report.report.stage != "final":
                     final_outcome = "failed"
                     lifecycle = "failed"
                     failure_code = "ARTIFACT_INCOMPLETE"
@@ -1974,6 +2026,7 @@ class OperationalTaskService:
             run_model = OperationalTaskRun.model_validate(run)
             await self.repo.update_run(_json(run_model), expected)
             if final_outcome != "succeeded":
+                await task_questions.cancel_pending(self, task_id, now)
                 await self._fail_run_messages(run_id, failure_code or "Run failed", now)
             task.update(
                 lifecycle_status=lifecycle,
@@ -1994,7 +2047,7 @@ class OperationalTaskService:
                     failure_detail or "工况任务执行失败",
                     now,
                 )
-            elif lifecycle == "waiting_human":
+            elif lifecycle == "waiting_human" and not pending_question:
                 pending_actions = any(
                     action.approval_status == "pending" for action in actions
                 )
@@ -2207,6 +2260,7 @@ class OperationalTaskService:
         task["agent_lease_expires_at"] = None
 
     async def _requeue_task(self, task: dict, now: datetime) -> None:
+        await task_questions.cancel_pending(self, UUID(str(task["task_id"])), now)
         if self._supports_runs:
             active_run = await self.repo.get_active_run(str(task["task_id"]))
             while active_run is not None:

app/services/task_questions.py

@@ -0,0 +1,120 @@
+"""Question lifecycle sharing the operational task transaction and notification bus."""
+from datetime import datetime
+from typing import TYPE_CHECKING
+from uuid import UUID, uuid4
+
+from app.auth.permissions import OPERATIONAL_TASK_EXECUTE, OPERATIONAL_TASK_WRITE
+from app.domain.domain_errors import require
+from app.domain.schemas.operational_task import OperationalTaskDetail, OperationalTaskSummary
+from app.domain.schemas.task_question import TaskQuestionCreate, TaskQuestionReply, TaskQuestionRequest
+
+if TYPE_CHECKING:
+    from app.services.operational_tasks import OperationalTaskService
+
+
+async def list_questions(service: "OperationalTaskService", task_id: UUID) -> list[dict]:
+    # Legacy test repositories do not implement question persistence.
+    reader = getattr(service.repo, "list_questions", None)
+    return await reader(str(task_id)) if reader else []
+
+
+async def cancel_pending(service: "OperationalTaskService", task_id: UUID, now: datetime) -> None:
+    for raw in await list_questions(service, task_id):
+        question = TaskQuestionRequest.model_validate(raw)
+        if question.status != "pending":
+            continue
+        question = question.model_copy(update={"status": "cancelled", "version": question.version + 1})
+        await service.repo.save_question(str(task_id), question.model_dump(mode="json"))
+        await service._resolve_notifications(task_id, kinds=["human_input_required"],
+            subject_type="question", subject_id=question.question_request_id,
+            now=now, reason="question_superseded")
+
+
+async def create_question(service: "OperationalTaskService", task_id: UUID,
+                          command: TaskQuestionCreate) -> TaskQuestionRequest:
+    require(OPERATIONAL_TASK_EXECUTE in service.permissions, "PERMISSION_DENIED",
+            "Current user cannot publish task questions", status=403)
+    body = command.model_dump(mode="json")
+    scope = f"task:{task_id}:question:create"
+    async with service.repo.transaction():
+        await service.repo.lock(f"task:{task_id}")
+        replay = await service._request_replay(scope, command.request_id, body)
+        if replay:
+            return TaskQuestionRequest.model_validate(await service.repo.get_question(
+                str(task_id), replay["question_request_id"]))
+        task = await service._task(task_id)
+        service._require_actionable_task(task)
+        require(task["task_type"] == "event_investigation", "INVALID_TASK_TYPE",
+                "Questions require an investigation task", status=409)
+        await service.repo.lock(f"run:{command.run_id}")
+        run = await service._run(task_id, command.run_id)
+        require(run["status"] in {"claimed", "running"}
+                and run["generation"] == task["execution_generation"]
+                and run.get("agent_session_id") == command.source_session_id,
+                "SESSION_MISMATCH", "Question must belong to the active investigation run", status=409)
+        now = service._now()
+        service._require_active_run_lease(run, now)
+        await service._require_no_pending_calculations(task_id, run)
+        require(not any(q["status"] == "pending" for q in await list_questions(service, task_id)),
+                "QUESTION_PENDING", "Answer the existing questions first", status=409)
+        question = TaskQuestionRequest(question_request_id=uuid4(), task_id=task_id,
+            run_id=command.run_id, execution_generation=run["generation"],
+            source_session_id=command.source_session_id, questions=command.questions,
+            status="pending", version=1, created_at=now)
+        await service.repo.save_question(str(task_id), question.model_dump(mode="json"))
+        task.update(revision=task["revision"] + 1, updated_at=now)
+        await service.repo.update_task(OperationalTaskSummary.model_validate(task).model_dump(mode="json", exclude={"active_run", "latest_run"}))
+        await service._task_event(task, now)
+        await service._notify(task_id, "human_input_required", task["title"],
+            "；".join(q.title for q in question.questions), now,
+            required_permission=OPERATIONAL_TASK_WRITE, subject_type="question",
+            subject_id=question.question_request_id)
+        await service._save_request(scope, command.request_id, body,
+            {"question_request_id": str(question.question_request_id)})
+        return question
+
+
+async def answer_question(service: "OperationalTaskService", task_id: UUID, question_id: UUID,
+                          command: TaskQuestionReply) -> OperationalTaskDetail:
+    require(OPERATIONAL_TASK_WRITE in service.permissions, "PERMISSION_DENIED",
+            "Current user cannot answer task questions", status=403)
+    body = command.model_dump(mode="json")
+    scope = f"task:{task_id}:question:{question_id}:answer"
+    async with service.repo.transaction():
+        await service.repo.lock(f"task:{task_id}")
+        if await service._request_replay(scope, command.request_id, body):
+            return await service.detail(task_id)
+        task = await service._task(task_id)
+        service._require_actionable_task(task)
+        raw = await service.repo.get_question(str(task_id), str(question_id))
+        require(raw is not None, "QUESTION_NOT_FOUND", "Question not found", status=404)
+        question = TaskQuestionRequest.model_validate(raw)
+        require(question.status == "pending" and question.version == command.expected_version
+                and question.execution_generation == task["execution_generation"],
+                "VERSION_CONFLICT", "Question is no longer pending in the current generation", status=409)
+        require(task["lifecycle_status"] == "waiting_human"
+                and await service.repo.get_active_run(str(task_id)) is None,
+                "RUN_NOT_SETTLED", "Wait until the investigation has entered waiting_human", status=409)
+        answers = {a.question_id: a for a in command.answers}
+        known = {q.question_id for q in question.questions}
+        require(len(answers) == len(command.answers) and set(answers) <= known,
+                "ANSWER_INVALID", "Unknown or duplicate questions", status=422)
+        for q in question.questions:
+            answer = answers.get(q.question_id)
+            selected = answer.selected_option_ids if answer else []
+            text = answer.custom_text.strip() if answer else ""
+            require(len(set(selected)) == len(selected) and set(selected) <= {o.id for o in q.options}
+                    and (q.mode == "multiple" or len(selected) <= 1)
+                    and (q.allow_custom or not text) and (not q.required or bool(selected or text)),
+                    "ANSWER_INVALID", "Answer does not satisfy the question", status=422)
+        now = service._now()
+        question = question.model_copy(update={"status": "answered", "version": question.version + 1,
+            "answers": command.answers, "answered_at": now, "answered_by": service._actor()})
+        await service.repo.save_question(str(task_id), question.model_dump(mode="json"))
+        await service._resolve_notifications(task_id, kinds=["human_input_required"],
+            subject_type="question", subject_id=question_id, now=now, reason="question_answered")
+        await service._requeue_task(task, now)
+        await service.repo.update_task(OperationalTaskSummary.model_validate(task).model_dump(mode="json", exclude={"active_run", "latest_run"}))
+        await service._task_event(task, now)
+        await service._save_request(scope, command.request_id, body, {"task_id": str(task_id)})
+        return await service.detail(task_id)

app/services/valve_isolation.py

@@ -1,13 +1,22 @@
 from typing import Any
 
-from app.algorithms.valve_isolation.topology_search import valve_isolation_analysis
-from app.native.wndb.gis.network_views import get_network_link_nodes
+from app.algorithms.valve_isolation.topology_search import (
+    AccidentElementType,
+    valve_isolation_analysis,
+)
+from app.native.wndb.gis.network_views import get_valve_isolation_topology
 
 
 def analyze_valve_isolation(
     network: str,
     accident_element: str | list[str],
-    disabled_valves: list[str] = None,
+    disabled_valves: list[str] | None = None,
+    accident_element_type: AccidentElementType = "auto",
 ) -> dict[str, Any]:
-    link_entries = get_network_link_nodes(network)
-    return valve_isolation_analysis(link_entries, accident_element, disabled_valves)
+    link_entries, source_nodes, closed_links = get_valve_isolation_topology(network)
+    return valve_isolation_analysis(
+        link_entries, accident_element, disabled_valves,
+        accident_element_type=accident_element_type,
+        water_source_nodes=source_nodes,
+        closed_links=closed_links,
+    )