"""Real spawned workers share a listener while established streams drain.""" from contextlib import contextmanager import json import multiprocessing from pathlib import Path from threading import Event, Thread import time from types import SimpleNamespace import httpx import pytest import uvicorn from scripts.server_handoff import HandoffSupervisor def wait_for(predicate, *, timeout=5.): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if predicate(): return time.sleep(0.02) raise AssertionError("test-owned worker did not reach the expected state") def next_event(lines, *, after_ns=0): for line in lines: if line.startswith("data: "): event = json.loads(line[len("data: "):]) if event.get("time_ns", 0) >= after_ns: return event raise AssertionError("SSE ended before its fixture release marker") def owned_workers(supervisor): return [worker for worker in [supervisor.current, *supervisor.retiring] if worker is not None] def close_owned_supervisor(supervisor, directory, original_children): """Release every fixture stream before waiting; bound cleanup even on failure.""" (directory / "release-all").touch() (directory / "allow-startup").touch() workers = owned_workers(supervisor) errors = [] def close(): try: supervisor.close() except BaseException as error: errors.append(error) closing = Thread(target=close, daemon=True) closing.start() closing.join(timeout=8) leaked = [process for process in multiprocessing.active_children() if process.pid not in original_children and process.is_alive()] if closing.is_alive() or leaked: # This is emergency teardown of processes created by this test, never # production reload policy or an operation on the developer's service. for process in leaked: process.terminate() for process in leaked: process.join(timeout=3) closing.join(timeout=3) pytest.fail("handoff cleanup left a test-owned worker running") for worker in workers: worker.process.join(timeout=1) assert not worker.process.is_alive() if errors: raise errors[0] @pytest.fixture def handoff(monkeypatch, tmp_path): monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "fixtures")) monkeypatch.setenv("SCADA_HANDOFF_FIXTURE_DIR", str(tmp_path)) (tmp_path / "version").write_text("v1") (tmp_path / "startup-mode").write_text("ready") (tmp_path / "readiness").write_text("true") original_children = {process.pid for process in multiprocessing.active_children()} @contextmanager def running(*, startup_timeout=5., readiness_check=None): config = uvicorn.Config( "server_handoff_app:app", host="127.0.0.1", port=0, lifespan="on", log_level="error", access_log=False, timeout_graceful_shutdown=None, ) supervisor = HandoffSupervisor(config, startup_timeout=startup_timeout, poll_interval=.02, readiness_check=readiness_check) try: yield supervisor, tmp_path finally: close_owned_supervisor(supervisor, tmp_path, original_children) return running def base_url(supervisor): return f"http://127.0.0.1:{supervisor.socket.getsockname()[1]}" def health(supervisor, *, timeout=3.): # A fresh TCP connection makes each assertion exercise current acceptance. with httpx.Client(timeout=timeout, trust_env=False) as client: response = client.get(base_url(supervisor) + "/health") response.raise_for_status() return response.json() def test_first_worker_is_spawned_and_ready_before_start_returns(handoff): with handoff() as (supervisor, _): assert supervisor.start() is True worker = supervisor.current assert worker.process._start_method == "spawn" assert worker.process.is_alive() assert health(supervisor) == {"pid": worker.process.pid, "version": "v1"} assert supervisor.retiring == [] def test_reload_keeps_old_sse_until_release_and_new_requests_use_new_worker(handoff): with handoff() as (supervisor, directory): assert supervisor.start() is True old = supervisor.current with httpx.Client(timeout=3., trust_env=False) as client: with client.stream("GET", base_url(supervisor) + "/events") as stream: stream.raise_for_status() lines = stream.iter_lines() assert next_event(lines)["pid"] == old.process.pid (directory / "version").write_text("v2") (directory / "startup-mode").write_text("delay") stop_probe = Event() samples, errors = [], [] def probe_health(): while not stop_probe.is_set(): try: samples.append(health(supervisor, timeout=.3)) except Exception as error: errors.append(repr(error)) stop_probe.wait(.01) probing = Thread(target=probe_health, daemon=True) probing.start() try: wait_for(lambda: samples) assert supervisor.reload() is True wait_for(lambda: any(sample["version"] == "v2" for sample in samples)) finally: stop_probe.set() probing.join(timeout=2) assert not probing.is_alive() assert errors == [], f"HTTP availability gap during reload: {errors}" assert {sample["version"] for sample in samples} == {"v1", "v2"} new = supervisor.current assert new.process.pid != old.process.pid assert old in supervisor.retiring assert old.process.is_alive() for _ in range(3): assert health(supervisor) == {"pid": new.process.pid, "version": "v2"} after_handoff = time.time_ns() event = next_event(lines, after_ns=after_handoff) assert (event["pid"], event["version"]) == (old.process.pid, "v1") assert not (directory / f"stopped-{old.process.pid}").exists() (directory / f"release-{old.process.pid}").touch() assert any(line == "event: complete" for line in lines) old.process.join(timeout=5) assert old.process.exitcode == 0 assert (directory / f"stopped-{old.process.pid}").read_text() == "v1" supervisor.reap() assert old not in supervisor.retiring assert health(supervisor) == {"pid": new.process.pid, "version": "v2"} @pytest.mark.parametrize("mode", ["fail", "wait"]) def test_candidate_startup_failure_keeps_previous_worker_serving(handoff, mode): with handoff() as (supervisor, directory): assert supervisor.start() is True old = supervisor.current (directory / "version").write_text("broken-v2") (directory / "startup-mode").write_text(mode) if mode == "wait": supervisor.startup_timeout = .75 assert supervisor.reload() is False assert supervisor.current is old assert old.process.is_alive() assert health(supervisor) == {"pid": old.process.pid, "version": "v1"} assert supervisor.retiring == [] assert len(list(directory.glob("started-*"))) == 2 def test_candidate_process_start_error_keeps_previous_worker_healthy(handoff, monkeypatch): def fail_start(): raise OSError("test process spawn rejected") with handoff() as (supervisor, _): assert supervisor.start() is True old = supervisor.current candidate = SimpleNamespace(pid=None, start=fail_start) monkeypatch.setattr("scripts.server_handoff.get_subprocess", lambda **_: candidate) assert supervisor.reload() is False assert supervisor.current is old assert supervisor.retiring == [] assert old.process.is_alive() assert health(supervisor) == {"pid": old.process.pid, "version": "v1"} def test_readiness_check_rejects_unusable_candidate_before_handoff(handoff): with handoff(readiness_check="server_handoff_app:readiness_check") as (supervisor, directory): assert supervisor.start() is True old = supervisor.current assert (directory / f"checked-{old.process.pid}").read_text() == "true" assert health(supervisor) == {"pid": old.process.pid, "version": "v1"} (directory / "version").write_text("not-ready-v2") (directory / "readiness").write_text("false") stop_probe = Event() samples, errors = [], [] def probe_health(): while not stop_probe.is_set(): try: samples.append(health(supervisor, timeout=.3)) except Exception as error: errors.append(repr(error)) stop_probe.wait(.01) probing = Thread(target=probe_health, daemon=True) probing.start() try: wait_for(lambda: samples) assert supervisor.reload() is False finally: stop_probe.set() probing.join(timeout=2) assert not probing.is_alive() assert errors == [] assert all(sample == {"pid": old.process.pid, "version": "v1"} for sample in samples) rejected = [path for path in directory.glob("checked-*") if path.name != f"checked-{old.process.pid}"] assert len(rejected) == 1 assert rejected[0].read_text() == "false" assert supervisor.current is old assert old.process.is_alive() assert supervisor.retiring == [] assert health(supervisor) == {"pid": old.process.pid, "version": "v1"} (directory / "version").write_text("ready-v3") (directory / "readiness").write_text("true") assert supervisor.reload() is True new = supervisor.current assert new.process.pid != old.process.pid assert (directory / f"checked-{new.process.pid}").read_text() == "true" assert health(supervisor) == {"pid": new.process.pid, "version": "ready-v3"} def test_close_waits_for_existing_stream_to_finish_without_forcing_worker(handoff): with handoff() as (supervisor, directory): assert supervisor.start() is True worker = supervisor.current with httpx.Client(timeout=3., trust_env=False) as client: with client.stream("GET", base_url(supervisor) + "/events") as stream: lines = stream.iter_lines() assert next_event(lines)["pid"] == worker.process.pid errors = [] def close(): try: supervisor.close() except BaseException as error: errors.append(error) closing = Thread(target=close, daemon=True) closing.start() try: after_close = time.time_ns() assert next_event(lines, after_ns=after_close)["pid"] == worker.process.pid assert closing.is_alive() assert worker.process.is_alive() (directory / f"release-{worker.process.pid}").touch() assert any(line == "event: complete" for line in lines) finally: (directory / "release-all").touch() closing.join(timeout=5) assert not closing.is_alive() assert errors == [] assert worker.process.exitcode == 0 assert (directory / f"stopped-{worker.process.pid}").exists() @pytest.mark.parametrize("active_alive,edited,expected_reloads", [(False, False, 1), (True, True, 0)]) def test_retiring_limit_coalesces_edits_but_never_blocks_crash_recovery( monkeypatch, active_alive, edited, expected_reloads, ): class OneWatchIteration: calls = 0 def wait(self, _timeout): self.calls += 1 return self.calls > 1 def set(self): self.calls = 2 supervisor = HandoffSupervisor.__new__(HandoffSupervisor) supervisor.current = SimpleNamespace(process=SimpleNamespace(is_alive=lambda: active_alive)) supervisor.retiring = [object(), object()] supervisor.should_exit = OneWatchIteration() supervisor.poll_interval = .001 supervisor._files = {} calls = [] monkeypatch.setattr(supervisor, "start", lambda: True) monkeypatch.setattr(supervisor, "reap", lambda: None) monkeypatch.setattr(supervisor, "_snapshot", lambda: {"edited.py": (1, 1)} if edited else {}) monkeypatch.setattr(supervisor, "reload", lambda: calls.append("reload")) monkeypatch.setattr(supervisor, "close", lambda: calls.append("close")) supervisor.run() assert calls.count("reload") == expected_reloads assert calls[-1] == "close"