320 lines
13 KiB
Python
320 lines
13 KiB
Python
"""Real spawned workers share a listener while established streams drain."""
|
|
from contextlib import contextmanager
|
|
import json
|
|
import multiprocessing
|
|
from pathlib import Path
|
|
from threading import Event, Thread
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
import httpx
|
|
import pytest
|
|
import uvicorn
|
|
|
|
from scripts.server_handoff import HandoffSupervisor
|
|
|
|
|
|
def wait_for(predicate, *, timeout=5.):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.02)
|
|
raise AssertionError("test-owned worker did not reach the expected state")
|
|
|
|
|
|
def next_event(lines, *, after_ns=0):
|
|
for line in lines:
|
|
if line.startswith("data: "):
|
|
event = json.loads(line[len("data: "):])
|
|
if event.get("time_ns", 0) >= after_ns:
|
|
return event
|
|
raise AssertionError("SSE ended before its fixture release marker")
|
|
|
|
|
|
def owned_workers(supervisor):
|
|
return [worker for worker in [supervisor.current, *supervisor.retiring] if worker is not None]
|
|
|
|
|
|
def close_owned_supervisor(supervisor, directory, original_children):
|
|
"""Release every fixture stream before waiting; bound cleanup even on failure."""
|
|
(directory / "release-all").touch()
|
|
(directory / "allow-startup").touch()
|
|
workers = owned_workers(supervisor)
|
|
errors = []
|
|
|
|
def close():
|
|
try:
|
|
supervisor.close()
|
|
except BaseException as error:
|
|
errors.append(error)
|
|
|
|
closing = Thread(target=close, daemon=True)
|
|
closing.start()
|
|
closing.join(timeout=8)
|
|
leaked = [process for process in multiprocessing.active_children()
|
|
if process.pid not in original_children and process.is_alive()]
|
|
if closing.is_alive() or leaked:
|
|
# This is emergency teardown of processes created by this test, never
|
|
# production reload policy or an operation on the developer's service.
|
|
for process in leaked:
|
|
process.terminate()
|
|
for process in leaked:
|
|
process.join(timeout=3)
|
|
closing.join(timeout=3)
|
|
pytest.fail("handoff cleanup left a test-owned worker running")
|
|
for worker in workers:
|
|
worker.process.join(timeout=1)
|
|
assert not worker.process.is_alive()
|
|
if errors:
|
|
raise errors[0]
|
|
|
|
|
|
@pytest.fixture
|
|
def handoff(monkeypatch, tmp_path):
|
|
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "fixtures"))
|
|
monkeypatch.setenv("SCADA_HANDOFF_FIXTURE_DIR", str(tmp_path))
|
|
(tmp_path / "version").write_text("v1")
|
|
(tmp_path / "startup-mode").write_text("ready")
|
|
(tmp_path / "readiness").write_text("true")
|
|
original_children = {process.pid for process in multiprocessing.active_children()}
|
|
|
|
@contextmanager
|
|
def running(*, startup_timeout=5., readiness_check=None):
|
|
config = uvicorn.Config(
|
|
"server_handoff_app:app", host="127.0.0.1", port=0,
|
|
lifespan="on", log_level="error", access_log=False,
|
|
timeout_graceful_shutdown=None,
|
|
)
|
|
supervisor = HandoffSupervisor(config, startup_timeout=startup_timeout, poll_interval=.02,
|
|
readiness_check=readiness_check)
|
|
try:
|
|
yield supervisor, tmp_path
|
|
finally:
|
|
close_owned_supervisor(supervisor, tmp_path, original_children)
|
|
|
|
return running
|
|
|
|
|
|
def base_url(supervisor):
|
|
return f"http://127.0.0.1:{supervisor.socket.getsockname()[1]}"
|
|
|
|
|
|
def health(supervisor, *, timeout=3.):
|
|
# A fresh TCP connection makes each assertion exercise current acceptance.
|
|
with httpx.Client(timeout=timeout, trust_env=False) as client:
|
|
response = client.get(base_url(supervisor) + "/health")
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
|
|
def test_first_worker_is_spawned_and_ready_before_start_returns(handoff):
|
|
with handoff() as (supervisor, _):
|
|
assert supervisor.start() is True
|
|
worker = supervisor.current
|
|
assert worker.process._start_method == "spawn"
|
|
assert worker.process.is_alive()
|
|
assert health(supervisor) == {"pid": worker.process.pid, "version": "v1"}
|
|
assert supervisor.retiring == []
|
|
|
|
|
|
def test_reload_keeps_old_sse_until_release_and_new_requests_use_new_worker(handoff):
|
|
with handoff() as (supervisor, directory):
|
|
assert supervisor.start() is True
|
|
old = supervisor.current
|
|
with httpx.Client(timeout=3., trust_env=False) as client:
|
|
with client.stream("GET", base_url(supervisor) + "/events") as stream:
|
|
stream.raise_for_status()
|
|
lines = stream.iter_lines()
|
|
assert next_event(lines)["pid"] == old.process.pid
|
|
(directory / "version").write_text("v2")
|
|
(directory / "startup-mode").write_text("delay")
|
|
stop_probe = Event()
|
|
samples, errors = [], []
|
|
|
|
def probe_health():
|
|
while not stop_probe.is_set():
|
|
try:
|
|
samples.append(health(supervisor, timeout=.3))
|
|
except Exception as error:
|
|
errors.append(repr(error))
|
|
stop_probe.wait(.01)
|
|
|
|
probing = Thread(target=probe_health, daemon=True)
|
|
probing.start()
|
|
try:
|
|
wait_for(lambda: samples)
|
|
assert supervisor.reload() is True
|
|
wait_for(lambda: any(sample["version"] == "v2" for sample in samples))
|
|
finally:
|
|
stop_probe.set()
|
|
probing.join(timeout=2)
|
|
assert not probing.is_alive()
|
|
assert errors == [], f"HTTP availability gap during reload: {errors}"
|
|
assert {sample["version"] for sample in samples} == {"v1", "v2"}
|
|
|
|
new = supervisor.current
|
|
assert new.process.pid != old.process.pid
|
|
assert old in supervisor.retiring
|
|
assert old.process.is_alive()
|
|
for _ in range(3):
|
|
assert health(supervisor) == {"pid": new.process.pid, "version": "v2"}
|
|
after_handoff = time.time_ns()
|
|
event = next_event(lines, after_ns=after_handoff)
|
|
assert (event["pid"], event["version"]) == (old.process.pid, "v1")
|
|
assert not (directory / f"stopped-{old.process.pid}").exists()
|
|
|
|
(directory / f"release-{old.process.pid}").touch()
|
|
assert any(line == "event: complete" for line in lines)
|
|
old.process.join(timeout=5)
|
|
assert old.process.exitcode == 0
|
|
assert (directory / f"stopped-{old.process.pid}").read_text() == "v1"
|
|
supervisor.reap()
|
|
assert old not in supervisor.retiring
|
|
assert health(supervisor) == {"pid": new.process.pid, "version": "v2"}
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["fail", "wait"])
|
|
def test_candidate_startup_failure_keeps_previous_worker_serving(handoff, mode):
|
|
with handoff() as (supervisor, directory):
|
|
assert supervisor.start() is True
|
|
old = supervisor.current
|
|
(directory / "version").write_text("broken-v2")
|
|
(directory / "startup-mode").write_text(mode)
|
|
if mode == "wait":
|
|
supervisor.startup_timeout = .75
|
|
assert supervisor.reload() is False
|
|
assert supervisor.current is old
|
|
assert old.process.is_alive()
|
|
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
|
assert supervisor.retiring == []
|
|
assert len(list(directory.glob("started-*"))) == 2
|
|
|
|
|
|
def test_candidate_process_start_error_keeps_previous_worker_healthy(handoff, monkeypatch):
|
|
def fail_start():
|
|
raise OSError("test process spawn rejected")
|
|
|
|
with handoff() as (supervisor, _):
|
|
assert supervisor.start() is True
|
|
old = supervisor.current
|
|
candidate = SimpleNamespace(pid=None, start=fail_start)
|
|
monkeypatch.setattr("scripts.server_handoff.get_subprocess", lambda **_: candidate)
|
|
assert supervisor.reload() is False
|
|
assert supervisor.current is old
|
|
assert supervisor.retiring == []
|
|
assert old.process.is_alive()
|
|
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
|
|
|
|
|
def test_readiness_check_rejects_unusable_candidate_before_handoff(handoff):
|
|
with handoff(readiness_check="server_handoff_app:readiness_check") as (supervisor, directory):
|
|
assert supervisor.start() is True
|
|
old = supervisor.current
|
|
assert (directory / f"checked-{old.process.pid}").read_text() == "true"
|
|
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
|
(directory / "version").write_text("not-ready-v2")
|
|
(directory / "readiness").write_text("false")
|
|
stop_probe = Event()
|
|
samples, errors = [], []
|
|
|
|
def probe_health():
|
|
while not stop_probe.is_set():
|
|
try:
|
|
samples.append(health(supervisor, timeout=.3))
|
|
except Exception as error:
|
|
errors.append(repr(error))
|
|
stop_probe.wait(.01)
|
|
|
|
probing = Thread(target=probe_health, daemon=True)
|
|
probing.start()
|
|
try:
|
|
wait_for(lambda: samples)
|
|
assert supervisor.reload() is False
|
|
finally:
|
|
stop_probe.set()
|
|
probing.join(timeout=2)
|
|
assert not probing.is_alive()
|
|
assert errors == []
|
|
assert all(sample == {"pid": old.process.pid, "version": "v1"} for sample in samples)
|
|
rejected = [path for path in directory.glob("checked-*") if path.name != f"checked-{old.process.pid}"]
|
|
assert len(rejected) == 1
|
|
assert rejected[0].read_text() == "false"
|
|
assert supervisor.current is old
|
|
assert old.process.is_alive()
|
|
assert supervisor.retiring == []
|
|
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
|
|
|
(directory / "version").write_text("ready-v3")
|
|
(directory / "readiness").write_text("true")
|
|
assert supervisor.reload() is True
|
|
new = supervisor.current
|
|
assert new.process.pid != old.process.pid
|
|
assert (directory / f"checked-{new.process.pid}").read_text() == "true"
|
|
assert health(supervisor) == {"pid": new.process.pid, "version": "ready-v3"}
|
|
|
|
|
|
def test_close_waits_for_existing_stream_to_finish_without_forcing_worker(handoff):
|
|
with handoff() as (supervisor, directory):
|
|
assert supervisor.start() is True
|
|
worker = supervisor.current
|
|
with httpx.Client(timeout=3., trust_env=False) as client:
|
|
with client.stream("GET", base_url(supervisor) + "/events") as stream:
|
|
lines = stream.iter_lines()
|
|
assert next_event(lines)["pid"] == worker.process.pid
|
|
errors = []
|
|
|
|
def close():
|
|
try:
|
|
supervisor.close()
|
|
except BaseException as error:
|
|
errors.append(error)
|
|
|
|
closing = Thread(target=close, daemon=True)
|
|
closing.start()
|
|
try:
|
|
after_close = time.time_ns()
|
|
assert next_event(lines, after_ns=after_close)["pid"] == worker.process.pid
|
|
assert closing.is_alive()
|
|
assert worker.process.is_alive()
|
|
(directory / f"release-{worker.process.pid}").touch()
|
|
assert any(line == "event: complete" for line in lines)
|
|
finally:
|
|
(directory / "release-all").touch()
|
|
closing.join(timeout=5)
|
|
assert not closing.is_alive()
|
|
assert errors == []
|
|
assert worker.process.exitcode == 0
|
|
assert (directory / f"stopped-{worker.process.pid}").exists()
|
|
|
|
|
|
@pytest.mark.parametrize("active_alive,edited,expected_reloads", [(False, False, 1), (True, True, 0)])
|
|
def test_retiring_limit_coalesces_edits_but_never_blocks_crash_recovery(
|
|
monkeypatch, active_alive, edited, expected_reloads,
|
|
):
|
|
class OneWatchIteration:
|
|
calls = 0
|
|
|
|
def wait(self, _timeout):
|
|
self.calls += 1
|
|
return self.calls > 1
|
|
|
|
def set(self):
|
|
self.calls = 2
|
|
|
|
supervisor = HandoffSupervisor.__new__(HandoffSupervisor)
|
|
supervisor.current = SimpleNamespace(process=SimpleNamespace(is_alive=lambda: active_alive))
|
|
supervisor.retiring = [object(), object()]
|
|
supervisor.should_exit = OneWatchIteration()
|
|
supervisor.poll_interval = .001
|
|
supervisor._files = {}
|
|
calls = []
|
|
monkeypatch.setattr(supervisor, "start", lambda: True)
|
|
monkeypatch.setattr(supervisor, "reap", lambda: None)
|
|
monkeypatch.setattr(supervisor, "_snapshot", lambda: {"edited.py": (1, 1)} if edited else {})
|
|
monkeypatch.setattr(supervisor, "reload", lambda: calls.append("reload"))
|
|
monkeypatch.setattr(supervisor, "close", lambda: calls.append("close"))
|
|
supervisor.run()
|
|
assert calls.count("reload") == expected_reloads
|
|
assert calls[-1] == "close"
|