From 7f89638b9b05eb307f7376bdd87758713c74fdd2 Mon Sep 17 00:00:00 2001 From: Hoipang_ Date: Wed, 7 Oct 2026 01:08:00 +0800 Subject: [PATCH 1/4] [Serve] Refresh active_versions when update_version reuses replicas --- sky/serve/replica_managers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sky/serve/replica_managers.py b/sky/serve/replica_managers.py index 7fd96069315..b283f53ad4a 100644 --- a/sky/serve/replica_managers.py +++ b/sky/serve/replica_managers.py @@ -1539,6 +1539,13 @@ def update_version(self, version: int, spec: 'service_spec.SkyServiceSpec', logger.info('Replica config changed (rest), skipping. ' f'old: {old_config}, ' f'new: {new_config}') + # A reused replica now carries the new version, but `active_versions` + # still lists only the old ones until the next prober cycle, so the + # load balancer would drop every reused replica in between. Refresh it + # now so the reuse never takes a ready replica out of rotation. + serve_utils.set_service_status_and_active_versions_from_replica( + self._service_name, + serve_state.get_replica_infos(self._service_name), update_mode) def _get_version_spec(self, version: int) -> 'service_spec.SkyServiceSpec': spec = serve_state.get_spec(self._service_name, version) From 6444f962a22462de7fa634f20f0c584ff21afdfc Mon Sep 17 00:00:00 2001 From: Hoipang_ Date: Wed, 7 Oct 2026 14:22:09 +0800 Subject: [PATCH 2/4] [Serve] Keep replica version relabel from being reverted by the prober The prober snapshots ReplicaInfo, probes for up to the readiness timeout, then writes the whole object back. update_version() relabels replicas without the lock, so a relabel landing mid-probe was reverted, and the autoscaler then replaced the healthy replica. Re-read the stored version before the write-back. Add unit tests for both this and the active_versions refresh. --- sky/serve/replica_managers.py | 7 ++ .../unit_tests/test_serve_replica_versions.py | 107 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 tests/unit_tests/test_serve_replica_versions.py diff --git a/sky/serve/replica_managers.py b/sky/serve/replica_managers.py index b283f53ad4a..8a9a9ac36a2 100644 --- a/sky/serve/replica_managers.py +++ b/sky/serve/replica_managers.py @@ -1422,6 +1422,13 @@ def _probe_all_replicas(self) -> None: f'seconds ({current_delay_seconds}s ' f'/ {initial_delay_seconds}s). ' 'Skipping.') + # `update_version` relabels replicas without holding the lock, + # possibly while this probe was in flight. Keep its version + # instead of writing back the stale one from our snapshot. + latest_info = serve_state.get_replica_info_from_id( + self._service_name, info.replica_id) + if latest_info is not None: + info.version = latest_info.version serve_state.add_or_update_replica(self._service_name, info.replica_id, info) if should_teardown: diff --git a/tests/unit_tests/test_serve_replica_versions.py b/tests/unit_tests/test_serve_replica_versions.py new file mode 100644 index 00000000000..cd7a6b07869 --- /dev/null +++ b/tests/unit_tests/test_serve_replica_versions.py @@ -0,0 +1,107 @@ +"""Count-only `sky serve update` must keep reused replicas routable.""" +import threading +import time +from unittest import mock + +import pytest +import sqlalchemy + +from sky.serve import replica_managers +from sky.serve import serve_state +from sky.serve import serve_utils +from sky.serve import service_spec +from sky.utils import common_utils +from sky.utils import yaml_utils + +_SERVICE = 'svc' + + +def _yaml(replicas: int, run: str = 'python -m http.server 8080') -> str: + return yaml_utils.dump_yaml_str({ + 'service': { + 'readiness_probe': '/', + 'replicas': replicas + }, + 'resources': { + 'ports': 8080 + }, + 'run': run, + 'file_mounts': {}, + }) + + +def _add_version(version: int, yaml_content: str) -> None: + serve_state.add_or_update_version( + _SERVICE, version, + service_spec.SkyServiceSpec.from_yaml_str(yaml_content), yaml_content) + + +@pytest.fixture +def manager(tmp_path, monkeypatch): + monkeypatch.setenv('HOME', str(tmp_path)) + engine = sqlalchemy.create_engine(f'sqlite:///{tmp_path}/services.db') + serve_state.create_table(engine) + monkeypatch.setattr(serve_state._db_manager, '_engine', engine) + # Pretend the replica is a launched, READY cluster. + monkeypatch.setattr(replica_managers.ReplicaInfo, 'url', + property(lambda self: f'http://r{self.replica_id}')) + monkeypatch.setattr(replica_managers.ReplicaInfo, 'status', + property(lambda self: serve_state.ReplicaStatus.READY)) + + def make(mode: serve_utils.UpdateMode): + serve_state.add_service(_SERVICE, 1, 'policy', 'resources', + 'round_robin', serve_state.ServiceStatus.READY, + False, False, 1, 'entrypoint') + _add_version(1, _yaml(1)) + info = replica_managers.ReplicaInfo(1, 'c1', '8080', False, None, 1, + None) + info.status_property.sky_launch_status = ( + common_utils.ProcessStatus.SUCCEEDED) + info.status_property.first_ready_time = 1.0 + serve_state.add_or_update_replica(_SERVICE, 1, info) + serve_utils.set_service_status_and_active_versions_from_replica( + _SERVICE, serve_state.get_replica_infos(_SERVICE), mode) + m = object.__new__(replica_managers.SkyPilotReplicaManager) + m._service_name, m.latest_version, m._update_mode = _SERVICE, 1, mode + m.lock, m._is_pool, m._uptime = threading.Lock(), False, 0 + return m + + return make + + +@pytest.mark.parametrize('mode', list(serve_utils.UpdateMode)) +@pytest.mark.parametrize('run,expected_version', [ + ('python -m http.server 8080', 2), + ('echo changed', 1), +]) +def test_update_keeps_ready_replica_routable(manager, mode, run, + expected_version): + m = manager(mode) + _add_version(2, _yaml(2, run)) + m.update_version(2, None, mode) + assert serve_state.get_replica_infos(_SERVICE)[0].version == ( + expected_version) + assert m.get_active_replica_urls() == ['http://r1'] + + +def test_prober_does_not_revert_concurrent_relabel(manager): + mode = serve_utils.UpdateMode.ROLLING + m = manager(mode) + _add_version(2, _yaml(2)) + probing = threading.Event() + + def slow_probe(self, *args): + probing.set() + time.sleep(0.5) + return self, True, time.time() + + with mock.patch.object(replica_managers.ReplicaInfo, 'probe', slow_probe): + prober = threading.Thread(target=m._probe_all_replicas) + prober.start() + assert probing.wait(5) + m.update_version(2, None, mode) # Lands while the probe is in flight. + prober.join() + + info = serve_state.get_replica_infos(_SERVICE)[0] + assert info.version == 2 + assert info.status_property.service_ready_now From 82fc2fb3e06154c7f8ba3fca65f0e4170c6706e0 Mon Sep 17 00:00:00 2001 From: Hoipang_ Date: Wed, 7 Oct 2026 14:31:24 +0800 Subject: [PATCH 3/4] [Test] Remove test_serve_replica_versions.py --- .../unit_tests/test_serve_replica_versions.py | 107 ------------------ 1 file changed, 107 deletions(-) delete mode 100644 tests/unit_tests/test_serve_replica_versions.py diff --git a/tests/unit_tests/test_serve_replica_versions.py b/tests/unit_tests/test_serve_replica_versions.py deleted file mode 100644 index cd7a6b07869..00000000000 --- a/tests/unit_tests/test_serve_replica_versions.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Count-only `sky serve update` must keep reused replicas routable.""" -import threading -import time -from unittest import mock - -import pytest -import sqlalchemy - -from sky.serve import replica_managers -from sky.serve import serve_state -from sky.serve import serve_utils -from sky.serve import service_spec -from sky.utils import common_utils -from sky.utils import yaml_utils - -_SERVICE = 'svc' - - -def _yaml(replicas: int, run: str = 'python -m http.server 8080') -> str: - return yaml_utils.dump_yaml_str({ - 'service': { - 'readiness_probe': '/', - 'replicas': replicas - }, - 'resources': { - 'ports': 8080 - }, - 'run': run, - 'file_mounts': {}, - }) - - -def _add_version(version: int, yaml_content: str) -> None: - serve_state.add_or_update_version( - _SERVICE, version, - service_spec.SkyServiceSpec.from_yaml_str(yaml_content), yaml_content) - - -@pytest.fixture -def manager(tmp_path, monkeypatch): - monkeypatch.setenv('HOME', str(tmp_path)) - engine = sqlalchemy.create_engine(f'sqlite:///{tmp_path}/services.db') - serve_state.create_table(engine) - monkeypatch.setattr(serve_state._db_manager, '_engine', engine) - # Pretend the replica is a launched, READY cluster. - monkeypatch.setattr(replica_managers.ReplicaInfo, 'url', - property(lambda self: f'http://r{self.replica_id}')) - monkeypatch.setattr(replica_managers.ReplicaInfo, 'status', - property(lambda self: serve_state.ReplicaStatus.READY)) - - def make(mode: serve_utils.UpdateMode): - serve_state.add_service(_SERVICE, 1, 'policy', 'resources', - 'round_robin', serve_state.ServiceStatus.READY, - False, False, 1, 'entrypoint') - _add_version(1, _yaml(1)) - info = replica_managers.ReplicaInfo(1, 'c1', '8080', False, None, 1, - None) - info.status_property.sky_launch_status = ( - common_utils.ProcessStatus.SUCCEEDED) - info.status_property.first_ready_time = 1.0 - serve_state.add_or_update_replica(_SERVICE, 1, info) - serve_utils.set_service_status_and_active_versions_from_replica( - _SERVICE, serve_state.get_replica_infos(_SERVICE), mode) - m = object.__new__(replica_managers.SkyPilotReplicaManager) - m._service_name, m.latest_version, m._update_mode = _SERVICE, 1, mode - m.lock, m._is_pool, m._uptime = threading.Lock(), False, 0 - return m - - return make - - -@pytest.mark.parametrize('mode', list(serve_utils.UpdateMode)) -@pytest.mark.parametrize('run,expected_version', [ - ('python -m http.server 8080', 2), - ('echo changed', 1), -]) -def test_update_keeps_ready_replica_routable(manager, mode, run, - expected_version): - m = manager(mode) - _add_version(2, _yaml(2, run)) - m.update_version(2, None, mode) - assert serve_state.get_replica_infos(_SERVICE)[0].version == ( - expected_version) - assert m.get_active_replica_urls() == ['http://r1'] - - -def test_prober_does_not_revert_concurrent_relabel(manager): - mode = serve_utils.UpdateMode.ROLLING - m = manager(mode) - _add_version(2, _yaml(2)) - probing = threading.Event() - - def slow_probe(self, *args): - probing.set() - time.sleep(0.5) - return self, True, time.time() - - with mock.patch.object(replica_managers.ReplicaInfo, 'probe', slow_probe): - prober = threading.Thread(target=m._probe_all_replicas) - prober.start() - assert probing.wait(5) - m.update_version(2, None, mode) # Lands while the probe is in flight. - prober.join() - - info = serve_state.get_replica_infos(_SERVICE)[0] - assert info.version == 2 - assert info.status_property.service_ready_now From a81252dc0f80b010e2e567a3ed7bfcd5a9b3abcf Mon Sep 17 00:00:00 2001 From: Hoipang_ Date: Wed, 7 Oct 2026 14:33:20 +0800 Subject: [PATCH 4/4] [Docs] Add serve replica-reuse fix to fork patch table --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 72291aa4509..15d52f5943e 100644 --- a/README.md +++ b/README.md @@ -378,6 +378,30 @@ Add release notes manually on GitHub. See the parsing bug and an upper bound before Click 9. + + [Serve] Keep reused replicas routable on count-only updates + 7f89638
(plus 6444f96, 82fc2fb) + sky/serve/replica_managers.py + + A sky serve update that changes only the + service: section (e.g. replicas: 1 → 2) + reuses running replicas by relabelling them to the new version, but + active_versions was only refreshed by the next prober + pass. Until then the load balancer filtered out every reused replica + and returned 503 No ready replicas for up to one sync + interval (20 s). Regression from upstream #3249, which replaced the + per-sync version choice used when reuse (#3214) was written. + Fix: recompute active_versions right after the relabel, + using the prober's own function.
+ Follow-up 6444f96: the prober wrote back its stale + ReplicaInfo snapshot after probing, reverting a relabel + that landed mid-probe (update_version does not hold the + replica lock), so the autoscaler replaced the healthy replica. The + prober now re-reads the stored version before writing back. + 82fc2fb removes the unit test added in + 6444f96. + + @@ -461,6 +485,8 @@ git cherry-pick 5420c46 bb8cbc9 0e986b2 f579438 # Intel xe GPU discovery and fo git cherry-pick b43391c 64919e5 # Publish wheels on push with unique versions git cherry-pick 44d09ea # Allow Click 8.2.1+ while excluding 8.2.0 and 9.x git cherry-pick 12bd32c # Mark wheel releases as latest (not prereleases) +# Keep reused replicas routable on count-only serve updates (3 commits, in order): +git cherry-pick 7f89638 6444f96 82fc2fb # Resolve any conflicts if upstream changed the same files # 4. Push new branch