vllm.model_executor.model_loader.weight_cache.daemon ¶

class WeightCacheDaemon:
    """Per-GPU process that loads one TP shard and serves CUDA IPC handles."""

    def __init__(
        self,
        vllm_config: VllmConfig,
        tp_rank: int,
        distributed_init_method: str,
        socket_dir: str | None = None,
    ):
        self.vllm_config = vllm_config
        self.tp_rank = tp_rank
        self.distributed_init_method = distributed_init_method
        self.socket_dir = socket_dir
        self.model: torch.nn.Module | None = None
        self.entries: dict[str, TensorEntry] = {}
        self.aliases: dict[str, str] = {}
        # Fingerprint before loading: process_weights_after_loading may
        # mutate hf_config.quantization_config.
        self.cache_config = WeightCacheKey.from_model_config(
            vllm_config.model_config,
            tp_size=vllm_config.parallel_config.tensor_parallel_size,
            tp_rank=tp_rank,
        )

    def load_model(self) -> None:
        from vllm.model_executor.model_loader import get_model

        tp_size = self.cache_config.tp_size
        torch.accelerator.set_device_index(self.tp_rank)
        init_distributed_environment(
            world_size=tp_size,
            rank=self.tp_rank,
            distributed_init_method=self.distributed_init_method,
            local_rank=self.tp_rank,
            backend=current_platform.dist_backend,
        )
        with set_current_vllm_config(self.vllm_config):
            ensure_model_parallel_initialized(tp_size, 1)
            self.model = get_model(vllm_config=self.vllm_config)
        self._export_entries()
        logger.info(
            "Weight cache daemon rank %d cached %d tensors",
            self.tp_rank,
            len(self.entries),
        )

    def _export_entries(self) -> None:
        assert self.model is not None
        self.entries, self.aliases = export_entries(self.model)

    def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
        """Serve requests until terminated.

        The socket is only bound once the model is fully cached, so clients
        get a connection error (and fall back to disk) until the daemon is
        ready.

        Args:
            ready_callback: Invoked once the socket is bound and listening,
                so the launcher can report overall readiness.
        """
        socket_path = self._socket_path()
        ensure_private_socket_dir(
            os.path.dirname(socket_path), strict_perms=self.socket_dir is None
        )
        # Hold an exclusive per-GPU lock for the daemon's lifetime so a second
        # daemon cannot remove this daemon's live socket and hijack the path.
        lock_fd = self._acquire_gpu_lock(socket_path)
        if os.path.exists(socket_path):
            os.unlink(socket_path)
        server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        server.bind(socket_path)
        os.chmod(socket_path, 0o600)
        server.listen()
        logger.info(
            "Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
        )
        if ready_callback is not None:
            ready_callback()
        try:
            while True:
                conn, _ = server.accept()
                with conn:
                    try:
                        verify_peer_is_owner(conn)
                        self._handle_connection(conn)
                    except (ConnectionError, EOFError):
                        logger.warning("Client disconnected mid-request")
                    except Exception as e:
                        # Report the error back instead of just closing the
                        # socket, but don't let it take the daemon down.
                        logger.exception(
                            "Error handling weight cache client; continuing"
                        )
                        with contextlib.suppress(OSError):
                            send_msg(conn, {"status": "error", "message": str(e)})
        finally:
            server.close()
            if os.path.exists(socket_path):
                os.unlink(socket_path)
            os.close(lock_fd)

    def _acquire_gpu_lock(self, socket_path: str) -> int:
        """Take an exclusive lock guarding this GPU's socket path.

        The lock is advisory and released automatically when the daemon exits
        (or crashes), so a stale socket is only ever removed by whoever owns
        the lock. A running daemon holding it makes a second daemon fail fast
        instead of clobbering the live socket.
        """
        lock_fd = os.open(f"{socket_path}.lock", os.O_CREAT | os.O_RDWR, 0o600)
        try:
            fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except OSError as e:
            os.close(lock_fd)
            raise WeightCacheUnavailableError(
                f"Another weight cache daemon already owns {socket_path}"
            ) from e
        return lock_fd

    def _socket_path(self) -> str:
        device_index = torch.accelerator.current_device_index()
        gpu_id = get_physical_device_id(device_index)
        if gpu_id is None:
            gpu_id = device_index
        return get_socket_path(gpu_id, self.socket_dir)

    def _handle_connection(self, conn: socket.socket) -> None:
        request = recv_msg(conn)
        cmd = request.get("cmd")
        if cmd == "get_state":
            self._handle_get_state(conn, request)
        elif cmd == "release":
            self._handle_release(conn)
        else:
            send_msg(conn, {"status": "error", "message": f"Unknown command {cmd!r}"})

    def _handle_get_state(self, conn: socket.socket, request: dict) -> None:
        client_config = request.get("cache_config")
        if not isinstance(client_config, WeightCacheKey):
            send_msg(conn, {"status": "error", "message": "Missing cache_config"})
            return
        mismatched = self.cache_config.mismatched_fields(client_config)
        if mismatched:
            logger.warning("WeightCacheKey mismatch on fields: %s", mismatched)
            send_msg(conn, {"status": "mismatch", "fields": mismatched})
            return
        if not self.entries:
            send_msg(conn, {"status": "error", "message": "Weights were released"})
            return
        send_msg(
            conn,
            {
                "status": "ok",
                "entries": self.entries,
                "aliases": self.aliases,
                "gpu_uuid": self._gpu_uuid(),
            },
        )
        logger.info_once(
            "Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
            self.tp_rank,
            len(self.entries),
            len(self.aliases),
        )

    def _handle_release(self, conn: socket.socket) -> None:
        self.entries.clear()
        self.aliases.clear()
        self.model = None
        torch.accelerator.empty_cache()
        logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
        send_msg(conn, {"status": "ok"})

    def _gpu_uuid(self) -> str:
        props = torch.cuda.get_device_properties(
            torch.accelerator.current_device_index()
        )
        return str(props.uuid)