host_cmd

host_cmd

jarvis host — download and serve a model locally with auto backend setup.

Functions

host

host(model: str, backend: Optional[str], port: Optional[int], trust_remote_code: bool) -> None

Download (if needed) and serve a model locally.

Examples:

 jarvis host mlx-community/Qwen2.5-7B-4bit --backend mlx jarvis host Qwen/Qwen3-8B --backend vllm jarvis host qwen3:8b --backend ollama jarvis host meta-llama/Llama-3-8B -b sglang

Source code in src/openjarvis/cli/host_cmd.py

@click.command()
@click.argument("model")
@click.option(
    "-b",
    "--backend",
    type=click.Choice(list(_BACKENDS.keys()), case_sensitive=False),
    default=None,
    help="Inference backend to use. Auto-detected if omitted.",
)
@click.option(
    "-p",
    "--port",
    type=int,
    default=None,
    help="Port to serve on (default depends on backend).",
)
@click.option(
    "--trust-remote-code",
    is_flag=True,
    default=False,
    help="Pass --trust-remote-code to the backend.",
)
def host(
    model: str,
    backend: Optional[str],
    port: Optional[int],
    trust_remote_code: bool,
) -> None:
    """Download (if needed) and serve a model locally.

    Examples:

    \b
      jarvis host mlx-community/Qwen2.5-7B-4bit --backend mlx
      jarvis host Qwen/Qwen3-8B --backend vllm
      jarvis host qwen3:8b --backend ollama
      jarvis host meta-llama/Llama-3-8B -b sglang
    """
    console = Console()

    if backend is None:
        detected = _detect_backend()
        if detected is None:
            console.print("[red]Could not auto-detect a suitable backend.[/red]")
            console.print("Specify one with [cyan]--backend[/cyan].")
            raise SystemExit(1)
        backend = detected
        name = _BACKENDS[backend]["display"]
        console.print(
            f"Auto-detected backend: [bold cyan]{name}[/bold cyan]"
        )

    info = _BACKENDS[backend]

    if not _is_package_available(backend):
        if not _install_backend(backend, console):
            raise SystemExit(1)
        if not _is_package_available(backend):
            console.print(
                f"[red]{info['display']} still not available after install.[/red]"
            )
            console.print(
                "You may need to restart your shell or "
                "activate the correct environment."
            )
            raise SystemExit(1)

    serve_port = port or info["default_port"]
    cmd = _build_serve_command(backend, model, serve_port)

    if trust_remote_code:
        if backend in ("vllm", "sglang"):
            cmd.append("--trust-remote-code")
        elif backend == "mlx":
            cmd.extend(["--trust-remote-code", "True"])

    host_url = f"http://localhost:{serve_port}"

    table = Table.grid(padding=(0, 2))
    table.add_row("[bold]Backend:[/bold]", info["display"])
    table.add_row("[bold]Model:[/bold]", model)
    table.add_row("[bold]Endpoint:[/bold]", host_url)
    table.add_row("[bold]Command:[/bold]", " ".join(cmd))

    console.print()
    console.print(Panel(table, title="Hosting Model", border_style="green"))
    console.print()

    if backend != "ollama":
        console.print(
            f"[dim]The model server will be available at {host_url}[/dim]"
        )
        console.print(
            "[dim]OpenJarvis will auto-discover it. "
            "Press Ctrl+C to stop.[/dim]\n"
        )

    try:
        proc = subprocess.Popen(cmd)
        proc.wait()
    except KeyboardInterrupt:
        console.print("\n[yellow]Shutting down model server...[/yellow]")
        proc.send_signal(signal.SIGTERM)
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()
        console.print("[green]Server stopped.[/green]")
    except FileNotFoundError:
        console.print(f"[red]Command not found:[/red] {cmd[0]}")
        console.print(f"Make sure {info['display']} is installed and on your PATH.")
        raise SystemExit(1)