Skip to content

Commit

Permalink
add --host and --port options to llamafile/serve.py
Browse files Browse the repository at this point in the history
  • Loading branch information
Pwuts committed Jun 21, 2024
1 parent 271e59b commit aecc363
Showing 1 changed file with 12 additions and 2 deletions.
14 changes: 12 additions & 2 deletions autogpt/scripts/llamafile/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,20 @@
help=f"Name of the llamafile to serve. Default: {LLAMAFILE.name}",
)
@click.option("--llamafile_url", help="Download URL for the llamafile you want to use")
@click.option(
"--host", help="Specify the address for the llamafile server to listen on"
)
@click.option(
"--port", type=int, help="Specify the port for the llamafile server to listen on"
)
@click.option(
"--use-gpu", is_flag=True, help="Use an AMD or Nvidia GPU to speed up inference"
)
def main(
llamafile: Optional[Path] = None,
llamafile_url: Optional[str] = None,
host: Optional[str] = None,
port: Optional[int] = None,
use_gpu: bool = False,
):
if not llamafile:
Expand Down Expand Up @@ -103,15 +111,17 @@ def main(

base_command = [f".\\{LLAMAFILE_EXE}", "-m", llamafile]

if host:
base_command.extend(["--host", host])
if port:
base_command.extend(["--port", str(port)])
if use_gpu:
base_command.extend(["-ngl", "9999"])

subprocess.run(
[
*base_command,
"--server",
"--host",
"0.0.0.0",
"--nobrowser",
"--ctx-size",
"0",
Expand Down

0 comments on commit aecc363

Please sign in to comment.