Implement ORB AWS EC2 Worker Adapter

.github/actions/run-test/action.yml

@@ -55,7 +55,14 @@ runs:
       run: |
         uv pip install --system -r examples/applications/requirements_applications.txt
         uv pip install --system -r examples/ray_compat/requirements.txt
+        touch examples/skip_examples.txt
+        readarray -t skip_examples < examples/skip_examples.txt
         for example in "./examples"/*.py; do
+          filename=$(basename "$example")
+          if [[ " ${skip_examples[*]} " =~ [[:space:]]${filename}[[:space:]] ]]; then
+            echo "Skipping $example"
+            continue
+          fi
           echo "Running $example"
           python $example
         done

.gitignore

@@ -34,3 +34,6 @@ CMakeFiles/
 .vs/
 src/scaler/protocol/capnp/*.c++
 src/scaler/protocol/capnp/*.h
+
+orb/logs/
+orb/metrics/

README.md

@@ -249,6 +249,8 @@ The following table maps each Scaler command to its corresponding section name i
 | `scaler_top`                     | `[top]`                     |
 | `scaler_worker_adapter_native`   | `[native_worker_adapter]`   |
 | `scaler_worker_adapter_symphony` | `[symphony_worker_adapter]` |
+| `scaler_worker_adapter_orb`      | `[orb_worker_adapter]`      |
+| `scaler_worker_adapter_ecs`      | `[ecs_worker_adapter]`      |
 
 ### Practical Scenarios & Examples
 

ami/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+set -x
+
+# Get the directory where the script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VERSION_FILE="$SCRIPT_DIR/../src/scaler/version.txt"
+
+if [ ! -f "$VERSION_FILE" ]; then
+    echo "Error: Version file not found at $VERSION_FILE"
+    exit 1
+fi
+
+VERSION=$(cat "$VERSION_FILE" | tr -d '[:space:]')
+
+echo "Building AMI for version: $VERSION"
+
+cd "$SCRIPT_DIR"
+packer build -var "version=$VERSION" opengris-scaler.pkr.hcl

ami/opengris-scaler.pkr.hcl

@@ -0,0 +1,69 @@
+packer {
+  required_plugins {
+    amazon = {
+      version = "~> 1"
+      source  = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+variable "aws_region" {
+  type    = string
+  default = "us-east-1"
+}
+
+variable "version" {
+  type    = string
+}
+
+variable "ami_regions" {
+  type    = list(string)
+  default = []
+  description = "A list of regions to copy the AMI to."
+}
+
+variable "ami_groups" {
+  type    = list(string)
+  default = ["all"]
+  description = "A list of groups to share the AMI with. Set to ['all'] to make public."
+}
+
+variable "python_version" {
+  type    = string
+  default = "3.13"
+}
+
+source "amazon-ebs" "opengris-scaler" {
+  ami_name      = "opengris-scaler-${var.version}-py${var.python_version}"
+  instance_type = "t2.small"
+  region        = var.aws_region
+  ami_regions   = var.ami_regions
+  ami_groups    = var.ami_groups
+  source_ami_filter {
+    filters = {
+      name                = "al2023-ami-2023.*-kernel-*-x86_64"
+      root-device-type    = "ebs"
+      virtualization-type = "hvm"
+    }
+    most_recent = true
+    owners      = ["amazon"]
+  }
+  ssh_username = "ec2-user"
+}
+
+build {
+  name    = "opengris-scaler-build"
+  sources = ["source.amazon-ebs.opengris-scaler"]
+
+  provisioner "shell" {
+    inline = [
+      "sudo dnf update -y",
+      "sudo dnf install -y python${var.python_version} python${var.python_version}-pip",
+      "sudo python${var.python_version} -m venv /opt/opengris-scaler",
+      "sudo /opt/opengris-scaler/bin/python -m pip install --upgrade pip",
+      "sudo /opt/opengris-scaler/bin/pip install opengris-scaler==${var.version}",
+      "sudo ln -sf /opt/opengris-scaler/bin/scaler_* /usr/local/bin/",
+      "sudo ln -sf /opt/opengris-scaler/bin/python /usr/local/bin/opengris-python"
+    ]
+  }
+}

docs/source/tutorials/examples.rst

@@ -15,6 +15,14 @@ Shows how to send a basic task to scheduler
 .. literalinclude:: ../../../examples/simple_client.py
    :language: python
 
+Submit Tasks
+~~~~~~~~~~~~
+
+Shows various ways to submit tasks (submit, map, starmap)
+
+.. literalinclude:: ../../../examples/submit_tasks.py
+   :language: python
+
 Client Mapping Tasks
 ~~~~~~~~~~~~~~~~~~~~
 

examples/readme.md

@@ -17,6 +17,8 @@ Ensure that the scheduler and cluster are set up before running clients.
     Shows how to send a nested task to scheduler
 - `simple_client.py`
     Shows how to send a basic task to scheduler
+- `submit_tasks.py`
+    Shows various ways to submit tasks (submit, map, starmap)
 - `task_capabilities.py`
     Shows how to use capabilities to route task to various workers
 - `ray_compat/`

examples/skip_examples.txt

@@ -0,0 +1 @@
+submit_tasks.py

examples/submit_tasks.py

@@ -0,0 +1,55 @@
+"""
+This example demonstrates various ways to submit tasks to a Scaler scheduler.
+It shows how to use the Client to:
+1. Submit a single task using .submit()
+2. Submit multiple tasks using .map()
+3. Submit tasks with multiple arguments using .map() and .starmap()
+"""
+
+import argparse
+import math
+
+from scaler import Client
+
+
+def square(value: int):
+    return value * value
+
+
+def add(x: int, y: int):
+    return x + y
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Submit tasks to a Scaler scheduler.")
+    parser.add_argument("url", help="The URL of the Scaler scheduler (e.g., tcp://127.0.0.1:2345)")
+    args = parser.parse_args()
+
+    print(f"Connecting to scheduler at {args.url}...")
+
+    # Use the Client as a context manager to ensure proper cleanup
+    with Client(address=args.url) as client:
+        print("Submitting a single task using .submit()...")
+        future = client.submit(square, 4)
+        print(f"Result of square(4): {future.result()}")
+
+        print("\nSubmitting multiple tasks using .map()...")
+        # client.map() works like Python's built-in map()
+        results = client.map(math.sqrt, range(1, 6))
+        print(f"Results of sqrt(1..5): {list(results)}")
+
+        print("\nSubmitting tasks with multiple arguments using .map()...")
+        # You can pass multiple iterables to map() for functions with multiple arguments
+        results_add = client.map(add, [1, 2, 3], [10, 20, 30])
+        print(f"Results of add([1,2,3], [10,20,30]): {list(results_add)}")
+
+        print("\nSubmitting tasks with multiple arguments using .starmap()...")
+        # starmap() takes an iterable of argument tuples
+        results_starmap = client.starmap(add, [(5, 5), (10, 10)])
+        print(f"Results of starmap(add, [(5,5), (10,10)]): {list(results_starmap)}")
+
+    print("\nAll tasks completed successfully.")
+
+
+if __name__ == "__main__":
+    main()