From 59227f2cd6cdb63c71a00ea64962f4f80fbdd847 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Fri, 13 Sep 2024 00:08:47 +0530
Subject: [PATCH 01/17] add justfile for local k8s

---
 justfile | 325 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 325 insertions(+)
 create mode 100644 justfile

diff --git a/justfile b/justfile
new file mode 100644
index 00000000000..5e798df7e46
--- /dev/null
+++ b/justfile
@@ -0,0 +1,325 @@
+cluster_default := "syft-dev"
+cluster_high := "syft-high"
+cluster_low := "syft-low"
+cluster_gw := "syft-gw"
+cluster_signoz := "signoz"
+
+port_default := "8080"
+port_high := port_default
+port_low := "8081"
+port_gw := "8082"
+port_signoz := "3301"
+port_registry := "5800"
+
+ns_default := "syft"
+ns_high := "high"
+ns_low := "low"
+ns_gw := "gw"
+
+profile_noop := ""
+
+signoz_url := "http://host.k3d.internal:" + port_signoz
+
+profiles := ""
+tracing := "true"
+
+_all_profiles := if tracing == "true" {
+    profiles + ",tracing"
+} else {
+    profiles
+}
+
+@default:
+    echo "This is Syft"
+    just --list
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Start a local registry on http://k3d-registry.localhost:{{port_registry}}
+[group('registry')]
+start-registry:
+    k3d --version
+    @-docker volume create k3d-registry-vol
+    @-k3d registry create registry.localhost --port {{port_registry}} -v k3d-registry-vol:/var/lib/registry --no-help
+
+    if ! grep -q k3d-registry.localhost /etc/hosts; then \
+        sudo python3 scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; \
+    fi
+
+    @curl --silent --retry 5 --retry-all-errors http://k3d-registry.localhost:{{port_registry}}/v2/_catalog | jq
+    @echo "\033[1;32mStarted registry at http://k3d-registry.localhost:{{port_registry}} \033[0m"
+
+[group('registry')]
+stop-registry:
+    -k3d registry delete registry.localhost
+    -docker volume rm k3d-registry-vol
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Launch Syft Datasite high-side cluster on http://localhost:{{port_high}}
+[group('highside')]
+start-high: (cluster-delete cluster_high) (cluster-create cluster_high port_high)
+    @echo "Started Syft Datasite (high-side) on http://localhost:{{port_high}}/"
+
+# Stop Syft Datasite high-side cluster
+[group('highside')]
+stop-high: (cluster-delete cluster_high)
+    @echo "Stopped Syft Datasite (high-side)"
+
+[group('highside')]
+deploy-high: (devspace-deploy cluster_high ns_default)
+    @echo "Done"
+
+[group('highside')]
+reset-high: (state-reset cluster_high ns_default)
+    @echo "Done"
+
+[group('highside')]
+cleanup-high: (devspace-purge cluster_high ns_default) && (ns-cleanup cluster_high ns_default)
+    @echo "Done"
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Launch Syft Datasite low-side cluster on http://localhost:{{port_low}}
+[group('lowside')]
+start-low: (cluster-create cluster_low port_low)
+    @echo "Started Syft Datasite (low-side) on http://localhost:{{port_low}}/"
+
+# Stop Syft Datasite low-side cluster
+[group('lowside')]
+stop-low: (cluster-delete cluster_low)
+    @echo "Stopped Syft Datasite (low-side)"
+
+[group('lowside')]
+deploy-low: (devspace-deploy cluster_low ns_default _all_profiles+",datasite-low")
+    @echo "Done"
+
+[group('lowside')]
+reset-low: (state-reset cluster_low ns_default)
+    @echo "Done"
+
+[group('lowside')]
+cleanup-low: (devspace-purge cluster_low ns_default) && (ns-cleanup cluster_low ns_default)
+    @echo "Done"
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Launch Syft Gateway cluster on http://localhost:{{port_gw}}
+[group('gateway')]
+start-gw: (cluster-create cluster_gw port_gw)
+    @echo "Started Syft Gateway on http://localhost:{{port_gw}}/"
+
+# Stop Syft Gateway cluster
+[group('gateway')]
+stop-gw: (cluster-delete cluster_gw)
+    @echo "Stopped Syft Gateway"
+
+[group('gateway')]
+deploy-gw: (devspace-deploy cluster_gw ns_default _all_profiles+",gateway")
+    @echo "Done"
+
+[group('gateway')]
+reset-gw: (state-reset cluster_gw ns_default)
+    @echo "Done"
+
+[group('gateway')]
+cleanup-gw: (devspace-purge cluster_gw ns_default) && (ns-cleanup cluster_gw ns_default)
+    @echo "Done"
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Launch a Syft beefy cluster on http://localhost:{{port_default}}
+[group('shared')]
+start-shared: (cluster-create cluster_default port_default "--servers 5")
+    @echo "Started Syft on http://localhost:{{port_default}}/"
+
+# Stop Syft shared cluster
+[group('shared')]
+stop-shared: (cluster-delete cluster_default)
+    @echo "Stopped Syft cluster"
+
+# Deploy to "high" namespace on the shared cluster
+[group('shared')]
+deploy-ns-high: (devspace-deploy cluster_default ns_high)
+    @echo "Deployed Syft Gateway on {{cluster_default}}"
+
+# Delete the "high" namespace
+[group('shared')]
+delete-ns-high: (ns-cleanup cluster_default ns_high)
+    @echo "Done"
+
+# Deploy to "low" namespace on the shared cluster
+[group('shared')]
+deploy-ns-low: (devspace-deploy cluster_default ns_low _all_profiles+",datasite-low")
+    @echo "Deployed Syft Gateway on {{cluster_default}}"
+
+# Delete the "low" namespace
+[group('shared')]
+delete-ns-low: (ns-cleanup cluster_default ns_low)
+    @echo "Done"
+
+# Deploy to "gw" namespace on the shared cluster
+[group('shared')]
+deploy-ns-gw: (devspace-deploy cluster_default ns_gw _all_profiles+",gateway")
+    @echo "Deployed Syft Gateway on {{cluster_default}}"
+
+# Delete the "gw" namespace
+[group('shared')]
+delete-ns-gw: (ns-cleanup cluster_default ns_gw)
+    @echo "Done"
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Launch SigNoz on http://localhost:{{port_signoz}}
+[group('signoz')]
+start-signoz: && apply-signoz setup-signoz
+    k3d cluster create {{cluster_signoz}} \
+        --port {{port_signoz}}:3301@loadbalancer \
+        --port 4317:4317@loadbalancer \
+        --k3s-arg "--disable=metrics-server@server:*"
+
+    @echo "Started SigNoz on http://localhost:{{port_signoz}}"
+
+[group('signoz')]
+delete-collector:
+    helm uninstall k8s-infra
+
+# Stop SigNoz cluster
+[group('signoz')]
+stop-signoz: (cluster-delete cluster_signoz)
+    @echo "Stopped SigNoz cluster"
+
+[private]
+[group('signoz')]
+apply-collector cluster=cluster_default:
+    @echo "Installing SigNoz OTel Collector"
+    helm install k8s-infra k8s-infra \
+        --repo https://charts.signoz.io \
+        --kube-context k3d-{{cluster}} \
+        --set global.deploymentEnvironment=local \
+        --set clusterName={{cluster}} \
+        --set otelCollectorEndpoint=http://{{signoz_url}}:4317 \
+        --set otelInsecure=true \
+        --set presets.otlpExporter.enabled=true \
+        --set presets.loggingExporter.enabled=true
+
+[private]
+[group('signoz')]
+apply-signoz:
+    @echo "Installing SigNoz on the cluster"
+    helm install signoz signoz \
+        --repo https://charts.signoz.io \
+        --namespace platform \
+        --create-namespace \
+        --version 0.52.0 \
+        --set frontend.service.type=LoadBalancer \
+        --set otelCollector.service.type=LoadBalancer \
+        --set otelCollectorMetrics.service.type=LoadBalancer
+
+[private]
+[group('signoz')]
+setup-signoz:
+    @echo "Waiting for SigNoz frontend to be available..."
+    @WAIT_TIME=5 ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context k3d-signoz &> /dev/null
+
+    @echo "Setting up SigNoz account"
+    @curl --retry 5 --retry-all-errors -X POST \
+        -H "Content-Type: application/json" \
+        --data '{"email":"admin@localhost","name":"admin","orgName":"openmined","password":"password"}' \
+        http://localhost:3301/api/v1/register
+
+    @printf '\nSignoz is running on http://localhost:3301\nEmail: \033[1;36madmin@localhost\033[0m\nPassword: \033[1;36mpassword\033[0m\n'
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+[private]
+[group('cluster')]
+cluster-create name=cluster_default port=port_default *args='': start-registry && (apply-coredns name) (apply-collector name)
+    k3d cluster create {{ name }} \
+        --port {{ port }}:80@loadbalancer \
+        --registry-use k3d-registry.localhost:5800 {{ args }}
+
+[private]
+[group('cluster')]
+cluster-delete *args=cluster_default:
+    k3d cluster delete {{ args }}
+
+[private]
+[group('cluster')]
+ns-cleanup name=cluster_default ns=ns_default:
+    kubectl delete namespace {{ns}} --force --grace-period=0 --context k3d-{{cluster_high}}
+
+[group('cluster')]
+cluster-list:
+    k3d cluster list
+
+# Stop all Syft clusters
+[group('cluster')]
+stop-all: (cluster-delete cluster_high cluster_low cluster_gw cluster_signoz)
+    @echo "Stopped all Syft clusters"
+
+[private]
+[group('cluster')]
+apply-coredns cluster=cluster_default:
+    @echo "Applying custom CoreDNS config"
+
+    kubectl apply -f ./scripts/k8s-coredns-custom.yml --context k3d-{{cluster}}
+    kubectl delete pod -n kube-system -l k8s-app=kube-dns --context k3d-{{cluster}}
+
+[group('cluster')]
+k9s-high:
+    k9s --context k3d-{{cluster_high}}
+
+[group('cluster')]
+k9s-low:
+    k9s --context k3d-{{cluster_low}}
+
+[group('cluster')]
+k9s-gw:
+    k9s --context k3d-{{cluster_gw}}
+
+[group('cluster')]
+k9s-signoz:
+    k9s --context k3d-{{cluster_signoz}}
+
+[group('cluster')]
+k9s-shared:
+    k9s --context k3d-{{cluster_default}}
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+[private]
+devspace-deploy cluster namespace profile=_all_profiles:
+    #!/bin/bash
+    set -euxo pipefail
+
+    cd packages/grid
+
+    PROFILE="{{profile}}"
+    if [ -n "$PROFILE" ]; then
+        PROFILE=$(echo "$PROFILE" | sed 's/^,*//; s/,*$//')
+        PROFILE="-p $PROFILE"
+    fi
+
+    echo devspace deploy -b \
+        --kube-context k3d-{{cluster}} \
+        --no-warn \
+        --namespace {{namespace}} $PROFILE \
+        --var CONTAINER_REGISTRY=k3d-registry.localhost:5800
+
+[private]
+devspace-purge cluster namespace:
+    #!/bin/bash
+    cd packages/grid
+    devspace purge --force-purge --kube-context k3d-{{cluster}} --no-warn --namespace {{namespace}}
+    sleep 3
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+[private]
+[group('syft')]
+state-reset name=cluster_default namespace=ns_default:
+    kubectl config use-context k3d-{{name}}
+    scripts/reset_k8s.sh

From 33b2107cee628c400bca8110309d8178c4eba0ea Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Fri, 13 Sep 2024 01:23:48 +0530
Subject: [PATCH 02/17] format

---
 justfile | 96 +++++++++++++++++++++++++-------------------------------
 1 file changed, 42 insertions(+), 54 deletions(-)

diff --git a/justfile b/justfile
index 5e798df7e46..d2766a0d777 100644
--- a/justfile
+++ b/justfile
@@ -3,31 +3,21 @@ cluster_high := "syft-high"
 cluster_low := "syft-low"
 cluster_gw := "syft-gw"
 cluster_signoz := "signoz"
-
 port_default := "8080"
 port_high := port_default
 port_low := "8081"
 port_gw := "8082"
 port_signoz := "3301"
 port_registry := "5800"
-
 ns_default := "syft"
 ns_high := "high"
 ns_low := "low"
 ns_gw := "gw"
-
 profile_noop := ""
-
 signoz_url := "http://host.k3d.internal:" + port_signoz
-
 profiles := ""
 tracing := "true"
-
-_all_profiles := if tracing == "true" {
-    profiles + ",tracing"
-} else {
-    profiles
-}
+_all_profiles := if tracing == "true" { profiles + ",tracing" } else { profiles }
 
 @default:
     echo "This is Syft"
@@ -40,14 +30,14 @@ _all_profiles := if tracing == "true" {
 start-registry:
     k3d --version
     @-docker volume create k3d-registry-vol
-    @-k3d registry create registry.localhost --port {{port_registry}} -v k3d-registry-vol:/var/lib/registry --no-help
+    @-k3d registry create registry.localhost --port {{ port_registry }} -v k3d-registry-vol:/var/lib/registry --no-help
 
     if ! grep -q k3d-registry.localhost /etc/hosts; then \
         sudo python3 scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; \
     fi
 
-    @curl --silent --retry 5 --retry-all-errors http://k3d-registry.localhost:{{port_registry}}/v2/_catalog | jq
-    @echo "\033[1;32mStarted registry at http://k3d-registry.localhost:{{port_registry}} \033[0m"
+    @curl --silent --retry 5 --retry-all-errors http://k3d-registry.localhost:{{ port_registry }}/v2/_catalog | jq
+    @echo "\033[1;32mStarted registry at http://k3d-registry.localhost:{{ port_registry }} \033[0m"
 
 [group('registry')]
 stop-registry:
@@ -59,7 +49,7 @@ stop-registry:
 # Launch Syft Datasite high-side cluster on http://localhost:{{port_high}}
 [group('highside')]
 start-high: (cluster-delete cluster_high) (cluster-create cluster_high port_high)
-    @echo "Started Syft Datasite (high-side) on http://localhost:{{port_high}}/"
+    @echo "Started Syft Datasite (high-side) on http://localhost:{{ port_high }}/"
 
 # Stop Syft Datasite high-side cluster
 [group('highside')]
@@ -83,7 +73,7 @@ cleanup-high: (devspace-purge cluster_high ns_default) && (ns-cleanup cluster_hi
 # Launch Syft Datasite low-side cluster on http://localhost:{{port_low}}
 [group('lowside')]
 start-low: (cluster-create cluster_low port_low)
-    @echo "Started Syft Datasite (low-side) on http://localhost:{{port_low}}/"
+    @echo "Started Syft Datasite (low-side) on http://localhost:{{ port_low }}/"
 
 # Stop Syft Datasite low-side cluster
 [group('lowside')]
@@ -91,7 +81,7 @@ stop-low: (cluster-delete cluster_low)
     @echo "Stopped Syft Datasite (low-side)"
 
 [group('lowside')]
-deploy-low: (devspace-deploy cluster_low ns_default _all_profiles+",datasite-low")
+deploy-low: (devspace-deploy cluster_low ns_default _all_profiles + ",datasite-low")
     @echo "Done"
 
 [group('lowside')]
@@ -107,7 +97,7 @@ cleanup-low: (devspace-purge cluster_low ns_default) && (ns-cleanup cluster_low
 # Launch Syft Gateway cluster on http://localhost:{{port_gw}}
 [group('gateway')]
 start-gw: (cluster-create cluster_gw port_gw)
-    @echo "Started Syft Gateway on http://localhost:{{port_gw}}/"
+    @echo "Started Syft Gateway on http://localhost:{{ port_gw }}/"
 
 # Stop Syft Gateway cluster
 [group('gateway')]
@@ -115,7 +105,7 @@ stop-gw: (cluster-delete cluster_gw)
     @echo "Stopped Syft Gateway"
 
 [group('gateway')]
-deploy-gw: (devspace-deploy cluster_gw ns_default _all_profiles+",gateway")
+deploy-gw: (devspace-deploy cluster_gw ns_default _all_profiles + ",gateway")
     @echo "Done"
 
 [group('gateway')]
@@ -131,7 +121,7 @@ cleanup-gw: (devspace-purge cluster_gw ns_default) && (ns-cleanup cluster_gw ns_
 # Launch a Syft beefy cluster on http://localhost:{{port_default}}
 [group('shared')]
 start-shared: (cluster-create cluster_default port_default "--servers 5")
-    @echo "Started Syft on http://localhost:{{port_default}}/"
+    @echo "Started Syft on http://localhost:{{ port_default }}/"
 
 # Stop Syft shared cluster
 [group('shared')]
@@ -141,7 +131,7 @@ stop-shared: (cluster-delete cluster_default)
 # Deploy to "high" namespace on the shared cluster
 [group('shared')]
 deploy-ns-high: (devspace-deploy cluster_default ns_high)
-    @echo "Deployed Syft Gateway on {{cluster_default}}"
+    @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "high" namespace
 [group('shared')]
@@ -150,8 +140,8 @@ delete-ns-high: (ns-cleanup cluster_default ns_high)
 
 # Deploy to "low" namespace on the shared cluster
 [group('shared')]
-deploy-ns-low: (devspace-deploy cluster_default ns_low _all_profiles+",datasite-low")
-    @echo "Deployed Syft Gateway on {{cluster_default}}"
+deploy-ns-low: (devspace-deploy cluster_default ns_low _all_profiles + ",datasite-low")
+    @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "low" namespace
 [group('shared')]
@@ -160,8 +150,8 @@ delete-ns-low: (ns-cleanup cluster_default ns_low)
 
 # Deploy to "gw" namespace on the shared cluster
 [group('shared')]
-deploy-ns-gw: (devspace-deploy cluster_default ns_gw _all_profiles+",gateway")
-    @echo "Deployed Syft Gateway on {{cluster_default}}"
+deploy-ns-gw: (devspace-deploy cluster_default ns_gw _all_profiles + ",gateway")
+    @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "gw" namespace
 [group('shared')]
@@ -173,12 +163,12 @@ delete-ns-gw: (ns-cleanup cluster_default ns_gw)
 # Launch SigNoz on http://localhost:{{port_signoz}}
 [group('signoz')]
 start-signoz: && apply-signoz setup-signoz
-    k3d cluster create {{cluster_signoz}} \
-        --port {{port_signoz}}:3301@loadbalancer \
+    k3d cluster create {{ cluster_signoz }} \
+        --port {{ port_signoz }}:3301@loadbalancer \
         --port 4317:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"
 
-    @echo "Started SigNoz on http://localhost:{{port_signoz}}"
+    @echo "Started SigNoz on http://localhost:{{ port_signoz }}"
 
 [group('signoz')]
 delete-collector:
@@ -189,22 +179,22 @@ delete-collector:
 stop-signoz: (cluster-delete cluster_signoz)
     @echo "Stopped SigNoz cluster"
 
-[private]
 [group('signoz')]
+[private]
 apply-collector cluster=cluster_default:
     @echo "Installing SigNoz OTel Collector"
     helm install k8s-infra k8s-infra \
         --repo https://charts.signoz.io \
-        --kube-context k3d-{{cluster}} \
+        --kube-context k3d-{{ cluster }} \
         --set global.deploymentEnvironment=local \
-        --set clusterName={{cluster}} \
-        --set otelCollectorEndpoint=http://{{signoz_url}}:4317 \
+        --set clusterName={{ cluster }} \
+        --set otelCollectorEndpoint=http://{{ signoz_url }}:4317 \
         --set otelInsecure=true \
         --set presets.otlpExporter.enabled=true \
         --set presets.loggingExporter.enabled=true
 
-[private]
 [group('signoz')]
+[private]
 apply-signoz:
     @echo "Installing SigNoz on the cluster"
     helm install signoz signoz \
@@ -216,8 +206,8 @@ apply-signoz:
         --set otelCollector.service.type=LoadBalancer \
         --set otelCollectorMetrics.service.type=LoadBalancer
 
-[private]
 [group('signoz')]
+[private]
 setup-signoz:
     @echo "Waiting for SigNoz frontend to be available..."
     @WAIT_TIME=5 ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context k3d-signoz &> /dev/null
@@ -230,26 +220,24 @@ setup-signoz:
 
     @printf '\nSignoz is running on http://localhost:3301\nEmail: \033[1;36madmin@localhost\033[0m\nPassword: \033[1;36mpassword\033[0m\n'
 
-
 # ---------------------------------------------------------------------------------------------------------------------
 
-
-[private]
 [group('cluster')]
+[private]
 cluster-create name=cluster_default port=port_default *args='': start-registry && (apply-coredns name) (apply-collector name)
     k3d cluster create {{ name }} \
         --port {{ port }}:80@loadbalancer \
         --registry-use k3d-registry.localhost:5800 {{ args }}
 
-[private]
 [group('cluster')]
+[private]
 cluster-delete *args=cluster_default:
     k3d cluster delete {{ args }}
 
-[private]
 [group('cluster')]
+[private]
 ns-cleanup name=cluster_default ns=ns_default:
-    kubectl delete namespace {{ns}} --force --grace-period=0 --context k3d-{{cluster_high}}
+    kubectl delete namespace {{ ns }} --force --grace-period=0 --context k3d-{{ cluster_high }}
 
 [group('cluster')]
 cluster-list:
@@ -260,33 +248,33 @@ cluster-list:
 stop-all: (cluster-delete cluster_high cluster_low cluster_gw cluster_signoz)
     @echo "Stopped all Syft clusters"
 
-[private]
 [group('cluster')]
+[private]
 apply-coredns cluster=cluster_default:
     @echo "Applying custom CoreDNS config"
 
-    kubectl apply -f ./scripts/k8s-coredns-custom.yml --context k3d-{{cluster}}
-    kubectl delete pod -n kube-system -l k8s-app=kube-dns --context k3d-{{cluster}}
+    kubectl apply -f ./scripts/k8s-coredns-custom.yml --context k3d-{{ cluster }}
+    kubectl delete pod -n kube-system -l k8s-app=kube-dns --context k3d-{{ cluster }}
 
 [group('cluster')]
 k9s-high:
-    k9s --context k3d-{{cluster_high}}
+    k9s --context k3d-{{ cluster_high }}
 
 [group('cluster')]
 k9s-low:
-    k9s --context k3d-{{cluster_low}}
+    k9s --context k3d-{{ cluster_low }}
 
 [group('cluster')]
 k9s-gw:
-    k9s --context k3d-{{cluster_gw}}
+    k9s --context k3d-{{ cluster_gw }}
 
 [group('cluster')]
 k9s-signoz:
-    k9s --context k3d-{{cluster_signoz}}
+    k9s --context k3d-{{ cluster_signoz }}
 
 [group('cluster')]
 k9s-shared:
-    k9s --context k3d-{{cluster_default}}
+    k9s --context k3d-{{ cluster_default }}
 
 # ---------------------------------------------------------------------------------------------------------------------
 
@@ -297,29 +285,29 @@ devspace-deploy cluster namespace profile=_all_profiles:
 
     cd packages/grid
 
-    PROFILE="{{profile}}"
+    PROFILE="{{ profile }}"
     if [ -n "$PROFILE" ]; then
         PROFILE=$(echo "$PROFILE" | sed 's/^,*//; s/,*$//')
         PROFILE="-p $PROFILE"
     fi
 
     echo devspace deploy -b \
-        --kube-context k3d-{{cluster}} \
+        --kube-context k3d-{{ cluster }} \
         --no-warn \
-        --namespace {{namespace}} $PROFILE \
+        --namespace {{ namespace }} $PROFILE \
         --var CONTAINER_REGISTRY=k3d-registry.localhost:5800
 
 [private]
 devspace-purge cluster namespace:
     #!/bin/bash
     cd packages/grid
-    devspace purge --force-purge --kube-context k3d-{{cluster}} --no-warn --namespace {{namespace}}
+    devspace purge --force-purge --kube-context k3d-{{ cluster }} --no-warn --namespace {{ namespace }}
     sleep 3
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-[private]
 [group('syft')]
+[private]
 state-reset name=cluster_default namespace=ns_default:
-    kubectl config use-context k3d-{{name}}
+    kubectl config use-context k3d-{{ name }}
     scripts/reset_k8s.sh

From d526312a64477d88306c8fc9666ffe2162a47afd Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Fri, 13 Sep 2024 02:51:32 +0530
Subject: [PATCH 03/17] add cloud deployments + fixes

---
 justfile | 127 +++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 85 insertions(+), 42 deletions(-)

diff --git a/justfile b/justfile
index d2766a0d777..1e6b4a95ad6 100644
--- a/justfile
+++ b/justfile
@@ -1,26 +1,32 @@
-cluster_default := "syft-dev"
-cluster_high := "syft-high"
-cluster_low := "syft-low"
-cluster_gw := "syft-gw"
-cluster_signoz := "signoz"
+cluster_default := "k3d-syft-dev"
+cluster_high := "k3d-syft-high"
+cluster_low := "k3d-syft-low"
+cluster_gw := "k3d-syft-gw"
+cluster_signoz := "k3d-signoz"
+
 port_default := "8080"
 port_high := port_default
 port_low := "8081"
 port_gw := "8082"
 port_signoz := "3301"
 port_registry := "5800"
+
 ns_default := "syft"
 ns_high := "high"
 ns_low := "low"
 ns_gw := "gw"
-profile_noop := ""
+
+registry_url := "k3d-registry.localhost:" + port_registry
 signoz_url := "http://host.k3d.internal:" + port_signoz
+
 profiles := ""
 tracing := "true"
-_all_profiles := if tracing == "true" { profiles + ",tracing" } else { profiles }
+
+_g_profiles := if tracing == "true" { profiles + ",tracing" } else { profiles }
+
+# ---------------------------------------------------------------------------------------------------------------------
 
 @default:
-    echo "This is Syft"
     just --list
 
 # ---------------------------------------------------------------------------------------------------------------------
@@ -37,7 +43,7 @@ start-registry:
     fi
 
     @curl --silent --retry 5 --retry-all-errors http://k3d-registry.localhost:{{ port_registry }}/v2/_catalog | jq
-    @echo "\033[1;32mStarted registry at http://k3d-registry.localhost:{{ port_registry }} \033[0m"
+    @echo "\033[1;32mRegistring running at http://k3d-registry.localhost:{{ port_registry }}\033[0m"
 
 [group('registry')]
 stop-registry:
@@ -81,7 +87,7 @@ stop-low: (cluster-delete cluster_low)
     @echo "Stopped Syft Datasite (low-side)"
 
 [group('lowside')]
-deploy-low: (devspace-deploy cluster_low ns_default _all_profiles + ",datasite-low")
+deploy-low: (devspace-deploy cluster_low ns_default "-p datasite-low")
     @echo "Done"
 
 [group('lowside')]
@@ -105,7 +111,7 @@ stop-gw: (cluster-delete cluster_gw)
     @echo "Stopped Syft Gateway"
 
 [group('gateway')]
-deploy-gw: (devspace-deploy cluster_gw ns_default _all_profiles + ",gateway")
+deploy-gw: (devspace-deploy cluster_gw ns_default "-p gateway")
     @echo "Done"
 
 [group('gateway')]
@@ -140,7 +146,7 @@ delete-ns-high: (ns-cleanup cluster_default ns_high)
 
 # Deploy to "low" namespace on the shared cluster
 [group('shared')]
-deploy-ns-low: (devspace-deploy cluster_default ns_low _all_profiles + ",datasite-low")
+deploy-ns-low: (devspace-deploy cluster_default ns_low "-p datasite-low")
     @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "low" namespace
@@ -150,7 +156,7 @@ delete-ns-low: (ns-cleanup cluster_default ns_low)
 
 # Deploy to "gw" namespace on the shared cluster
 [group('shared')]
-deploy-ns-gw: (devspace-deploy cluster_default ns_gw _all_profiles + ",gateway")
+deploy-ns-gw: (devspace-deploy cluster_default ns_gw "-p gateway")
     @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "gw" namespace
@@ -163,7 +169,7 @@ delete-ns-gw: (ns-cleanup cluster_default ns_gw)
 # Launch SigNoz on http://localhost:{{port_signoz}}
 [group('signoz')]
 start-signoz: && apply-signoz setup-signoz
-    k3d cluster create {{ cluster_signoz }} \
+    k3d cluster create signoz \
         --port {{ port_signoz }}:3301@loadbalancer \
         --port 4317:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"
@@ -181,11 +187,11 @@ stop-signoz: (cluster-delete cluster_signoz)
 
 [group('signoz')]
 [private]
-apply-collector cluster=cluster_default:
+apply-collector cluster:
     @echo "Installing SigNoz OTel Collector"
     helm install k8s-infra k8s-infra \
         --repo https://charts.signoz.io \
-        --kube-context k3d-{{ cluster }} \
+        --kube-context {{ cluster }} \
         --set global.deploymentEnvironment=local \
         --set clusterName={{ cluster }} \
         --set otelCollectorEndpoint=http://{{ signoz_url }}:4317 \
@@ -199,6 +205,7 @@ apply-signoz:
     @echo "Installing SigNoz on the cluster"
     helm install signoz signoz \
         --repo https://charts.signoz.io \
+        --kube-context {{ cluster_signoz }} \
         --namespace platform \
         --create-namespace \
         --version 0.52.0 \
@@ -210,7 +217,7 @@ apply-signoz:
 [private]
 setup-signoz:
     @echo "Waiting for SigNoz frontend to be available..."
-    @WAIT_TIME=5 ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context k3d-signoz &> /dev/null
+    @WAIT_TIME=5 ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context {{ cluster_signoz }} &> /dev/null
 
     @echo "Setting up SigNoz account"
     @curl --retry 5 --retry-all-errors -X POST \
@@ -224,20 +231,31 @@ setup-signoz:
 
 [group('cluster')]
 [private]
-cluster-create name=cluster_default port=port_default *args='': start-registry && (apply-coredns name) (apply-collector name)
-    k3d cluster create {{ name }} \
+cluster-create cluster port *args='': start-registry && (apply-coredns cluster) (apply-collector cluster)
+    #!/bin/bash
+    set -euox pipefail
+
+    # remove the k3d- prefix
+    CLUSTER_NAME=$(echo "{{ cluster }}" | sed -e 's/k3d-//g')
+
+    k3d cluster create $CLUSTER_NAME \
         --port {{ port }}:80@loadbalancer \
         --registry-use k3d-registry.localhost:5800 {{ args }}
 
 [group('cluster')]
 [private]
-cluster-delete *args=cluster_default:
-    k3d cluster delete {{ args }}
+cluster-delete *args='':
+    #!/bin/bash
+    set -euox pipefail
+
+    # remove the k3d- prefix
+    ARGS=$(echo "{{ args }}" | sed -e 's/k3d-//g')
+    k3d cluster delete $ARGS
 
 [group('cluster')]
 [private]
-ns-cleanup name=cluster_default ns=ns_default:
-    kubectl delete namespace {{ ns }} --force --grace-period=0 --context k3d-{{ cluster_high }}
+ns-cleanup context namespace:
+    kubectl delete ns {{ namespace }} --force --grace-period=0 --context {{ context }}
 
 [group('cluster')]
 cluster-list:
@@ -250,64 +268,89 @@ stop-all: (cluster-delete cluster_high cluster_low cluster_gw cluster_signoz)
 
 [group('cluster')]
 [private]
-apply-coredns cluster=cluster_default:
+apply-coredns cluster:
     @echo "Applying custom CoreDNS config"
 
-    kubectl apply -f ./scripts/k8s-coredns-custom.yml --context k3d-{{ cluster }}
-    kubectl delete pod -n kube-system -l k8s-app=kube-dns --context k3d-{{ cluster }}
+    kubectl apply -f ./scripts/k8s-coredns-custom.yml --context {{ cluster }}
+    kubectl delete pod -n kube-system -l k8s-app=kube-dns --context {{ cluster }}
 
 [group('cluster')]
 k9s-high:
-    k9s --context k3d-{{ cluster_high }}
+    k9s --context {{ cluster_high }}
 
 [group('cluster')]
 k9s-low:
-    k9s --context k3d-{{ cluster_low }}
+    k9s --context {{ cluster_low }}
 
 [group('cluster')]
 k9s-gw:
-    k9s --context k3d-{{ cluster_gw }}
+    k9s --context {{ cluster_gw }}
 
 [group('cluster')]
 k9s-signoz:
-    k9s --context k3d-{{ cluster_signoz }}
+    k9s --context {{ cluster_signoz }}
 
 [group('cluster')]
 k9s-shared:
-    k9s --context k3d-{{ cluster_default }}
+    k9s --context {{ cluster_default }}
 
 # ---------------------------------------------------------------------------------------------------------------------
 
 [private]
-devspace-deploy cluster namespace profile=_all_profiles:
+devspace-deploy cluster namespace *args='':
     #!/bin/bash
-    set -euxo pipefail
+    set -euo pipefail
 
     cd packages/grid
 
-    PROFILE="{{ profile }}"
+    PROFILE="{{ _g_profiles }}"
+    PROFILE=$(echo "$PROFILE" | sed -E 's/^,*|,*$//g')
     if [ -n "$PROFILE" ]; then
-        PROFILE=$(echo "$PROFILE" | sed 's/^,*//; s/,*$//')
         PROFILE="-p $PROFILE"
     fi
 
-    echo devspace deploy -b \
-        --kube-context k3d-{{ cluster }} \
+    devspace deploy -b \
         --no-warn \
-        --namespace {{ namespace }} $PROFILE \
-        --var CONTAINER_REGISTRY=k3d-registry.localhost:5800
+        --kube-context {{ cluster }} \
+        --namespace {{ namespace }} \
+        $PROFILE \
+        {{ args }} \
+        --var CONTAINER_REGISTRY={{ registry_url }}
 
 [private]
 devspace-purge cluster namespace:
     #!/bin/bash
+    set -euo pipefail
+
     cd packages/grid
-    devspace purge --force-purge --kube-context k3d-{{ cluster }} --no-warn --namespace {{ namespace }}
+    devspace purge --force-purge --kube-context {{ cluster }} --no-warn --namespace {{ namespace }}
     sleep 3
 
 # ---------------------------------------------------------------------------------------------------------------------
 
+[group('cloud')]
+[private]
+deploy-cloud cluster registry namespace profile:
+    #!/bin/bash
+    set -euo pipefail
+
+    kubectl config get-contexts {{ cluster }} > /dev/null
+    # cloud deployments always have tracing false + platform=amd64
+    just tracing=false registry_url={{ registry }} devspace-deploy {{ cluster }} {{ namespace }} "-p {{ profile }} --var PLATFORM=amd64"
+
+[group('cloud')]
+deploy-gcp-high gke_cluster gcp_registry namespace="syft": (deploy-cloud gke_cluster gcp_registry namespace "gcp")
+
+[group('cloud')]
+deploy-gcp-low gke_cluster gcp_registry namespace="syft": (deploy-cloud gke_cluster gcp_registry namespace "gcp-low")
+
+[group('cloud')]
+deploy-az-high aks_cluster az_registry namespace="syft": (deploy-cloud aks_cluster az_registry namespace "azure")
+
+# ---------------------------------------------------------------------------------------------------------------------
+
 [group('syft')]
 [private]
-state-reset name=cluster_default namespace=ns_default:
-    kubectl config use-context k3d-{{ name }}
+state-reset name namespace:
+    kubectl config use-context {{ name }}
     scripts/reset_k8s.sh

From d924b54498bc79623f8c34a51a945cae2e98fcca Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Fri, 13 Sep 2024 14:32:29 +0530
Subject: [PATCH 04/17] Linux + OTEL fixes

---
 justfile                    | 31 +++++++++++++++++--------------
 packages/grid/devspace.yaml |  2 +-
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/justfile b/justfile
index 1e6b4a95ad6..544f3937256 100644
--- a/justfile
+++ b/justfile
@@ -8,7 +8,8 @@ port_default := "8080"
 port_high := port_default
 port_low := "8081"
 port_gw := "8082"
-port_signoz := "3301"
+port_signoz_ui := "3301"
+port_signoz_otel := "4317"
 port_registry := "5800"
 
 ns_default := "syft"
@@ -17,13 +18,15 @@ ns_low := "low"
 ns_gw := "gw"
 
 registry_url := "k3d-registry.localhost:" + port_registry
-signoz_url := "http://host.k3d.internal:" + port_signoz
+signoz_otel_url := "http://host.k3d.internal:" + port_signoz_otel
 
 profiles := ""
 tracing := "true"
 
 _g_profiles := if tracing == "true" { profiles + ",tracing" } else { profiles }
 
+python_path := `which python || which python3`
+
 # ---------------------------------------------------------------------------------------------------------------------
 
 @default:
@@ -39,7 +42,7 @@ start-registry:
     @-k3d registry create registry.localhost --port {{ port_registry }} -v k3d-registry-vol:/var/lib/registry --no-help
 
     if ! grep -q k3d-registry.localhost /etc/hosts; then \
-        sudo python3 scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; \
+        sudo {{python_path}} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; \
     fi
 
     @curl --silent --retry 5 --retry-all-errors http://k3d-registry.localhost:{{ port_registry }}/v2/_catalog | jq
@@ -124,9 +127,9 @@ cleanup-gw: (devspace-purge cluster_gw ns_default) && (ns-cleanup cluster_gw ns_
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-# Launch a Syft beefy cluster on http://localhost:{{port_default}}
+# Launch a Syft shared cluster on http://localhost:{{port_default}}
 [group('shared')]
-start-shared: (cluster-create cluster_default port_default "--servers 5")
+start-shared: (cluster-create cluster_default port_default "--agents 2 --servers 1")
     @echo "Started Syft on http://localhost:{{ port_default }}/"
 
 # Stop Syft shared cluster
@@ -166,15 +169,15 @@ delete-ns-gw: (ns-cleanup cluster_default ns_gw)
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-# Launch SigNoz on http://localhost:{{port_signoz}}
+# Launch SigNoz on http://localhost:{{port_signoz_ui}}
 [group('signoz')]
 start-signoz: && apply-signoz setup-signoz
     k3d cluster create signoz \
-        --port {{ port_signoz }}:3301@loadbalancer \
-        --port 4317:4317@loadbalancer \
+        --port {{ port_signoz_ui }}:3301@loadbalancer \
+        --port {{ port_signoz_otel }}:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"
 
-    @echo "Started SigNoz on http://localhost:{{ port_signoz }}"
+    @printf "Started SigNoz\nDashboard: http://localhost:{{ port_signoz_ui }}\nOTEL Endpoint: http://localhost:{{ port_signoz_otel }}\n"
 
 [group('signoz')]
 delete-collector:
@@ -194,7 +197,7 @@ apply-collector cluster:
         --kube-context {{ cluster }} \
         --set global.deploymentEnvironment=local \
         --set clusterName={{ cluster }} \
-        --set otelCollectorEndpoint=http://{{ signoz_url }}:4317 \
+        --set otelCollectorEndpoint={{ signoz_otel_url }} \
         --set otelInsecure=true \
         --set presets.otlpExporter.enabled=true \
         --set presets.loggingExporter.enabled=true
@@ -217,7 +220,7 @@ apply-signoz:
 [private]
 setup-signoz:
     @echo "Waiting for SigNoz frontend to be available..."
-    @WAIT_TIME=5 ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context {{ cluster_signoz }} &> /dev/null
+    @bash ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context {{ cluster_signoz }} &> /dev/null
 
     @echo "Setting up SigNoz account"
     @curl --retry 5 --retry-all-errors -X POST \
@@ -233,7 +236,7 @@ setup-signoz:
 [private]
 cluster-create cluster port *args='': start-registry && (apply-coredns cluster) (apply-collector cluster)
     #!/bin/bash
-    set -euox pipefail
+    set -euo pipefail
 
     # remove the k3d- prefix
     CLUSTER_NAME=$(echo "{{ cluster }}" | sed -e 's/k3d-//g')
@@ -246,7 +249,7 @@ cluster-create cluster port *args='': start-registry && (apply-coredns cluster)
 [private]
 cluster-delete *args='':
     #!/bin/bash
-    set -euox pipefail
+    set -euo pipefail
 
     # remove the k3d- prefix
     ARGS=$(echo "{{ args }}" | sed -e 's/k3d-//g')
@@ -263,7 +266,7 @@ cluster-list:
 
 # Stop all Syft clusters
 [group('cluster')]
-stop-all: (cluster-delete cluster_high cluster_low cluster_gw cluster_signoz)
+stop-all: (cluster-delete cluster_default cluster_high cluster_low cluster_gw cluster_signoz)
     @echo "Stopped all Syft clusters"
 
 [group('cluster')]
diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml
index ae6245df2c1..0e60e893a4b 100644
--- a/packages/grid/devspace.yaml
+++ b/packages/grid/devspace.yaml
@@ -132,7 +132,7 @@ profiles:
         value:
           tracing:
             enabled: true
-            otlpEndpoint: "http://syft-signoz-otel-collector.platform:4317"
+            otlpEndpoint: "http://host.k3d.internal:4317"
             otelProtocol: "grpc"
 
   - name: bigquery-scenario-tests

From f911b3194579ef6133d2a658ed7bbbdab853f173 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Fri, 13 Sep 2024 14:45:10 +0530
Subject: [PATCH 05/17] drop echos

---
 justfile | 29 ++---------------------------
 1 file changed, 2 insertions(+), 27 deletions(-)

diff --git a/justfile b/justfile
index 544f3937256..c08c6fd6d7f 100644
--- a/justfile
+++ b/justfile
@@ -22,7 +22,6 @@ signoz_otel_url := "http://host.k3d.internal:" + port_signoz_otel
 
 profiles := ""
 tracing := "true"
-
 _g_profiles := if tracing == "true" { profiles + ",tracing" } else { profiles }
 
 python_path := `which python || which python3`
@@ -42,7 +41,7 @@ start-registry:
     @-k3d registry create registry.localhost --port {{ port_registry }} -v k3d-registry-vol:/var/lib/registry --no-help
 
     if ! grep -q k3d-registry.localhost /etc/hosts; then \
-        sudo {{python_path}} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; \
+        sudo {{ python_path }} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; \
     fi
 
     @curl --silent --retry 5 --retry-all-errors http://k3d-registry.localhost:{{ port_registry }}/v2/_catalog | jq
@@ -58,114 +57,91 @@ stop-registry:
 # Launch Syft Datasite high-side cluster on http://localhost:{{port_high}}
 [group('highside')]
 start-high: (cluster-delete cluster_high) (cluster-create cluster_high port_high)
-    @echo "Started Syft Datasite (high-side) on http://localhost:{{ port_high }}/"
 
 # Stop Syft Datasite high-side cluster
 [group('highside')]
 stop-high: (cluster-delete cluster_high)
-    @echo "Stopped Syft Datasite (high-side)"
 
 [group('highside')]
 deploy-high: (devspace-deploy cluster_high ns_default)
-    @echo "Done"
 
 [group('highside')]
 reset-high: (state-reset cluster_high ns_default)
-    @echo "Done"
 
 [group('highside')]
 cleanup-high: (devspace-purge cluster_high ns_default) && (ns-cleanup cluster_high ns_default)
-    @echo "Done"
 
 # ---------------------------------------------------------------------------------------------------------------------
 
 # Launch Syft Datasite low-side cluster on http://localhost:{{port_low}}
 [group('lowside')]
 start-low: (cluster-create cluster_low port_low)
-    @echo "Started Syft Datasite (low-side) on http://localhost:{{ port_low }}/"
 
 # Stop Syft Datasite low-side cluster
 [group('lowside')]
 stop-low: (cluster-delete cluster_low)
-    @echo "Stopped Syft Datasite (low-side)"
 
 [group('lowside')]
 deploy-low: (devspace-deploy cluster_low ns_default "-p datasite-low")
-    @echo "Done"
 
 [group('lowside')]
 reset-low: (state-reset cluster_low ns_default)
-    @echo "Done"
 
 [group('lowside')]
 cleanup-low: (devspace-purge cluster_low ns_default) && (ns-cleanup cluster_low ns_default)
-    @echo "Done"
 
 # ---------------------------------------------------------------------------------------------------------------------
 
 # Launch Syft Gateway cluster on http://localhost:{{port_gw}}
 [group('gateway')]
 start-gw: (cluster-create cluster_gw port_gw)
-    @echo "Started Syft Gateway on http://localhost:{{ port_gw }}/"
 
 # Stop Syft Gateway cluster
 [group('gateway')]
 stop-gw: (cluster-delete cluster_gw)
-    @echo "Stopped Syft Gateway"
 
 [group('gateway')]
 deploy-gw: (devspace-deploy cluster_gw ns_default "-p gateway")
-    @echo "Done"
 
 [group('gateway')]
 reset-gw: (state-reset cluster_gw ns_default)
-    @echo "Done"
 
 [group('gateway')]
 cleanup-gw: (devspace-purge cluster_gw ns_default) && (ns-cleanup cluster_gw ns_default)
-    @echo "Done"
 
 # ---------------------------------------------------------------------------------------------------------------------
 
 # Launch a Syft shared cluster on http://localhost:{{port_default}}
 [group('shared')]
-start-shared: (cluster-create cluster_default port_default "--agents 2 --servers 1")
-    @echo "Started Syft on http://localhost:{{ port_default }}/"
+start-shared: (cluster-create cluster_default port_default "--agents 2")
 
 # Stop Syft shared cluster
 [group('shared')]
 stop-shared: (cluster-delete cluster_default)
-    @echo "Stopped Syft cluster"
 
 # Deploy to "high" namespace on the shared cluster
 [group('shared')]
 deploy-ns-high: (devspace-deploy cluster_default ns_high)
-    @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "high" namespace
 [group('shared')]
 delete-ns-high: (ns-cleanup cluster_default ns_high)
-    @echo "Done"
 
 # Deploy to "low" namespace on the shared cluster
 [group('shared')]
 deploy-ns-low: (devspace-deploy cluster_default ns_low "-p datasite-low")
-    @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "low" namespace
 [group('shared')]
 delete-ns-low: (ns-cleanup cluster_default ns_low)
-    @echo "Done"
 
 # Deploy to "gw" namespace on the shared cluster
 [group('shared')]
 deploy-ns-gw: (devspace-deploy cluster_default ns_gw "-p gateway")
-    @echo "Deployed Syft Gateway on {{ cluster_default }}"
 
 # Delete the "gw" namespace
 [group('shared')]
 delete-ns-gw: (ns-cleanup cluster_default ns_gw)
-    @echo "Done"
 
 # ---------------------------------------------------------------------------------------------------------------------
 
@@ -186,7 +162,6 @@ delete-collector:
 # Stop SigNoz cluster
 [group('signoz')]
 stop-signoz: (cluster-delete cluster_signoz)
-    @echo "Stopped SigNoz cluster"
 
 [group('signoz')]
 [private]

From 410fe3847701ba2b9c3b4530c0f9b7e2e0b39bd5 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Sun, 15 Sep 2024 16:04:08 +0530
Subject: [PATCH 06/17] refactor + comments

---
 justfile | 246 +++++++++++++++++++++++++++++++++----------------------
 1 file changed, 150 insertions(+), 96 deletions(-)

diff --git a/justfile b/justfile
index c08c6fd6d7f..d7a7036d08e 100644
--- a/justfile
+++ b/justfile
@@ -3,6 +3,12 @@ cluster_high := "k3d-syft-high"
 cluster_low := "k3d-syft-low"
 cluster_gw := "k3d-syft-gw"
 cluster_signoz := "k3d-signoz"
+ns_default := "syft"
+ns_high := "high"
+ns_low := "low"
+ns_gw := "gw"
+
+# ---------------------------------------------------------------------------------------------------------------------
 
 port_default := "8080"
 port_high := port_default
@@ -12,18 +18,23 @@ port_signoz_ui := "3301"
 port_signoz_otel := "4317"
 port_registry := "5800"
 
-ns_default := "syft"
-ns_high := "high"
-ns_low := "low"
-ns_gw := "gw"
-
 registry_url := "k3d-registry.localhost:" + port_registry
 signoz_otel_url := "http://host.k3d.internal:" + port_signoz_otel
 
+# ---------------------------------------------------------------------------------------------------------------------
+
+# devspace profiles (comma-separated)
 profiles := ""
+
+# enable tracing by adding "tracing" profile in devspace
 tracing := "true"
+
 _g_profiles := if tracing == "true" { profiles + ",tracing" } else { profiles }
 
+# ---------------------------------------------------------------------------------------------------------------------
+
+# this might break if you have alias python = python3 or either of the executable not pointing to the correct one
+# just fix your system instead of making of fixing this
 python_path := `which python || which python3`
 
 # ---------------------------------------------------------------------------------------------------------------------
@@ -48,100 +59,102 @@ start-registry:
     @echo "\033[1;32mRegistring running at http://k3d-registry.localhost:{{ port_registry }}\033[0m"
 
 [group('registry')]
-stop-registry:
+delete-registry:
     -k3d registry delete registry.localhost
     -docker volume rm k3d-registry-vol
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-# Launch Syft Datasite high-side cluster on http://localhost:{{port_high}}
+# Launch a Datasite high-side cluster on http://localhost:{{port_high}}
 [group('highside')]
-start-high: (cluster-delete cluster_high) (cluster-create cluster_high port_high)
+start-high: (delete-cluster cluster_high) (create-cluster cluster_high port_high)
 
-# Stop Syft Datasite high-side cluster
+# Stop the Datasite high-side cluster
 [group('highside')]
-stop-high: (cluster-delete cluster_high)
+delete-high: (delete-cluster cluster_high)
 
 [group('highside')]
-deploy-high: (devspace-deploy cluster_high ns_default)
+deploy-high: (deploy-devspace cluster_high ns_default)
 
 [group('highside')]
-reset-high: (state-reset cluster_high ns_default)
+reset-high: (reset-syft cluster_high ns_default)
 
+# Remove devpsace deployment + namespace from the high-side cluster
 [group('highside')]
-cleanup-high: (devspace-purge cluster_high ns_default) && (ns-cleanup cluster_high ns_default)
+cleanup-high: (purge-devspace cluster_high ns_default) (delete-ns cluster_high ns_default)
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-# Launch Syft Datasite low-side cluster on http://localhost:{{port_low}}
+# Launch a Datasite low-side cluster on http://localhost:{{port_low}}
 [group('lowside')]
-start-low: (cluster-create cluster_low port_low)
+start-low: (create-cluster cluster_low port_low)
 
-# Stop Syft Datasite low-side cluster
+# Stop the Datasite low-side cluster
 [group('lowside')]
-stop-low: (cluster-delete cluster_low)
+delete-low: (delete-cluster cluster_low)
 
+# Deploy Syft to the low-side cluster
 [group('lowside')]
-deploy-low: (devspace-deploy cluster_low ns_default "-p datasite-low")
+deploy-low: (deploy-devspace cluster_low ns_default "-p datasite-low")
 
+# Reset Syft DB state in the low-side cluster
 [group('lowside')]
-reset-low: (state-reset cluster_low ns_default)
+reset-low: (reset-syft cluster_low ns_default)
 
+# Remove devpsace deployment + namespace from the low-side cluster
 [group('lowside')]
-cleanup-low: (devspace-purge cluster_low ns_default) && (ns-cleanup cluster_low ns_default)
+cleanup-low: (purge-devspace cluster_low ns_default) (delete-ns cluster_low ns_default)
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-# Launch Syft Gateway cluster on http://localhost:{{port_gw}}
+# Launch a Gateway cluster on http://localhost:{{port_gw}}
 [group('gateway')]
-start-gw: (cluster-create cluster_gw port_gw)
+start-gw: (create-cluster cluster_gw port_gw)
 
-# Stop Syft Gateway cluster
+# Delete the Gateway cluster
 [group('gateway')]
-stop-gw: (cluster-delete cluster_gw)
+delete-gw: (delete-cluster cluster_gw)
 
+# Deploy Syft to the gateway cluster
 [group('gateway')]
-deploy-gw: (devspace-deploy cluster_gw ns_default "-p gateway")
+deploy-gw: (deploy-devspace cluster_gw ns_default "-p gateway")
 
+# Reset Syft DB state in the gateway cluster
 [group('gateway')]
-reset-gw: (state-reset cluster_gw ns_default)
+reset-gw: (reset-syft cluster_gw ns_default)
 
+# Remove devpsace deployment + namespace from the gateway cluster
 [group('gateway')]
-cleanup-gw: (devspace-purge cluster_gw ns_default) && (ns-cleanup cluster_gw ns_default)
+cleanup-gw: (purge-devspace cluster_gw ns_default) (delete-ns cluster_gw ns_default)
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-# Launch a Syft shared cluster on http://localhost:{{port_default}}
-[group('shared')]
-start-shared: (cluster-create cluster_default port_default "--agents 2")
+# TODO - multi-namespace -> unique k3d ports
+# # Launch a multi-agent cluster on http://localhost:{{port_default}}
+# [group('shared')]
+# start-shared: (create-cluster cluster_default port_default "--agents 2")
 
-# Stop Syft shared cluster
-[group('shared')]
-stop-shared: (cluster-delete cluster_default)
+# # Stop the multi-agent cluster
+# [group('shared')]
+# delete-shared: (delete-cluster cluster_default)
 
-# Deploy to "high" namespace on the shared cluster
-[group('shared')]
-deploy-ns-high: (devspace-deploy cluster_default ns_high)
+# [group('shared')]
+# deploy-ns-high: (deploy-devspace cluster_default ns_high)
 
-# Delete the "high" namespace
-[group('shared')]
-delete-ns-high: (ns-cleanup cluster_default ns_high)
+# [group('shared')]
+# delete-ns-high: (delete-ns cluster_default ns_high)
 
-# Deploy to "low" namespace on the shared cluster
-[group('shared')]
-deploy-ns-low: (devspace-deploy cluster_default ns_low "-p datasite-low")
+# [group('shared')]
+# deploy-ns-low: (deploy-devspace cluster_default ns_low "-p datasite-low")
 
-# Delete the "low" namespace
-[group('shared')]
-delete-ns-low: (ns-cleanup cluster_default ns_low)
+# [group('shared')]
+# delete-ns-low: (delete-ns cluster_default ns_low)
 
-# Deploy to "gw" namespace on the shared cluster
-[group('shared')]
-deploy-ns-gw: (devspace-deploy cluster_default ns_gw "-p gateway")
+# [group('shared')]
+# deploy-ns-gw: (deploy-devspace cluster_default ns_gw "-p gateway")
 
-# Delete the "gw" namespace
-[group('shared')]
-delete-ns-gw: (ns-cleanup cluster_default ns_gw)
+# [group('shared')]
+# delete-ns-gw: (delete-ns cluster_default ns_gw)
 
 # ---------------------------------------------------------------------------------------------------------------------
 
@@ -153,7 +166,9 @@ start-signoz: && apply-signoz setup-signoz
         --port {{ port_signoz_otel }}:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"
 
-    @printf "Started SigNoz\nDashboard: http://localhost:{{ port_signoz_ui }}\nOTEL Endpoint: http://localhost:{{ port_signoz_otel }}\n"
+    @printf "Started SigNoz\n\
+        Dashboard: \033[1;36mhttp://localhost:{{ port_signoz_ui }}\033[0m\n\
+        OTEL Endpoint: \033[1;36mhttp://localhost:{{ port_signoz_otel }}\033[0m\n"
 
 [group('signoz')]
 delete-collector:
@@ -161,7 +176,7 @@ delete-collector:
 
 # Stop SigNoz cluster
 [group('signoz')]
-stop-signoz: (cluster-delete cluster_signoz)
+delete-signoz: (delete-cluster cluster_signoz)
 
 [group('signoz')]
 [private]
@@ -195,7 +210,8 @@ apply-signoz:
 [private]
 setup-signoz:
     @echo "Waiting for SigNoz frontend to be available..."
-    @bash ./packages/grid/scripts/wait_for.sh service signoz-frontend --namespace platform --context {{ cluster_signoz }} &> /dev/null
+    @bash ./packages/grid/scripts/wait_for.sh service signoz-frontend \
+        --namespace platform --context {{ cluster_signoz }} &> /dev/null
 
     @echo "Setting up SigNoz account"
     @curl --retry 5 --retry-all-errors -X POST \
@@ -203,13 +219,26 @@ setup-signoz:
         --data '{"email":"admin@localhost","name":"admin","orgName":"openmined","password":"password"}' \
         http://localhost:3301/api/v1/register
 
-    @printf '\nSignoz is running on http://localhost:3301\nEmail: \033[1;36madmin@localhost\033[0m\nPassword: \033[1;36mpassword\033[0m\n'
+    @printf '\nSignoz is running on http://localhost:3301\n\
+        Email: \033[1;36madmin@localhost\033[0m\n\
+        Password: \033[1;36mpassword\033[0m\n'
 
 # ---------------------------------------------------------------------------------------------------------------------
 
+# List all clusters
+[group('cluster')]
+list-clusters:
+    k3d cluster list
+
+# Stop all clusters
+[group('cluster')]
+[confirm('Confirm delete all clusters?')]
+delete-clusters:
+    k3d cluster delete --all
+
 [group('cluster')]
 [private]
-cluster-create cluster port *args='': start-registry && (apply-coredns cluster) (apply-collector cluster)
+create-cluster cluster port *args='': start-registry && (apply-coredns cluster) (apply-collector cluster)
     #!/bin/bash
     set -euo pipefail
 
@@ -222,7 +251,7 @@ cluster-create cluster port *args='': start-registry && (apply-coredns cluster)
 
 [group('cluster')]
 [private]
-cluster-delete *args='':
+delete-cluster *args='':
     #!/bin/bash
     set -euo pipefail
 
@@ -232,18 +261,9 @@ cluster-delete *args='':
 
 [group('cluster')]
 [private]
-ns-cleanup context namespace:
+delete-ns context namespace:
     kubectl delete ns {{ namespace }} --force --grace-period=0 --context {{ context }}
 
-[group('cluster')]
-cluster-list:
-    k3d cluster list
-
-# Stop all Syft clusters
-[group('cluster')]
-stop-all: (cluster-delete cluster_default cluster_high cluster_low cluster_gw cluster_signoz)
-    @echo "Stopped all Syft clusters"
-
 [group('cluster')]
 [private]
 apply-coredns cluster:
@@ -252,30 +272,11 @@ apply-coredns cluster:
     kubectl apply -f ./scripts/k8s-coredns-custom.yml --context {{ cluster }}
     kubectl delete pod -n kube-system -l k8s-app=kube-dns --context {{ cluster }}
 
-[group('cluster')]
-k9s-high:
-    k9s --context {{ cluster_high }}
-
-[group('cluster')]
-k9s-low:
-    k9s --context {{ cluster_low }}
-
-[group('cluster')]
-k9s-gw:
-    k9s --context {{ cluster_gw }}
-
-[group('cluster')]
-k9s-signoz:
-    k9s --context {{ cluster_signoz }}
-
-[group('cluster')]
-k9s-shared:
-    k9s --context {{ cluster_default }}
-
 # ---------------------------------------------------------------------------------------------------------------------
 
+[group('devspace')]
 [private]
-devspace-deploy cluster namespace *args='':
+deploy-devspace cluster namespace *args='':
     #!/bin/bash
     set -euo pipefail
 
@@ -295,8 +296,9 @@ devspace-deploy cluster namespace *args='':
         {{ args }} \
         --var CONTAINER_REGISTRY={{ registry_url }}
 
+[group('devspace')]
 [private]
-devspace-purge cluster namespace:
+purge-devspace cluster namespace:
     #!/bin/bash
     set -euo pipefail
 
@@ -314,21 +316,73 @@ deploy-cloud cluster registry namespace profile:
 
     kubectl config get-contexts {{ cluster }} > /dev/null
     # cloud deployments always have tracing false + platform=amd64
-    just tracing=false registry_url={{ registry }} devspace-deploy {{ cluster }} {{ namespace }} "-p {{ profile }} --var PLATFORM=amd64"
+    just tracing=false registry_url={{ registry }} \
+        deploy-devspace {{ cluster }} {{ namespace }} "-p {{ profile }} --var PLATFORM=amd64"
 
 [group('cloud')]
-deploy-gcp-high gke_cluster gcp_registry namespace="syft": (deploy-cloud gke_cluster gcp_registry namespace "gcp")
+[private]
+purge-cloud cluster namespace:
+    #!/bin/bash
+    set -euo pipefail
 
-[group('cloud')]
-deploy-gcp-low gke_cluster gcp_registry namespace="syft": (deploy-cloud gke_cluster gcp_registry namespace "gcp-low")
+    kubectl config get-contexts {{ cluster }} > /dev/null
+    just purge-devspace {{ cluster }} {{ namespace }}
 
-[group('cloud')]
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Auth all components required for deploying Syft to Google Cloud
+[group('cloud-gcp')]
+auth-gcloud gcp_project gcp_region gcp_cluster gcp_registry:
+    # get artifact registry registry
+    gcloud auth configure-docker {{ gcp_registry }} --quiet
+    # get cluster credentials
+    gcloud container clusters get-credentials {{ gcp_cluster }} --region {{ gcp_region }} --project {{ gcp_project }}
+
+# Deploy local code as datasite-high to Google Kubernetes Engine
+[group('cloud-gcp')]
+deploy-gcp-high gcp_cluster gcp_registry namespace="syft": (deploy-cloud gcp_cluster gcp_registry namespace "gcp")
+
+# Deploy local code as datasite-high to Google Kubernetes Engine
+[group('cloud-gcp')]
+deploy-gcp-low gcp_cluster gcp_registry namespace="syft": (deploy-cloud gcp_cluster gcp_registry namespace "gcp-low")
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Deploy local code as datasite-high to Azure Kubernetes Service
+[group('cloud-az')]
 deploy-az-high aks_cluster az_registry namespace="syft": (deploy-cloud aks_cluster az_registry namespace "azure")
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-[group('syft')]
+# Reset Syft state in a cluster
+[group('utils')]
 [private]
-state-reset name namespace:
+reset-syft name namespace:
     kubectl config use-context {{ name }}
     scripts/reset_k8s.sh
+
+# K9s into the Datasite High cluster
+[group('utils')]
+k9s-high:
+    k9s --context {{ cluster_high }}
+
+# K9s into the Datesite Low cluster
+[group('utils')]
+k9s-low:
+    k9s --context {{ cluster_low }}
+
+# K9s into the Gateway cluster
+[group('utils')]
+k9s-gw:
+    k9s --context {{ cluster_gw }}
+
+# K9s into the Signoz cluster
+[group('utils')]
+k9s-signoz:
+    k9s --context {{ cluster_signoz }}
+
+# Stop all Syft clusters + registry
+[group('utils')]
+[confirm('Confirm delete all clusters, registry and volume?')]
+delete-all: delete-clusters delete-registry
+    @echo "Stopped all Syft components"

From d136903e7ae6651bb1893441d0f02366fdbf2aae Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Sun, 15 Sep 2024 19:25:06 +0530
Subject: [PATCH 07/17] add some more commands

---
 justfile | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/justfile b/justfile
index d7a7036d08e..c6cea2e8586 100644
--- a/justfile
+++ b/justfile
@@ -73,9 +73,11 @@ start-high: (delete-cluster cluster_high) (create-cluster cluster_high port_high
 [group('highside')]
 delete-high: (delete-cluster cluster_high)
 
+# Deploy Syft to the high-side cluster
 [group('highside')]
 deploy-high: (deploy-devspace cluster_high ns_default)
 
+# Reset Syft DB state in the high-side cluster
 [group('highside')]
 reset-high: (reset-syft cluster_high ns_default)
 
@@ -162,6 +164,7 @@ cleanup-gw: (purge-devspace cluster_gw ns_default) (delete-ns cluster_gw ns_defa
 [group('signoz')]
 start-signoz: && apply-signoz setup-signoz
     k3d cluster create signoz \
+        --no-image-volume \
         --port {{ port_signoz_ui }}:3301@loadbalancer \
         --port {{ port_signoz_otel }}:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"
@@ -170,11 +173,12 @@ start-signoz: && apply-signoz setup-signoz
         Dashboard: \033[1;36mhttp://localhost:{{ port_signoz_ui }}\033[0m\n\
         OTEL Endpoint: \033[1;36mhttp://localhost:{{ port_signoz_otel }}\033[0m\n"
 
+# Remove SigNoz from the cluster
 [group('signoz')]
 delete-collector:
     helm uninstall k8s-infra
 
-# Stop SigNoz cluster
+# Remove SigNoz from the cluster
 [group('signoz')]
 delete-signoz: (delete-cluster cluster_signoz)
 
@@ -232,7 +236,6 @@ list-clusters:
 
 # Stop all clusters
 [group('cluster')]
-[confirm('Confirm delete all clusters?')]
 delete-clusters:
     k3d cluster delete --all
 
@@ -312,7 +315,7 @@ purge-devspace cluster namespace:
 [private]
 deploy-cloud cluster registry namespace profile:
     #!/bin/bash
-    set -euo pipefail
+    set -euox pipefail
 
     kubectl config get-contexts {{ cluster }} > /dev/null
     # cloud deployments always have tracing false + platform=amd64
@@ -383,6 +386,20 @@ k9s-signoz:
 
 # Stop all Syft clusters + registry
 [group('utils')]
-[confirm('Confirm delete all clusters, registry and volume?')]
 delete-all: delete-clusters delete-registry
     @echo "Stopped all Syft components"
+
+[group('utils')]
+[confirm('Confirm prune all docker resources?')]
+prune-docker:
+    -docker container prune -f
+    -docker volume prune -af
+    -docker image prune -af
+    -docker builder prune -af
+    -docker buildx prune -af
+    -docker system prune -af --volumes
+
+[group('utils')]
+yank-ns namespace:
+    -kubectl delete ns {{ namespace }} --now --timeout=5s
+    kubectl get ns {{ namespace }} -o json | jq '.spec.finalizers = []' | kubectl replace --raw /api/v1/namespaces/{{ namespace }}/finalize -f -

From 74355feae3ddcb0e8937c8e15552e4b05ed87617 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Mon, 16 Sep 2024 00:56:18 +0530
Subject: [PATCH 08/17] print macos x-plat build warning

---
 justfile | 49 +++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/justfile b/justfile
index c6cea2e8586..ce253500739 100644
--- a/justfile
+++ b/justfile
@@ -291,6 +291,8 @@ deploy-devspace cluster namespace *args='':
         PROFILE="-p $PROFILE"
     fi
 
+    echo "Deploying to {{ cluster }}"
+
     devspace deploy -b \
         --no-warn \
         --kube-context {{ cluster }} \
@@ -313,14 +315,40 @@ purge-devspace cluster namespace:
 
 [group('cloud')]
 [private]
-deploy-cloud cluster registry namespace profile:
+check-platform:
     #!/bin/bash
-    set -euox pipefail
+    set -euo pipefail
+
+    OSTYPE=$(uname -sm)
+    MSG="==================================================================================================\n\
+    Deploying dev->cloud k8s (x64 nodes) requires images to be built with --platform=linux/amd64\n\
+    On Apple Silicon, cross-platform image is unstable on different providers\n\n\
+    Current status:\n\
+    ✅ | Docker Desktop | 4.34.0+ | *Enable* containerd and *uncheck* 'Use Rosetta for x86_64/amd64...'\n\
+    ❌ | OrbStack       | 1.7.2   | Rosetta: gets stuck & qemu: errors with 'illegal instruction'\n\
+    ❌ | Lima VM/Colima | 0.23.2  | Rosetta: gets stuck & qemu: errors with 'illegal instruction'\n\
+    =================================================================================================="
+
+    if [[ "$OSTYPE" == "Darwin arm64" ]]; then
+        echo -e $MSG
+    fi
+
+[group('cloud')]
+[private]
+deploy-cloud cluster registry namespace profile: check-platform
+    #!/bin/bash
+    set -euo pipefail
+
+    CONTEXT_NAME=$(kubectl config get-contexts -o=name | grep "{{ cluster }}")
+
+    if [ -z "$CONTEXT_NAME" ]; then
+        echo "No context found for cluster '{{ cluster }}'"
+        exit 1
+    fi
 
-    kubectl config get-contexts {{ cluster }} > /dev/null
     # cloud deployments always have tracing false + platform=amd64
     just tracing=false registry_url={{ registry }} \
-        deploy-devspace {{ cluster }} {{ namespace }} "-p {{ profile }} --var PLATFORM=amd64"
+        deploy-devspace $CONTEXT_NAME {{ namespace }} "-p {{ profile }} --var PLATFORM=amd64"
 
 [group('cloud')]
 [private]
@@ -328,8 +356,14 @@ purge-cloud cluster namespace:
     #!/bin/bash
     set -euo pipefail
 
-    kubectl config get-contexts {{ cluster }} > /dev/null
-    just purge-devspace {{ cluster }} {{ namespace }}
+    CONTEXT_NAME=$(kubectl config get-contexts -o=name | grep "{{ cluster }}")
+
+    if [ -z "$CONTEXT_NAME" ]; then
+        echo "No context found for cluster '{{ cluster }}'"
+        exit 1
+    fi
+
+    just purge-devspace $CONTEXT_NAME {{ namespace }}
 
 # ---------------------------------------------------------------------------------------------------------------------
 
@@ -345,6 +379,9 @@ auth-gcloud gcp_project gcp_region gcp_cluster gcp_registry:
 [group('cloud-gcp')]
 deploy-gcp-high gcp_cluster gcp_registry namespace="syft": (deploy-cloud gcp_cluster gcp_registry namespace "gcp")
 
+[group('cloud-gcp')]
+purge-gcp-high gcp_cluster namespace: (purge-cloud gcp_cluster namespace)
+
 # Deploy local code as datasite-high to Google Kubernetes Engine
 [group('cloud-gcp')]
 deploy-gcp-low gcp_cluster gcp_registry namespace="syft": (deploy-cloud gcp_cluster gcp_registry namespace "gcp-low")

From ac9c59a5912ad55b75f02a8126abaf2b9dafee60 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Mon, 16 Sep 2024 01:09:22 +0530
Subject: [PATCH 09/17] gcloud login

---
 justfile                                      | 12 +++++
 .../{gcp.nosync.yaml => gcp.bucketsync.yaml}  | 51 ++++++++++++++++---
 packages/grid/helm/examples/gcp/gcp.high.yaml | 51 +++----------------
 3 files changed, 63 insertions(+), 51 deletions(-)
 rename packages/grid/helm/examples/gcp/{gcp.nosync.yaml => gcp.bucketsync.yaml} (56%)

diff --git a/justfile b/justfile
index ce253500739..cc0be98e173 100644
--- a/justfile
+++ b/justfile
@@ -370,8 +370,20 @@ purge-cloud cluster namespace:
 # Auth all components required for deploying Syft to Google Cloud
 [group('cloud-gcp')]
 auth-gcloud gcp_project gcp_region gcp_cluster gcp_registry:
+    #!/bin/bash
+    set -euo pipefail
+
+    ACCOUNT=$(gcloud config get-value account)
+    if [ -z "$ACCOUNT" ]; then
+        gcloud auth login
+    fi
+
+    # install gke-gcloud-auth-plugin
+    gcloud components install gke-gcloud-auth-plugin
+
     # get artifact registry registry
     gcloud auth configure-docker {{ gcp_registry }} --quiet
+
     # get cluster credentials
     gcloud container clusters get-credentials {{ gcp_cluster }} --region {{ gcp_region }} --project {{ gcp_project }}
 
diff --git a/packages/grid/helm/examples/gcp/gcp.nosync.yaml b/packages/grid/helm/examples/gcp/gcp.bucketsync.yaml
similarity index 56%
rename from packages/grid/helm/examples/gcp/gcp.nosync.yaml
rename to packages/grid/helm/examples/gcp/gcp.bucketsync.yaml
index 02935edfd8f..efdbbe72e68 100644
--- a/packages/grid/helm/examples/gcp/gcp.nosync.yaml
+++ b/packages/grid/helm/examples/gcp/gcp.bucketsync.yaml
@@ -1,14 +1,16 @@
 # =================================================================================
 # Syft on GKE Cluster
 #
-# Server side  : high
-# Automount  : NO
-# Ingress    : gce
+# Server side : high
+# Automount : YES. 1 GCS bucket mounted to seaweedfs
+# Ingress   : gce
 # Extras:
 #  - BackendConfig for increased timeout
+#  - Secret for seaweedfs mount
 # =================================================================================
 
 server:
+  # Basic Server Config
   name: syft-gcp
   side: high
 
@@ -16,7 +18,16 @@ server:
   # Useful when workload identity is setup
   useInternalRegistry: false
 
-  # Resources set inline with c3-standard-4 machine type
+  # Force backend to write results to this bucket
+  # should be same as mountApi.mounts.local_bucket
+  defaultBucketName: syft-bucket-high-gcs
+
+  # For autopilot clusters with GKE 1.28+, uncomment this
+  # nodeSelector:
+  #   cloud.google.com/compute-class: Performance
+  #   cloud.google.com/machine-family: c3
+
+  # Pod resources set inline with c3-standard-4 machine type
   resources:
     requests:
       cpu: 2
@@ -28,9 +39,28 @@ server:
 # =================================================================================
 
 seaweedfs:
+  # SeaweedFS PVC size
   storageSize: 100Gi
 
-  # Resources set inline with c3-standard-4 machine type
+  # For autopilot clusters with GKE 1.28+, uncomment this
+  # nodeSelector:
+  #   cloud.google.com/compute-class: Performance
+  #   cloud.google.com/machine-family: c3
+
+  # Automount Config
+  # -- Mounts GCS bucket "syft-bucket-high" to SeaweedFS bucket "syft-bucket-high-gcs"
+  # -- "gcs_creds.json" must exist in "seaweedfs-mount-secret"
+  # -- "seaweedfs-mount-secret" must be provisioned externally or in extraResources
+  mountApi:
+    mounts:
+      - local_bucket: syft-bucket-high-gcs
+        remote_bucket:
+          type: gcs
+          bucket_name: syft-bucket-high
+          creds: /run/secrets/mount/gcs_creds.json
+    secretKeyName: seaweedfs-mount-secret
+
+  # Pod resources set inline with c3-standard-4 machine type
   resources:
     requests:
       cpu: 2
@@ -73,7 +103,7 @@ mongo:
 # =================================================================================
 
 extraResources:
-  # Configure load balancer backend service
+  # Configure GCE load balancer backend
   # https://cloud.google.com/kubernetes-engine/docs/how-to/ingress-configuration#configuring_ingress_features_through_backendconfig_parameters
   - apiVersion: cloud.google.com/v1
     kind: BackendConfig
@@ -81,3 +111,12 @@ extraResources:
       name: custom-backend-config
     spec:
       timeoutSec: 1800
+
+  # Secret to mount GCS bucket in seaweedfs
+  - apiVersion: v1
+    kind: Secret
+    metadata:
+      name: seaweedfs-mount-secret
+    type: Opaque
+    data:
+      gcs_creds.json: base64 encoded value
diff --git a/packages/grid/helm/examples/gcp/gcp.high.yaml b/packages/grid/helm/examples/gcp/gcp.high.yaml
index efdbbe72e68..02935edfd8f 100644
--- a/packages/grid/helm/examples/gcp/gcp.high.yaml
+++ b/packages/grid/helm/examples/gcp/gcp.high.yaml
@@ -1,16 +1,14 @@
 # =================================================================================
 # Syft on GKE Cluster
 #
-# Server side : high
-# Automount : YES. 1 GCS bucket mounted to seaweedfs
-# Ingress   : gce
+# Server side  : high
+# Automount  : NO
+# Ingress    : gce
 # Extras:
 #  - BackendConfig for increased timeout
-#  - Secret for seaweedfs mount
 # =================================================================================
 
 server:
-  # Basic Server Config
   name: syft-gcp
   side: high
 
@@ -18,16 +16,7 @@ server:
   # Useful when workload identity is setup
   useInternalRegistry: false
 
-  # Force backend to write results to this bucket
-  # should be same as mountApi.mounts.local_bucket
-  defaultBucketName: syft-bucket-high-gcs
-
-  # For autopilot clusters with GKE 1.28+, uncomment this
-  # nodeSelector:
-  #   cloud.google.com/compute-class: Performance
-  #   cloud.google.com/machine-family: c3
-
-  # Pod resources set inline with c3-standard-4 machine type
+  # Resources set inline with c3-standard-4 machine type
   resources:
     requests:
       cpu: 2
@@ -39,28 +28,9 @@ server:
 # =================================================================================
 
 seaweedfs:
-  # SeaweedFS PVC size
   storageSize: 100Gi
 
-  # For autopilot clusters with GKE 1.28+, uncomment this
-  # nodeSelector:
-  #   cloud.google.com/compute-class: Performance
-  #   cloud.google.com/machine-family: c3
-
-  # Automount Config
-  # -- Mounts GCS bucket "syft-bucket-high" to SeaweedFS bucket "syft-bucket-high-gcs"
-  # -- "gcs_creds.json" must exist in "seaweedfs-mount-secret"
-  # -- "seaweedfs-mount-secret" must be provisioned externally or in extraResources
-  mountApi:
-    mounts:
-      - local_bucket: syft-bucket-high-gcs
-        remote_bucket:
-          type: gcs
-          bucket_name: syft-bucket-high
-          creds: /run/secrets/mount/gcs_creds.json
-    secretKeyName: seaweedfs-mount-secret
-
-  # Pod resources set inline with c3-standard-4 machine type
+  # Resources set inline with c3-standard-4 machine type
   resources:
     requests:
       cpu: 2
@@ -103,7 +73,7 @@ mongo:
 # =================================================================================
 
 extraResources:
-  # Configure GCE load balancer backend
+  # Configure load balancer backend service
   # https://cloud.google.com/kubernetes-engine/docs/how-to/ingress-configuration#configuring_ingress_features_through_backendconfig_parameters
   - apiVersion: cloud.google.com/v1
     kind: BackendConfig
@@ -111,12 +81,3 @@ extraResources:
       name: custom-backend-config
     spec:
       timeoutSec: 1800
-
-  # Secret to mount GCS bucket in seaweedfs
-  - apiVersion: v1
-    kind: Secret
-    metadata:
-      name: seaweedfs-mount-secret
-    type: Opaque
-    data:
-      gcs_creds.json: base64 encoded value

From 097748fbcadad9558162e1f7982604b284d1536b Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Mon, 16 Sep 2024 01:14:47 +0530
Subject: [PATCH 10/17] gcloud check plugin

---
 justfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/justfile b/justfile
index cc0be98e173..d12a24e1ef2 100644
--- a/justfile
+++ b/justfile
@@ -373,13 +373,17 @@ auth-gcloud gcp_project gcp_region gcp_cluster gcp_registry:
     #!/bin/bash
     set -euo pipefail
 
+    # login to gcloud
     ACCOUNT=$(gcloud config get-value account)
     if [ -z "$ACCOUNT" ]; then
         gcloud auth login
     fi
 
     # install gke-gcloud-auth-plugin
-    gcloud components install gke-gcloud-auth-plugin
+    gke_installed=$(gcloud components list --only-local-state --filter gke-gcloud-auth-plugin --format=list 2>/dev/null)
+    if [ -z "$gke_installed" ]; then
+        gcloud components install gke-gcloud-auth-plugin
+    fi
 
     # get artifact registry registry
     gcloud auth configure-docker {{ gcp_registry }} --quiet

From 4ceb38c52e0c44ef5567d2829cc92e803c089c17 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Mon, 16 Sep 2024 12:12:56 +0530
Subject: [PATCH 11/17] fix signoz cluster start error

---
 justfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/justfile b/justfile
index d12a24e1ef2..d9427d98359 100644
--- a/justfile
+++ b/justfile
@@ -164,7 +164,6 @@ cleanup-gw: (purge-devspace cluster_gw ns_default) (delete-ns cluster_gw ns_defa
 [group('signoz')]
 start-signoz: && apply-signoz setup-signoz
     k3d cluster create signoz \
-        --no-image-volume \
         --port {{ port_signoz_ui }}:3301@loadbalancer \
         --port {{ port_signoz_otel }}:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"

From 26bd19cf40bb132d2bf006d8e15290c538dca653 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Wed, 18 Sep 2024 17:42:03 +0530
Subject: [PATCH 12/17] fix secrets error on bucketsync deployment

---
 packages/grid/helm/examples/gcp/gcp.bucketsync.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/grid/helm/examples/gcp/gcp.bucketsync.yaml b/packages/grid/helm/examples/gcp/gcp.bucketsync.yaml
index 2a430807fac..3a106824897 100644
--- a/packages/grid/helm/examples/gcp/gcp.bucketsync.yaml
+++ b/packages/grid/helm/examples/gcp/gcp.bucketsync.yaml
@@ -119,4 +119,5 @@ extraResources:
       name: seaweedfs-mount-secret
     type: Opaque
     data:
-      gcs_creds.json: base64 encoded value
+      # base 64 encoded value
+      gcs_creds.json: e30=

From 02c45614a8d682b6c8e0b675ee026f9bb256ff85 Mon Sep 17 00:00:00 2001
From: Yash Gorana <yash.gorana@hotmail.com>
Date: Wed, 18 Sep 2024 18:45:49 +0530
Subject: [PATCH 13/17] final set of changes

---
 justfile | 59 ++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 38 insertions(+), 21 deletions(-)

diff --git a/justfile b/justfile
index d9427d98359..8df4a9637b5 100644
--- a/justfile
+++ b/justfile
@@ -1,3 +1,7 @@
+set dotenv-load
+
+# ---------------------------------------------------------------------------------------------------------------------
+
 cluster_default := "k3d-syft-dev"
 cluster_high := "k3d-syft-high"
 cluster_low := "k3d-syft-low"
@@ -334,41 +338,44 @@ check-platform:
 
 [group('cloud')]
 [private]
-deploy-cloud cluster registry namespace profile: check-platform
+deploy-cloud cluster_ctx registry_url namespace profile: check-platform
     #!/bin/bash
-    set -euo pipefail
 
-    CONTEXT_NAME=$(kubectl config get-contexts -o=name | grep "{{ cluster }}")
+    CONTEXT_NAME=$(kubectl config get-contexts -o=name | grep "{{ cluster_ctx }}")
 
     if [ -z "$CONTEXT_NAME" ]; then
-        echo "No context found for cluster '{{ cluster }}'"
+        echo "Context not found: {{ cluster_ctx }}. Authorized with cloud providers to get relevant K8s cluster contexts"
         exit 1
     fi
 
+    set -euo pipefail
+
     # cloud deployments always have tracing false + platform=amd64
-    just tracing=false registry_url={{ registry }} \
+    just tracing=false registry_url={{ registry_url }} \
         deploy-devspace $CONTEXT_NAME {{ namespace }} "-p {{ profile }} --var PLATFORM=amd64"
 
 [group('cloud')]
 [private]
-purge-cloud cluster namespace:
+purge-cloud cluster_ctx namespace:
     #!/bin/bash
-    set -euo pipefail
 
-    CONTEXT_NAME=$(kubectl config get-contexts -o=name | grep "{{ cluster }}")
+    CONTEXT_NAME=$(kubectl config get-contexts -o=name | grep "{{ cluster_ctx }}")
 
     if [ -z "$CONTEXT_NAME" ]; then
-        echo "No context found for cluster '{{ cluster }}'"
+        echo "Context not found: {{ cluster_ctx }}. Authorized with cloud providers to get relevant K8s cluster contexts"
         exit 1
     fi
 
+    set -euo pipefail
+
     just purge-devspace $CONTEXT_NAME {{ namespace }}
+    kubectl delete ns {{ namespace }} --force --grace-period=0 --context $CONTEXT_NAME
 
 # ---------------------------------------------------------------------------------------------------------------------
 
 # Auth all components required for deploying Syft to Google Cloud
 [group('cloud-gcp')]
-auth-gcloud gcp_project gcp_region gcp_cluster gcp_registry:
+auth-gcloud:
     #!/bin/bash
     set -euo pipefail
 
@@ -378,31 +385,41 @@ auth-gcloud gcp_project gcp_region gcp_cluster gcp_registry:
         gcloud auth login
     fi
 
+    echo "Logged in as \"$(gcloud config get-value account)\""
+
     # install gke-gcloud-auth-plugin
     gke_installed=$(gcloud components list --only-local-state --filter gke-gcloud-auth-plugin --format=list 2>/dev/null)
     if [ -z "$gke_installed" ]; then
         gcloud components install gke-gcloud-auth-plugin
+        echo "Installed gke-gcloud-auth-plugin"
     fi
 
-    # get artifact registry registry
-    gcloud auth configure-docker {{ gcp_registry }} --quiet
-
-    # get cluster credentials
-    gcloud container clusters get-credentials {{ gcp_cluster }} --region {{ gcp_region }} --project {{ gcp_project }}
-
 # Deploy local code as datasite-high to Google Kubernetes Engine
 [group('cloud-gcp')]
-deploy-gcp-high gcp_cluster gcp_registry namespace="syft": (deploy-cloud gcp_cluster gcp_registry namespace "gcp")
+deploy-gcp-high gcp_cluster gcp_registry_url namespace="syft": (deploy-cloud gcp_cluster gcp_registry_url namespace "gcp")
 
+# Deploy local code as datasite-high to Google Kubernetes Engine
 [group('cloud-gcp')]
-purge-gcp-high gcp_cluster namespace: (purge-cloud gcp_cluster namespace)
+deploy-gcp-low gcp_cluster gcp_registry_url namespace="syft": (deploy-cloud gcp_cluster gcp_registry_url namespace "gcp-low")
 
-# Deploy local code as datasite-high to Google Kubernetes Engine
+# Purge deployment from a cluster
 [group('cloud-gcp')]
-deploy-gcp-low gcp_cluster gcp_registry namespace="syft": (deploy-cloud gcp_cluster gcp_registry namespace "gcp-low")
+purge-gcp gcp_cluster namespace="syft": (purge-cloud gcp_cluster namespace)
 
 # ---------------------------------------------------------------------------------------------------------------------
 
+[group('cloud-az')]
+auth-az tenant="creditsopenmined.onmicrosoft.com":
+    #!/bin/bash
+
+    # login to azure
+    ACCOUNT=$(az account show --query user.name)
+    if [ -z "$ACCOUNT" ]; then
+        az login --tenant {{ tenant }}
+    fi
+
+    echo "Logged in as $(az account show --query user.name)"
+
 # Deploy local code as datasite-high to Azure Kubernetes Service
 [group('cloud-az')]
 deploy-az-high aks_cluster az_registry namespace="syft": (deploy-cloud aks_cluster az_registry namespace "azure")
@@ -441,8 +458,8 @@ k9s-signoz:
 delete-all: delete-clusters delete-registry
     @echo "Stopped all Syft components"
 
-[group('utils')]
 [confirm('Confirm prune all docker resources?')]
+[group('utils')]
 prune-docker:
     -docker container prune -f
     -docker volume prune -af

From da5dc2c628740b163e1733ae7e323ddd7c548cc8 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Thu, 19 Sep 2024 16:48:11 +0530
Subject: [PATCH 14/17] renamed k3d create clusters to original name to prevent
 confusion on running individual commands

---
 justfile | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/justfile b/justfile
index 8df4a9637b5..937f6050fe7 100644
--- a/justfile
+++ b/justfile
@@ -2,11 +2,11 @@ set dotenv-load
 
 # ---------------------------------------------------------------------------------------------------------------------
 
-cluster_default := "k3d-syft-dev"
-cluster_high := "k3d-syft-high"
-cluster_low := "k3d-syft-low"
-cluster_gw := "k3d-syft-gw"
-cluster_signoz := "k3d-signoz"
+cluster_default := "syft-dev"
+cluster_high := "syft-high"
+cluster_low := "syft-low"
+cluster_gw := "syft-gw"
+cluster_signoz := "signoz"
 ns_default := "syft"
 ns_high := "high"
 ns_low := "low"
@@ -248,21 +248,20 @@ create-cluster cluster port *args='': start-registry && (apply-coredns cluster)
     #!/bin/bash
     set -euo pipefail
 
-    # remove the k3d- prefix
-    CLUSTER_NAME=$(echo "{{ cluster }}" | sed -e 's/k3d-//g')
-
-    k3d cluster create $CLUSTER_NAME \
+    k3d cluster create {{cluster}} \
         --port {{ port }}:80@loadbalancer \
         --registry-use k3d-registry.localhost:5800 {{ args }}
 
+    # Since k3d adds k3d- prefix to the cluster name
+    # we create a new context without the prefix
+    kubectl config rename-context k3d-{{cluster}} {{cluster}}
+
 [group('cluster')]
 [private]
 delete-cluster *args='':
     #!/bin/bash
     set -euo pipefail
 
-    # remove the k3d- prefix
-    ARGS=$(echo "{{ args }}" | sed -e 's/k3d-//g')
     k3d cluster delete $ARGS
 
 [group('cluster')]

From 2575f51f9499eba6e7bb51070c8126435ae938c6 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Thu, 19 Sep 2024 17:04:43 +0530
Subject: [PATCH 15/17] explicit set k3d context

---
 justfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/justfile b/justfile
index 937f6050fe7..27122b38c2f 100644
--- a/justfile
+++ b/justfile
@@ -254,7 +254,7 @@ create-cluster cluster port *args='': start-registry && (apply-coredns cluster)
 
     # Since k3d adds k3d- prefix to the cluster name
     # we create a new context without the prefix
-    kubectl config rename-context k3d-{{cluster}} {{cluster}}
+    kubectl config set-context {{ cluster }} --cluster=k3d-{{ cluster}} --user=admin@k3d-{{ cluster}}
 
 [group('cluster')]
 [private]

From dd398f5d81df9596cc19f838c24b275119db5e12 Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Thu, 19 Sep 2024 17:06:36 +0530
Subject: [PATCH 16/17] fix delete-cluster

---
 justfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/justfile b/justfile
index 27122b38c2f..3c0ccc58a82 100644
--- a/justfile
+++ b/justfile
@@ -262,7 +262,7 @@ delete-cluster *args='':
     #!/bin/bash
     set -euo pipefail
 
-    k3d cluster delete $ARGS
+    k3d cluster delete {{ args }}
 
 [group('cluster')]
 [private]

From c7c3928313705b88aca616ef50d111a52d405f2a Mon Sep 17 00:00:00 2001
From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com>
Date: Thu, 19 Sep 2024 17:50:10 +0530
Subject: [PATCH 17/17] rename signoz variables, add alias for signoz k3d
 cluster

---
 justfile | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/justfile b/justfile
index 3c0ccc58a82..90feb202155 100644
--- a/justfile
+++ b/justfile
@@ -167,18 +167,23 @@ cleanup-gw: (purge-devspace cluster_gw ns_default) (delete-ns cluster_gw ns_defa
 # Launch SigNoz on http://localhost:{{port_signoz_ui}}
 [group('signoz')]
 start-signoz: && apply-signoz setup-signoz
-    k3d cluster create signoz \
+    k3d cluster create {{ cluster_signoz }} \
         --port {{ port_signoz_ui }}:3301@loadbalancer \
         --port {{ port_signoz_otel }}:4317@loadbalancer \
         --k3s-arg "--disable=metrics-server@server:*"
 
+    # Since k3d adds k3d- prefix to the cluster name
+    # we create a new context without the prefix
+    kubectl config set-context {{ cluster_signoz }} --cluster=k3d-{{ cluster_signoz }} \
+        --user=admin@k3d-{{ cluster_signoz }}
+
     @printf "Started SigNoz\n\
         Dashboard: \033[1;36mhttp://localhost:{{ port_signoz_ui }}\033[0m\n\
         OTEL Endpoint: \033[1;36mhttp://localhost:{{ port_signoz_otel }}\033[0m\n"
 
 # Remove SigNoz from the cluster
 [group('signoz')]
-delete-collector:
+delete-signoz-agent:
     helm uninstall k8s-infra
 
 # Remove SigNoz from the cluster
@@ -187,8 +192,8 @@ delete-signoz: (delete-cluster cluster_signoz)
 
 [group('signoz')]
 [private]
-apply-collector cluster:
-    @echo "Installing SigNoz OTel Collector"
+apply-signoz-agent cluster:
+    @echo "Installing SigNoz OTel Agent"
     helm install k8s-infra k8s-infra \
         --repo https://charts.signoz.io \
         --kube-context {{ cluster }} \
@@ -244,7 +249,7 @@ delete-clusters:
 
 [group('cluster')]
 [private]
-create-cluster cluster port *args='': start-registry && (apply-coredns cluster) (apply-collector cluster)
+create-cluster cluster port *args='': start-registry && (apply-coredns cluster) (apply-signoz-agent cluster)
     #!/bin/bash
     set -euo pipefail
 
@@ -426,6 +431,7 @@ deploy-az-high aks_cluster az_registry namespace="syft": (deploy-cloud aks_clust
 # ---------------------------------------------------------------------------------------------------------------------
 
 # Reset Syft state in a cluster
+# TODO: make reset_k8s.sh take in context and namespace as args
 [group('utils')]
 [private]
 reset-syft name namespace: