Integrate selkies-gstreamer to Jupyter and Coder (just like noVNC)

[ehfd]

Update:

We currently have this Coder configuration working self-hosted:

entrypoint.sh for Coder:

set -e
supervisord

Open main.tf

terraform {
  required_providers {
    coder = {
      source  = "coder/coder"
      version = "~> 0.6.17"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.18"
    }
  }
}

provider "coder" {
  feature_use_managed_variables = true
}

provider "kubernetes" {
  config_path = var.use_kubeconfig == true ? "~/.kube/config" : null
}

data "coder_workspace" "me" {}

data "coder_parameter" "cpu" {
  name    = "CPU (cores)"
  default = "2"
  icon    = "/icon/memory.svg"
  mutable = true
  option {
    name  = "2 Cores"
    value = "2"
  }
  option {
    name  = "4 Cores"
    value = "4"
  }
  option {
    name  = "6 Cores"
    value = "6"
  }
  option {
    name  = "8 Cores"
    value = "8"
  }
  option {
    name  = "16 Cores"
    value = "16"
  }
}

data "coder_parameter" "memory" {
  name    = "Memory (GB)"
  default = "2"
  icon    = "/icon/memory.svg"
  mutable = true
  option {
    name  = "2 GB"
    value = "2"
  }
  option {
    name  = "4 GB"
    value = "4"
  }
  option {
    name  = "6 GB"
    value = "6"
  }
  option {
    name  = "8 GB"
    value = "8"
  }
    option {
    name  = "16 GB"
    value = "16"
  }
  option {
    name  = "32 GB"
    value = "32"
  }
  option {
    name  = "64 GB"
    value = "64"
  }
}

data "coder_parameter" "password" {
  name         = "password"
  display_name = "Selkies Password"
  description  = "The Selkies password for authentication. User is ubuntu."
  icon         = "/emojis/1f511.png"
  mutable      = false
  option {
    name  = "mypasswd"
    value = "mypasswd"
  }
}

data "coder_parameter" "home_disk_size" {
  name         = "home_disk_size"
  display_name = "Home disk size"
  description  = "The size of the home disk in GB"
  default      = "100"
  type         = "number"
  icon         = "/emojis/1f4be.png"
  mutable      = false
  validation {
    min = 1
    max = 99999
  }
}



variable "use_kubeconfig" {
  type        = bool
  description = "Use host kubeconfig? (true/false)"
  default     = false
}

variable "namespace" {
  type        = string
  description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces)"
  default     = "coder"
}


resource "kubernetes_pod" "main" {
  count = data.coder_workspace.me.start_count
  metadata {
    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
    namespace = var.namespace
    labels = {
      "app.kubernetes.io/name"     = "coder-workspace"
      "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
      "app.kubernetes.io/part-of"  = "coder"
      // Coder specific labels.
      "com.coder.resource"       = "true"
      "com.coder.workspace.id"   = data.coder_workspace.me.id
      "com.coder.workspace.name" = data.coder_workspace.me.name
      "com.coder.user.id"        = data.coder_workspace.me.owner_id
      "com.coder.user.username"  = data.coder_workspace.me.owner
    }
    annotations = {
      "com.coder.user.email" = data.coder_workspace.me.owner_email
    }
  }

  spec {

    container {
      name  = "dev"
      image = "ghcr.io/selkies-project/nvidia-glx-desktop:latest"
      image_pull_policy = "Always"
      command           = ["sh", "-c", coder_agent.main.init_script]
      # security_context {
      #   run_as_user = "1000"
      #   privileged = true
      # }

      env {
        name  = "TZ"
        value = "UTC"
      }
      env {
        name  = "DISPLAY_SIZEW"
        value = "1920"
      }
      env {
        name  = "DISPLAY_SIZEH"
        value = "1080"
      }
      env {
        name  = "DISPLAY_REFRESH"
        value = "60"
      }
      env {
        name  = "DISPLAY_DPI"
        value = "96"
      }
      env {
        name  = "DISPLAY_CDEPTH"
        value = "24"
      }
      env {
        name  = "VIDEO_PORT"
        value = "DFP"
      }
      env {
        name  = "PASSWD"
        value = "${data.coder_parameter.password.value}"
      }
      env {
        name  = "SELKIES_ENCODER"
        value = "nvh264enc"
      }
      env {
        name  = "SELKIES_ENABLE_RESIZE"
        value = "false"
      }
      env {
        name  = "SELKIES_VIDEO_BITRATE"
        value = "8000"
      }
      env {
        name  = "SELKIES_FRAMERATE"
        value = "60"
      }
      env {
        name  = "SELKIES_AUDIO_BITRATE"
        value = "128000"
      }
      env {
        name  = "SELKIES_ENABLE_BASIC_AUTH"
        value = "true"
      }
      env {
        name  = "SELKIES_BASIC_AUTH_PASSWORD"
        value = "${data.coder_parameter.password.value}"
      }
      env {
        name  = "SELKIES_TURN_REST_URI"
        value = "http://turn-rest.nrp-nautilus.io"
      }
      env {
        name  = "SELKIES_TURN_PROTOCOL"
        value = "tcp"
      }
      env {
        name  = "SELKIES_TURN_TLS"
        value = "false"
      }
      env {
        name  = "CODER_AGENT_TOKEN"
        value = coder_agent.main.token
      }

      stdin = true
      tty   = true

      port {
        name           = "http"
        container_port = 8080
        protocol       = "TCP"
      }

      resources {
        limits = {
          "cpu"    = "${data.coder_parameter.cpu.value}"
          "memory" = "${data.coder_parameter.memory.value}Gi"
          "nvidia.com/gpu" = 1
        }
        requests = {
          "cpu"    = "${data.coder_parameter.cpu.value}"
          "memory" = "${data.coder_parameter.memory.value}Gi"
          "nvidia.com/gpu" = 1
        }
      }

      volume_mount {
        mount_path = "/home/ubuntu/persistent"
        name       = "home"
        read_only  = false
        sub_path   = "home"
      }

      volume_mount {
        mount_path = "/dev/shm"
        name       = "dshm"
      }
    }

    dns_policy = "None"
    dns_config {
      nameservers = ["8.8.8.8", "8.8.4.4"]
    }

    volume {
      name = "dshm"
      empty_dir {}
    }

    volume {
      name = "home"
      persistent_volume_claim {
        claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
        read_only  = false
      }
    }

    affinity {
      node_affinity {
        required_during_scheduling_ignored_during_execution {
          node_selector_term {
            match_expressions {
              key      = "topology.kubernetes.io/zone"
              operator = "NotIn"
              values   = ["myzone"]
            }
          }
        }
      }
    }
  }
}

resource "coder_agent" "main" {
  os                     = "linux"
  arch                   = "amd64"
  login_before_ready     = false
  startup_script_timeout = 180
  startup_script         = <<-EOT
    set -e

    # install and start code-server
    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/tmp/code-server --version 4.8.3
    /tmp/code-server/bin/code-server --auth none --port 13337 >/tmp/code-server.log 2>&1 &

    echo "Initializing Supervisor..."
    nohup supervisord
  EOT
}

resource "coder_app" "code-server" {
  agent_id     = coder_agent.main.id
  slug         = "code-server"
  display_name = "code-server"
  icon         = "/icon/code.svg"
  url          = "http://localhost:13337?folder=/home/coder"
  subdomain    = false
  share        = "owner"

  healthcheck {
    url       = "http://localhost:13337/healthz"
    interval  = 3
    threshold = 10
  }
}

resource "kubernetes_persistent_volume_claim" "home" {
  metadata {
    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}-home"
    namespace = var.namespace
    labels = {
      "app.kubernetes.io/name"     = "coder-pvc"
      "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
      "app.kubernetes.io/part-of"  = "coder"
      // Coder specific labels.
      "com.coder.resource"       = "true"
      "com.coder.workspace.id"   = data.coder_workspace.me.id
      "com.coder.workspace.name" = data.coder_workspace.me.name
      "com.coder.user.id"        = data.coder_workspace.me.owner_id
      "com.coder.user.username"  = data.coder_workspace.me.owner
    }
    annotations = {
      "com.coder.user.email" = data.coder_workspace.me.owner_email
    }
  }
  wait_until_bound = false
  spec {
    access_modes = ["ReadWriteOnce"]
    storage_class_name = "rook-ceph-block"
    resources {
      requests = {
        storage = "${data.coder_parameter.home_disk_size.value}Gi"
      }
    }
  }
}

resource "coder_app" "selkies" {
  agent_id      = coder_agent.main.id
  slug          = "selkies"  
  display_name  = "Selkies"
  icon          = "/emojis/1f3ae.png"
  url           = "http://localhost:8080"
  subdomain     = true
  share         = "owner"
}

---

Self-explanatory. Just like VS Code Server and noVNC, a button click in Jupyter should lead to a window with Selkies.
This will help greatly in robotics, simulations, and other kinds of research.

External contribution dearly expected.

It can be a separate project, a PR, or any other form of contribution.
Integration with Jupyter Docker containers should also be possible.

Creating a template for Coder would also be of interest.

[Inrixia]

Messing around trying to get this working with coder.

Can access the web endpoint fine but webrtc fails.

When using coder port-forward to open needed ports for a direct connection webrtc also fails as the signaling handshake specifies the private host ip of the container which is not resolvable eg 10.x.x.x vs 127.0.0.1.

Ideally there would be a way to manually specify the ports/ip used to get around this, though I'm not sure how tenable that is.

I haven't tested using a turn server because if I have to fall back to that I'll just end up going with something like KasmVNC which works seamlessly over a single port.

[Inrixia]

This looks super promising. Will check it out tomorrow, thanks!

[Inrixia]

Hey, have messed around a bit with this. Had to switch to using the egl container as I am not using a gpu for testing.

While this boots and works fine. It still requires a turn server which does not work due to firewall restrictions.

Plus I cannot justify needing to run a seperate turn server.

How difficult would it be to be able to pin ports needed so that a direct connection could be established using coder port-forward

[ehfd]

https://github.com/selkies-project/docker-nvidia-egl-desktop#running-with-kubernetes

Maybe the internal TURN server option could work here.

[Inrixia]

That might work since can limit it to only two ports. Will give it a try and see!

Though actually may run into the issue earlier where the internal ip of the pod doesn't match the forwarded (127.0.0.1). Will update how it goes

[Inrixia]

Yea so when doing ice negotiation it is advertising the internal pod ips (10.x.x.x) so ice still fails as that address is not resolvable. It has to advertise something like 127.0.0.1 or local host etc so that the client attempts to connect via the port forward.

I ran into this issue when originally testing as mentioned above.

Having a option to override what ip is advertised would probably fix this.

[Inrixia]

Actually I don't seem to be able to get it to use the internal turn server.

Checking the logs its still advertising defaults

[Inrixia]

Wait I'm being dumb. Was missing the turn host env var, it's further down the docs and didn't notice it.

[Inrixia]

Hmm, now I'm getting proper advertising but even though the internal server advertises 65535 as a option it is still listening on the old ports and not 65535

[ehfd]

@Inrixia What's your precise settings?

[Inrixia]

Same as the template you gave.

But only turn settings are min port, max port, turn port, turn host, turn protocol

[Inrixia]

The turn port is listened on.

But the min and max ports (65534-65535) are not. So when they are advertised they are unreachable afik

[ehfd]

Very strange. You specified TURN_MIN_PORT and TURN_MAX_PORT?

[Inrixia]

Yep. If you have the chance can you share a working config? Maybe I'm missing something obvious.

This is running on a normal coder deployment on aks and I can see what ports the container is listening on.

I'm using the coder cli to port forward and it works fine. Just need to get the ice negotiation to use the right ports.

Setting the turn host ip fixed the issue with it not using 127... Too so afik it's just the underlying server not listening on the range that's the issue.

[ehfd]

If you do not specify anything on SELKIES_TURN_HOST, it will automatically resolve to the external IP of that instance.

Let me check something quickly now about the port range.

[Inrixia]

Yep, the ip issues were resolved by specifying the host arg. Just the ports listening issue now afik

[ehfd]

@Inrixia In top or htop, could you get the PID of any of the turnserver process and do cat /proc/${PID}/cmdline | tr '\000' ' ' for ${PID}?

[ehfd]

I need to see the input for:

# Configure coTURN script
RUN echo "#!/bin/bash\n\
set -e\n\
turnserver \
    --verbose \
    --listening-ip=\"0.0.0.0\" \
    --listening-ip=\"::\" \
    --listening-port=\"\${SELKIES_TURN_PORT:-3478}\" \
    --realm=\"\${TURN_REALM:-example.com}\" \
    --external-ip=\"\${TURN_EXTERNAL_IP:-\$(dig TXT +short @ns1.google.com o-o.myaddr.l.google.com 2>/dev/null | { read output; if [ -z \"\$output\" ] || echo \"\$output\" | grep -q '^;;'; then exit 1; else echo \"\$(echo \$output | sed 's,\\\",,g')\"; fi } || dig -6 TXT +short @ns1.google.com o-o.myaddr.l.google.com 2>/dev/null | { read output; if [ -z \"\$output\" ] || echo \"\$output\" | grep -q '^;;'; then exit 1; else echo \"\$(echo \$output | sed 's,\\\",,g')\"; fi } || hostname -I 2>/dev/null | awk '{print \$1; exit}' || echo '127.0.0.1')}\" \
    --min-port=\"\${TURN_MIN_PORT:-49152}\" \
    --max-port=\"\${TURN_MAX_PORT:-65535}\" \
    --channel-lifetime=\"\${TURN_CHANNEL_LIFETIME:--1}\" \
    --lt-cred-mech \
    --user \"selkies:\${TURN_RANDOM_PASSWORD}\" \
    --no-cli \
    --cli-password=\"\${TURN_RANDOM_PASSWORD:-\$(tr -dc 'A-Za-z0-9' < /dev/urandom 2>/dev/null | head -c 24)}\" \
    --allow-loopback-peers \
    \${TURN_EXTRA_ARGS} \$@\
" > /etc/start-turnserver.sh && chmod -f 755 /etc/start-turnserver.sh

[Inrixia]

turnserver --verbose --listening-ip=0.0.0.0 --listening-ip=:: --listening-port=3478 --realm=example.com --external-ip=127.0.0.1 --min-port=65534 --max-port=65535 --channel-lifetime=-1 --lt-cred-mech --user selkies:REDACTED --no-cli --cli-password=REDACTED --allow-loopback-peers

[ehfd]

External IP SHOULD NOT be 127.0.0.1. It should be your real IP that clients should know (public IP if over the internet, private IP if confined to LAN).

Otherwise, I don't see why opening 65534 and 65535 in the Coder configuration shouldn't work.

[Inrixia]

I'm accessing the coder service via coder port-forward <workspace> ...ports... which lets me hit the webui for example via https://127.0.0.1:8081

So that should be fine. I have ports 8081, 3478 & 65534-65535 forwarded on tcp/udp. Coder reports ports that it sees as open and 65534-65535 is not listed but 8081, 3478 and other ports are. When attempting to connect other ports like 58xxx are also opened so clearly the server is trying to negotiate but using the wrong ports.

[ehfd]

Port should be 8080. It should go through NGINX because the interfaces themselves only allow loopback access for security.

[Inrixia]

Tried 8080 same thing. Though I did notice that the advertisements do still have the inaccessible internal ip in them.

candidate:9 1 UDP 337658111 127.0.0.1 65534 typ relay raddr 10.224.1.158 rport 9

10.224.1.158 is a pod ip that is not accessible.

[ehfd]

Then, you should change 10.224.1.158 to an accessible public IP in SELKIES_TURN_HOST.

[ehfd]

Need the logs in chrome://webrtc-internals

[PMohanJ]

As the discussion seems to be around TURN server, could you please test your Public turn server from: https://icetest.info/
Just provide your turn details and start the test. Let us know what it outputs.

Ideally it should output ICE candidates of type host, srflx and relay, and we are especially looking for relay type. If you don't see relay candidate then your TURN server isn't accessible publicly

[Inrixia]

This is what I get testing the public ice server in icetest.com

ICE server list 
URL: turn:turn.anyfirewall.com:443?transport=tcp Username: webrtc Credential: webrtc 
 

Results 
IceGatheringState: complete 
host 
udp 
48825b31-0a22-4298-add6-5a0776d729e3.local:63651 
N/A

[Inrixia]

I'll test my private one when I get the time

[PMohanJ]

IceGatheringState: complete
host
udp
48825b31-0a22-4298-add6-5a0776d729e3.local:63651
N/A

I don't see any relay candidate nor srflx candidate. I don't think that TURN server is accessible from public internet. Thus your connection is failing.

[Inrixia]

Quick question regarding having a turn server hosted. If it's hosted in a private network but all machines have access to it then that should work right?

Or is it impossible to utilize a turn server over a private network that isn't internet facing?

[PMohanJ]

If the device from which you're accessing is part of that private network then it should work.

[Inrixia]

Yea, I'll post the results using the private turn server once I spin it back up.

Its accessible from both the client and container so should work. But I was getting the same issue as above with it so maybe I'm missing something. Will see.

[PMohanJ]

If the device from which you're accessing is part of that private network then it should work.

I was wrong.

Unfortunately, CoTURN in a private network isn't working for TURN functionality. I just verified it in a private network.

The browser isn't even generating any ICE candidate.

[Inrixia]

Yea that's pretty much exactly the issue I'm running into.

Is there no solution to this?

[ehfd]

Likely requires settings from the coTURN side. Should be possible but I do not have a specific answer.

[PMohanJ]

@Inrixia could you please share your private turn server setup? Like how you're running CoTURN? As a container or in K8s? If k8s then mind sharing the manifest?

Also, have you tried STUNner? I've been trying it out, but for some reasons unable to make it work as my K8s is self hosted on a single private VM.

[Inrixia]

I've been using stunner on aks but have had similar issues. Just using the default udp config. But I'll try post the info about it's connectivity when I get the chance.

[PMohanJ]

Finally, I got the private CoTURN to work. Here's the docker command:

docker run --name coturn -d -p 3478:3478 --network host -p 3478:3478/udp -p 49160-49200:49160-49200/udp coturn/coturn -n --min-port=49160 --max-port=49200 --lt-cred-mech --user=yourusername:yourpassword --realm=example.com --relay-ip=192.168.100.95

Change the relay-ip to the machine IP. Also yeah, it has to be running on host network. Or else the turnserver can't relay over the machine address. https://github.com/coturn/coturn/blob/master/examples/etc/turnserver.conf#L112

[ehfd]

Please do a PR in docs about this. @PMohanJ I will review personally.

Make sure to search all the docs related to coTURN.

[PMohanJ]

@Inrixia could you please confirm us if this coTURN setup is working in your case?

[Inrixia]

It's a docker config so wouldn't apply to my environment.

[PMohanJ]

Could you explain your environment?

This is what I was asking yesterday..

could you please share your private turn server setup? Like how you're running CoTURN? As a container or in K8s? If k8s then mind sharing the manifest?

[Inrixia]

It's running on aks, for turn I'm using stunner.

Services aren't exposed on a public ip, the client used to connect is doing so via private ip. Similar to hosting a k8s cluster yourself.

[PMohanJ]

Oh, so far your private/public TURN server is based on STUNner, but not coTURN.
For STUNner in private network, we need some resolution from the STUNner community. I've opened a thread for the same in STUNner discord server, if you'd like check it: https://discord.com/channels/945255818494902282/1275695495490175050

[Inrixia]

Ah, good to know. I can try running coturn on the cluster instead to see if that works if stunner is broken for private address space.

Will update when I get the time to mess with it more.

[PMohanJ]

@Inrixia I had a conversation with the maintainer.
As per STUNner, there are two ICE mode, as of now Selkies-gstreamer doesn't provide any configurable way to support Asymmetric mode. So for time being you can edit UDPRoute manifest to have it include the Stunner service as a backend endpoint.
For example:

apiVersion: stunner.l7mp.io/v1
kind: UDPRoute
metadata:
  name: media-plane
  namespace: stunner
spec:
  parentRefs:
    - name: udp-gateway
  rules:
    - backendRefs:
        - name: media-plane  # Selkies pod service 
          namespace: default # namespace in which selkies pod is running
        - name: udp-gateway # Stunner's service name that gets created when we create gateway resource.
          namespace: stunner

But the maintainer recommend to go with Asymmetric mode as symmetric might have some performance issues.
I'd make necessary changes to repo after getting approval from @ehfd. Meanwhile please test this and let us know the results when you get time.

[Inrixia]

Awesome! Thanks will try this and update how it goes

[Inrixia]

So I've tried using that config but still getting an endless Waiting for stream.
Looking at webrtc-internals I'm still seeing icecandidateerrors specifically code 701: TURN allocate request timed out.

[PMohanJ]

Could you open the DevTools of the browser and check for the ICE candidates being exchanged over ws /signalling request?
I'm expecting relay candidates both from browser and server.

Also if you could come over discord and share this config setup along with stunner pod logs, it'd be helpful to further debug this.

[ehfd]

@Inrixia #165 (comment)
This could have fixed the potential underlying issue.

[Inrixia]

I'll give it a shot if I get time. Thanks for keeping me up to date

[gitizenss]

Has anyone done this? Wanting to use this with Coder, no clue where to start, if its feasible, etc. Would be nice to have a quick way of doing so.

[ehfd]

@gitizenss
Check Open main.tf in #64 (comment)