diff --git a/.gitignore b/.gitignore index 31e86e221..42d69573d 100644 --- a/.gitignore +++ b/.gitignore @@ -71,9 +71,14 @@ target/ _site/ .sass-cache/ - # Debug-related .kube/ -# vscode +# vscode ide stuff +*.code-workspace +.history/ .vscode/ + +# jetbrains ide stuff +*.iml +.idea/ diff --git a/etc/kernel-launchers/bootstrap/bootstrap-kernel.sh b/etc/kernel-launchers/bootstrap/bootstrap-kernel.sh index 99032f05f..57e0f7ab8 100755 --- a/etc/kernel-launchers/bootstrap/bootstrap-kernel.sh +++ b/etc/kernel-launchers/bootstrap/bootstrap-kernel.sh @@ -12,7 +12,7 @@ launch_python_kernel() { export JPY_PARENT_PID=$$ # Force reset of parent pid since we're detached set -x - python ${KERNEL_LAUNCHERS_DIR}/python/scripts/launch_ipykernel.py --RemoteProcessProxy.kernel-id ${KERNEL_ID} --RemoteProcessProxy.response-address ${EG_RESPONSE_ADDRESS} --RemoteProcessProxy.spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} + python ${KERNEL_LAUNCHERS_DIR}/python/scripts/launch_ipykernel.py --RemoteProcessProxy.kernel-id ${KERNEL_ID} --RemoteProcessProxy.port-range ${EG_PORT_RANGE} --RemoteProcessProxy.response-address ${EG_RESPONSE_ADDRESS} --RemoteProcessProxy.spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} { set +x; } 2>/dev/null } @@ -21,7 +21,7 @@ launch_R_kernel() { # and shutdown requests from Enterprise Gateway. set -x - Rscript ${KERNEL_LAUNCHERS_DIR}/R/scripts/launch_IRkernel.R --RemoteProcessProxy.kernel-id ${KERNEL_ID} --RemoteProcessProxy.response-address ${EG_RESPONSE_ADDRESS} --RemoteProcessProxy.spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} + Rscript ${KERNEL_LAUNCHERS_DIR}/R/scripts/launch_IRkernel.R --RemoteProcessProxy.kernel-id ${KERNEL_ID} --RemoteProcessProxy.port-range ${EG_PORT_RANGE} --RemoteProcessProxy.response-address ${EG_RESPONSE_ADDRESS} --RemoteProcessProxy.spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE} { set +x; } 2>/dev/null } @@ -59,14 +59,14 @@ launch_scala_kernel() { --class launcher.ToreeLauncher \ "${LAUNCHER_APP}" \ "${TOREE_OPTS}" \ - "--RemoteProcessProxy.kernel-id ${KERNEL_ID} --RemoteProcessProxy.response-address ${EG_RESPONSE_ADDRESS} --RemoteProcessProxy.spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE}" + "--RemoteProcessProxy.kernel-id ${KERNEL_ID} --RemoteProcessProxy.port-range ${EG_PORT_RANGE} --RemoteProcessProxy.response-address ${EG_RESPONSE_ADDRESS} --RemoteProcessProxy.spark-context-initialization-mode ${KERNEL_SPARK_CONTEXT_INIT_MODE}" { set +x; } 2>/dev/null } # Ensure that required envs are present, check language before the dynamic values if [ -z "${KERNEL_LANGUAGE+x}" ] then - echo "KERNEL_LANGUAGE is required. Set this value in the image or when starting container." + echo "KERNEL_LANGUAGE is required. Set this value in the image or when starting container." exit 1 fi if [ -z "${KERNEL_ID+x}" ] || [ -z "${EG_RESPONSE_ADDRESS+x}" ] diff --git a/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2 b/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2 index 1404a0cd6..261c55521 100644 --- a/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2 +++ b/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2 @@ -38,20 +38,22 @@ spec: {% endif %} containers: - env: + - name: EG_PORT_RANGE + value: "{{ eg_port_range }}" - name: EG_RESPONSE_ADDRESS value: "{{ eg_response_address }}" + - name: KERNEL_ID + value: "{{ kernel_id }}" - name: KERNEL_LANGUAGE value: "{{ kernel_language }}" - - name: KERNEL_SPARK_CONTEXT_INIT_MODE - value: "{{ kernel_spark_context_init_mode }}" - name: KERNEL_NAME value: "{{ kernel_name }}" - - name: KERNEL_USERNAME - value: "{{ kernel_username }}" - - name: KERNEL_ID - value: "{{ kernel_id }}" - name: KERNEL_NAMESPACE value: "{{ kernel_namespace }}" + - name: KERNEL_SPARK_CONTEXT_INIT_MODE + value: "{{ kernel_spark_context_init_mode }}" + - name: KERNEL_USERNAME + value: "{{ kernel_username }}" image: "{{ kernel_image }}" name: "{{ kernel_pod_name }}" {% if kernel_cpus is defined or kernel_memory is defined or kernel_gpus is defined or kernel_cpus_limit is defined or kernel_memory_limit is defined or kernel_gpus_limit is defined %} diff --git a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py index d4cd957bd..13c3278be 100644 --- a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py +++ b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py @@ -26,7 +26,7 @@ def generate_kernel_pod_yaml(keywords): return k8s_yaml -def launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode): +def launch_kubernetes_kernel(kernel_id, port_range, response_addr, spark_context_init_mode): # Launches a containerized kernel as a kubernetes pod. config.load_incluster_config() @@ -37,9 +37,10 @@ def launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode): # Factory values... # Since jupyter lower cases the kernel directory as the kernel-name, we need to capture its case-sensitive # value since this is used to locate the kernel launch script within the image. - keywords['kernel_name'] = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - keywords['kernel_id'] = kernel_id + keywords['eg_port_range'] = port_range keywords['eg_response_address'] = response_addr + keywords['kernel_id'] = kernel_id + keywords['kernel_name'] = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) keywords['kernel_spark_context_init_mode'] = spark_context_init_mode # Walk env variables looking for names prefixed with KERNEL_. When found, set corresponding keyword value @@ -82,7 +83,7 @@ def launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode): if __name__ == '__main__': """ - Usage: launch_kubernetes_kernel + Usage: launch_kubernetes_kernel [--RemoteProcessProxy.kernel-id ] [--RemoteProcessProxy.response-address ] [--RemoteProcessProxy.spark-context-initialization-mode ] @@ -91,6 +92,8 @@ def launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode): parser = argparse.ArgumentParser() parser.add_argument('--RemoteProcessProxy.kernel-id', dest='kernel_id', nargs='?', help='Indicates the id associated with the launched kernel.') + parser.add_argument('--RemoteProcessProxy.port-range', dest='port_range', nargs='?', + metavar='..', help='Port range to impose for kernel ports') parser.add_argument('--RemoteProcessProxy.response-address', dest='response_address', nargs='?', metavar=':', help='Connection address (:) for returning connection file') parser.add_argument('--RemoteProcessProxy.spark-context-initialization-mode', dest='spark_context_init_mode', @@ -99,7 +102,8 @@ def launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode): arguments = vars(parser.parse_args()) kernel_id = arguments['kernel_id'] + port_range = arguments['port_range'] response_addr = arguments['response_address'] spark_context_init_mode = arguments['spark_context_init_mode'] - launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode) + launch_kubernetes_kernel(kernel_id, port_range, response_addr, spark_context_init_mode)