diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 8b2def8cd..000000000 --- a/.flake8 +++ /dev/null @@ -1,16 +0,0 @@ -[flake8] -ignore = E501, W503, E402 -builtins = c, get_config -exclude = - .cache, - .github, - docs -enable-extensions = G -extend-ignore = - G001, G002, G004, G200, G201, G202, - # black adds spaces around ':' - E203, -per-file-ignores = - # B011: Do not call assert False since python -O removes these calls - # F841 local variable 'foo' is assigned to but never used - enterprise_gateway/tests/*: B011, F841 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 5e4379eda..174372db7 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -10,6 +10,6 @@ To generate better logs, please run the gateway with `--debug` command line para ## Environment -- Enterprise Gateway Version [e.g. 1.x, 2.x, ...] -- Platform: [e.g. YARN, Kubernetes ...] -- Others [e.g. Jupyter Server 5.7, JupyterHub 1.0, etc] +- Enterprise Gateway Version \[e.g. 1.x, 2.x, ...\] +- Platform: \[e.g. YARN, Kubernetes ...\] +- Others \[e.g. Jupyter Server 5.7, JupyterHub 1.0, etc\] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7564b174e..90b61ef1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-beta - 3.11.0"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] steps: - name: Checkout uses: actions/checkout@v2 @@ -61,8 +61,14 @@ jobs: docker logs itest-yarn - name: Bump versions run: | - pip install tbump - tbump --dry-run --no-tag --no-push 100.100.100rc0 + pipx run tbump --dry-run --no-tag --no-push 100.100.100rc0 + + pre_commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - uses: jupyterlab/maintainer-tools/.github/actions/pre-commit@v1 link_check: runs-on: ubuntu-latest @@ -116,3 +122,20 @@ jobs: steps: - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - uses: jupyterlab/maintainer-tools/.github/actions/test-sdist@v1 + + + python_tests_check: # This job does nothing and is only used for the branch protection + if: always() + needs: + - build + - link_check + - test_minimum_versions + - build_docs + - pre_commit + - test_sdist + runs-on: ubuntu-latest + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1f8193d4..3d787de4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,15 +31,16 @@ repos: files: \.py$ args: [--profile=black] - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.0 + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.10.1 hooks: - - id: prettier - exclude: | - (?x)^( - etc/kubernetes/.*.yaml| - website/.* - )$ + - id: validate-pyproject + stages: [manual] + + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.16 + hooks: + - id: mdformat - repo: https://github.com/asottile/pyupgrade rev: v2.38.2 @@ -53,12 +54,14 @@ repos: - id: doc8 args: [--max-line-length=200] - - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 + - repo: https://github.com/john-hen/Flake8-pyproject + rev: 1.0.1 hooks: - - id: flake8 + - id: Flake8-pyproject + alias: flake8 additional_dependencies: ["flake8-bugbear==22.6.22", "flake8-implicit-str-concat==0.2.0"] + stages: [manual] - repo: https://github.com/sirosen/check-jsonschema rev: 0.18.3 diff --git a/readthedocs.yml b/.readthedocs.yaml similarity index 100% rename from readthedocs.yml rename to .readthedocs.yaml diff --git a/LICENSE.md b/LICENSE.md index 7ab7a0ef6..08ca808fc 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -57,5 +57,7 @@ change to one of the Jupyter repositories. With this in mind, the following banner should be used in any source code file to indicate the copyright and license terms: - # Copyright (c) Jupyter Development Team. - # Distributed under the terms of the Modified BSD License. +``` +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +``` diff --git a/README.md b/README.md index 90514fefa..4c7d9bf66 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ an enterprise. Inspired by Jupyter Kernel Gateway, Jupyter Enterprise Gateway pr - Adds support for remote kernels hosted throughout the enterprise where kernels can be launched in the following ways: - _ Local to the Enterprise Gateway server (today's Kernel Gateway behavior) - _ On specific nodes of the cluster utilizing a round-robin algorithm \* On nodes identified by an associated resource manager + \_ Local to the Enterprise Gateway server (today's Kernel Gateway behavior) + \_ On specific nodes of the cluster utilizing a round-robin algorithm * On nodes identified by an associated resource manager - Provides support for Apache Spark managed by YARN, IBM Spectrum Conductor, Kubernetes or Docker Swarm out of the box. Others can be configured via Enterprise Gateway's extensible framework. - Secure communication from the client, through the Enterprise Gateway server, to the kernels - Multi-tenant capabilities diff --git a/docs/source/contributors/system-architecture.md b/docs/source/contributors/system-architecture.md index 4e65bb4fc..f5b824663 100644 --- a/docs/source/contributors/system-architecture.md +++ b/docs/source/contributors/system-architecture.md @@ -32,9 +32,9 @@ environment variables (also described in the kernel specification), is passed to a process class. This class supports four basic methods following its creation: 1. `poll()` to determine if the process is still running -2. `wait()` to block the caller until the process has terminated -3. `send_signal(signum)` to send a signal to the process -4. `kill()` to terminate the process +1. `wait()` to block the caller until the process has terminated +1. `send_signal(signum)` to send a signal to the process +1. `kill()` to terminate the process As you can see, other forms of process communication can be achieved by abstracting the launch mechanism. @@ -83,7 +83,7 @@ See the [Process Proxy](#process-proxy) section for more details on process prox `RemoteMappingKernelManager` is a subclass of Jupyter Server's [`AsyncMappingKernelManager`](https://github.com/jupyter-server/jupyter_server/blob/745f5ba3f00280c1e1900326a7e08463d48a3912/jupyter_server/services/kernels/kernelmanager.py#L633) and provides two functions. 1. It provides the vehicle for making the `RemoteKernelManager` class known and available. -2. It overrides `start_kernel` to look at the target kernel's kernel spec to see if it contains a remote process proxy class entry. If so, it records the name of the class in its member variable to be made available to the kernel start logic. +1. It overrides `start_kernel` to look at the target kernel's kernel spec to see if it contains a remote process proxy class entry. If so, it records the name of the class in its member variable to be made available to the kernel start logic. ## Remote Kernel Manager @@ -404,9 +404,9 @@ Kernel launchers provide a means of normalizing behaviors across kernels while a There are four primary tasks of a kernel launcher: 1. Creation of the connection file and ZMQ ports on the remote (target) system along with a _gateway listener_ socket -2. Conveyance of the connection (and listener socket) information back to the Enterprise Gateway process -3. Invocation of the target kernel -4. Listen for interrupt and shutdown requests from Enterprise Gateway and carry out the action when appropriate +1. Conveyance of the connection (and listener socket) information back to the Enterprise Gateway process +1. Invocation of the target kernel +1. Listen for interrupt and shutdown requests from Enterprise Gateway and carry out the action when appropriate Kernel launchers are minimally invoked with three parameters (all of which are conveyed by the `argv` stanza of the corresponding `kernel.json` file) - the kernel's ID as created by the server and conveyed via the placeholder `{kernel_id}`, a response address consisting of the Enterprise Gateway server IP and port on which to return the connection information similarly represented by the placeholder `{response_address}`, and a public-key used by the launcher to encrypt an AES key that encrypts the kernel's connection information back to the server and represented by the placeholder `{public_key}`. @@ -447,6 +447,7 @@ Here's a [kernel.json](https://github.com/jupyter-server/enterprise_gateway/blob Other options supported by launchers include: - `--RemoteProcessProxy.port-range {port_range}` - passes configured port-range to launcher where launcher applies that range to kernel ports. The port-range may be configured globally or on a per-kernel specification basis, as previously described. + - `--RemoteProcessProxy.spark-context-initialization-mode [lazy|eager|none]` - indicates the _timeframe_ in which the spark context will be created. - `lazy` (default) attempts to defer initialization as late as possible - although this can vary depending on the @@ -479,11 +480,11 @@ eval exec \ Theoretically speaking, enabling a kernel for use in other frameworks amounts to the following: 1. Build a kernel specification file that identifies the process proxy class to be used. -2. Implement the process proxy class such that it supports the four primitive functions of +1. Implement the process proxy class such that it supports the four primitive functions of `poll()`, `wait()`, `send_signal(signum)` and `kill()` along with `launch_process()`. -3. If the process proxy corresponds to a remote process, derive the process proxy class from +1. If the process proxy corresponds to a remote process, derive the process proxy class from `RemoteProcessProxy` and implement `confirm_remote_startup()` and `handle_timeout()`. -4. Insert invocation of a launcher (if necessary) which builds the connection file and +1. Insert invocation of a launcher (if necessary) which builds the connection file and returns its contents on the `{response_address}` socket and following the encryption protocol set forth in the other launchers. ```{seealso} diff --git a/docs/source/developers/custom-images.md b/docs/source/developers/custom-images.md index 62b2a3e88..1a3690eac 100644 --- a/docs/source/developers/custom-images.md +++ b/docs/source/developers/custom-images.md @@ -21,7 +21,7 @@ USER $NB_UID # switch back to the jovyan user Users that do not wish to extend an existing kernel image must be cognizant of a couple of things. 1. Requirements of a kernel-based image to be used by Enterprise Gateway. -2. Is the base image one from [Jupyter Docker-stacks](https://github.com/jupyter/docker-stacks)? +1. Is the base image one from [Jupyter Docker-stacks](https://github.com/jupyter/docker-stacks)? ### Requirements for Custom Kernel Images diff --git a/docs/source/developers/dev-process-proxy.md b/docs/source/developers/dev-process-proxy.md index 3d0d55159..c18bed0c7 100644 --- a/docs/source/developers/dev-process-proxy.md +++ b/docs/source/developers/dev-process-proxy.md @@ -19,10 +19,10 @@ That said, if you and your organization plan to stay on Enterprise Gateway 2.x o Please refer to the [Process Proxy section](../contributors/system-architecture.md#process-proxy) in the System Architecture pages for descriptions and structure of existing process proxies. Here is the general guideline for the process of implementing a process proxy. 1. Identify and understand how to _decorate_ your "job" within the resource manager. In Hadoop YARN, this is done by using the kernel's ID as the _application name_ by setting the [`--name` parameter to `${KERNEL_ID}`](https://github.com/jupyter-server/enterprise_gateway/blob/54c8e31d9b17418f35454b49db691d2ce5643c22/etc/kernelspecs/spark_python_yarn_cluster/kernel.json#L14). In Kubernetes, we apply the kernel's ID to the [`kernel-id` label on the POD](https://github.com/jupyter-server/enterprise_gateway/blob/54c8e31d9b17418f35454b49db691d2ce5643c22/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2#L16). -2. Today, all invocations of kernels into resource managers use a shell or python script mechanism configured into the `argv` stanza of the kernelspec. If you take this approach, you need to apply the necessary changes to integrate with your resource manager. -3. Determine how to interact with the resource manager's API to _discover_ the kernel and determine on which host it's running. This interaction should occur immediately following Enterprise Gateway's receipt of the kernel's connection information in its response from the kernel launcher. This extra step, performed within `confirm_remote_startup()`, is necessary to get the appropriate host name as reflected in the resource manager's API. -4. Determine how to monitor the "job" using the resource manager API. This will become part of the `poll()` implementation to determine if the kernel is still running. This should be as quick as possible since it occurs every 3 seconds. If this is an expensive call, you may need to make some adjustments like skip the call every so often. -5. Determine how to terminate "jobs" using the resource manager API. This will become part of the termination sequence, but probably only necessary if the message-based shutdown does not work (i.e., a last resort). +1. Today, all invocations of kernels into resource managers use a shell or python script mechanism configured into the `argv` stanza of the kernelspec. If you take this approach, you need to apply the necessary changes to integrate with your resource manager. +1. Determine how to interact with the resource manager's API to _discover_ the kernel and determine on which host it's running. This interaction should occur immediately following Enterprise Gateway's receipt of the kernel's connection information in its response from the kernel launcher. This extra step, performed within `confirm_remote_startup()`, is necessary to get the appropriate host name as reflected in the resource manager's API. +1. Determine how to monitor the "job" using the resource manager API. This will become part of the `poll()` implementation to determine if the kernel is still running. This should be as quick as possible since it occurs every 3 seconds. If this is an expensive call, you may need to make some adjustments like skip the call every so often. +1. Determine how to terminate "jobs" using the resource manager API. This will become part of the termination sequence, but probably only necessary if the message-based shutdown does not work (i.e., a last resort). ```{tip} Because kernel IDs are globally unique, they serve as ideal identifiers for discovering where in the cluster the kernel is running. diff --git a/docs/source/developers/kernel-launcher.md b/docs/source/developers/kernel-launcher.md index b5018b1c9..d8a252dee 100644 --- a/docs/source/developers/kernel-launcher.md +++ b/docs/source/developers/kernel-launcher.md @@ -7,9 +7,9 @@ Its generally recommended that the launcher be written in the language of the ke To reiterate, the four tasks of a kernel launcher are: 1. Create the necessary connection information based on the 5 zero-mq ports, a signature key and algorithm specifier, along with a _gateway listener_ socket. -2. Conveyance of the connection (and listener socket) information back to the Enterprise Gateway process after encrypting the information using AES, then encrypting the AES key using the provided public key. -3. Invocation of the target kernel. -4. Listen for interrupt and shutdown requests from Enterprise Gateway on the communication socket and carry out the action when appropriate. +1. Conveyance of the connection (and listener socket) information back to the Enterprise Gateway process after encrypting the information using AES, then encrypting the AES key using the provided public key. +1. Invocation of the target kernel. +1. Listen for interrupt and shutdown requests from Enterprise Gateway on the communication socket and carry out the action when appropriate. ## Creating the connection information diff --git a/docs/source/operators/config-cli.md b/docs/source/operators/config-cli.md index c3c64069c..39c44ae4e 100644 --- a/docs/source/operators/config-cli.md +++ b/docs/source/operators/config-cli.md @@ -297,11 +297,11 @@ WebhookKernelSessionManager(KernelSessionManager) options --------------------------------------------------------- --WebhookKernelSessionManager.enable_persistence= Enable kernel session persistence (True or False). Default = False - (EG_KERNEL_SESSION_PERSISTENCE env var) + (EG_KERNEL_SESSION_PERSISTENCE env var) Default: False --WebhookKernelSessionManager.persistence_root= Identifies the root 'directory' under which the 'kernel_sessions' node will - reside. This directory should exist. (EG_PERSISTENCE_ROOT env var) + reside. This directory should exist. (EG_PERSISTENCE_ROOT env var) Default: None --WebhookKernelSessionManager.webhook_url= URL endpoint for webhook kernel session manager diff --git a/docs/source/operators/config-dynamic.md b/docs/source/operators/config-dynamic.md index 1e2a85763..7149c15a6 100644 --- a/docs/source/operators/config-dynamic.md +++ b/docs/source/operators/config-dynamic.md @@ -12,11 +12,11 @@ values, but does have the following caveats: 1. Any configuration variables set on the command line (CLI) or via environment variables are NOT eligible for dynamic updates. This is because Jupyter gives those values priority over file-based configuration variables. -2. Any configuration variables tied to background processing may not reflect their update if +1. Any configuration variables tied to background processing may not reflect their update if the variable is not _observed_ for changes. For example, the code behind `RemoteKernelManager.cull_idle_timeout` may not reflect changes to the timeout period if that variable is not monitored (i.e., observed) for changes. -3. Only `Configurables` registered by Enterprise Gateway are eligible for dynamic updates. +1. Only `Configurables` registered by Enterprise Gateway are eligible for dynamic updates. Currently, that list consists of the following (and their subclasses): EnterpriseGatewayApp, RemoteKernelManager, KernelSpecManager, and KernelSessionManager. diff --git a/docs/source/operators/config-security.md b/docs/source/operators/config-security.md index 6f98a1fb5..3ee61f1f7 100644 --- a/docs/source/operators/config-security.md +++ b/docs/source/operators/config-security.md @@ -111,8 +111,8 @@ script and requires the following: 1. The gateway user (i.e., the user in which Enterprise Gateway is running) must be enabled to perform sudo operations on each potential host. This enablement must also be done to prevent password prompts since Enterprise Gateway runs in the background. Refer to your operating system documentation for details. -2. Each user identified by `KERNEL_USERNAME` must be associated with an actual operating system user on each host. -3. Once the gateway user is configured for `sudo` privileges it is **strongly recommended** that that user be included +1. Each user identified by `KERNEL_USERNAME` must be associated with an actual operating system user on each host. +1. Once the gateway user is configured for `sudo` privileges it is **strongly recommended** that that user be included in the set of `unauthorized_users`. Otherwise, kernels not configured for impersonation, or those requests that do not include `KERNEL_USERNAME`, will run as the, now, highly privileged gateway user! @@ -152,43 +152,43 @@ The list of [additional supported environment variables](config-add-env.md#addit Enterprise Gateway supports Secure Sockets Layer (SSL) communication with its clients. With SSL enabled, all the communication between the server and client are encrypted and highly secure. -1. You can start Enterprise Gateway to communicate via a secure protocol mode by setting the `certfile` and `keyfile` - options with the command: +1. You can start Enterprise Gateway to communicate via a secure protocol mode by setting the `certfile` and `keyfile` + options with the command: - ``` - jupyter enterprisegateway --ip=0.0.0.0 --port_retries=0 --certfile=mycert.pem --keyfile=mykey.key - ``` + ``` + jupyter enterprisegateway --ip=0.0.0.0 --port_retries=0 --certfile=mycert.pem --keyfile=mykey.key + ``` - As server starts up, the log should reflect the following, + As server starts up, the log should reflect the following, - ``` - [EnterpriseGatewayApp] Jupyter Enterprise Gateway at https://localhost:8888 - ``` + ``` + [EnterpriseGatewayApp] Jupyter Enterprise Gateway at https://localhost:8888 + ``` - Note: Enterprise Gateway server is started with `HTTPS` instead of `HTTP`, meaning server side SSL is enabled. + Note: Enterprise Gateway server is started with `HTTPS` instead of `HTTP`, meaning server side SSL is enabled. - TIP: - A self-signed certificate can be generated with openssl. For example, the following command will create a - certificate valid for 365 days with both the key and certificate data written to the same file: + ````{tip} + A self-signed certificate can be generated with openssl. For example, the following command will create a + certificate valid for 365 days with both the key and certificate data written to the same file: - ```bash - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mykey.key -out mycert.pem - ``` + ```bash + openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mykey.key -out mycert.pem + ```` -2. With Enterprise Gateway server SSL enabled, now you need to configure the client side SSL, which is accomplished via the Gateway configuration options embedded in Notebook server. +1. With Enterprise Gateway server SSL enabled, now you need to configure the client side SSL, which is accomplished via the Gateway configuration options embedded in Notebook server. - During Jupyter Notebook server startup, export the following environment variables where the gateway-enabled server has access - during runtime: + During Jupyter Notebook server startup, export the following environment variables where the gateway-enabled server has access + during runtime: - ```bash - export JUPYTER_GATEWAY_CLIENT_CERT=${PATH_TO_PEM_FILE} - export JUPYTER_GATEWAY_CLIENT_KEY=${PATH_TO_KEY_FILE} - export JUPYTER_GATEWAY_CA_CERTS=${PATH_TO_SELFSIGNED_CA} - ``` + ```bash + export JUPYTER_GATEWAY_CLIENT_CERT=${PATH_TO_PEM_FILE} + export JUPYTER_GATEWAY_CLIENT_KEY=${PATH_TO_KEY_FILE} + export JUPYTER_GATEWAY_CA_CERTS=${PATH_TO_SELFSIGNED_CA} + ``` - ```{note} - If using a self-signed certificate, you can set `JUPYTER_GATEWAY_CA_CERTS` same as `JUPYTER_GATEWAY_CLIENT_CERT`. - ``` + ```{note} + If using a self-signed certificate, you can set `JUPYTER_GATEWAY_CA_CERTS` same as `JUPYTER_GATEWAY_CLIENT_CERT`. + ``` ### Using Enterprise Gateway configuration file @@ -196,10 +196,10 @@ You can also utilize the [Enterprise Gateway configuration file](config-file.md# To enable SSL from the configuration file, modify the corresponding parameter to the appropriate value. - ``` - c.EnterpriseGatewayApp.certfile = '/absolute/path/to/your/certificate/fullchain.pem' - c.EnterpriseGatewayApp.keyfile = '/absolute/path/to/your/certificate/privatekey.key' - ``` +``` +c.EnterpriseGatewayApp.certfile = '/absolute/path/to/your/certificate/fullchain.pem' +c.EnterpriseGatewayApp.keyfile = '/absolute/path/to/your/certificate/privatekey.key' +``` Using configuration file achieves the same result as starting the server with `--certfile` and `--keyfile`, this way provides better readability and maintainability. diff --git a/docs/source/operators/deploy-distributed.md b/docs/source/operators/deploy-distributed.md index b6f97c9d7..0fa46b191 100644 --- a/docs/source/operators/deploy-distributed.md +++ b/docs/source/operators/deploy-distributed.md @@ -5,9 +5,9 @@ This section describes how to deploy Enterprise Gateway to manage kernels across Steps required to complete deployment on a distributed cluster are: 1. [Install Enterprise Gateway](installing-eg.md) on the "primary node" of the cluster. -2. [Install the desired kernels](installing-kernels.md) -3. Install and configure the server and desired kernel specifications (see below) -4. [Launch Enterprise Gateway](launching-eg.md) +1. [Install the desired kernels](installing-kernels.md) +1. Install and configure the server and desired kernel specifications (see below) +1. [Launch Enterprise Gateway](launching-eg.md) The `DistributedProcessProxy` simply uses a fixed set of host names and selects the _next_ host using a simple round-robin algorithm (see the [Roadmap](../contributors/roadmap.md) for making this pluggable). In this case, you can still experience bottlenecks on a given node that receives requests to start "large" kernels, but otherwise, you will be better off compared to when all kernels are started on a single node or as local processes, which is the default for Jupyter Notebook and JupyterLab when not configured to use Enterprise Gateway. diff --git a/docs/source/operators/deploy-docker.md b/docs/source/operators/deploy-docker.md index 0bed20c1b..972148ffc 100644 --- a/docs/source/operators/deploy-docker.md +++ b/docs/source/operators/deploy-docker.md @@ -63,8 +63,8 @@ When the process proxy class is `DockerProcessProxy` the `launch_docker.py` scri Items worth noting: 1. The Swarm service or Docker container name will be composed of the launching username (`KERNEL_USERNAME`) and kernel-id. -2. The service/container will have 3 labels applied: "kernel_id=", "component=kernel", and "app=enterprise-gateway" - similar to Kubernetes. -3. The service/container will be launched within the same docker network as Enterprise Gateway. +1. The service/container will have 3 labels applied: "kernel_id=", "component=kernel", and "app=enterprise-gateway" - similar to Kubernetes. +1. The service/container will be launched within the same docker network as Enterprise Gateway. ## DockerSwarmProcessProxy diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md index 2426dee06..e58785f5d 100644 --- a/docs/source/operators/deploy-kubernetes.md +++ b/docs/source/operators/deploy-kubernetes.md @@ -26,7 +26,7 @@ We are using helm templates to manage Kubernetes resource configurations, which There are two main deployment scenarios if RBAC is enabled in your Kubernetes cluster: 1. Deployment user has **_Cluster Administrator Access_**. In this scenario, you have full access to the cluster and can deploy all components as needed. -2. Deployment user has **_Namespace Administrator Access_**. This is typical for shared multi-tenant environments where each Team has control over their namespace, but not the cluster. In this scenario, your cluster Administrator can deploy the RBAC resources and Kernel Image Puller and you can deploy Enterprise Gateway. +1. Deployment user has **_Namespace Administrator Access_**. This is typical for shared multi-tenant environments where each Team has control over their namespace, but not the cluster. In this scenario, your cluster Administrator can deploy the RBAC resources and Kernel Image Puller and you can deploy Enterprise Gateway. ## Prerequisites @@ -56,7 +56,7 @@ helm upgrade --install enterprise-gateway \ ``` -Alternatively, the helm chart tarball is also accessible as an asset on our [release](https://github.com/jupyter-server/enterprise_gateway/releases) page, replace [VERSION] with specific release version you want to use: +Alternatively, the helm chart tarball is also accessible as an asset on our [release](https://github.com/jupyter-server/enterprise_gateway/releases) page, replace \[VERSION\] with specific release version you want to use: ```bash helm upgrade --install enterprise-gateway \ @@ -78,7 +78,7 @@ If you do not have a Kubernetes Ingress configured on your cluster the easiest w - Ingress controller deployed on your Kubernetes cluster. Review the Kubernetes [documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) for available options. - Wildcard DNS record is configured to point to the IP of the LoadBalancer, which frontends your ingress controller -- Review specific Ingress controller configuration to enable wildcard path support if you are using Kubernetes version < v1.18 +- Review specific Ingress controller configuration to enable wildcard path support if you are using Kubernetes version \< v1.18 - With Kubernetes v1.18 Ingress uses `PathType` parameter which is set to `Prefix` in the helm chart by default, so no additional configuration is required - Refer to your ingress controller documentation on how to set up TLS with your ingress @@ -94,7 +94,7 @@ ingress: ``` -Add this file to your helm command and apply to the cluster replacing [PLACEHOLDER] with appropriate values for your environment: +Add this file to your helm command and apply to the cluster replacing \[PLACEHOLDER\] with appropriate values for your environment: ```bash helm upgrade --install enterprise-gateway \ @@ -251,7 +251,7 @@ ingress: ### Deploy with helm -Add values file to your helm command and apply to the cluster replacing [PLACEHOLDER] with appropriate values for your environment: +Add values file to your helm command and apply to the cluster replacing \[PLACEHOLDER\] with appropriate values for your environment: ```bash helm upgrade --install enterprise-gateway @@ -268,7 +268,7 @@ if you are using private registry add setting base64 encoded secret value to you Choose this deployment option if you want to deploy directly from Kubernetes template files with kubectl, rather than using a package manager like helm. -Add values file to your helm command and generate `yaml` files replacing [PLACEHOLDER] with appropriate values for your environment: +Add values file to your helm command and generate `yaml` files replacing \[PLACEHOLDER\] with appropriate values for your environment: ```bash helm template \ @@ -444,10 +444,10 @@ can override them with helm's `--set` or `--values` options. Always use `--set` | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------ | | `global.rbac` | Create Kubernetes RBAC resources | `true` | | `global.commonLabels` | Common labels to apply to daemonset and deployment resources | `{}` | -| `global.imagePullSecrets` | Optional array of image pull secrets for Service Account for pulling images from private service registries | [] | +| `global.imagePullSecrets` | Optional array of image pull secrets for Service Account for pulling images from private service registries | \[\] | | `imagePullSecretsCreate.enabled` | Optional enable creation of the Kubernetes secrets to access private registries. | 'false' | | `imagePullSecretsCreate.annotations` | Annotations for Kubernetes secrets | '{}' | -| `imagePullSecretsCreate.secrets` | Array of Kubernetes secrets to create with the following structure: `name` - secret name and `data` - base64 encoded Secret value. Example: `{ name: "myregistrykey", data: "SGVsbG8gc2VjcmV0Cg==" }` | '[]' | +| `imagePullSecretsCreate.secrets` | Array of Kubernetes secrets to create with the following structure: `name` - secret name and `data` - base64 encoded Secret value. Example: `{ name: "myregistrykey", data: "SGVsbG8gc2VjcmV0Cg==" }` | '\[\]' | | `image` | Enterprise Gateway image name and tag to use. Ensure the tag is updated to the version of Enterprise Gateway you wish to run. | `elyra/enterprise-gateway:VERSION`, where `VERSION` is the release being used | | `imagePullPolicy` | Enterprise Gateway image pull policy. Use `IfNotPresent` policy so that dev-based systems don't automatically update. This provides more control. Since formal tags will be release-specific this policy should be sufficient for them as well. | `IfNotPresent` | | `service.type` | Kubernetes Service Type - Nodeport,ClusterIP,LoadBalancer | `ClusterIP` | @@ -736,9 +736,13 @@ There are a number of items worth noting: Note that since kernels run in isolated namespaces by default, it's often helpful to include the clause `--all-namespaces` on commands that will span namespaces. To isolate commands to a given namespace, you'll need to add the namespace clause `--namespace `. 1. Each kernel pod is named by the invoking user (via the `KERNEL_USERNAME` env) and its kernel_id (env `KERNEL_ID`). This identifier also applies to those kernels launched within `spark-on-kubernetes`. + 1. Kernel pods use the specified `securityContext`. If env `KERNEL_UID` is not specified in the kernel creation request a default value of `1000` (the jovyan user) will be used. Similarly, for `KERNEL_GID`, whose default is `100` (the users group). In addition, Enterprise Gateway enforces a list of prohibited UID and GID values. By default, this list is initialized to the 0 (root) UID and GID. Administrators can configure the `EG_PROHIBITED_UIDS` and `EG_PROHIBITED_GIDS` environment variables via the `deployment.yaml` file with comma-separated values to alter the set of user and group ids to be prevented. + 1. As noted above, if `KERNEL_NAMESPACE` is not provided in the request, Enterprise Gateway will create a namespace using the same naming algorithm for the pod. In addition, the `kernel-controller` cluster role will be bound to a namespace-scoped role binding of the same name using the namespace's default service account as its subject. Users wishing to use their own kernel namespaces must provide **both** `KERNEL_NAMESPACE` and `KERNEL_SERVICE_ACCOUNT_NAME` as these are both used in the `kernel-pod.yaml.j2` as `{{ kernel_namespace }}` and `{{ kernel_service_account_name }}`, respectively. + 1. Kernel pods have restart policies of `Never`. This is because the Jupyter framework already has built-in logic for auto-restarting failed kernels and any other restart policy would likely interfere with the built-in behaviors. + 1. The parameters to the launcher that is built into the image are communicated via environment variables as noted in the `env:` section above. ## Unconditional Volume Mounts @@ -980,7 +984,7 @@ https://elyra-kube1.foo.bar.com:30000/dashboard/#!/overview?namespace=default From there, logs can be accessed by selecting the `Pods` option in the left-hand pane followed by the _lined_ icon on the far right. -- User \"system:serviceaccount:default:default\" cannot list pods in the namespace \"default\" +- User "system:serviceaccount:default:default" cannot list pods in the namespace "default" On a recent deployment, Enterprise Gateway was not able to create or list kernel pods. Found the following command was necessary. (Kubernetes security relative to Enterprise Gateway is still under construction.) diff --git a/docs/source/operators/deploy-single.md b/docs/source/operators/deploy-single.md index df611ffba..5c8be927e 100644 --- a/docs/source/operators/deploy-single.md +++ b/docs/source/operators/deploy-single.md @@ -5,9 +5,9 @@ Single-server deployment can be useful for development and is not meant to be ru Steps to deploy a single server are: 1. [Install Enterprise Gateway](installing-eg.md) -2. [Install the desired kernels](installing-kernels.md) -3. Install and configure the server and desired kernel specifications (see below) -4. [Launch Enterprise Gateway](launching-eg.md) +1. [Install the desired kernels](installing-kernels.md) +1. Install and configure the server and desired kernel specifications (see below) +1. [Launch Enterprise Gateway](launching-eg.md) If you just want to try Enterprise Gateway in a single-server setup, you can use the following kernels specification (no need for a kernel launcher since the kernel runs locally): diff --git a/docs/source/operators/deploy-yarn-cluster.md b/docs/source/operators/deploy-yarn-cluster.md index 764ba410b..21935ecd3 100644 --- a/docs/source/operators/deploy-yarn-cluster.md +++ b/docs/source/operators/deploy-yarn-cluster.md @@ -11,9 +11,9 @@ The following sample kernelspecs are currently available on YARN cluster: Steps required to complete deployment on a Hadoop YARN cluster are: 1. [Install Enterprise Gateway](installing-eg.md) on the primary node of the Hadoop YARN cluster. Note, this location is not a hard-requirement, but recommended. If installed remotely, some extra configuration will be necessary relative to the Hadoop configuration. -2. [Install the desired kernels](installing-kernels.md) -3. Install and configure the server and desired kernel specifications (see below) -4. [Launch Enterprise Gateway](launching-eg.md) +1. [Install the desired kernels](installing-kernels.md) +1. Install and configure the server and desired kernel specifications (see below) +1. [Launch Enterprise Gateway](launching-eg.md) The distributed capabilities are currently based on an Apache Spark cluster utilizing Hadoop YARN as the resource manager and thus require the following environment variables to be set to facilitate the integration between Apache Spark and Hadoop YARN components: diff --git a/docs/source/other/troubleshooting.md b/docs/source/other/troubleshooting.md index a797478b2..0a355d346 100644 --- a/docs/source/other/troubleshooting.md +++ b/docs/source/other/troubleshooting.md @@ -13,14 +13,14 @@ be validated independently. The following items can be used as a checklist to co 1. Confirm that Enterprise Gateway is servicing general requests. This can be accomplished using the following `curl` command, which should produce the json corresponding to the configured kernelspecs: `bash curl http://:/api/kernelspecs ` -2. Independently validate any resource manager you're running against. Various resource managers usually provide +1. Independently validate any resource manager you're running against. Various resource managers usually provide examples for how to go about validating their configuration. -3. Confirm that the Enterprise Gateway arguments for contacting the configured resource manager are in place. These +1. Confirm that the Enterprise Gateway arguments for contacting the configured resource manager are in place. These should be covered in the deployment section of our Operators Guide. -4. If using a Notebook server as your front-end, ensure that the Gateway configuration options or NB2KG extension settings are properly configured. +1. If using a Notebook server as your front-end, ensure that the Gateway configuration options or NB2KG extension settings are properly configured. Once the notebook has started, a refresh on the tree view should issue the same `kernelspecs` request in step 1 and the drop-down menu items for available kernels should reflect an entry for each kernelspec returned. -5. **Always** consult your Enterprise Gateway log file. If you have not redirected `stdout` and `stderr` to a +1. **Always** consult your Enterprise Gateway log file. If you have not redirected `stdout` and `stderr` to a file you are highly encouraged to do so. In addition, you should enable `DEBUG` logging at least until your configuration is stable. Please note, however, that you may be asked to produce an Enterprise Gateway log with `DEBUG` enabled when reporting issues. An example of output redirection and `DEBUG` logging is also provided in our @@ -35,8 +35,8 @@ a "Kernel error" and State: 'FAILED'.** generated, make a note of it. For example, you can locate the applicationId `application_1506552273380_0011` from the following snippet of message: `[D 2017-09-28 17:13:22.675 EnterpriseGatewayApp] 13: State: 'ACCEPTED', Host: 'burna2.yourcompany.com', KernelID: '28a5e827-4676-4415-bbfc-ac30a0dcc4c3', ApplicationID: 'application_1506552273380_0011' 17/09/28 17:13:22 INFO YarnClientImpl: Submitted application application_1506552273380_0011 17/09/28 17:13:22 INFO Client: Application report for application_1506552273380_0011 (state: ACCEPTED) 17/09/28 17:13:22 INFO Client: client token: N/A diagnostics: AM container is launched, waiting for AM container to Register with RM ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: default start time: 1506644002471 final status: UNDEFINED tracking URL: http://burna1.yourcompany.com:8088/proxy/application_1506552273380_0011/` -2. Lookup the YARN log for that applicationId in the YARN ResourceManager UI: ![YARN ResourceManager UI](../images/yarnui.jpg) -3. Drill down from the applicationId to find logs for the failed attempts and take appropriate +1. Lookup the YARN log for that applicationId in the YARN ResourceManager UI: ![YARN ResourceManager UI](../images/yarnui.jpg) +1. Drill down from the applicationId to find logs for the failed attempts and take appropriate actions. For example, for the error below, ``` Traceback (most recent call last): @@ -182,7 +182,7 @@ RuntimeError: Invalid port range '1000..2000' specified. Range for valid port nu ``` To address this issue, make sure that the specified port range does not overlap with TCP's well-known -port range of (0, 1024]. +port range of (0, 1024\]. ## Hadoop YARN Timeout @@ -243,9 +243,9 @@ Scenario: **Running Jupyter Enterprise Gateway on OpenShift Kubernetes Environme As described [in the OpenShift Admin Guide](https://docs.openshift.com/container-platform/4.10/openshift_images/create-images.html) there is a need to issue the following command to enable running with `USER` in Dockerfile. - ```bash - oc adm policy add-scc-to-group anyuid system:authenticated - ``` +```bash +oc adm policy add-scc-to-group anyuid system:authenticated +``` ## Opening an issue @@ -257,11 +257,11 @@ already reported. If found, please add a comment to the issue so that we can get issues are important to us). If not found, please provide the following information if possible in a **new issue**. 1. Describe the issue in as much detail as possible. This should include configuration information about your environment. -2. Gather and _attach_ the following files to the issue. If possible, please archive the files first. +1. Gather and _attach_ the following files to the issue. If possible, please archive the files first. 1. The **complete** Enterprise Gateway log file. If possible, please enable `DEBUG` logging that encompasses the issue. You can refer to this section of our [Operators Guide](../operators/launching-eg.md#launching-enterprise-gateway-common) for redirection and `DEBUG` enablement. - 2. The log file(s) produced from the corresponding kernel. This is primarily a function of the underlying resource + 1. The log file(s) produced from the corresponding kernel. This is primarily a function of the underlying resource manager. - For containerized installations like Kubernetes or Docker Swarm, kernel log output can be captured by running the appropriate `logs` command against the pod or container, respectively. The names of the @@ -270,9 +270,9 @@ issues are important to us). If not found, please provide the following informat you'll need to navigate to the appropriate log directory relative the application ID associated with the kernel. The application ID can be located in the Enterprise Gateway log. If you have access to an administrative console, you can usually navigate to the application logs more easily. - 3. Although unlikely, the notebook log may also be helpful. If we find that the issue is more client-side + 1. Although unlikely, the notebook log may also be helpful. If we find that the issue is more client-side related, we may ask for `DEBUG` logging there as well. -3. If you have altered or created new kernel specifications, the files corresponding to the failing kernels would be +1. If you have altered or created new kernel specifications, the files corresponding to the failing kernels would be helpful. These files could also be added to the attached archive or attached separately. Please know that we understand that some information cannot be provided due to its sensitivity. In such cases, just diff --git a/enterprise_gateway/services/sessions/kernelsessionmanager.py b/enterprise_gateway/services/sessions/kernelsessionmanager.py index bd5d90234..4d28f2570 100644 --- a/enterprise_gateway/services/sessions/kernelsessionmanager.py +++ b/enterprise_gateway/services/sessions/kernelsessionmanager.py @@ -47,7 +47,7 @@ class KernelSessionManager(LoggingConfigurable): session_persistence_default_value, config=True, help="""Enable kernel session persistence (True or False). Default = False - (EG_KERNEL_SESSION_PERSISTENCE env var)""", +(EG_KERNEL_SESSION_PERSISTENCE env var)""", ) @default("enable_persistence") @@ -64,7 +64,7 @@ def session_persistence_default(self) -> bool: persistence_root = Unicode( config=True, help="""Identifies the root 'directory' under which the 'kernel_sessions' node will - reside. This directory should exist. (EG_PERSISTENCE_ROOT env var)""", +reside. This directory should exist. (EG_PERSISTENCE_ROOT env var)""", ) @default("persistence_root") diff --git a/pyproject.toml b/pyproject.toml index f5b96e5f0..8639bad0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,3 +110,23 @@ filterwarnings = [ "ignore:Passing unrecognized arguments to super:DeprecationWarning:jupyter_client", "ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning", ] + +[tool.flake8] +ignore = "E501, W503, E402" +builtins = "c, get_config" +exclude = [ + ".cache", + ".github", + "docs", +] +enable-extensions = "G" +extend-ignore = [ + "G001", "G002", "G004", "G200", "G201", "G202", + # black adds spaces around ':' + "E203", +] +per-file-ignores = [ + # B011: Do not call assert False since python -O removes these calls + # F841 local variable 'foo' is assigned to but never used + "enterprise_gateway/tests/*: B011", "F841", +] diff --git a/website/index.md b/website/index.md index e4d427d21..72df6b6e5 100644 --- a/website/index.md +++ b/website/index.md @@ -1,3 +1,3 @@ ---- -layout: home ---- +______________________________________________________________________ + +## layout: home diff --git a/website/platform-kubernetes.md b/website/platform-kubernetes.md index e276e3282..5306ded63 100644 --- a/website/platform-kubernetes.md +++ b/website/platform-kubernetes.md @@ -1,7 +1,6 @@ ---- -layout: page -title: Jupyter Enterprise Gateway and Kubernetes ---- +______________________________________________________________________ + +## layout: page title: Jupyter Enterprise Gateway and Kubernetes Recently, we have experienced various advances in AI, in particular around Deep Learning. This have increased the popularity of Deep Learning use cases and also the proliferation of several development @@ -38,7 +37,6 @@ Jupyter Enterprise Gateway can easily be deployed into your Kubernetes cluster: 
kubectl apply -f https://raw.githubusercontent.com/jupyter/enterprise_gateway/main/etc/kubernetes/enterprise-gateway.yaml
- #### Deployment Scripts
@@ -47,4 +45,4 @@ The Jupyter Enterprise Gateway development team uses some Ansible scripts for pr test environments, these scripts might be useful for users trying to get started with the gateway on a Kubernetes environment. -* Ansible Deployment scripts : ansible-kubernetes-cluster +- Ansible Deployment scripts : ansible-kubernetes-cluster diff --git a/website/platform-spark.md b/website/platform-spark.md index 335021340..f93deb35c 100644 --- a/website/platform-spark.md +++ b/website/platform-spark.md @@ -1,7 +1,6 @@ ---- -layout: page -title: Jupyter Enterprise Gateway and Apache Spark ---- +______________________________________________________________________ + +## layout: page title: Jupyter Enterprise Gateway and Apache Spark The Big Data Analytics use cases require processing large data sets which are not containable by the resources available on a single machine. @@ -29,7 +28,6 @@ security enable on the cluster, then each notebook kernel will be running as the from the users that requested the notebook kernel, thus leveraging all configured ACLs when accessing HDFS, and other secured resources. -
@@ -48,4 +46,4 @@ Jupyter Enterprise Gateway in a vanilla deployment of Spark and YARN. The Jupyter Enterprise Gateway development team uses some Ansible scripts for provisioning test environments, these scripts might be useful for users trying to get started with the gateway. -* Ansible Deployment scripts : ansible-spark-cluster +- Ansible Deployment scripts : ansible-spark-cluster diff --git a/website/privacy-policy.md b/website/privacy-policy.md index ad06966e5..44efe9c2e 100644 --- a/website/privacy-policy.md +++ b/website/privacy-policy.md @@ -1,18 +1,17 @@ ---- -layout: page -title: Jupyter Enterprise Gateway Privacy Policy ---- +______________________________________________________________________ + +## layout: page title: Jupyter Enterprise Gateway Privacy Policy ## Jupyter Enterprise Gateway Privacy Policy Information about your use of this website is collected using server access logs and a tracking cookie. The collected information consists of the following: -* The IP address from which you access the website; -* The type of browser and operating system you use to access our site; -* The date and time you access our site; -* The pages you visit; and -* The addresses of pages from where you followed a link to our site. +- The IP address from which you access the website; +- The type of browser and operating system you use to access our site; +- The date and time you access our site; +- The pages you visit; and +- The addresses of pages from where you followed a link to our site. Part of this information is gathered using a tracking cookie set by the [Google Analytics](https://www.google.com/analytics/) service and handled by Google as described in their [privacy policy](https://www.google.com/privacy.html).