From d6b7efd9c4a9891b2e873f62b53e333b6591ee0b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 7 Aug 2024 13:00:19 -0500 Subject: [PATCH 1/6] Use `/run` as the default for PID/UNIX-domain sockets --- docs/src/graceful_termination.md | 32 +++++++++++++++++++++++++++++--- src/graceful_termination.jl | 29 ++++++++++++----------------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/docs/src/graceful_termination.md b/docs/src/graceful_termination.md index 3a1d1de..5378d6f 100644 --- a/docs/src/graceful_termination.md +++ b/docs/src/graceful_termination.md @@ -69,7 +69,32 @@ Finally, the entrypoint for the container should also not directly use the Julia ### Read-only Filesystem -If you have a read-only filesystem on your container you'll need to configure a writeable volume mount for K8sDeputy.jl. The `DEPUTY_IPC_DIR` environmental variable can be used to instruct K8sDeputy.jl where to store the named pipes it creates for interprocess communication: +If you have a read-only filesystem on your container you'll need to configure a writeable `/run` directory for K8sDeputy.jl so it can create UNIX-domain sockets for interprocess communication: + +```yaml +apiVersion: v1 +kind: Pod +spec: + containers: + - name: app + # command: ["/bin/sh", "-c", "julia entrypoint.jl; sleep 1"] + lifecycle: + preStop: + exec: + command: ["julia", "-e", "using K8sDeputy; graceful_terminate()"] + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - name: ipc + mountPath: /run + subPath: app # Set the `subPath` to the container name to ensure per-container isolation + volumes: + - name: ipc + emptyDir: + medium: Memory +``` + +Alternatively, if you don't want to specify the write location for these IPC files you can use the `DEPUTY_IPC_DIR` environmental variable: ```yaml apiVersion: v1 @@ -88,8 +113,9 @@ spec: securityContext: readOnlyRootFilesystem: true volumeMounts: - - mountPath: /mnt/deputy-ipc - name: deputy-ipc + - name: deputy-ipc + mountPath: /mnt/deputy-ipc + subPath: app # Set the `subPath` to the container name to ensure per-container isolation volumes: - name: deputy-ipc emptyDir: diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index b15b116..0ce2cce 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -9,23 +9,18 @@ # Linux typically stores PID files in `/run` which requires root access. For systems with # read-only file systems we need to support a user specified writable volume. -_deputy_ipc_dir() = get(tempdir, ENV, "DEPUTY_IPC_DIR") +_deputy_ipc_dir() = get(ENV, "DEPUTY_IPC_DIR", "/run") -# Prefer using UNIX domain sockets but if the `DEPUTY_IPC_DIR` is set assume the file -# system is read-only and use a named pipe instead. +# Write transient UNIX-domain sockets to the IPC directory. function _graceful_terminator_socket_path(pid::Int32) - name = "graceful-terminator.$pid" - return haskey(ENV, "DEPUTY_IPC_DIR") ? joinpath(_deputy_ipc_dir(), name) : name + name = "graceful-terminator.$pid.sock" + return joinpath(_deputy_ipc_dir(), name) end # Following the Linux convention for pid files: # https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s15.html entrypoint_pid_file() = joinpath(_deputy_ipc_dir(), "julia-entrypoint.pid") -function set_entrypoint_pid(pid::Integer) - file = entrypoint_pid_file() - mkpath(dirname(file)) - return write(file, string(pid) * "\n") -end +set_entrypoint_pid(pid::Integer) = write(entrypoint_pid_file(), string(pid) * "\n") function entrypoint_pid() pid_file = entrypoint_pid_file() @@ -85,15 +80,15 @@ process and the `preStop` process to cleanly terminate. function graceful_terminator(f; set_entrypoint::Bool=true) set_entrypoint && set_entrypoint_pid(getpid()) - # Utilize UNIX domain sockets or named pipes for the IPC. Avoid using network sockets - # here as we don't want to allow access to this functionality from outside of the - # localhost. Each process uses a distinct socket name allowing for multiple Julia - # processes to allow independent use of the graceful terminator. + # Utilize UNIX-domain sockets (Linux) or named pipes (Windows) for the IPC. Avoid using + # network sockets here as we don't want to allow access to this functionality from + # outside of the localhost. Each process uses a distinct socket name allowing for + # multiple Julia processes to allow independent use of the graceful terminator. socket_path = _graceful_terminator_socket_path(getpid()) - # Remove any pre-existing named pipe as otherwise this will cause our `listen` call to - # fail. Should be safe to remove this file as it has been reserved for this PID. Only - # should be needed in the scenario where the K8s pod has been restarted and the + # Remove any pre-existing UNIX-domain socket as otherwise this will cause our `listen` + # call to fail. Should be safe to remove this file as it has been reserved for this PID. + # Only should be needed in the scenario where the K8s pod has been restarted and the # location of the socket exists in a K8s volume. ispath(socket_path) && rm(socket_path) From 9b87bb61c02d9df079816888e1e455258c6e45fd Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 7 Aug 2024 13:21:29 -0500 Subject: [PATCH 2/6] Keep DEPUTY_IPC_DIR for testing --- docs/src/quickstart.md | 10 +++--- src/graceful_termination.jl | 9 +++--- test/graceful_termination.jl | 63 ++++++++++++++++++++---------------- test/health.jl | 6 +++- 4 files changed, 50 insertions(+), 38 deletions(-) diff --git a/docs/src/quickstart.md b/docs/src/quickstart.md index 4704bd2..38c9ed0 100644 --- a/docs/src/quickstart.md +++ b/docs/src/quickstart.md @@ -27,9 +27,6 @@ For users who want to get started quickly you can use the following template to containers: - name: app command: ["/bin/sh", "-c", "julia entrypoint.jl; sleep 1"] - env: - - name: DEPUTY_IPC_DIR - value: /mnt/deputy-ipc ports: - name: health-check containerPort: 8081 # Default K8sDeputy.jl heath check port @@ -54,11 +51,12 @@ For users who want to get started quickly you can use the following template to - all readOnlyRootFilesystem: true volumeMounts: - - mountPath: /mnt/deputy-ipc - name: deputy-ipc + - name: ipc + mountPath: /run + subPath: app # Set the `subPath` to the container name to ensure per-container isolation terminationGracePeriodSeconds: 30 volumes: - - name: deputy-ipc + - name: ipc emptyDir: medium: Memory ``` diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index 0ce2cce..2fbbf45 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -7,14 +7,15 @@ # from experimenting with this there are a few issues such as being unable to use locks or # printing (`jl_safe_printf` does work). -# Linux typically stores PID files in `/run` which requires root access. For systems with -# read-only file systems we need to support a user specified writable volume. +# Linux stores PID files and UNIX-domain sockets in `/run`. Users with K8s containers +# utilizing read-only file systems should make use of a volume mount to allow K8sDeputy.jl +# to write to `/run`. Users can change the IPC directory by specifying `DEPUTY_IPC_DIR` but +# this is mainly just used for testing. _deputy_ipc_dir() = get(ENV, "DEPUTY_IPC_DIR", "/run") # Write transient UNIX-domain sockets to the IPC directory. function _graceful_terminator_socket_path(pid::Int32) - name = "graceful-terminator.$pid.sock" - return joinpath(_deputy_ipc_dir(), name) + return joinpath(_deputy_ipc_dir(), "graceful-terminator.$pid.sock") end # Following the Linux convention for pid files: diff --git a/test/graceful_termination.jl b/test/graceful_termination.jl index a4196de..7775c09 100644 --- a/test/graceful_termination.jl +++ b/test/graceful_termination.jl @@ -1,4 +1,6 @@ @testset "graceful_terminator" begin + deputy_ipc_dir = mktempdir() + @testset "Julia entrypoint" begin code = quote using K8sDeputy @@ -12,6 +14,7 @@ end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) buffer = IOBuffer() p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) @test timedwait(() -> process_running(p), Second(5)) === :ok @@ -21,7 +24,9 @@ # When no PID is passed in the process ID is read from the Julia entrypoint file. # Blocks untils the process terminates. - @test graceful_terminate() === nothing + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + @test graceful_terminate() === nothing + end @test process_exited(p) @test p.exitcode == 2 @@ -47,6 +52,7 @@ end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) buffer1 = IOBuffer() buffer2 = IOBuffer() p1 = run(pipeline(cmd; stdout=buffer1, stderr=buffer1); wait=false) @@ -57,8 +63,10 @@ sleep(3) # Blocks untils the process terminates - @test graceful_terminate(getpid(p1)) === nothing - @test graceful_terminate(getpid(p2)) === nothing + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + @test graceful_terminate(getpid(p1)) === nothing + @test graceful_terminate(getpid(p2)) === nothing + end @test process_exited(p1) @test process_exited(p2) @@ -73,10 +81,9 @@ end # When users set `DEPUTY_IPC_DIR` they may be using a K8s volume. As even `emptyDir` - # volumes persist for the lifetime of the pod we may have a named pipe already present - # from a previous restart. + # volumes persist for the lifetime of the pod we may have a UNIX-domain socket already + # present from a previous restart. @testset "bind after restart" begin - deputy_ipc_dir = mktempdir() code = quote using K8sDeputy using Sockets: listen @@ -99,31 +106,33 @@ end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) + buffer = IOBuffer() + p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) + @test timedwait(() -> process_running(p), Second(5)) === :ok - withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do - buffer = IOBuffer() - p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) - @test timedwait(() -> process_running(p), Second(5)) === :ok - - # Allow some time for Julia to startup and the graceful terminator to be registered. - sleep(3) + # Allow some time for Julia to startup and the graceful terminator to be registered. + sleep(3) - # Socket exists as a named pipe - socket_path = K8sDeputy._graceful_terminator_socket_path(getpid(p)) - @test ispath(socket_path) - @test !isfile(socket_path) + # Socket exists as a UNIX-domain socket + socket_path = withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + K8sDeputy._graceful_terminator_socket_path(getpid(p)) + end + @test ispath(socket_path) + @test !isfile(socket_path) - # Blocks untils the process terminates + # Blocks untils the process terminates + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do @test graceful_terminate(getpid(p)) === nothing - @test process_exited(p) - @test p.exitcode == 2 - - output = String(take!(buffer)) - expected = """ - [ Info: GRACEFUL TERMINATION HANDLER - [ Info: SHUTDOWN COMPLETE - """ - @test output == expected end + @test process_exited(p) + @test p.exitcode == 2 + + output = String(take!(buffer)) + expected = """ + [ Info: GRACEFUL TERMINATION HANDLER + [ Info: SHUTDOWN COMPLETE + """ + @test output == expected end end diff --git a/test/health.jl b/test/health.jl index a2c9f22..1b61731 100644 --- a/test/health.jl +++ b/test/health.jl @@ -192,6 +192,7 @@ end end @testset "graceful termination" begin + deputy_ipc_dir = mktempdir() port = rand(EPHEMERAL_PORT_RANGE) code = quote using K8sDeputy, Sockets @@ -211,6 +212,7 @@ end end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) buffer = IOBuffer() p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) @test timedwait(() -> process_running(p), Second(5)) === :ok @@ -220,7 +222,9 @@ end end === :ok # Blocks untils the process terminates - graceful_terminate(getpid(p)) + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + graceful_terminate(getpid(p)) + end @test process_exited(p) @test p.exitcode == 1 From f74f029317a9bcb8ee89888c655a3a2e6fccd34f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 7 Aug 2024 13:21:48 -0500 Subject: [PATCH 3/6] Set project version to 0.1.4 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 15b58b3..016dd24 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "K8sDeputy" uuid = "2481ae95-212f-4650-bb21-d53ea3caf09f" authors = ["Beacon Biosignals, Inc"] -version = "0.1.3" +version = "0.1.4" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 4fc649a62ea3d0fb6c3dd33efa81cfcdd7734562 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 7 Aug 2024 13:28:28 -0500 Subject: [PATCH 4/6] Drop instructions on using DEPUTY_IPC_DIR --- docs/src/graceful_termination.md | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/docs/src/graceful_termination.md b/docs/src/graceful_termination.md index 5378d6f..c4f7a44 100644 --- a/docs/src/graceful_termination.md +++ b/docs/src/graceful_termination.md @@ -93,31 +93,3 @@ spec: emptyDir: medium: Memory ``` - -Alternatively, if you don't want to specify the write location for these IPC files you can use the `DEPUTY_IPC_DIR` environmental variable: - -```yaml -apiVersion: v1 -kind: Pod -spec: - containers: - - name: app - # command: ["/bin/sh", "-c", "julia entrypoint.jl; sleep 1"] - env: - - name: DEPUTY_IPC_DIR - value: /mnt/deputy-ipc - lifecycle: - preStop: - exec: - command: ["julia", "-e", "using K8sDeputy; graceful_terminate()"] - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - name: deputy-ipc - mountPath: /mnt/deputy-ipc - subPath: app # Set the `subPath` to the container name to ensure per-container isolation - volumes: - - name: deputy-ipc - emptyDir: - medium: Memory -``` From 7ebc730b8050756631686651792982ce744b496f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 7 Aug 2024 13:35:15 -0500 Subject: [PATCH 5/6] Formatting Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/graceful_termination.jl | 2 +- test/health.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/graceful_termination.jl b/test/graceful_termination.jl index 7775c09..1042005 100644 --- a/test/graceful_termination.jl +++ b/test/graceful_termination.jl @@ -116,7 +116,7 @@ # Socket exists as a UNIX-domain socket socket_path = withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do - K8sDeputy._graceful_terminator_socket_path(getpid(p)) + return K8sDeputy._graceful_terminator_socket_path(getpid(p)) end @test ispath(socket_path) @test !isfile(socket_path) diff --git a/test/health.jl b/test/health.jl index 1b61731..87ced77 100644 --- a/test/health.jl +++ b/test/health.jl @@ -223,7 +223,7 @@ end # Blocks untils the process terminates withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do - graceful_terminate(getpid(p)) + return graceful_terminate(getpid(p)) end @test process_exited(p) @test p.exitcode == 1 From 18cec0006a55a1eea398b43a5a7051a7a7cd051b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 7 Aug 2024 13:48:01 -0500 Subject: [PATCH 6/6] Use socket extension --- src/graceful_termination.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index 2fbbf45..7732b5f 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -15,7 +15,7 @@ _deputy_ipc_dir() = get(ENV, "DEPUTY_IPC_DIR", "/run") # Write transient UNIX-domain sockets to the IPC directory. function _graceful_terminator_socket_path(pid::Int32) - return joinpath(_deputy_ipc_dir(), "graceful-terminator.$pid.sock") + return joinpath(_deputy_ipc_dir(), "graceful-terminator.$pid.socket") end # Following the Linux convention for pid files: