diff --git a/Project.toml b/Project.toml index 15b58b3..016dd24 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "K8sDeputy" uuid = "2481ae95-212f-4650-bb21-d53ea3caf09f" authors = ["Beacon Biosignals, Inc"] -version = "0.1.3" +version = "0.1.4" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/docs/src/graceful_termination.md b/docs/src/graceful_termination.md index 3a1d1de..c4f7a44 100644 --- a/docs/src/graceful_termination.md +++ b/docs/src/graceful_termination.md @@ -69,7 +69,7 @@ Finally, the entrypoint for the container should also not directly use the Julia ### Read-only Filesystem -If you have a read-only filesystem on your container you'll need to configure a writeable volume mount for K8sDeputy.jl. The `DEPUTY_IPC_DIR` environmental variable can be used to instruct K8sDeputy.jl where to store the named pipes it creates for interprocess communication: +If you have a read-only filesystem on your container you'll need to configure a writeable `/run` directory for K8sDeputy.jl so it can create UNIX-domain sockets for interprocess communication: ```yaml apiVersion: v1 @@ -78,9 +78,6 @@ spec: containers: - name: app # command: ["/bin/sh", "-c", "julia entrypoint.jl; sleep 1"] - env: - - name: DEPUTY_IPC_DIR - value: /mnt/deputy-ipc lifecycle: preStop: exec: @@ -88,10 +85,11 @@ spec: securityContext: readOnlyRootFilesystem: true volumeMounts: - - mountPath: /mnt/deputy-ipc - name: deputy-ipc + - name: ipc + mountPath: /run + subPath: app # Set the `subPath` to the container name to ensure per-container isolation volumes: - - name: deputy-ipc + - name: ipc emptyDir: medium: Memory ``` diff --git a/docs/src/quickstart.md b/docs/src/quickstart.md index 4704bd2..38c9ed0 100644 --- a/docs/src/quickstart.md +++ b/docs/src/quickstart.md @@ -27,9 +27,6 @@ For users who want to get started quickly you can use the following template to containers: - name: app command: ["/bin/sh", "-c", "julia entrypoint.jl; sleep 1"] - env: - - name: DEPUTY_IPC_DIR - value: /mnt/deputy-ipc ports: - name: health-check containerPort: 8081 # Default K8sDeputy.jl heath check port @@ -54,11 +51,12 @@ For users who want to get started quickly you can use the following template to - all readOnlyRootFilesystem: true volumeMounts: - - mountPath: /mnt/deputy-ipc - name: deputy-ipc + - name: ipc + mountPath: /run + subPath: app # Set the `subPath` to the container name to ensure per-container isolation terminationGracePeriodSeconds: 30 volumes: - - name: deputy-ipc + - name: ipc emptyDir: medium: Memory ``` diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index b15b116..7732b5f 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -7,25 +7,21 @@ # from experimenting with this there are a few issues such as being unable to use locks or # printing (`jl_safe_printf` does work). -# Linux typically stores PID files in `/run` which requires root access. For systems with -# read-only file systems we need to support a user specified writable volume. -_deputy_ipc_dir() = get(tempdir, ENV, "DEPUTY_IPC_DIR") +# Linux stores PID files and UNIX-domain sockets in `/run`. Users with K8s containers +# utilizing read-only file systems should make use of a volume mount to allow K8sDeputy.jl +# to write to `/run`. Users can change the IPC directory by specifying `DEPUTY_IPC_DIR` but +# this is mainly just used for testing. +_deputy_ipc_dir() = get(ENV, "DEPUTY_IPC_DIR", "/run") -# Prefer using UNIX domain sockets but if the `DEPUTY_IPC_DIR` is set assume the file -# system is read-only and use a named pipe instead. +# Write transient UNIX-domain sockets to the IPC directory. function _graceful_terminator_socket_path(pid::Int32) - name = "graceful-terminator.$pid" - return haskey(ENV, "DEPUTY_IPC_DIR") ? joinpath(_deputy_ipc_dir(), name) : name + return joinpath(_deputy_ipc_dir(), "graceful-terminator.$pid.socket") end # Following the Linux convention for pid files: # https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s15.html entrypoint_pid_file() = joinpath(_deputy_ipc_dir(), "julia-entrypoint.pid") -function set_entrypoint_pid(pid::Integer) - file = entrypoint_pid_file() - mkpath(dirname(file)) - return write(file, string(pid) * "\n") -end +set_entrypoint_pid(pid::Integer) = write(entrypoint_pid_file(), string(pid) * "\n") function entrypoint_pid() pid_file = entrypoint_pid_file() @@ -85,15 +81,15 @@ process and the `preStop` process to cleanly terminate. function graceful_terminator(f; set_entrypoint::Bool=true) set_entrypoint && set_entrypoint_pid(getpid()) - # Utilize UNIX domain sockets or named pipes for the IPC. Avoid using network sockets - # here as we don't want to allow access to this functionality from outside of the - # localhost. Each process uses a distinct socket name allowing for multiple Julia - # processes to allow independent use of the graceful terminator. + # Utilize UNIX-domain sockets (Linux) or named pipes (Windows) for the IPC. Avoid using + # network sockets here as we don't want to allow access to this functionality from + # outside of the localhost. Each process uses a distinct socket name allowing for + # multiple Julia processes to allow independent use of the graceful terminator. socket_path = _graceful_terminator_socket_path(getpid()) - # Remove any pre-existing named pipe as otherwise this will cause our `listen` call to - # fail. Should be safe to remove this file as it has been reserved for this PID. Only - # should be needed in the scenario where the K8s pod has been restarted and the + # Remove any pre-existing UNIX-domain socket as otherwise this will cause our `listen` + # call to fail. Should be safe to remove this file as it has been reserved for this PID. + # Only should be needed in the scenario where the K8s pod has been restarted and the # location of the socket exists in a K8s volume. ispath(socket_path) && rm(socket_path) diff --git a/test/graceful_termination.jl b/test/graceful_termination.jl index a4196de..1042005 100644 --- a/test/graceful_termination.jl +++ b/test/graceful_termination.jl @@ -1,4 +1,6 @@ @testset "graceful_terminator" begin + deputy_ipc_dir = mktempdir() + @testset "Julia entrypoint" begin code = quote using K8sDeputy @@ -12,6 +14,7 @@ end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) buffer = IOBuffer() p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) @test timedwait(() -> process_running(p), Second(5)) === :ok @@ -21,7 +24,9 @@ # When no PID is passed in the process ID is read from the Julia entrypoint file. # Blocks untils the process terminates. - @test graceful_terminate() === nothing + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + @test graceful_terminate() === nothing + end @test process_exited(p) @test p.exitcode == 2 @@ -47,6 +52,7 @@ end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) buffer1 = IOBuffer() buffer2 = IOBuffer() p1 = run(pipeline(cmd; stdout=buffer1, stderr=buffer1); wait=false) @@ -57,8 +63,10 @@ sleep(3) # Blocks untils the process terminates - @test graceful_terminate(getpid(p1)) === nothing - @test graceful_terminate(getpid(p2)) === nothing + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + @test graceful_terminate(getpid(p1)) === nothing + @test graceful_terminate(getpid(p2)) === nothing + end @test process_exited(p1) @test process_exited(p2) @@ -73,10 +81,9 @@ end # When users set `DEPUTY_IPC_DIR` they may be using a K8s volume. As even `emptyDir` - # volumes persist for the lifetime of the pod we may have a named pipe already present - # from a previous restart. + # volumes persist for the lifetime of the pod we may have a UNIX-domain socket already + # present from a previous restart. @testset "bind after restart" begin - deputy_ipc_dir = mktempdir() code = quote using K8sDeputy using Sockets: listen @@ -99,31 +106,33 @@ end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) + buffer = IOBuffer() + p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) + @test timedwait(() -> process_running(p), Second(5)) === :ok - withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do - buffer = IOBuffer() - p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) - @test timedwait(() -> process_running(p), Second(5)) === :ok - - # Allow some time for Julia to startup and the graceful terminator to be registered. - sleep(3) + # Allow some time for Julia to startup and the graceful terminator to be registered. + sleep(3) - # Socket exists as a named pipe - socket_path = K8sDeputy._graceful_terminator_socket_path(getpid(p)) - @test ispath(socket_path) - @test !isfile(socket_path) + # Socket exists as a UNIX-domain socket + socket_path = withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + return K8sDeputy._graceful_terminator_socket_path(getpid(p)) + end + @test ispath(socket_path) + @test !isfile(socket_path) - # Blocks untils the process terminates + # Blocks untils the process terminates + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do @test graceful_terminate(getpid(p)) === nothing - @test process_exited(p) - @test p.exitcode == 2 - - output = String(take!(buffer)) - expected = """ - [ Info: GRACEFUL TERMINATION HANDLER - [ Info: SHUTDOWN COMPLETE - """ - @test output == expected end + @test process_exited(p) + @test p.exitcode == 2 + + output = String(take!(buffer)) + expected = """ + [ Info: GRACEFUL TERMINATION HANDLER + [ Info: SHUTDOWN COMPLETE + """ + @test output == expected end end diff --git a/test/health.jl b/test/health.jl index a2c9f22..87ced77 100644 --- a/test/health.jl +++ b/test/health.jl @@ -192,6 +192,7 @@ end end @testset "graceful termination" begin + deputy_ipc_dir = mktempdir() port = rand(EPHEMERAL_PORT_RANGE) code = quote using K8sDeputy, Sockets @@ -211,6 +212,7 @@ end end cmd = `$(Base.julia_cmd()) --color=no -e $code` + cmd = addenv(cmd, "DEPUTY_IPC_DIR" => deputy_ipc_dir) buffer = IOBuffer() p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) @test timedwait(() -> process_running(p), Second(5)) === :ok @@ -220,7 +222,9 @@ end end === :ok # Blocks untils the process terminates - graceful_terminate(getpid(p)) + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + return graceful_terminate(getpid(p)) + end @test process_exited(p) @test p.exitcode == 1