From cb7795134fbf56efbaa3299fe9f2922c2066ae02 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 2 Jul 2024 13:50:53 -0500 Subject: [PATCH 1/5] Restrict `pid` to `Int32` --- src/K8sDeputy.jl | 1 + src/deprecated.jl | 3 +++ src/graceful_termination.jl | 6 +++--- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 src/deprecated.jl diff --git a/src/K8sDeputy.jl b/src/K8sDeputy.jl index 16daeea..b6c4631 100644 --- a/src/K8sDeputy.jl +++ b/src/K8sDeputy.jl @@ -10,5 +10,6 @@ export Deputy, graceful_terminator, readied!, shutdown!, graceful_terminate include("graceful_termination.jl") include("health.jl") include("server.jl") +include("deprecated.jl") end # module K8sDeputy diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 0000000..34b6aca --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,3 @@ +using Base: @deprecate + +@deprecate graceful_terminate(pid::Integer; wait::Bool=true) graceful_terminate(Int32(pid); wait) true diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index f3605aa..82c7eee 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -24,7 +24,7 @@ set_entrypoint_pid(pid::Integer) = write(entrypoint_pid_file(), string(pid) * "\ function entrypoint_pid() pid_file = entrypoint_pid_file() - return isfile(pid_file) ? parse(Int, readchomp(pid_file)) : 1 + return isfile(pid_file) ? parse(Int32, readchomp(pid_file)) : Int32(1) end # https://docs.libuv.org/en/v1.x/process.html#c.uv_kill @@ -115,12 +115,12 @@ function graceful_terminator(f; set_entrypoint::Bool=true) end """ - graceful_terminate(pid::Integer=entrypoint_pid(); wait::Bool=true) -> Nothing + graceful_terminate(pid::Int32=entrypoint_pid(); wait::Bool=true) -> Nothing Initiates the execution of the `graceful_terminator` user callback in the process `pid`. See `graceful_terminator` for more details. """ -function graceful_terminate(pid::Integer=entrypoint_pid(); wait::Bool=true) +function graceful_terminate(pid::Int32=entrypoint_pid(); wait::Bool=true) # Note: The follow dead code has been left here purposefully as an example of how to # view output when running via `preStop`. # From 30ec4e09cfe3804d0f113d1aaefdd54933eb90ee Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 2 Jul 2024 13:51:26 -0500 Subject: [PATCH 2/5] Remove old named pipe after pod restart --- src/graceful_termination.jl | 23 ++++++++++----- test/graceful_termination.jl | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index 82c7eee..6bfca40 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -13,7 +13,8 @@ _deputy_ipc_dir() = get(tempdir, ENV, "DEPUTY_IPC_DIR") # Prefer using UNIX domain sockets but if the `DEPUTY_IPC_DIR` is set assume the file # system is read-only and use a named pipe instead. -function _socket_path(name) +function _graceful_terminator_socket_path(pid::Int32) + name = "graceful-terminator.$pid" return haskey(ENV, "DEPUTY_IPC_DIR") ? joinpath(_deputy_ipc_dir(), name) : name end @@ -80,11 +81,19 @@ process and the `preStop` process to cleanly terminate. function graceful_terminator(f; set_entrypoint::Bool=true) set_entrypoint && set_entrypoint_pid(getpid()) - # Utilize UNIX domain sockets for the IPC. Avoid using network sockets here as we don't - # want to allow access to this functionality from outside of the localhost. Each process - # uses a distinct socket name allowing for multiple Julia processes to allow independent - # use of the graceful terminator. - server = listen(_socket_path("graceful-terminator.$(getpid())")) + # Utilize UNIX domain sockets or named pipes for the IPC. Avoid using network sockets + # here as we don't want to allow access to this functionality from outside of the + # localhost. Each process uses a distinct socket name allowing for multiple Julia + # processes to allow independent use of the graceful terminator. + socket_path = _graceful_terminator_socket_path(getpid()) + + # Remove any pre-existing named pipe as otherwise this will cause our `listen` call to + # fail. Should be safe to remove this file as it has been reserved for this PID. Only + # should be needed in the scenarion where the K8s pod has been restarted and the + # location of the socket exists in a K8s volume. + ispath(socket_path) && rm(socket_path) + + server = listen(socket_path) t = Threads.@spawn begin while isopen(server) @@ -131,7 +140,7 @@ function graceful_terminate(pid::Int32=entrypoint_pid(); wait::Bool=true) # println(io, "preStop called") # end - sock = connect(_socket_path("graceful-terminator.$pid")) + sock = connect(_graceful_terminator_socket_path(pid)) println(sock, "terminate") close(sock) diff --git a/test/graceful_termination.jl b/test/graceful_termination.jl index 075ae91..a4196de 100644 --- a/test/graceful_termination.jl +++ b/test/graceful_termination.jl @@ -71,4 +71,59 @@ @test output1 == expected @test output2 == expected end + + # When users set `DEPUTY_IPC_DIR` they may be using a K8s volume. As even `emptyDir` + # volumes persist for the lifetime of the pod we may have a named pipe already present + # from a previous restart. + @testset "bind after restart" begin + deputy_ipc_dir = mktempdir() + code = quote + using K8sDeputy + using Sockets: listen + + # Generate the socket as if the K8s pod had restarted and this path remained. + # One difference with this setup is the process which created the socket is + # still running so there may be a write lock on the file which definitely + # wouldn't exist if the pod had restarted. Note that closing the socket will + # remove the path. + socket_path = K8sDeputy._graceful_terminator_socket_path(getpid()) + server = listen(socket_path) + + atexit(() -> @info "SHUTDOWN COMPLETE") + graceful_terminator(; set_entrypoint=false) do + @info "GRACEFUL TERMINATION HANDLER" + exit(2) + return nothing + end + sleep(60) + end + + cmd = `$(Base.julia_cmd()) --color=no -e $code` + + withenv("DEPUTY_IPC_DIR" => deputy_ipc_dir) do + buffer = IOBuffer() + p = run(pipeline(cmd; stdout=buffer, stderr=buffer); wait=false) + @test timedwait(() -> process_running(p), Second(5)) === :ok + + # Allow some time for Julia to startup and the graceful terminator to be registered. + sleep(3) + + # Socket exists as a named pipe + socket_path = K8sDeputy._graceful_terminator_socket_path(getpid(p)) + @test ispath(socket_path) + @test !isfile(socket_path) + + # Blocks untils the process terminates + @test graceful_terminate(getpid(p)) === nothing + @test process_exited(p) + @test p.exitcode == 2 + + output = String(take!(buffer)) + expected = """ + [ Info: GRACEFUL TERMINATION HANDLER + [ Info: SHUTDOWN COMPLETE + """ + @test output == expected + end + end end From b0906c9d6ed11be1739f592d244ab706565c11dd Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 2 Jul 2024 13:51:34 -0500 Subject: [PATCH 3/5] Set project version to 0.1.1 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 85bb45d..9d44130 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "K8sDeputy" uuid = "2481ae95-212f-4650-bb21-d53ea3caf09f" authors = ["Beacon Biosignals, Inc"] -version = "0.1.0" +version = "0.1.1" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 422328f09c062bc093eb8c86db066c4ea186ebbd Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 3 Jul 2024 09:21:54 -0500 Subject: [PATCH 4/5] Fix formatting with deprecation --- src/deprecated.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index 34b6aca..1cdd9af 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,3 +1,5 @@ using Base: @deprecate -@deprecate graceful_terminate(pid::Integer; wait::Bool=true) graceful_terminate(Int32(pid); wait) true +@deprecate(graceful_terminate(pid::Integer; wait::Bool=true), + graceful_terminate(Int32(pid); wait), + true) From 8aa6585a43ddb0643d8c5b309051039b5f69f277 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 3 Jul 2024 14:00:25 -0500 Subject: [PATCH 5/5] Comment fix --- src/graceful_termination.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graceful_termination.jl b/src/graceful_termination.jl index 6bfca40..be6d49b 100644 --- a/src/graceful_termination.jl +++ b/src/graceful_termination.jl @@ -89,7 +89,7 @@ function graceful_terminator(f; set_entrypoint::Bool=true) # Remove any pre-existing named pipe as otherwise this will cause our `listen` call to # fail. Should be safe to remove this file as it has been reserved for this PID. Only - # should be needed in the scenarion where the K8s pod has been restarted and the + # should be needed in the scenario where the K8s pod has been restarted and the # location of the socket exists in a K8s volume. ispath(socket_path) && rm(socket_path)