diff --git a/cargo-shuttle/src/provisioner_server.rs b/cargo-shuttle/src/provisioner_server.rs index 69e1d7161..f5b789c8a 100644 --- a/cargo-shuttle/src/provisioner_server.rs +++ b/cargo-shuttle/src/provisioner_server.rs @@ -244,8 +244,15 @@ impl LocalProvisioner { } async fn wait_for_ready(&self, container_name: &str, is_ready_cmd: Vec) -> Result<()> { - loop { - trace!("waiting for '{container_name}' to be ready for connections"); + const MAX_RETRIES: u32 = 60; + const RETRY_DELAY_MS: u64 = 500; + + for attempt in 0..MAX_RETRIES { + trace!( + "waiting for '{container_name}' to be ready for connections (attempt {}/{})", + attempt + 1, + MAX_RETRIES + ); let config = CreateExecOptions { cmd: Some(is_ready_cmd.clone()), @@ -254,32 +261,64 @@ impl LocalProvisioner { ..Default::default() }; - let CreateExecResults { id } = self - .docker - .create_exec(container_name, config) - .await - .expect("failed to create exec to check if container is ready"); + let exec_result = self.docker.create_exec(container_name, config).await; + + let exec_id = match exec_result { + Ok(CreateExecResults { id }) => id, + Err(bollard::errors::Error::DockerResponseServerError { + status_code: 409, + message, + }) => { + trace!("container '{container_name}' not ready yet: {message}"); + + match self.docker.inspect_container(container_name, None).await { + Ok(container) => { + let state = + container.state.as_ref().context("container has no state")?; + + if !state.running.unwrap_or(false) { + bail!("container '{container_name}' has stopped unexpectedly with status: {:?}, exit code: {:?}", + state.status, state.exit_code); + } + + sleep(Duration::from_millis(RETRY_DELAY_MS)).await; + continue; + } + Err(e) => { + bail!("failed to inspect container '{container_name}': {e}"); + } + } + } + Err(e) => { + bail!("failed to create exec to check if container '{container_name}' is ready: {e}"); + } + }; let ready_result = self .docker - .start_exec(&id, None) + .start_exec(&exec_id, None) .await - .expect("failed to execute ready command"); + .context("failed to execute ready command")?; if let bollard::exec::StartExecResults::Attached { mut output, .. } = ready_result { while let Some(line) = output.next().await { trace!("line: {:?}", line); if let bollard::container::LogOutput::StdOut { .. } = - line.expect("output to have a log line") + line.context("output to have a log line")? { return Ok(()); } } } - sleep(Duration::from_millis(500)).await; + sleep(Duration::from_millis(RETRY_DELAY_MS)).await; } + + bail!( + "container '{container_name}' did not become ready within {} seconds", + (MAX_RETRIES as u64 * RETRY_DELAY_MS) / 1000 + ) } async fn pull_image(&self, image: &str) -> Result<(), String> {