Merge pull request #3215 from arnaldo2792/additional-ecs-settings

Add container images cleanup settings to API

README.md

@@ -656,6 +656,13 @@ Valid time units include `s`, `m`, and `h`, e.g. `1h`, `1m1s`.
 * `settings.ecs.metadata-service-burst`: The burst rate limit of the throttling configurations set for the task metadata service.
 * `settings.ecs.reserved-memory`: The amount of memory, in MiB, reserved for critical system processes.
 * `settings.ecs.task-cleanup-wait`: Time to wait before the task's containers are removed after they are stopped.
+Valid time units are `s`, `m`, and `h`, e.g. `1h`, `1m1s`.
+* `settings.ecs.image-cleanup-wait`: Time to wait between image cleanup cycles.
+Valid time units are `s`, `m`, and `h`, e.g. `1h`, `1m1s`.
+* `settings.ecs.image-cleanup-delete-per-cycle`: Number of images to delete in a single image cleanup cycle.
+* `settings.ecs.image-cleanup-enabled`: Enable automatic images clean up after the tasks have been removed.
+Defaults to `false`
+* `settings.ecs.image-cleanup-age`: Time since the image was pulled to be considered for clean up.
 Valid time units are `s`, `m`, and `h`, e.g. `1h`, `1m1s`.
 
   **Note**: `metadata-service-rps` and `metadata-service-burst` directly map to the values set by the `ECS_TASK_METADATA_RPS_LIMIT` environment variable.

packages/ecs-agent/ecs.config

@@ -22,3 +22,9 @@ ECS_CONTAINER_STOP_TIMEOUT="{{settings.ecs.container-stop-timeout}}"
 {{#if settings.ecs.task-cleanup-wait}}
 ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION="{{settings.ecs.task-cleanup-wait}}"
 {{/if}}
+{{#if settings.ecs.image-cleanup-wait}}
+ECS_IMAGE_CLEANUP_INTERVAL="{{settings.ecs.image-cleanup-wait}}"
+{{/if}}
+{{# if settings.ecs.image-cleanup-age}}
+ECS_IMAGE_MINIMUM_CLEANUP_AGE="{{settings.ecs.image-cleanup-age}}"
+{{/if}}

sources/Cargo.toml

@@ -52,6 +52,7 @@ members = [
     "api/migration/migrations/v1.14.0/public-admin-container-v0-10-1",
     "api/migration/migrations/v1.14.0/aws-control-container-v0-7-2",
     "api/migration/migrations/v1.14.0/public-control-container-v0-7-2",
+    "api/migration/migrations/v1.14.2/ecs-images-cleanup",
 
     "bloodhound",
 

sources/api/ecs-settings-applier/src/ecs.rs

@@ -74,6 +74,14 @@ struct ECSConfig {
 
     #[serde(rename = "ReservedMemory", skip_serializing_if = "Option::is_none")]
     reserved_memory: Option<u16>,
+
+    #[serde(
+        rename = "NumImagesToDeletePerCycle",
+        skip_serializing_if = "Option::is_none"
+    )]
+    image_cleanup_delete_per_cycle: Option<i64>,
+
+    image_cleanup_disabled: bool,
 }
 
 // Returning a Result from main makes it print a Debug representation of the error, but with Snafu
@@ -141,6 +149,8 @@ async fn run() -> Result<()> {
         reserved_memory: ecs.reserved_memory,
         metadata_service_rps: ecs.metadata_service_rps,
         metadata_service_burst: ecs.metadata_service_burst,
+        image_cleanup_delete_per_cycle: ecs.image_cleanup_delete_per_cycle,
+        image_cleanup_disabled: !ecs.image_cleanup_enabled.unwrap_or(true),
         ..Default::default()
     };
     if let Some(os) = settings.os {

sources/api/migration/migrations/v1.14.2/ecs-images-cleanup/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "ecs-images-cleanup"
+version = "0.1.0"
+edition = "2018"
+authors = ["Arnaldo Garcia Rincon <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers", version = "0.1.0"}

sources/api/migration/migrations/v1.14.2/ecs-images-cleanup/src/main.rs

@@ -0,0 +1,23 @@
+use migration_helpers::common_migrations::AddSettingsMigration;
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// We added additional configurations for the ECS agent
+fn run() -> Result<()> {
+    migrate(AddSettingsMigration(&[
+        "settings.ecs.image-cleanup-wait",
+        "settings.ecs.image-cleanup-delete-per-cycle",
+        "settings.ecs.image-cleanup-enabled",
+        "settings.ecs.image-cleanup-age",
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}

sources/models/src/lib.rs

@@ -294,6 +294,10 @@ struct ECSSettings {
     metadata_service_rps: i64,
     metadata_service_burst: i64,
     reserved_memory: u16,
+    image_cleanup_wait: ECSDurationValue,
+    image_cleanup_delete_per_cycle: i64,
+    image_cleanup_enabled: bool,
+    image_cleanup_age: ECSDurationValue,
 }
 
 #[model]