Merge pull request #319 from Yelp/u/wilmerrafael/CLUSTERMAN-808_adding_alert_option_on_max_cap_pools

Adding alert on maximum capacity option for pools

Author: wilmer05

acceptance/srv-configs/clusterman-clusters/local-dev/default.kubernetes

@@ -19,3 +19,5 @@ autoscale_signal:
 autoscaling:
   prevent_scale_down_after_capacity_loss: true
   instance_loss_threshold: 3
+
+alert_on_max_capacity: false

acceptance/srv-configs/clusterman-clusters/local-dev/default.mesos

@@ -27,3 +27,5 @@ autoscale_signal:
     - name: disk_allocated
       type: system_metrics
       minute_range: 10
+
+alert_on_max_capacity: false

clusterman/autoscaler/autoscaler.py

@@ -179,7 +179,10 @@ def run(self, dry_run: bool = False, timestamp: Optional[arrow.Arrow] = None) ->
             capacity_offset = get_capacity_offset(self.cluster, self.pool, self.scheduler, timestamp)
             new_target_capacity = self._compute_target_capacity(resource_request) + capacity_offset
             self.target_capacity_gauge.set(new_target_capacity, {"dry_run": dry_run})
-            self.max_capacity_gauge.set(self.pool_manager.max_capacity, {"dry_run": dry_run})
+            self.max_capacity_gauge.set(
+                self.pool_manager.max_capacity,
+                {"dry_run": dry_run, "alert_on_max_capacity": self.pool_manager.alert_on_max_capacity},
+            )
             self.setpoint_gauge.set(self.autoscaling_config.setpoint, {"dry_run": dry_run})
             self._emit_requested_resource_metrics(resource_request, dry_run=dry_run)
 

clusterman/autoscaler/pool_manager.py

@@ -85,6 +85,7 @@ def __init__(
         self.killable_nodes_prioritizing_v2 = self.pool_config.read_bool(
             "autoscaling.killable_nodes_prioritizing_v2", default=False
         )
+        self.alert_on_max_capacity = self.pool_config.read_bool("alert_on_max_capacity", default=True)
         monitoring_info = {"cluster": cluster, "pool": pool}
         self.killable_nodes_counter = get_monitoring_client().create_counter(SFX_KILLABLE_NODES_COUNT, monitoring_info)
 

clusterman/simulator/simulated_pool_manager.py

@@ -58,6 +58,7 @@ def __init__(
             self.pool_config.read_int("scaling_limits.min_node_scalein_uptime_seconds", default=-1),
             MAX_MIN_NODE_SCALEIN_UPTIME_SECONDS,
         )
+        self.alert_on_max_capacity = self.pool_config.read_bool("alert_on_max_capacity", default=True)
         self.killable_nodes_prioritizing_v2 = self.pool_config.read_bool(
             "autoscaling.killable_nodes_prioritizing_v2", default=False
         )

examples/schemas/pool.json

@@ -63,7 +63,8 @@
             },
             "additionalProperties": false
         },
-        "sensu_config": {"$ref": "definitions.json#sensu_config"}
+        "sensu_config": {"$ref": "definitions.json#sensu_config"},
+        "alert_on_max_capacity": {"type": "boolean"}
     },
     "additionalProperties": false
 }

itests/environment.py

@@ -120,6 +120,7 @@ def setup_configurations(context):
                 {"name": "cost", "type": APP_METRICS, "minute_range": 30},
             ],
         },
+        "alert_on_max_capacity": True,
     }
     kube_pool_config = {
         "resource_groups": [
@@ -142,6 +143,7 @@ def setup_configurations(context):
             "internal": True,
             "period_minutes": 7,
         },
+        "alert_on_max_capacity": True,
     }
     with staticconf.testing.MockConfiguration(
         boto_config, namespace=CREDENTIALS_NAMESPACE

tests/autoscaler/autoscaler_test.py

@@ -48,6 +48,7 @@ def pool_configs():
                 "max_weight_to_add": 200,
                 "max_weight_to_remove": 10,
             },
+            "alert_on_max_capacity": True,
         },
         namespace=POOL_NAMESPACE.format(pool="bar", scheduler="mesos"),
     ):
@@ -86,6 +87,10 @@ def mock_autoscaler():
         "scaling_limits.max_capacity",
         namespace=POOL_NAMESPACE.format(pool="bar", scheduler="mesos"),
     )
+    mock_autoscaler.pool_manager.alert_on_max_capacity = staticconf.read_bool(
+        "alert_on_max_capacity",
+        namespace=POOL_NAMESPACE.format(pool="bar", scheduler="mesos"),
+    )
     mock_autoscaler.pool_manager.non_orphan_fulfilled_capacity = 0
 
     mock_autoscaler.target_capacity_gauge = mock.Mock(spec=GaugeProtocol)
@@ -155,7 +160,7 @@ def test_autoscaler_run(dry_run, mock_autoscaler, run_timestamp):
 
     assert mock_autoscaler.target_capacity_gauge.set.call_args == mock.call(100, {"dry_run": dry_run})
     assert mock_autoscaler.max_capacity_gauge.set.call_args == mock.call(
-        mock_autoscaler.pool_manager.max_capacity, {"dry_run": dry_run}
+        mock_autoscaler.pool_manager.max_capacity, {"dry_run": dry_run, "alert_on_max_capacity": True}
     )
     assert mock_autoscaler.setpoint_gauge.set.call_args == mock.call(0.7, {"dry_run": dry_run})
     assert mock_autoscaler._compute_target_capacity.call_args == mock.call(resource_request)

tests/conftest.py

@@ -144,6 +144,7 @@ def clusterman_pool_config():
                 {"name": "cost", "type": APP_METRICS, "minute_range": 30},
             ],
         },
+        "alert_on_max_capacity": True,
     }
     with staticconf.testing.MockConfiguration(config, namespace="bar.mesos_config"):
         yield
@@ -200,6 +201,7 @@ def clusterman_k8s_pool_config():
             },
             "disable_autoscaling": False,
         },
+        "alert_on_max_capacity": False,
     }
     with staticconf.testing.MockConfiguration(config, namespace="bar.kubernetes_config"):
         yield