Merge pull request #35 from globocom/release/candidate#0.6.0

Release/candidate#0.6.0

.python-version

@@ -0,0 +1 @@
+3.9.6

Pipfile

@@ -15,5 +15,8 @@ kopf = "==0.28.1"
 pydantic = "==1.7.3"
 kubernetes = "==12.0.1"
 coverage = "==5.3.1"
+
+jinja2 = "*"
+
 [requires]
 python_version = "3.8"

README.md

@@ -104,6 +104,74 @@ spec:
 ```
 The rancher.filters, rancher.labels and rancher.ignore fields are specific to Rancher. Other cluster sources may have other values. You can get all the examples of ClusterRules objects [here](https://github.com/globocom/enforcement-service/tree/master/examples/sourcers).
 
+A ClusterRule also supports dynamic configurations using the Jinja expression language. You can create dynamic models 
+using cluster fields returned by cluster source, or any other valid Python code.
+
+```yaml
+apiVersion: enforcement.globo.com/v1beta1
+kind: ClusterRule
+metadata:
+  name: dynamic-rule
+spec:
+  enforcements: 
+      #The variable name references the cluster name defined in the Cluster source.
+    - name: ${% if name=='cluster1' %} guestbook-cluster1 ${% else %} guestbook-other ${% endif %} 
+      repo: https://github.com/argoproj/argocd-example-apps #Git repository
+      path: helm-guestbook 
+      namespace: ${{ name }} #Cluster name
+      helm:
+        parameters: 
+          replicaCount: ${{ 2*5 }}
+          clusterURL: ${{ url }} #Cluster URL
+  source:
+    rancher: {}
+```
+The ***name***, ***url*** and ***id*** fields are available for all cluster sources, however there are also specific fields for each 
+Cluster source, see the list [here](https://github.com/globocom/enforcement-service/tree/master/examples/dynamic/dynamic.md).
+
+You can also configure your ClusterRule using HTTP request returns using the Python [requests](https://docs.python-requests.org/en/master/) library.
+
+```yaml
+apiVersion: enforcement.globo.com/v1beta1
+kind: ClusterRule
+metadata:
+  name: dynamic-rule
+spec:
+  enforcements: 
+    - name: guestbook
+      repo: https://github.com/argoproj/argocd-example-apps 
+      path: helm-guestbook 
+      helm:
+        parameters: 
+          uuid: ${{ requests.get('https://httpbin.org/uuid').json()['uuid'] }}
+  source:
+    rancher: {}
+```
+
+## Triggers
+
+You can configure triggers to be notified every time enforcement is installed on a cluster. Triggers are HTTP requests that Enforcement will make every time a package is installed on some cluster selected by cluster rule.
+
+```yaml
+apiVersion: enforcement.globo.com/v1beta1
+kind: ClusterRule
+metadata:
+  name: dev-rule #Rule name
+spec:
+  enforcements:
+   - name: guestbook #Name
+     repo: https://github.com/argoproj/argocd-example-apps #Git repository
+     path: guestbook #Package folder within the repository
+  source:
+    rancher: {}
+  triggers:
+    beforeInstall:
+      endpoint: http://myendpoint.com/before
+      timeout: 5 #Optional. 5 seconds is the default
+    afterInstall:
+      endpoint: http://myendpoint.com/after
+      timeout: 15 #Optional. 5 seconds is the default
+```
 ### Creating a Secret
 
 Enforcement obtains the credentials to connect to the cluster source through a secret that must be previously created.

app/config/__init__.py

@@ -2,5 +2,6 @@
 from app.config.domain_module import DomainModule
 from app.config.infra_module import InfraModule
 from app.config.use_case_module import UseCaseModule
+from app.config.service_module import ServiceModule
 
-__all__ = ['DataModule', 'UseCaseModule', 'DomainModule', 'InfraModule']
+__all__ = ['DataModule', 'UseCaseModule', 'DomainModule', 'InfraModule', 'ServiceModule']

app/config/domain_module.py

@@ -4,10 +4,27 @@
 from app.domain.enforcement_change_detector_builder import EnforcementChangeDetectorBuilder
 from app.domain.enforcement_installer_builder import EnforcementInstallerBuilder
 from app.domain.repositories import ClusterRepository, ProjectRepository, EnforcementRepository
+from app.domain.enforcement_dynamic_mapper import EnforcementDynamicMapper
+from app.domain.triggers import TriggerBase, TriggerService, TriggerBuilder
 
 
 class DomainModule(Module):
 
+    @provider
+    @singleton
+    def provide_enforcement_dynamic_mapper(self) -> EnforcementDynamicMapper:
+        return EnforcementDynamicMapper()
+
+    @provider
+    @singleton
+    def provide_trigger_base(self, trigger_service: TriggerService) -> TriggerBase:
+        return TriggerBase(trigger_service=trigger_service)
+
+    @provider
+    @singleton
+    def provide_trigger_builder(self, trigger_base: TriggerBase) -> TriggerBuilder:
+        return TriggerBuilder(trigger_base=trigger_base)
+
     @provider
     @singleton
     def provide_cluster_group_builder(self, cluster_repository: ClusterRepository,
@@ -17,8 +34,14 @@ def provide_cluster_group_builder(self, cluster_repository: ClusterRepository,
     @provider
     @singleton
     def provide_enforcement_installer_builder(self,
-                                              enforcement_repository: EnforcementRepository) -> EnforcementInstallerBuilder:
-        return EnforcementInstallerBuilder(enforcement_repository=enforcement_repository)
+                                              enforcement_repository: EnforcementRepository,
+                                              enforcement_dynamic_mapper: EnforcementDynamicMapper,
+                                              trigger_builder: TriggerBuilder) -> EnforcementInstallerBuilder:
+        return EnforcementInstallerBuilder(
+            enforcement_repository=enforcement_repository,
+            enforcement_dynamic_mapper=enforcement_dynamic_mapper,
+            trigger_builder=trigger_builder
+        )
 
     @provider
     @singleton

app/config/service_module.py

@@ -0,0 +1,12 @@
+from injector import Module, provider, singleton
+from app.domain.triggers import TriggerService
+from app.service.installation_notifier import InstallationNotifier
+
+
+class ServiceModule(Module):
+    @provider
+    @singleton
+    def provide_installation_notifier(self) -> TriggerService:
+        return InstallationNotifier()
+
+

app/config/use_case_module.py

@@ -3,7 +3,7 @@
 from app.domain.cluster_group_builder import ClusterGroupBuilder
 from app.domain.enforcement_change_detector_builder import EnforcementChangeDetectorBuilder
 from app.domain.enforcement_installer_builder import EnforcementInstallerBuilder
-from app.domain.repositories import EnforcementRepository
+from app.domain.triggers import TriggerBuilder
 from app.domain.source_locator import SourceLocator
 from app.domain.use_case import ApplyRulesUseCase, SyncRulesUseCase, UpdateRulesUseCase
 
@@ -19,7 +19,7 @@ def provider_apply_rules(
         return ApplyRulesUseCase(
             source_locator=locator,
             cluster_group_builder=cluster_group_builder,
-            enforcement_installer_builder=enforcement_installer_builder
+            enforcement_installer_builder=enforcement_installer_builder,
         )
 
     @provider

app/data/argo/application.py

@@ -132,5 +132,6 @@ def _make_enforcement_by_application(self, application: V1alpha1Application) ->
             name=application.metadata.name,
             repo=application.spec.source.repo_url,
             path=application.spec.source.path,
-            helm=helm
+            helm=helm,
+            labels=application.metadata.labels or []
         )

app/data/argo/cluster.py

@@ -21,7 +21,7 @@ def list_clusters_info(self) -> List[Dict[str, str]]:
         argo_clusters: V1alpha1ClusterList = self._cluster_service.list()
 
         info = [
-            {"name": item.name, "url": item.server}
+            {"name": item.name, "url": item.server, "connection": item.connection_state.status}
             for item in argo_clusters.items if item.name != 'in-cluster'
         ]
 

app/data/source/rancher.py

@@ -54,6 +54,7 @@ def _build_cluster(self, cluster_map: dict) -> Cluster:
             id=cluster_map['id'],
             token=self.secret.token,
             url=f'{self.secret.url}/k8s/clusters/{cluster_map["id"]}',
+            additional_data=cluster_map,
         )
 
 

app/domain/cluster_group.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import List
+from typing import List, Dict
 from argocd_client import ApiException
 
 import attr
@@ -9,6 +9,9 @@
 from app.domain.repositories import ClusterRepository, ProjectRepository
 
 
+CLUSTER_FAILED_STATE = "Failed"
+
+
 @attr.s(auto_attribs=True)
 class ClusterGroup:
     _clusters: List[Cluster]
@@ -20,9 +23,25 @@ def clusters(self):
         return self._clusters
 
     def register(self):
+        clusters_saved = self._cluster_repository.list_clusters_info()
+        clusters_health = self._get_clusters_health(clusters_saved)
+
+        cluster_saved_names = [cluster["name"] for cluster in clusters_saved]
+        clusters_healthy = []
+
         for cluster in self._clusters:
-            self._cluster_repository.register_cluster(cluster)
-            self._project_repository.create_project(cluster)
+            if cluster.name not in cluster_saved_names:
+                try:
+                    self._cluster_repository.register_cluster(cluster)
+                    self._project_repository.create_project(cluster)
+                    clusters_healthy.append(cluster)
+                finally:
+                    continue
+            else:
+                if clusters_health.get(cluster.name) != CLUSTER_FAILED_STATE:
+                    clusters_healthy.append(cluster)
+
+        self._clusters = clusters_healthy
 
     def unregister(self):
         for cluster in self._clusters:
@@ -33,6 +52,10 @@ def unregister(self):
                 if e.status != 404:
                     raise e
 
+    @classmethod
+    def _get_clusters_health(cls, clusters: List[Dict[str, str]]) -> Dict[str, str]:
+        return {cluster['name']: cluster['connection'] for cluster in clusters}
+
     def __sub__(self, other: ClusterGroup) -> ClusterGroup:
         cluster_names = {cluster.name: cluster for cluster in other.clusters}
         result_clusters = list(

app/domain/enforcement_dynamic_mapper.py

@@ -0,0 +1,22 @@
+from app.domain.entities import Cluster, Enforcement
+
+from jinja2 import Template
+import yaml
+import requests
+
+
+class EnforcementDynamicMapper:
+    def __call__(self, cluster: Cluster, enforcement: Enforcement) -> Enforcement:
+        yaml_enforcement = yaml.dump(enforcement.dict())
+        yaml_enforcement = yaml_enforcement.replace("$", "", -1)
+
+        t = Template(yaml_enforcement)
+        cluster_dict = cluster.dict()
+        cluster_dict.pop("additional_data")
+        cluster_dict.update(cluster.additional_data)
+        cluster_dict["requests"] = requests
+        yaml_enforcement = t.render(**cluster_dict)
+        enforcement_dict = yaml.load(yaml_enforcement)
+        return Enforcement(**enforcement_dict)
+
+

app/domain/enforcement_installer.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Callable
 
 import attr
 from argocd_client import ApiException
@@ -7,26 +7,36 @@
 from app.domain.entities import Enforcement, Cluster
 from app.domain.repositories import EnforcementRepository
 from app.domain.exceptions import EnforcementInvalidException
+from app.domain.enforcement_dynamic_mapper import EnforcementDynamicMapper
 
 
 @attr.s(auto_attribs=True)
 class EnforcementInstaller:
     _enforcements: List[Enforcement]
     _cluster_group: ClusterGroup
     _enforcement_repository: EnforcementRepository
+    _enforcement_dynamic_mapper: EnforcementDynamicMapper
+    _before_install_trigger: Callable[[Cluster, Enforcement], None]
+    _after_install_trigger: Callable[[Cluster, Enforcement], None]
 
     def install(self) -> List[Enforcement]:
 
         enforcements_error: List[Enforcement] = []
 
+        installed_enforcements = self._enforcement_repository.list_installed_enforcements()
+        enforcements_by_cluster = EnforcementInstaller._make_enforcements_by_cluster(installed_enforcements)
+
         for cluster in self._cluster_group.clusters:
-            installed_enforcements = self._enforcement_repository.list_installed_enforcements(cluster_name=cluster.name)
+            installed_enforcements = enforcements_by_cluster.get(cluster.name, [])
             installed_enforcements_names = self._get_enforcements_name(installed_enforcements)
             for enforcement in self._enforcements:
-                instance_name = self._make_enforcement_name(cluster, enforcement)
                 try:
+                    enforcement = self._enforcement_dynamic_mapper(cluster, enforcement)
+                    instance_name = self._make_enforcement_name(cluster, enforcement)
                     if instance_name not in installed_enforcements_names:
+                        self._before_install_trigger(cluster, enforcement)
                         self._enforcement_repository.create_enforcement(cluster.name, instance_name, enforcement)
+                        self._after_install_trigger(cluster, enforcement)
                     else:
                         self._enforcement_repository.update_enforcement(cluster.name, instance_name, enforcement)
                 except EnforcementInvalidException:
@@ -63,3 +73,14 @@ def _get_enforcements_name(cls, enforcements: List[Enforcement]):
     @classmethod
     def _make_enforcement_name(cls, cluster: Cluster, enforcement: Enforcement) -> str:
         return f"{cluster.name}-{enforcement.name}"
+
+    @classmethod
+    def _make_enforcements_by_cluster(cls, enforcements) -> dict:
+        by_cluster: dict = {}
+        for enforcement in enforcements:
+            cluster_name = enforcement.labels.get("cluster_name")
+            if not cluster_name:
+                continue
+            by_cluster[cluster_name] = by_cluster.get(cluster_name, []) + [enforcement]
+        return by_cluster
+

app/domain/enforcement_installer_builder.py

@@ -4,17 +4,29 @@
 
 from app.domain.cluster_group import ClusterGroup
 from app.domain.enforcement_installer import EnforcementInstaller
-from app.domain.entities import Enforcement
+from app.domain.entities import Enforcement, TriggersConfig
 from app.domain.repositories import EnforcementRepository
+from app.domain.enforcement_dynamic_mapper import EnforcementDynamicMapper
+from app.domain.triggers import TriggerBuilder
 
 
 @attr.s(auto_attribs=True)
 class EnforcementInstallerBuilder:
     _enforcement_repository: EnforcementRepository
+    _enforcement_dynamic_mapper: EnforcementDynamicMapper
+    _trigger_builder: TriggerBuilder
+
+    def build(self, enforcements: List[Enforcement], cluster_group: ClusterGroup,
+              triggers_config: TriggersConfig = None) -> EnforcementInstaller:
+
+        before_install_trigger = self._trigger_builder.build_before_install(triggers_config)
+        after_install_trigger = self._trigger_builder.build_after_install(triggers_config)
 
-    def build(self, enforcements: List[Enforcement], cluster_group: ClusterGroup) -> EnforcementInstaller:
         return EnforcementInstaller(
             enforcements=enforcements,
             cluster_group=cluster_group,
-            enforcement_repository=self._enforcement_repository
+            enforcement_repository=self._enforcement_repository,
+            enforcement_dynamic_mapper=self._enforcement_dynamic_mapper,
+            before_install_trigger=before_install_trigger,
+            after_install_trigger=after_install_trigger,
         )