Merge branch 'release/0.31.0' into main

airyhq · Sep 23, 2021 · 57853e3 · 57853e3
2 parents bf64315 + 3a18ce6
commit 57853e3
Show file tree

Hide file tree

Showing 107 changed files with 2,921 additions and 1,084 deletions.
diff --git a/.github/ISSUE_TEMPLATE/---docs.md b/.github/ISSUE_TEMPLATE/---docs.md
@@ -1,5 +1,5 @@
 ---
-name: "\U0001F4DA Bug report"
+name: "\U0001F4DA Doc change"
 about: Request a doc change in Airy.
 title: ""
 labels: docs

diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,18 @@ node_modules/
 yarn-error.log
 package-lock.json
 
+# Terraform
+.terraform
+*.tfstate
+*.tfstate.backup
+*.tfvars
+*.tfstate.*.backup
+.terraform.tfstate.lock.info
+.terraform.lock.hcl
+
+# Kubernetes
+.kubeconfig
+
 # Misc
 .DS_Store
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.30.0
+0.31.0
diff --git a/cli/pkg/helm/BUILD b/cli/pkg/helm/BUILD
@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//cli/pkg/kube",
+        "//infrastructure/lib/go/k8s/util",
         "@io_k8s_api//batch/v1:go_default_library",
         "@io_k8s_api//core/v1:go_default_library",
         "@io_k8s_api//rbac/v1:go_default_library",

diff --git a/cli/pkg/helm/helm.go b/cli/pkg/helm/helm.go
@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"strings"
 
+	"github.com/airyhq/airy/infrastructure/lib/go/k8s/util"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -152,6 +153,7 @@ func (h *Helm) runHelm(args []string) error {
 					ServiceAccountName: serviceAccountName,
 				},
 			},
+			BackoffLimit: util.Int32Ptr(0),
 		},
 	}
 

diff --git a/cli/pkg/providers/aws/aws.go b/cli/pkg/providers/aws/aws.go
@@ -50,43 +50,6 @@ func (p *provider) GetOverrides() tmpl.Variables {
 }
 
 func (p *provider) PostInstallation(providerConfig map[string]string, dir workspace.ConfigDir) error {
-	if providerConfig["hostUpdate"] != "false" {
-		conf, err := dir.LoadAiryYaml()
-		if err != nil {
-			return err
-		}
-
-		clientset, err := p.context.GetClientSet()
-		if err != nil {
-			return err
-		}
-
-		ingressService, err := clientset.CoreV1().Services("kube-system").Get(context.TODO(), "traefik", metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		loadBalancerUrl := ingressService.Status.LoadBalancer.Ingress[0].Hostname
-
-		if err = p.updateIngress("airy-core", loadBalancerUrl, conf.Kubernetes.Namespace); err != nil {
-			return err
-		}
-		if err = p.updateIngress("airy-core-ui", loadBalancerUrl, conf.Kubernetes.Namespace); err != nil {
-			return err
-		}
-		if err = p.updateIngress("airy-core-redirect", loadBalancerUrl, conf.Kubernetes.Namespace); err != nil {
-			return err
-		}
-
-		if err = p.updateHostsConfigMap(loadBalancerUrl, conf.Kubernetes.Namespace); err != nil {
-			return err
-		}
-
-		return dir.UpdateAiryYaml(func(conf workspace.AiryConf) workspace.AiryConf {
-			conf.Kubernetes.Host = loadBalancerUrl
-			return conf
-		})
-	}
 	return nil
 }
 

diff --git a/cli/pkg/workspace/airy_yaml.go b/cli/pkg/workspace/airy_yaml.go
@@ -5,10 +5,10 @@ type KubernetesConf struct {
 	ContainerRegistry string `yaml:"containerRegistry"`
 	Namespace         string `yaml:"namespace"`
 	NgrokEnabled      bool   `yaml:"ngrokEnabled"`
-	Host              string `yaml:"host"`
 }
 
 type IngressConf struct {
+	Host                    string            `yaml:"host,omitempty"`
 	Https                   bool              `yaml:"https,omitempty"`
 	LetsencryptEmail        string            `yaml:"letsencryptEmail,omitempty"`
 	LoadbalancerAnnotations map[string]string `yaml:"loadbalancerAnnotations,omitempty"`

diff --git a/cli/pkg/workspace/template/src/airy.yaml b/cli/pkg/workspace/template/src/airy.yaml
@@ -3,10 +3,10 @@ kubernetes:
   containerRegistry: ghcr.io/airyhq
   namespace: {{ .Namespace }}
   ngrokEnabled: {{ default false .NgrokEnabled }}
+ingress:
 {{- if .Host }}
   host: {{ default "airy.core" .Host }}
 {{- end }}
-ingress:
   https: {{ default false .Https}}
   letsencryptEmail: {{ .LetsencryptEmail}}
   {{- if .LoadbalancerAnnotations }}

diff --git a/docs/docs/changelog.md b/docs/docs/changelog.md
diff --git a/docs/docs/concepts/architecture.md b/docs/docs/concepts/architecture.md
@@ -21,6 +21,20 @@ of the [Chat Plugin](sources/chatplugin/overview.md).
 Having that in mind, these are the docker containers – or the `Airy Components` –
 which run as part of Airy Core:
 
+## Sensitive data
+
+### Application data
+
+All the application data is stored in the `Kafka` cluster and the topics which are created for the components. As the Kafka cluster is backed up by persistent EBS storage, the data is stored on the PersistentVolumes defined in the Kubernetes cluster. At the moment, the data in the Kafka topics is not encrypted.
+
+### Configuration data
+
+All the credentials, keys and secrets which the user can overwrite can be configured in the `airy.yaml` file. When running `airy create` or `airy config apply` these values are mapped into the following ConfigMaps inside the kafka cluster:
+
+- `security` ConfigMap - holding the necessary security parameters.
+- `{component-type}-{component}` ConfigMap - holding the configuration for individual sources and components
+- `airy-config-map` ConfigMap - storing a copy of the `airy.yaml` config file, inside the Kubernetes cluster.
+
 ## Sources
 
 - sources-`SOURCE_NAME`-webhook - Ingest events from the `SOURCE_NAME` source

diff --git a/docs/docs/getting-started/installation/aws.md b/docs/docs/getting-started/installation/aws.md
@@ -53,13 +53,31 @@ Default region name [None]: us-west-2
 Default output format [None]: json
 ```
 
-Apart from an EKS cluster, `airy create` will take care of all the necessary AWS
-resources, such as:
+## Services used
 
-- VPC resources (VPC, subnets, route tables, public gateways)
-- IAM roles and policy attachments
-- EKS cluster and EKS node groups
-- EC2 instances, as part of the created node group
+Apart from an EKS cluster, `airy create` will create of all the necessary AWS
+resources for Airy Core to run:
+
+|                        Service & pricing                        | Resources created by default                                                                        | Description                                                                                                                            | Overwrite [^1] |
+| :-------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------- | :------------: |
+|         [**VPC**](https://aws.amazon.com/vpc/pricing/)          | 1 VPC, 2 subnets with allowed Public IPs, 1 additional route table, 1 Internet gateway, DNS enabled | VPC which will contain all the created compute and network resources                                                                   |      Yes       |
+|                          **IAM** [^2]                           | 1 IAM role with attached policies [^3]                                                              | IAM role used for managing the EKS cluster and the node groups                                                                         |       No       |
+|    [**EKS**](https://calculator.aws/#/createCalculator/EKS)     | 1 EKS cluster                                                                                       | Kubernetes cluster to store all the Airy Core resources                                                                                |       No       |
+|    [**EC2**](https://calculator.aws/#/createCalculator/EC2)     | 2 EC2 instances, 4 EBS Volumes (10GB gp2 each)                                                      | The instances are a part of the `Node group` attached to the EKS cluster. The default instance type is: `c5.xlarge`, os type: `Linux`. |      Yes       |
+|     [**S3**](https://calculator.aws/#/createCalculator/S3)      | /                                                                                                   | Optional for the "Media resolver" component. Should be created independently. [^4]                                                     |      Yes       |
+| [**ELB**](https://aws.amazon.com/elasticloadbalancing/pricing/) | 1 Elastic Load Balancer                                                                             | Network Load Balancer created by the ingress controller Kubernetes service                                                             |       No       |
+
+[^1]: Options which can be overwritten with flags to the `airy create` command.
+[^2]: IAM roles are free of charge.
+[^3]: Attached policies: "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy".
+[^4]: The s3 bucket will should have PublicRead privileges. For writing to the S3 bucket, AWS credentials must be [configured](/getting-started/installation/configuration.md#components) in the `airy.yaml` file.
+
+Airy Core doesn't require extensive resources to run. However, you should consider the `AWS Service Limits` or `AWS Service Quotas` when deploying on AWS. If some of the resources cannot be created due to existing quotas in your AWS account, refer to the [following dashboard](https://eu-central-1.console.aws.amazon.com/servicequotas/home/) to modify them.
+
+Refer to the following links for more information on AWS Service Limits:
+
+- [AWS Service Quotas](https://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html)
+- [AWS Well-Architected Framework Concepts](https://wa.aws.amazon.com/wellarchitected/2020-07-02T19-33-23/wat.concept.service-limits.en.html)
 
 ## Create a cluster
 
@@ -195,8 +213,8 @@ export HOSTNAME="public-FQDN"
 Modify the existing ingress service to reconfigure the AWS LoadBalancer:
 
 ```sh
-kubectl -n kube-system annotate service traefik "service.beta.kubernetes.io/aws-load-balancer-ssl-ports=443" "service.beta.kubernetes.io/aws-load-balancer-ssl-cert=${ARN}"
-kubectl -n kube-system patch service traefik --patch '{"spec": { "ports": [ { "name": "https", "port": 443, "protocol": "TCP", "targetPort": 80 } ] } }'
+kubectl -n kube-system annotate service ingress-nginx-controller "service.beta.kubernetes.io/aws-load-balancer-ssl-ports=443" "service.beta.kubernetes.io/aws-load-balancer-ssl-cert=${ARN}"
+kubectl -n kube-system patch service ingress-nginx-controller --patch '{"spec": { "ports": [ { "name": "https", "port": 443, "protocol": "TCP", "targetPort": 80 } ] } }'
 ```
 
 Update the `hostnames` configMap with the new https endpoint:
@@ -218,7 +236,7 @@ kubectl get ingress airy-core-redirect -o json | jq "(.spec.rules[0].host=\"${HO
 You should create a CNAME DNS record for the specified public FQDN to point to the hostname of the LoadBalancer, created by AWS for the ingress service:
 
 ```sh
-kubectl get --namespace kube-system service traefik --output jsonpath='{.status.loadBalancer.ingress[0].hostname}{"\n"}'
+kubectl get --namespace kube-system service ingress-nginx-controller --output jsonpath='{.status.loadBalancer.ingress[0].hostname}{"\n"}'
 ```
 
 #### Print HTTPS endpoint
@@ -231,7 +249,7 @@ airy api endpoint
 
 ### HTTPS using Let's Encrypt
 
-You can customize your installation of `Airy Core` to install a custom Traefik ingress controller which has an enabled `Let's Encrypt` capability. The ingress controller will register and renew the certificates for you automatically.
+You can customize your installation of `Airy Core` to install an ingress controller which has an enabled `Let's Encrypt` capability. The ingress controller will register and renew the certificates for you automatically.
 
 #### Customize your Airy Core installation
 
@@ -244,47 +262,40 @@ airy create --provider aws --init-only
 Then edit your `airy.yaml` file and add the following configuration
 
 ```sh
-kubernetes:
-  host: myairy.myhostname.com
 ingress:
+  host: myairy.myhostname.com
   https: true
   letsencryptEmail: "[email protected]"
 
 ```
 
-The `kubernets.host` value should be set to your desired hostname. Configure the e-mail address you want to use for your Let's Encrypt registration under `ingress.letsencryptEmail`.
+The `ingress.host` value should be set to your desired hostname. Configure the e-mail address you want to use for your Let's Encrypt registration under `ingress.letsencryptEmail`.
 
 After setting these parameters, create your `Airy Core` instance with the following option:
 
 ```sh
-airy create --provider aws --provider-config hostUpdate=false
+airy create --provider aws
 ```
 
 :::note
-In case you have created your Airy Core instance without Let's Encrypt and want to add it later, you must use the `airy upgrade` command.
-
-Even if you don't upgrade to a new version, just modify your `airy.yaml` file as explained in this section and run
-
-`airy upgrade`
-
-After the upgrade is done, continue with setting up your DNS and starting the ingress controller.
+In case you have created your Airy Core instance without Let's Encrypt and want to add it later, modify your `airy.yaml` file accordingly and continue with the process from the next section.
 :::
 
 #### Setup your DNS
 
-You should create a CNAME DNS record for the hostname that you set under `kubernetes.host` in the previous step to point to the hostname of the LoadBalancer, created by AWS for the ingress service:
+You should create a CNAME DNS record for the hostname that you set under `ingress.host` in the previous step to point to the hostname of the LoadBalancer, created by AWS for the ingress service:
 
 ```sh
 export KUBECONFIG="PATH/TO/DIR/kube.conf"
-kubectl get --namespace kube-system service traefik --output jsonpath='{.status.loadBalancer.ingress[0].hostname}{"\n"}'
+kubectl get --namespace kube-system service ingress-nginx-controller --output jsonpath='{.status.loadBalancer.ingress[0].hostname}{"\n"}'
 ```
 
-#### Start the ingress controller
+#### Run airy upgrade
 
-If the ingress controller is started before the DNS record is added, the Let's Encrypt servers will block and throttle the registration attempts. That is why we recommend starting the ingress controller after the DNS record is added.
+If the ingress controller is started before the DNS record is added, the initial Let's Encrypt requests will fail and then all the following registration attempts will be blocked and throttled. That is why the generation of the Let's Encrypt certificates is disabled by default. In order to complete the setup, you must run the upgrade command.
 
 ```sh
-kubectl -n kube-system scale statefulset -l k8s-app=traefik-ingress-lb --replicas=1
+airy upgrade
 ```
 
 After this, your `Airy Core` will be reachable under HTTPS and on your desired hostname (for example https://myairy.myhostname.com).

diff --git a/docs/docs/getting-started/installation/configuration.md b/docs/docs/getting-started/installation/configuration.md
@@ -37,8 +37,6 @@ are looking for.
 
 - `namespace` the Kubernetes namespace that the **Airy Core** will use
 
-- `host` the hostname which will be used to access your `Airy Core` instance, outside of the Kubernetes cluster (default: airy.core)
-
 ### Prerequisites
 
 These settings are used to connect the **Airy Components** to your Kafka
@@ -54,6 +52,7 @@ cluster and Redis.
 
 - `ingress`
 
+  - `host` the hostname which will be used to access your `Airy Core` instance, outside of the Kubernetes cluster (default: airy.core)
   - `https` set to `true` to enable HTTPS
   - `loadbalancerAnnotations` list of annotations used to configure the LoadBalancer pointing to the ingress controller, in cloud environment (for AWS the following annotation is added by default: `service.beta.kubernetes.io/aws-load-balancer-type: nlb` )
   - `letsencryptEmail` the e-mail address used for Let's Encrypt registration, when using HTTPS.
@@ -91,7 +90,7 @@ cluster and Redis.
 - `media`
   - `storage`
     - `s3Key` set this to your AWS S3 access key id
-    - `s3Secret` set this to your AWS S3 secret access key
+    - `s3Secret` set this to your AWS S3 secret access key (The bucket needs to have PublicRead privileges)
     - `s3Bucket` set this to your AWS S3 bucket
     - `s3Region` set this to your AWS region
     - `s3Path` set this to your AWS S3 path

diff --git a/docs/docs/getting-started/installation/minikube.md b/docs/docs/getting-started/installation/minikube.md
@@ -16,7 +16,7 @@ your local machine using [minikube](https://minikube.sigs.k8s.io/).
 
 ## Create a minikube cluster
 
-First install the latest version of minikube using [their documentation](https://kubernetes.io/de/docs/tasks/tools/install-minikube/). If you already have it installed make sure that it's at least on version v1.19 and upgrade it if not.
+First install minikube using [their documentation](https://kubernetes.io/de/docs/tasks/tools/install-minikube/). Currently supported versions are: v1.19.0, v1.20.0, v1.21.0 and v1.22.0. If you already have it installed make sure that your version is in line with the supported versions.
 
 Next you also need to install the [Airy CLI](cli/introduction.md). Now you can run this command, which will create a new
 minikube cluster on your system and install Airy core on it:

diff --git a/docs/docs/getting-started/installation/security.md b/docs/docs/getting-started/installation/security.md
@@ -91,4 +91,4 @@ The redirect Uri to configure with your provider will always be of the form `{ai
 
 By default the deployed ingress resources don't have HTTPS enabled, so this needs to be configured depending on the provider where you are running `Airy Core`.
 
-We advise you to refer to the documentation of your cloud provider on how to enable HTTPS on the LoadBalancer which routes to the Traefik ingress controller.
+We advise you to refer to the documentation of your cloud provider on how to enable HTTPS on the LoadBalancer which routes to the installed ingress controller.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -91,4 +91,4 @@ The redirect Uri to configure with your provider will always be of the form `{ai

		By default the deployed ingress resources don't have HTTPS enabled, so this needs to be configured depending on the provider where you are running `Airy Core`.

		We advise you to refer to the documentation of your cloud provider on how to enable HTTPS on the LoadBalancer which routes to the Traefik ingress controller.
		We advise you to refer to the documentation of your cloud provider on how to enable HTTPS on the LoadBalancer which routes to the installed ingress controller.