M3l4 DevOps

otuskotlin · Apr 19, 2024 · 2da9fc2 · 2da9fc2
1 parent 6414121
commit 2da9fc2
Show file tree

Hide file tree

Showing 41 changed files with 7,061 additions and 5 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,51 @@
+# This is a basic workflow to help you get started with Actions
+
+name: 'CI-build'
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    branches:
+      - 'main'
+      - 'master'
+      - 'v-*'
+      - 'release/*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    runs-on: ubuntu-22.04
+    #    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Validate Gradle wrapper
+        uses: gradle/wrapper-validation-action@v1
+
+      # https://github.com/actions/runner-images/issues/675
+      - name: Hack sources.list
+        run: sudo sed -i 's|http://azure.archive.ubuntu.com/ubuntu/|http://mirror.arizona.edu/ubuntu/|g' /etc/apt/sources.list
+
+      - name: Install curl-dev
+        run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+
+      - name: Run Tests
+        uses: gradle/gradle-build-action@v2
+        with:
+          gradle-version: 8.6
+          arguments: check -i
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,49 @@
+# This is a basic workflow to help you get started with Actions
+
+name: 'CI-test'
+
+# Controls when the workflow will run
+on:
+  push:
+    branches-ignore:
+      - 'main'
+      - 'master'
+      - 'v-*'
+      - 'release/*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+#    runs-on: macos-latest
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Validate Gradle wrapper
+        uses: gradle/wrapper-validation-action@v1
+
+      - name: Install curl-dev
+        run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+
+      - name: Run Tests
+        uses: gradle/gradle-build-action@v2
+        with:
+          gradle-version: 8.6
+          arguments: check -i
+
+#      - name: Report unit tests
+#        uses: dorny/test-reporter@v1
+#        if: always()
+#        with:
+#          name: Gradle Tests
+#          path: '**/build/test-results/*/TEST-*.xml'
+#          reporter: java-junit
+#          fail-on-error: true
diff --git a/README.md b/README.md
@@ -22,10 +22,11 @@ Marketplace -- это площадка, на которой пользовате
     1. [Функциональные требования](./docs/02-analysis/01-functional-requiremens.md)
     2. [Нефункциональные требования](./docs/02-analysis/02-nonfunctional-requirements.md)
 3. DevOps
+   1. [Файлы сборки](./deploy)
 4. Архитектура
-    1. [ADR](./docs/04-architecture/01-adrs.md)
-    2. [Описание API](./docs/04-architecture/02-api.md)
-    3. [Компонентная схема](./docs/04-architecture/03-arch.md)
+    1. [ADR](docs/03-architecture/01-adrs.md)
+    2. [Описание API](docs/03-architecture/02-api.md)
+    3. [Компонентная схема](docs/03-architecture/03-arch.md)
 5. Тесты
 
 # Структура проекта

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -17,3 +17,10 @@ subprojects {
     group = rootProject.group
     version = rootProject.version
 }
+
+tasks {
+    create("check") {
+        group = "verification"
+        dependsOn(gradle.includedBuild("ok-marketplace-be").task(":check"))
+    }
+}
diff --git a/deploy/Dockerfile b/deploy/Dockerfile
@@ -0,0 +1,14 @@
+# Pull the minimal Ubuntu image
+FROM ubuntu:22.04
+
+# Install Nginx
+RUN apt-get -y update && apt-get -y install nginx
+
+# Copy the Nginx config
+COPY volumes/nginx /etc/nginx/sites-available/default
+
+# Expose the port for access
+EXPOSE 80/tcp
+
+# Run the Nginx server
+ENTRYPOINT ["/usr/sbin/nginx", "-g", "daemon off;"]
diff --git a/deploy/README.md b/deploy/README.md
@@ -0,0 +1,9 @@
+# deploy
+
+## Настройка
+Для корректного запуска файлов, необходимо добавить в настройки ядра Linux следующий параметр:
+
+**/etc/sysctl.d/20-opensearch.conf:**
+```
+vm.max_map_count = 262144
+```
diff --git a/deploy/call-envoy.sh b/deploy/call-envoy.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#TOKEN=""
+TOKEN=$(./keycloak-tokens.sh)
+
+#curl -H "Authorization: Bearer ${TOKEN}" \
+#  -H "X-Request-ID: 1234" \
+#  -H "x-client-request-id: 1235" \
+#  -H "Content-Type: application/json" \
+#  http://localhost:8090/v1/create \
+#  -d '{"debug":{"mode":"stub","stub":"success"},"ad":{"title":"my title","description":"my description","adType":"demand","visibility":"public","productId":"23423423"}}'
+
+curl -H "Authorization: Bearer ${TOKEN}" \
+  -H "X-Request-ID: 1234" \
+  -H "x-client-request-id: 1235" \
+  http://localhost:8090/
diff --git a/deploy/cert-gen.sh b/deploy/cert-gen.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+JKS_PASS=app123456
+
+DIR_CA=volumes/ca
+DIR_ENVOY=volumes/envoy/certs
+DIR_KC=volumes/keycloak/certs
+
+# Create directories
+mkdir -p $DIR_CA
+mkdir -p $DIR_ENVOY
+mkdir -p $DIR_APP
+mkdir -p $DIR_KC
+
+# Generate CA private key
+openssl genpkey -algorithm RSA -out $DIR_CA/ca.key
+
+# Generate CA certificate
+openssl req -x509 -new -nodes -key $DIR_CA/ca.key -sha256 -days 365 -out $DIR_CA/ca.crt -subj "/CN=my_ca"
+
+# envoy certificates ----------------------------------------------------
+
+# Generate envoy_sidecar private key
+openssl genpkey -algorithm RSA -out $DIR_ENVOY/envoy_sidecar.key
+
+# Generate a certificate signing request (CSR) for envoy_sidecar
+openssl req -new -key $DIR_ENVOY/envoy_sidecar.key -out $DIR_ENVOY/envoy_sidecar.csr -subj "/CN=envoy_sidecar"
+
+# Sign the CSR with the CA to get the envoy_sidecar certificate
+openssl x509 -req -in $DIR_ENVOY/envoy_sidecar.csr -CA $DIR_CA/ca.crt -CAkey $DIR_CA/ca.key -CAcreateserial -out $DIR_ENVOY/envoy_sidecar.crt -days 365 -sha256
+
+# keycloack certificates ----------------------------------------------------
+
+# Generate Keycloak private key and certificate
+openssl genpkey -algorithm RSA -out $DIR_KC/keycloak.key
+openssl req -new -key $DIR_KC/keycloak.key -out $DIR_KC/keycloak.csr -subj "/CN=keycloak"
+openssl x509 -req -in $DIR_KC/keycloak.csr -CA $DIR_CA/ca.crt -CAkey $DIR_CA/ca.key -CAcreateserial -out $DIR_KC/keycloak.crt -days 365 -sha256
+
+# Convert Keycloak certificates to PKCS12 format
+openssl pkcs12 -export -out $DIR_KC/keycloak.p12 -inkey $DIR_KC/keycloak.key -in $DIR_KC/keycloak.crt -name keycloak -passout pass:$JKS_PASS
+
+# Convert PKCS12 to JKS
+keytool -importkeystore -srckeystore $DIR_KC/keycloak.p12 -srcstoretype pkcs12 -destkeystore $DIR_KC/keycloak.jks -deststoretype JKS -srcstorepass $JKS_PASS -deststorepass $JKS_PASS
+
+echo "Certificates generated successfully!"
diff --git a/deploy/docker-compose-fbos-cluster.yml b/deploy/docker-compose-fbos-cluster.yml
@@ -0,0 +1,100 @@
+# Кластерная версия инфраструктуры на базе
+# OpenSearch (2 узла)
+# OpenSearch Dashboards
+# Fluent Bit
+
+version: '3'
+services:
+
+  app:
+    image: nginx:latest
+    ports:
+      - "8080:80"
+    depends_on:
+      - fluent-bit
+    volumes:
+      - ./volumes/nginx/default.conf:/etc/nginx/conf.d/default.conf
+      - ./volumes/nginx/nginx.conf:/etc/nginx/nginx.conf
+    logging:
+      # используемый драйвер логгирования
+      driver: "fluentd"
+      options:
+        # куда посылать лог-сообщения, необходимо чтобы адрес
+        # совпадал с настройками плагина forward
+        fluentd-address: localhost:24224
+        # теги используются для маршрутизации лог-сообщений, тема
+        # маршрутизации будет рассмотрена ниже
+        tag: app.logs
+
+  fluent-bit:
+    container_name: fluent-bit
+    image: fluent/fluent-bit
+    ports:
+      # необходимо открыть порты, которые используются плагином forward
+      - "24224:24224"
+      - "24224:24224/udp"
+      - "2020:2020"
+    environment:
+      - opensearch_host=opensearch
+      - opensearch_user=admin
+      - opensearch_pass=admin
+    volumes:
+      - ./volumes/fluent-bit-etc/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+      - ./volumes/fluent-bit-etc/parsers.conf:/fluent-bit/etc/parsers.conf
+
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    environment:
+      - cluster.name=opensearch-cluster # Name the cluster
+      - node.name=opensearch # Name the node that will run in this container
+      - discovery.seed_hosts=opensearch,opensearch2 # Nodes to look for when discovering the cluster
+      - cluster.initial_cluster_manager_nodes=opensearch,opensearch2 # Nodes eligible to serve as cluster manager
+      - bootstrap.memory_lock=true
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    ports:
+      - '9200:9200'
+      - '9600:9600'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data
+
+  opensearch2:
+    image: opensearchproject/opensearch:latest
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch2
+      - discovery.seed_hosts=opensearch,opensearch2
+      - cluster.initial_cluster_manager_nodes=opensearch,opensearch2
+      - bootstrap.memory_lock=true
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data2:/usr/share/opensearch/data
+
+  dashboards:
+    image: opensearchproject/opensearch-dashboards:latest
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_HOSTS: '["https://opensearch:9200","https://opensearch2:9200"]'
+    depends_on:
+      - opensearch
+
+volumes:
+  opensearch-data1:
+  opensearch-data2:
diff --git a/deploy/docker-compose-fbos.yml b/deploy/docker-compose-fbos.yml
@@ -0,0 +1,79 @@
+# Минимальная версия инфраструктуры на базе
+# OpenSearch
+# OpenSearch Dashboards
+# Fluent Bit
+
+version: '3'
+services:
+
+  app:
+    image: nginx:latest
+    ports:
+      - "8080:80"
+    depends_on:
+      - fluent-bit
+    volumes:
+      - ./volumes/nginx/default.conf:/etc/nginx/conf.d/default.conf
+      - ./volumes/nginx/nginx.conf:/etc/nginx/nginx.conf
+    logging:
+      # используемый драйвер логгирования
+      driver: "fluentd"
+      options:
+        # куда посылать лог-сообщения, необходимо чтобы адрес
+        # совпадал с настройками плагина forward
+        fluentd-address: localhost:24224
+        # теги используются для маршрутизации лог-сообщений, тема
+        # маршрутизации будет рассмотрена ниже
+        tag: app.logs
+
+  fluent-bit:
+    container_name: fluent-bit
+    image: fluent/fluent-bit
+    ports:
+      # необходимо открыть порты, которые используются плагином forward
+      - "24224:24224"
+      - "24224:24224/udp"
+      - "2020:2020"
+    environment:
+      - opensearch_host=opensearch
+      - opensearch_user=admin
+      - opensearch_pass=admin
+    volumes:
+      - ./volumes/fluent-bit-etc/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+      - ./volumes/fluent-bit-etc/parsers.conf:/fluent-bit/etc/parsers.conf
+
+  opensearch:
+    container_name: opensearch
+    image: opensearchproject/opensearch:latest
+    environment:
+      - discovery.type=single-node
+      - http.port=9200
+      - bootstrap.memory_lock=true
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    ports:
+      - '9200:9200'
+      - '9600:9600'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+
+  dashboards:
+    image: opensearchproject/opensearch-dashboards:latest
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
+    depends_on:
+      - opensearch
+
+volumes:
+  opensearch-data: