Skip to content

Commit

Permalink
M4l4 monitoring (#19)
Browse files Browse the repository at this point in the history
  • Loading branch information
svok authored Nov 11, 2024
1 parent a34fc85 commit 354ea2b
Show file tree
Hide file tree
Showing 40 changed files with 1,531 additions and 42 deletions.
48 changes: 29 additions & 19 deletions deploy/docker-compose-fbos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,25 @@
version: '3'
services:

app:
image: nginx:latest
ports:
- "8080:80"
depends_on:
- fluent-bit
volumes:
- ./volumes/nginx/default.conf:/etc/nginx/conf.d/default.conf
- ./volumes/nginx/nginx.conf:/etc/nginx/nginx.conf
logging:
# используемый драйвер логгирования
driver: "fluentd"
options:
# куда посылать лог-сообщения, необходимо чтобы адрес
# совпадал с настройками плагина forward
fluentd-address: localhost:24224
# теги используются для маршрутизации лог-сообщений, тема
# маршрутизации будет рассмотрена ниже
tag: app.logs
# app:
# image: nginx:latest
# ports:
# - "8080:80"
# depends_on:
# - fluent-bit
# volumes:
# - ./volumes/nginx/default.conf:/etc/nginx/conf.d/default.conf
# - ./volumes/nginx/nginx.conf:/etc/nginx/nginx.conf
# logging:
# # используемый драйвер логгирования
# driver: "fluentd"
# options:
# # куда посылать лог-сообщения, необходимо чтобы адрес
# # совпадал с настройками плагина forward
# fluentd-address: localhost:24224
# # теги используются для маршрутизации лог-сообщений, тема
# # маршрутизации будет рассмотрена ниже
# tag: app.logs

fluent-bit:
container_name: fluent-bit
Expand All @@ -50,9 +50,19 @@ services:
- http.port=9200
- bootstrap.memory_lock=true
- ES_JAVA_OPTS=-Xms512m -Xmx512m
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=adm-Password0
ports:
- '9200:9200'
- '9600:9600'
healthcheck:
test:
[
"CMD-SHELL",
"curl -ku admin:admin https://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
]
interval: 5s
timeout: 5s
retries: 24
ulimits:
memlock:
soft: -1
Expand Down
53 changes: 31 additions & 22 deletions deploy/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,30 +8,29 @@
# Адреса:
# Приложение доступно по http://localhost:8080 (envoy прокси)
# Панели мониторинга http://localhost:5601 (opensearch dashboards, kibana)
# Управление пользователями http://localhost:8081 (keycloak)
# Управление пользователями http://localhost:8081 (keycloak)
# Управление пользователями http://localhost:8080/admin (keycloak)

services:

app:
image: nginx:latest
depends_on:
- fluent-bit
- envoy
volumes:
- ./volumes/nginx/default.conf:/etc/nginx/conf.d/default.conf
- ./volumes/nginx/nginx.conf:/etc/nginx/nginx.conf
- ./volumes/nginx/html:/usr/share/nginx/html
logging:
# используемый драйвер логгирования
driver: "fluentd"
options:
# куда посылать лог-сообщения, необходимо чтобы адрес
# совпадал с настройками плагина forward
fluentd-address: localhost:24224
# теги используются для маршрутизации лог-сообщений, тема
# маршрутизации будет рассмотрена ниже
tag: app.logs
# app:
# image: nginx:latest
# depends_on:
# - fluent-bit
# - envoy
# volumes:
# - ./volumes/nginx/default.conf:/etc/nginx/conf.d/default.conf
# - ./volumes/nginx/nginx.conf:/etc/nginx/nginx.conf
# - ./volumes/nginx/html:/usr/share/nginx/html
# logging:
# # используемый драйвер логгирования
# driver: "fluentd"
# options:
# # куда посылать лог-сообщения, необходимо чтобы адрес
# # совпадал с настройками плагина forward
# fluentd-address: localhost:24224
# # теги используются для маршрутизации лог-сообщений, тема
# # маршрутизации будет рассмотрена ниже
# tag: app.logs

fluent-bit:
container_name: fluent-bit
Expand Down Expand Up @@ -61,6 +60,15 @@ services:
ports:
- '9200:9200'
- '9600:9600'
healthcheck:
test:
[
"CMD-SHELL",
"curl -ku admin:adm-Password0 https://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
]
interval: 5s
timeout: 5s
retries: 24
ulimits:
memlock:
soft: -1
Expand All @@ -81,7 +89,8 @@ services:
environment:
OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
depends_on:
- opensearch
opensearch:
condition: service_healthy

envoy:
image: envoyproxy/envoy:v1.29.0 # Use the official Envoy proxy image
Expand Down
2 changes: 2 additions & 0 deletions deploy/volumes/envoy/envoy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ static_resources:
route: { cluster: app }
- match: { prefix: "/v2/ad/" }
route: { cluster: app }
- match: { prefix: "/admin/" }
route: { cluster: keycloak }
- match: { prefix: "/realms/" }
route: { cluster: keycloak }
- match: { prefix: "/resources/" }
Expand Down
5 changes: 5 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ jvm-language = "21"
plugin-kotlin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }
plugin-binaryCompatibilityValidator = { module = "org.jetbrains.kotlinx:binary-compatibility-validator", version.ref = "binaryCompabilityValidator" }
kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime" }
kotlinx-atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version = "0.23.2" }
kotlinx-serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinx-serialization" }
kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" }
coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines" }
Expand All @@ -40,11 +41,15 @@ jackson-datatype = { module = "com.fasterxml.jackson.datatype:jackson-datatype-j

# Logging
logback = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
logback-appenders = { module = "com.sndyuk:logback-more-appenders", version = "1.8.8" }
logback-logstash = { module = "net.logstash.logback:logstash-logback-encoder", version = "7.4" }
kermit = { module = "co.touchlab:kermit", version.ref = "kermit" }
logger-fluentd = { module = "org.fluentd:fluent-logger", version = "0.3.4" }

# Ktor
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
ktor-client-okhttp = { module = "io.ktor:ktor-client-okhttp", version.ref = "ktor" }
ktor-network = { module = "io.ktor:ktor-network", version.ref = "ktor" }

# Testing
kotest-junit5 = { module = "io.kotest:kotest-runner-junit5", version.ref = "kotest" }
Expand Down
1 change: 1 addition & 0 deletions ok-marketplace-be/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ ext {
val specDir = layout.projectDirectory.dir("../specs")
set("spec-v1", specDir.file("specs-ad-v1.yaml").toString())
set("spec-v2", specDir.file("specs-ad-v2.yaml").toString())
set("spec-log1", specDir.file("specs-ad-log1.yaml").toString())
}

tasks {
Expand Down
58 changes: 58 additions & 0 deletions ok-marketplace-be/ok-marketplace-api-log1/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import org.openapitools.generator.gradle.plugin.tasks.GenerateTask

plugins {
id("build-kmp")
alias(libs.plugins.crowdproj.generator)
alias(libs.plugins.kotlinx.serialization)
}

crowdprojGenerate {
packageName.set("${project.group}.api.log1")
inputSpec.set(rootProject.ext["spec-log1"] as String)
}

kotlin {
sourceSets {
val commonMain by getting {
kotlin.srcDirs(layout.buildDirectory.dir("generate-resources/src/commonMain/kotlin"))
dependencies {
implementation(kotlin("stdlib-common"))
implementation(libs.coroutines.core)
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)

implementation(project(":ok-marketplace-common"))
}
}
val commonTest by getting {
dependencies {
implementation(kotlin("test-common"))
implementation(kotlin("test-annotations-common"))
}
}
val jvmMain by getting {
dependencies {
implementation(kotlin("stdlib-jdk8"))
}
}
val jvmTest by getting {
dependencies {
implementation(kotlin("test-junit"))
}
}

all {
languageSettings.optIn("kotlin.RequiresOptIn")
}
}
}

tasks {
val openApiGenerateTask: GenerateTask = getByName("openApiGenerate", GenerateTask::class) {
outputDir.set(layout.buildDirectory.file("generate-resources").get().toString())
finalizedBy("compileCommonMainKotlinMetadata")
}
filter { it.name.startsWith("compile") }.forEach {
it.dependsOn(openApiGenerateTask)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package ru.otus.otuskotlin.marketplace.api.log1.mapper

import kotlinx.datetime.Clock
import ru.otus.otuskotlin.marketplace.api.log1.models.*
import ru.otus.otuskotlin.marketplace.common.MkplContext
import ru.otus.otuskotlin.marketplace.common.models.*

fun MkplContext.toLog(logId: String) = CommonLogModel(
messageTime = Clock.System.now().toString(),
logId = logId,
source = "ok-marketplace",
ad = toMkplLog(),
errors = errors.map { it.toLog() },
)

private fun MkplContext.toMkplLog(): MkplLogModel? {
val adNone = MkplAd()
return MkplLogModel(
requestId = requestId.takeIf { it != MkplRequestId.NONE }?.asString(),
requestAd = adRequest.takeIf { it != adNone }?.toLog(),
responseAd = adResponse.takeIf { it != adNone }?.toLog(),
responseAds = adsResponse.takeIf { it.isNotEmpty() }?.filter { it != adNone }?.map { it.toLog() },
requestFilter = adFilterRequest.takeIf { it != MkplAdFilter() }?.toLog(),
).takeIf { it != MkplLogModel() }
}

private fun MkplAdFilter.toLog() = AdFilterLog(
searchString = searchString.takeIf { it.isNotBlank() },
ownerId = ownerId.takeIf { it != MkplUserId.NONE }?.asString(),
dealSide = dealSide.takeIf { it != MkplDealSide.NONE }?.name,
)

private fun MkplError.toLog() = ErrorLogModel(
message = message.takeIf { it.isNotBlank() },
field = field.takeIf { it.isNotBlank() },
code = code.takeIf { it.isNotBlank() },
level = level.name,
)

private fun MkplAd.toLog() = AdLog(
id = id.takeIf { it != MkplAdId.NONE }?.asString(),
title = title.takeIf { it.isNotBlank() },
description = description.takeIf { it.isNotBlank() },
adType = adType.takeIf { it != MkplDealSide.NONE }?.name,
visibility = visibility.takeIf { it != MkplVisibility.NONE }?.name,
ownerId = ownerId.takeIf { it != MkplUserId.NONE }?.asString(),
productId = productId.takeIf { it != MkplProductId.NONE }?.asString(),
permissions = permissionsClient.takeIf { it.isNotEmpty() }?.map { it.name }?.toSet(),
)
37 changes: 37 additions & 0 deletions ok-marketplace-be/ok-marketplace-app-common/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
plugins {
id("build-kmp")
}

kotlin {
sourceSets {
val coroutinesVersion: String by project
commonMain {
dependencies {
implementation(kotlin("stdlib-jdk8"))

// transport models
implementation(project(":ok-marketplace-common"))
implementation(project(":ok-marketplace-api-log1"))
}
}
commonTest {
dependencies {
implementation(kotlin("test-common"))
implementation(kotlin("test-annotations-common"))

implementation(libs.coroutines.core)
}
}

jvmTest {
dependencies {
implementation(kotlin("test"))
}
}
nativeTest {
dependencies {
implementation(kotlin("test"))
}
}
}
}
18 changes: 18 additions & 0 deletions ok-marketplace-be/ok-marketplace-app-tmp/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
plugins {
id("build-jvm")
application
}

application {
mainClass.set("ru.otus.otuskotlin.marketplace.app.tmp.MainKt")
}

dependencies {
implementation(project(":ok-marketplace-api-log1"))
implementation("ru.otus.otuskotlin.marketplace.libs:ok-marketplace-lib-logging-common")
implementation("ru.otus.otuskotlin.marketplace.libs:ok-marketplace-lib-logging-logback")

implementation(project(":ok-marketplace-common"))

implementation(libs.coroutines.core)
}
Loading

0 comments on commit 354ea2b

Please sign in to comment.