M7l3 postgres

.github/workflows/build.yml

@@ -27,6 +27,9 @@ jobs:
     #    runs-on: macos-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          token: ${{ secrets.PAT_TOKEN }}
 
       - name: Set up JDK 21
         uses: actions/setup-java@v3
@@ -37,6 +40,13 @@ jobs:
       - name: Validate Gradle wrapper
         uses: gradle/wrapper-validation-action@v1
 
+      # https://github.com/actions/runner-images/issues/675
+      - name: Hack sources.list
+        run: sudo sed -i 's|http://azure.archive.ubuntu.com/ubuntu/|http://mirror.arizona.edu/ubuntu/|g' /etc/apt/sources.list
+
+      - name: Install curl-dev
+        run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libpq-dev libpq5
+
       - name: Run Tests
         uses: gradle/gradle-build-action@v2
         with:

.gitignore

@@ -1,9 +1,10 @@
-.gradle/
-.kotlin/
+.gradle
 **/build/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/**/build/
 
+**/.kotlin/
+
 /.idea/
 /kotlin-js-store/
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pgkn"]
+	path = pgkn
+	url = [email protected]:crowdproj/pgkn.git

build-plugin/build.gradle.kts

@@ -12,6 +12,10 @@ gradlePlugin {
             id = "build-kmp"
             implementationClass = "ru.otus.otuskotlin.marketplace.plugin.BuildPluginMultiplatform"
         }
+        register("build-pgContainer") {
+            id = "build-pgContainer"
+            implementationClass = "ru.otus.otuskotlin.marketplace.plugin.BuildPluginPgContainer"
+        }
     }
 }
 
@@ -28,5 +32,9 @@ dependencies {
     implementation(libs.plugin.kotlin)
 //    implementation(libs.plugin.dokka)
     implementation(libs.plugin.binaryCompatibilityValidator)
-//    implementation(libs.plugin.mavenPublish)
+
+    implementation(libs.testcontainers.postgres)
+    implementation(libs.testcontainers.core)
+    implementation(libs.db.postgres)
+//    implementation("com.github.docker-java:docker-java-core:3.3.6")
 }

build-plugin/src/main/kotlin/BuildPluginPgContainer.kt

@@ -0,0 +1,34 @@
+package ru.otus.otuskotlin.marketplace.plugin
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.testcontainers.containers.PostgreSQLContainer
+import org.testcontainers.containers.wait.strategy.Wait
+
+@Suppress("unused")
+internal class BuildPluginPgContainer : Plugin<Project> {
+    val pgDbName = "marketplace_ads"
+    val pgUsername = "postgres"
+    val pgPassword = "marketplace-pass"
+
+    private val pgContainer = PostgreSQLContainer<Nothing>("postgres:latest").apply {
+        withUsername(pgUsername)
+        withPassword(pgPassword)
+        withDatabaseName(pgDbName)
+        waitingFor(Wait.forLogMessage("database system is ready to accept connections", 1))
+    }
+
+
+    override fun apply(project: Project): Unit = with(project) {
+        val stopTask = tasks.register("pgStop") {
+            group = "containers"
+            pgContainer.stop()
+        }
+        tasks.register("pgStart", PgContainerStartTask::class.java) {
+            pgContainer.start()
+//            port = pgContainer.getMappedPort(5432)
+            pgUrl = pgContainer.jdbcUrl
+            finalizedBy(stopTask)
+        }
+    }
+}

build-plugin/src/main/kotlin/PgContainerStartTask.kt

@@ -0,0 +1,19 @@
+package ru.otus.otuskotlin.marketplace.plugin
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.TaskAction
+import org.gradle.work.InputChanges
+
+open class PgContainerStartTask : DefaultTask() {
+    override fun getGroup(): String = "containers"
+
+//    var port: Int = 5432
+    var pgUrl: String = ""
+
+    @TaskAction
+    fun execute(inputs: InputChanges) { // InputChanges parameter
+        val msg = if (inputs.isIncremental) "CHANGED inputs are out of date"
+        else "ALL inputs are out of date"
+        println(msg)
+    }
+}

build.gradle.kts

@@ -19,8 +19,18 @@ subprojects {
 }
 
 tasks {
+    val buildImages: Task by creating {
+        dependsOn(gradle.includedBuild("ok-marketplace-be").task(":buildImages"))
+    }
+    val e2eTests: Task by creating {
+        dependsOn(gradle.includedBuild("ok-marketplace-tests").task(":e2eTests"))
+        mustRunAfter(buildImages)
+    }
+
     create("check") {
         group = "verification"
-        dependsOn(gradle.includedBuild("ok-marketplace-be").task(":check"))
+//        dependsOn(gradle.includedBuild("ok-marketplace-be").task(":check"))
+        dependsOn(buildImages)
+        dependsOn(e2eTests)
     }
 }

deploy/docker-compose-postgres.yml

@@ -0,0 +1,24 @@
+version: '3'
+services:
+  psql:
+    image: postgres
+    container_name: postgresql
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    expose:
+      - "5432"
+    environment:
+      POSTGRES_PASSWORD: marketplace-pass
+      POSTGRES_USER: postgres
+      POSTGRES_DB: marketplace_ads
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  postgres_data:
+

gradle/libs.versions.toml

@@ -18,6 +18,9 @@ kermit = "2.0.3"
 ktor = "2.3.9"
 spring-boot = "3.2.0"
 
+liquibase = "4.27.0"
+exposed = "0.50.0"
+
 # Docker
 testcontainers = "1.19.7"
 muschko = "9.4.0"
@@ -83,6 +86,16 @@ kafka-client = { module = "org.apache.kafka:kafka-clients", version = "3.7.0" }
 
 # Databases
 db-cache4k = "io.github.reactivecircus.cache4k:cache4k:0.13.0"
+db-postgres = "org.postgresql:postgresql:42.7.3"
+db-hikari = "com.zaxxer:HikariCP:5.1.0"
+db-exposed-core = { module = "org.jetbrains.exposed:exposed-core", version.ref = "exposed" }
+db-exposed-dao = { module = "org.jetbrains.exposed:exposed-dao", version.ref = "exposed" }
+db-exposed-jdbc = { module = "org.jetbrains.exposed:exposed-jdbc", version.ref = "exposed" }
+
+# Liquidbase
+liquibase-core = {module = "org.liquibase:liquibase-core", version.ref = "liquibase"}
+liquibase-picocli = "info.picocli:picocli:4.7.5"
+liquibase-snakeyml = "org.yaml:snakeyaml:1.33"
 
 # Testing
 kotest-junit5 = { module = "io.kotest:kotest-runner-junit5", version.ref = "kotest" }
@@ -93,9 +106,11 @@ mockito-kotlin = { module = "org.mockito.kotlin:mockito-kotlin", version = "5.2.
 
 testcontainers-core = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" }
 testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq", version.ref = "testcontainers" }
+testcontainers-postgres = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" }
 
 [bundles]
 kotest = ["kotest-junit5", "kotest-core", "kotest-datatest", "kotest-property"]
+exposed = ["db-exposed-core", "db-exposed-dao", "db-exposed-jdbc"]
 
 [plugins]
 kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" }
@@ -104,6 +119,7 @@ openapi-generator = { id = "org.openapi.generator", version.ref = "openapi-gener
 crowdproj-generator = { id = "com.crowdproj.generator", version = "0.2.0" }
 kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }
 shadowJar = { id = "com.github.johnrengelman.shadow", version = "8.1.1" }
+liquibase = { id = "org.liquibase.gradle", version = "2.2.2" }
 
 # Spring
 spring-boot = { id = "org.springframework.boot", version.ref = "spring-boot" }

ok-marketplace-be/build.gradle.kts

@@ -33,4 +33,10 @@ tasks {
             dependsOn(subprojects.map {  it.getTasksByName(tsk,false)})
         }
     }
+
+    create("buildImages") {
+        dependsOn(project("ok-marketplace-app-spring").tasks.getByName("bootBuildImage"))
+        dependsOn(project("ok-marketplace-app-ktor").tasks.getByName("publishImageToLocalRegistry"))
+        dependsOn(project("ok-marketplace-app-ktor").tasks.getByName("dockerBuildX64Image"))
+    }
 }

ok-marketplace-be/gradle.properties

@@ -1,6 +1,6 @@
 kotlin.code.style=official
 kotlin.native.ignoreDisabledTargets=true
-#kotlin.native.cacheKind.linuxX64=none
+kotlin.native.cacheKind.linuxX64=none
 org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8
 
 

ok-marketplace-be/ok-marketplace-api-v1-mappers/src/main/kotlin/MappersV1ToTransport.kt

@@ -54,7 +54,8 @@ fun MkplContext.toTransportSearch() = AdSearchResponse(
 fun MkplContext.toTransportOffers() = AdOffersResponse(
     result = state.toResult(),
     errors = errors.toTransportErrors(),
-    ads = adsResponse.toTransportAd()
+    ad = adResponse.toTransportAd(),
+    ads = adsResponse.toTransportAd(),
 )
 
 fun MkplContext.toTransportInit() = AdInitResponse(
@@ -75,6 +76,7 @@ private fun MkplAd.toTransportAd(): AdResponseObject = AdResponseObject(
     adType = adType.toTransportAd(),
     visibility = visibility.toTransportAd(),
     permissions = permissionsClient.toTransportAd(),
+    lock = lock.takeIf { it != MkplAdLock.NONE }?.asString()
 )
 
 private fun Set<MkplAdPermissionClient>.toTransportAd(): Set<AdPermissions>? = this

ok-marketplace-be/ok-marketplace-api-v2-kmp/src/commonMain/kotlin/MkplAdApiSerializer.kt

@@ -2,13 +2,17 @@
 
 package ru.otus.otuskotlin.marketplace.api.v2
 
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import ru.otus.otuskotlin.marketplace.api.v2.models.IRequest
 import ru.otus.otuskotlin.marketplace.api.v2.models.IResponse
 
+@OptIn(ExperimentalSerializationApi::class)
 @Suppress("JSON_FORMAT_REDUNDANT_DEFAULT")
 val apiV2Mapper = Json {
 //    ignoreUnknownKeys = true
+    allowTrailingComma = true
 }
 
 @Suppress("UNCHECKED_CAST")
@@ -22,6 +26,13 @@ fun apiV2ResponseSerialize(obj: IResponse): String =
 fun <T : IResponse> apiV2ResponseDeserialize(json: String) =
     apiV2Mapper.decodeFromString<IResponse>(json) as T
 
+inline fun <reified T : IResponse> apiV2ResponseSimpleDeserialize(json: String) =
+    apiV2Mapper.decodeFromString<T>(json)
+
 @Suppress("unused")
 fun apiV2RequestSerialize(obj: IRequest): String =
     apiV2Mapper.encodeToString(IRequest.serializer(), obj)
+
+@Suppress("unused")
+inline fun <reified T: IRequest> apiV2RequestSimpleSerialize(obj: T): String =
+    apiV2Mapper.encodeToString<T>(obj)

ok-marketplace-be/ok-marketplace-api-v2-kmp/src/commonMain/kotlin/mappers/MappersV2ToTransport.kt

@@ -50,7 +50,8 @@ fun MkplContext.toTransportSearch() = AdSearchResponse(
 fun MkplContext.toTransportOffers() = AdOffersResponse(
     result = state.toResult(),
     errors = errors.toTransportErrors(),
-    ads = adsResponse.toTransportAd()
+    ad = adResponse.toTransportAd(),
+    ads = adsResponse.toTransportAd(),
 )
 
 fun MkplContext.toTransportInit() = AdInitResponse(

ok-marketplace-be/ok-marketplace-app-ktor/build.gradle.kts

@@ -75,8 +75,11 @@ kotlin {
                 implementation(libs.ktor.serialization.json)
 
                 // DB
+                implementation(libs.uuid)
+                implementation(projects.okMarketplaceRepoCommon)
                 implementation(projects.okMarketplaceRepoStubs)
                 implementation(projects.okMarketplaceRepoInmemory)
+                implementation(projects.okMarketplaceRepoPostgres)
 
                 // logging
                 implementation(project(":ok-marketplace-api-log1"))
@@ -116,7 +119,7 @@ kotlin {
                 implementation(project(":ok-marketplace-api-v1-mappers"))
 
                 implementation("ru.otus.otuskotlin.marketplace.libs:ok-marketplace-lib-logging-logback")
-
+                implementation(libs.testcontainers.postgres)
             }
         }
 
@@ -155,6 +158,7 @@ tasks {
                 into("${[email protected]()}")
             }
         }
+        runCommand("apt-get update && apt-get install -y libpq5 && rm -rf /var/lib/apt/lists/*")
         copyFile(nativeFileX64.name, "/app/")
         copyFile("application.yaml", "/app/")
         exposePort(8080)

ok-marketplace-be/ok-marketplace-app-ktor/src/appleMain/kotlin/plugins/getDatabaseConf.apple.kt

@@ -0,0 +1,18 @@
+package ru.otus.otuskotlin.marketplace.app.ktor.plugins
+
+import io.ktor.server.application.*
+import ru.otus.otuskotlin.marketplace.app.ktor.configs.ConfigPaths
+import ru.otus.otuskotlin.marketplace.common.repo.IRepoAd
+
+actual fun Application.getDatabaseConf(type: AdDbType): IRepoAd {
+    val dbSettingPath = "${ConfigPaths.repository}.${type.confName}"
+    val dbSetting = environment.config.propertyOrNull(dbSettingPath)?.getString()?.lowercase()
+    return when (dbSetting) {
+        "in-memory", "inmemory", "memory", "mem" -> initInMemory()
+//        "postgres", "postgresql", "pg", "sql", "psql" -> initPostgres()
+        else -> throw IllegalArgumentException(
+            "$dbSettingPath must be set in application.yml to one of: " +
+                    "'inmemory', 'postgres', 'cassandra', 'gremlin'"
+        )
+    }
+}

ok-marketplace-be/ok-marketplace-app-ktor/src/commonMain/kotlin/configs/ConfigPaths.kt

@@ -0,0 +1,6 @@
+package ru.otus.otuskotlin.marketplace.app.ktor.configs
+
+object ConfigPaths {
+    const val mkplRoot = "marketplace"
+    const val repository = "$mkplRoot.repository"
+}

ok-marketplace-be/ok-marketplace-app-ktor/src/commonMain/kotlin/configs/PostgresConfig.kt

@@ -0,0 +1,25 @@
+package ru.otus.otuskotlin.marketplace.app.ktor.configs
+
+import io.ktor.server.config.*
+
+data class PostgresConfig(
+    val host: String = "localhost",
+    val port: Int = 5432,
+    val user: String = "postgres",
+    val password: String = "marketplace-pass",
+    val database: String = "marketplace-ads",
+    val schema: String = "public",
+) {
+    constructor(config: ApplicationConfig): this(
+        host = config.propertyOrNull("$PATH.host")?.getString() ?: "localhost",
+        port = config.propertyOrNull("$PATH.port")?.getString()?.toIntOrNull() ?: 5432,
+        user = config.propertyOrNull("$PATH.user")?.getString() ?: "postgres",
+        password = config.property("$PATH.password").getString(),
+        database = config.propertyOrNull("$PATH.database")?.getString() ?: "marketplace-ads",
+        schema = config.propertyOrNull("$PATH.schema")?.getString() ?: "public",
+    )
+
+    companion object {
+        const val PATH = "${ConfigPaths.repository}.psql"
+    }
+}

ok-marketplace-be/ok-marketplace-app-ktor/src/commonMain/kotlin/plugins/InitAppSettings.kt

@@ -6,14 +6,13 @@ import ru.otus.otuskotlin.marketplace.app.ktor.base.KtorWsSessionRepo
 import ru.otus.otuskotlin.marketplace.backend.repository.inmemory.AdRepoStub
 import ru.otus.otuskotlin.marketplace.biz.MkplAdProcessor
 import ru.otus.otuskotlin.marketplace.common.MkplCorSettings
-import ru.otus.otuskotlin.marketplace.repo.inmemory.AdRepoInMemory
 
 fun Application.initAppSettings(): MkplAppSettings {
     val corSettings = MkplCorSettings(
         loggerProvider = getLoggerProviderConf(),
         wsSessions = KtorWsSessionRepo(),
-        repoTest = AdRepoInMemory(),
-        repoProd = AdRepoInMemory(),
+        repoTest = getDatabaseConf(AdDbType.TEST),
+        repoProd = getDatabaseConf(AdDbType.PROD),
         repoStub = AdRepoStub(),
     )
     return MkplAppSettings(