build.gradle

/*
 * This file was generated by the Gradle 'init' task.
 *
 * This is a general purpose Gradle build.
 * Learn how to create Gradle builds at https://guides.gradle.org/creating-new-gradle-builds
 */
buildscript {
    repositories {
        jcenter() // this applies only to the Gradle 'Shadow' plugin
        gradlePluginPortal()
    }
    dependencies {
        classpath 'com.github.jengelman.gradle.plugins:shadow:5.2.0'
        classpath 'ru.vyarus:gradle-quality-plugin:4.3.0'
    }
}

plugins {
    id 'idea'

    // https://docs.gradle.org/current/userguide/application_plugin.html
    id 'application'

    // provide pretty output to the console for tests
    id 'com.adarshr.test-logger' version '3.0.0'

    // plugin to produce an uber jar https://imperceptiblethoughts.com/shadow/introduction/
    id 'com.github.johnrengelman.shadow' version '5.2.0'

    // http://xvik.github.io/gradle-quality-plugin/4.3.0/getting-started/
    id 'ru.vyarus.quality' version '4.6.0' // no config out of the box

    // https://github.com/davidmc24/gradle-avro-plugin
    id 'com.github.davidmc24.gradle.plugin.avro' version '1.0.0'
}


// artifact properties
group = 'com.examples.flinkml.streaming'
version = '1.0'
mainClassName = 'com.examples.flinkml.StreamingJob'
description = """Flink PMML Example"""

application {
  applicationDefaultJvmArgs = [
    "-Dconfig.resource=application.local.conf"
    ,"-Dlog4j2.configurationFile=conf/flink/log4j-local.properties"
    ,"-Dlog.file=./data/flink/logs" // log path for the flink webui when running dockerless
  ]
}

ext {
    javaVersion = '1.11'
    junitVersion = '5.6.2'
    flinkVersion = '1.13.1'
    scalaBinaryVersion = '2.11'
    slf4jVersion = '1.7.15'
    log4jVersion = '2.12.1'
}

task stopJob(type: Exec) {
  commandLine "docker-compose", "-p", "flink", "-f", "docker/flink-job-cluster.yml", "down"
}

task startJob(type: Exec, dependsOn: 'stopJob') {
  commandLine "docker-compose", "-p", "flink", "-f", "docker/flink-job-cluster.yml", "up", "-d"
}

task kafkaUp(type: Exec) {
  commandLine "docker-compose", "-p", "kafka", "-f", "docker/kafka-cluster.yml", "up"
}

task kafkaDown(type: Exec) {
  commandLine "docker-compose", "-p", "kafka", "-f", "docker/kafka-cluster.yml", "down"
}


task deleteTopics(type: Exec) {
  commandLine "./scripts/delete-topics.sh", "-b", "broker-1:19092", "fishes", "weight-predictions"
}

task createTopics(type: Exec) {
  commandLine "./scripts/create-topics.sh", "-b", "broker-1:19092", "fishes:1:1", "weight-predictions:1:1"
}

task recreateTopics(type: GradleBuild) {
  tasks = ['deleteTopics', 'createTopics']
}

task grafanaUp(type: Exec) {
  commandLine "docker-compose", "-p", "grafana", "-f", "docker/grafana-elastic-logstash.yml", "up"
}

task grafanaDown(type: Exec) {
  commandLine "docker-compose", "-p", "grafana", "-f", "docker/grafana-elastic-logstash.yml", "down"
}


sourceCompatibility = javaVersion
targetCompatibility = javaVersion
tasks.withType(JavaCompile) {
    options.encoding = 'UTF-8'
}


wrapper {
    gradleVersion = '6.5.1'
}

// declare where to find the dependencies of your project
repositories {
    mavenCentral()
    maven {
        url "https://repository.apache.org/content/repositories/snapshots/"
    }
}

// NOTE: We cannot use "compileOnly" or "shadow" configurations since then we could not run code
// in the IDE or with "gradle run". We also cannot exclude transitive dependencies from the
// shadowJar yet (see https://github.com/johnrengelman/shadow/issues/159).
// -> Explicitly define the // libraries we want to be included in the "flinkShadowJar" configuration!
configurations {
    flinkShadowJar // dependencies which go into the shadowJar

    // always exclude these (also from transitive dependencies) since they are provided by Flink
    flinkShadowJar.exclude group: 'org.apache.flink', module: 'force-shading'
    flinkShadowJar.exclude group: 'com.google.code.findbugs', module: 'jsr305'
    flinkShadowJar.exclude group: 'org.slf4j'
    flinkShadowJar.exclude group: 'org.apache.logging.log4j'

    //https://logging.apache.org/log4j/2.x/faq.html#exclusions
    //Good Explanation: https://stackoverflow.com/questions/42348755/class-path-contains-multiple-slf4j-bindings-error
    all*.exclude group: 'log4j', module: 'log4j'
    all*.exclude group: 'org.slf4j', module: 'slf4j-log4j12'
}


// For details see https://github.com/radarsh/gradle-test-logger-plugin
testlogger {
    theme 'mocha'
    slowThreshold 5000
    logLevel 'QUIET'
}

test {
    useJUnitPlatform()

    testLogging {
      events "passed", "skipped", "failed"
    }
}

// declare the dependencies for your production and test code
dependencies {
    // --------------------------------------------------------------
    // Compile-time dependencies that should NOT be part of the
    // shadow jar and are provided in the lib folder of Flink
    // --------------------------------------------------------------
    compile "org.apache.flink:flink-streaming-java_$scalaBinaryVersion:$flinkVersion"
    compile "org.apache.flink:flink-runtime-web_$scalaBinaryVersion:$flinkVersion"

    // --------------------------------------------------------------
    // Dependencies that should be part of the shadow jar, e.g.
    // connectors. These must be in the flinkShadowJar configuration!
    // --------------------------------------------------------------
    flinkShadowJar group: 'org.apache.flink', name: "flink-connector-kafka_$scalaBinaryVersion", version:"$flinkVersion"
    flinkShadowJar group: 'com.typesafe', name: 'config', version: '1.4.0'
    flinkShadowJar group: 'joda-time', name: 'joda-time', version: '2.7'
    flinkShadowJar group: 'org.jpmml', name: 'pmml-evaluator-metro', version: '1.5.15'
    flinkShadowJar group: 'com.google.code.gson', name: 'gson', version: '2.8.7'

    compile "org.apache.logging.log4j:log4j-api:$log4jVersion"
    compile "org.apache.logging.log4j:log4j-core:$log4jVersion"
    compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4jVersion"
    compile "org.slf4j:slf4j-log4j12:${slf4jVersion}"

    compile group: 'org.apache.flink', name: "flink-connector-kafka_$scalaBinaryVersion", version: "$flinkVersion"
    compile group: 'org.apache.avro', name: 'avro', version: '1.10.0'
    compile group: 'org.jpmml', name: 'pmml-evaluator-metro', version: '1.5.15'

    implementation group:'com.google.code.gson', name: 'gson', version: '2.8.7'
    compile group: 'joda-time', name: 'joda-time', version: '2.7'
    compile group: 'com.typesafe', name: 'config', version: '1.4.0'

    // Add test dependencies here.
    testImplementation "org.junit.jupiter:junit-jupiter-api:$junitVersion"
    testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitVersion"
    testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: "$junitVersion"
    testCompile group: 'org.mockito', name: 'mockito-core', version: '3.4.6'
    testCompile group: 'org.assertj', name:'assertj-core', version: '3.16.1'


    testCompile "org.apache.flink:flink-test-utils_$scalaBinaryVersion:$flinkVersion"
    testCompile "org.apache.flink:flink-runtime_$scalaBinaryVersion:$flinkVersion:tests"
    testCompile "org.apache.flink:flink-streaming-java_$scalaBinaryVersion:$flinkVersion:tests"
}

// make compileOnly dependencies available for tests:
sourceSets {
    main.compileClasspath += configurations.flinkShadowJar
    main.runtimeClasspath += configurations.flinkShadowJar

    test.compileClasspath += configurations.flinkShadowJar
    test.runtimeClasspath += configurations.flinkShadowJar

    javadoc.classpath += configurations.flinkShadowJar
}

run.classpath = sourceSets.main.runtimeClasspath

jar {
    manifest {
        attributes 'Built-By': System.getProperty('user.name'),
                'Build-Jdk': System.getProperty('java.version')
    }
}

shadowJar {
    baseName 'flink-stream-job'
    zip64 true
    configurations = [project.configurations.flinkShadowJar]
}