Skip to content

Commit f17bf05

Browse files
sarahchen6amarziali
sarahchen6
authored and
amarziali
committed
Convert ci_jobs.gradle to a convention plugin and extension (#9838)
* Turn ci_jobs into a convention plugin and extension * Remove ci_jobs file * Address review comments * Address review comments pt 2
1 parent 2377f0a commit f17bf05

File tree

4 files changed

+255
-198
lines changed

4 files changed

+255
-198
lines changed

‎build.gradle.kts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import com.diffplug.gradle.spotless.SpotlessExtension
2+
import datadog.gradle.plugin.ci.testAggregate
23

34
plugins {
45
id("datadog.gradle-debug")
56
id("datadog.dependency-locking")
67
id("datadog.tracer-version")
78
id("datadog.dump-hanged-test")
9+
id("datadog.ci-jobs")
810

911
id("com.diffplug.spotless") version "6.13.0"
1012
id("com.github.spotbugs") version "5.0.14"
@@ -137,4 +139,16 @@ allprojects {
137139
}
138140
}
139141

140-
apply(from = "$rootDir/gradle/ci_jobs.gradle")
142+
testAggregate("smoke", listOf(":dd-smoke-tests"), emptyList())
143+
testAggregate("instrumentation", listOf(":dd-java-agent:instrumentation"), emptyList())
144+
testAggregate("profiling", listOf(":dd-java-agent:agent-profiling"), emptyList())
145+
testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), forceCoverage = true)
146+
testAggregate(
147+
"base", listOf(":"),
148+
listOf(
149+
":dd-java-agent:instrumentation",
150+
":dd-smoke-tests",
151+
":dd-java-agent:agent-profiling",
152+
":dd-java-agent:agent-debugger"
153+
)
154+
)

‎buildSrc/src/main/kotlin/CIJobsExtensions.kt‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package datadog.gradle.plugin.ci
2+
3+
import org.gradle.api.Project
4+
import org.gradle.api.Task
5+
import org.gradle.kotlin.dsl.extra
6+
7+
/**
8+
* Checks if a task is affected by git changes
9+
*/
10+
internal fun isAffectedBy(baseTask: Task, affectedProjects: Map<Project, Set<String>>): String? {
11+
val visited = mutableSetOf<Task>()
12+
val queue = mutableListOf(baseTask)
13+
14+
while (queue.isNotEmpty()) {
15+
val t = queue.removeAt(0)
16+
if (visited.contains(t)) {
17+
continue
18+
}
19+
visited.add(t)
20+
21+
val affectedTasks = affectedProjects[t.project]
22+
if (affectedTasks != null) {
23+
if (affectedTasks.contains("all")) {
24+
return "${t.project.path}:${t.name}"
25+
}
26+
if (affectedTasks.contains(t.name)) {
27+
return "${t.project.path}:${t.name}"
28+
}
29+
}
30+
31+
t.taskDependencies.getDependencies(t).forEach { queue.add(it) }
32+
}
33+
return null
34+
}
35+
36+
/**
37+
* Creates a single aggregate root task that depends on matching subproject tasks
38+
*/
39+
private fun Project.createRootTask(
40+
rootTaskName: String,
41+
subProjTaskName: String,
42+
includePrefixes: List<String>,
43+
excludePrefixes: List<String>,
44+
forceCoverage: Boolean
45+
) {
46+
val coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
47+
tasks.register(rootTaskName) {
48+
subprojects.forEach { subproject ->
49+
val activePartition = subproject.extra.get("activePartition") as Boolean
50+
if (activePartition &&
51+
includePrefixes.any { subproject.path.startsWith(it) } &&
52+
!excludePrefixes.any { subproject.path.startsWith(it) }) {
53+
54+
val testTask = subproject.tasks.findByName(subProjTaskName)
55+
var isAffected = true
56+
57+
if (testTask != null) {
58+
val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean
59+
if (useGitChanges) {
60+
@Suppress("UNCHECKED_CAST")
61+
val affectedProjects = rootProject.extra.get("affectedProjects") as Map<Project, Set<String>>
62+
val fileTrigger = isAffectedBy(testTask, affectedProjects)
63+
if (fileTrigger != null) {
64+
logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)")
65+
} else {
66+
logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)")
67+
isAffected = false
68+
}
69+
}
70+
if (isAffected) {
71+
dependsOn(testTask)
72+
}
73+
}
74+
75+
if (isAffected && coverage) {
76+
val coverageTask = subproject.tasks.findByName("jacocoTestReport")
77+
if (coverageTask != null) {
78+
dependsOn(coverageTask)
79+
}
80+
val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
81+
if (verificationTask != null) {
82+
dependsOn(verificationTask)
83+
}
84+
}
85+
}
86+
}
87+
}
88+
}
89+
90+
/**
91+
* Creates aggregate test tasks for CI using createRootTask() above
92+
*
93+
* Creates three subtasks for the given base task name:
94+
* - ${baseTaskName}Test - runs allTests
95+
* - ${baseTaskName}LatestDepTest - runs allLatestDepTests
96+
* - ${baseTaskName}Check - runs check
97+
*/
98+
fun Project.testAggregate(
99+
baseTaskName: String,
100+
includePrefixes: List<String>,
101+
excludePrefixes: List<String> = emptyList(),
102+
forceCoverage: Boolean = false
103+
) {
104+
createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage)
105+
createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage)
106+
createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage)
107+
}
108+

‎buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
* This plugin defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize
3+
* jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes
4+
* with -PgitBaseRef.
5+
*/
6+
7+
import datadog.gradle.plugin.ci.isAffectedBy
8+
import java.io.File
9+
import kotlin.math.abs
10+
11+
// Set up activePartition property on all projects
12+
allprojects {
13+
extra.set("activePartition", true)
14+
15+
val shouldUseTaskPartitions = rootProject.hasProperty("taskPartitionCount") && rootProject.hasProperty("taskPartition")
16+
if (shouldUseTaskPartitions) {
17+
val taskPartitionCount = rootProject.property("taskPartitionCount") as String
18+
val taskPartition = rootProject.property("taskPartition") as String
19+
val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt())
20+
extra.set("activePartition", currentTaskPartition == taskPartition.toInt())
21+
}
22+
}
23+
24+
fun relativeToGitRoot(f: File): File {
25+
return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile()
26+
}
27+
28+
fun getChangedFiles(baseRef: String, newRef: String): List<File> {
29+
val stdout = StringBuilder()
30+
val stderr = StringBuilder()
31+
32+
val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef"))
33+
proc.inputStream.bufferedReader().use { stdout.append(it.readText()) }
34+
proc.errorStream.bufferedReader().use { stderr.append(it.readText()) }
35+
proc.waitFor()
36+
require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" }
37+
38+
val out = stdout.toString().trim()
39+
if (out.isEmpty()) {
40+
return emptyList()
41+
}
42+
43+
logger.debug("git diff output: $out")
44+
return out.split("\n").map { File(rootProject.projectDir, it.trim()) }
45+
}
46+
47+
// Initialize git change tracking
48+
rootProject.extra.set("useGitChanges", false)
49+
50+
if (rootProject.hasProperty("gitBaseRef")) {
51+
val baseRef = rootProject.property("gitBaseRef") as String
52+
val newRef = if (rootProject.hasProperty("gitNewRef")) {
53+
rootProject.property("gitNewRef") as String
54+
} else {
55+
"HEAD"
56+
}
57+
58+
val changedFiles = getChangedFiles(baseRef, newRef)
59+
rootProject.extra.set("changedFiles", changedFiles)
60+
rootProject.extra.set("useGitChanges", true)
61+
62+
val ignoredFiles = fileTree(rootProject.projectDir) {
63+
include(".gitignore", ".editorconfig")
64+
include("*.md", "**/*.md")
65+
include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd")
66+
include("NOTICE")
67+
include("static-analysis.datadog.yml")
68+
}
69+
70+
changedFiles.forEach { f ->
71+
if (ignoredFiles.contains(f)) {
72+
logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}")
73+
}
74+
}
75+
76+
val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) }
77+
rootProject.extra.set("changedFiles", filteredChangedFiles)
78+
79+
val globalEffectFiles = fileTree(rootProject.projectDir) {
80+
include(".gitlab/**")
81+
include("build.gradle")
82+
include("gradle/**")
83+
}
84+
85+
for (f in filteredChangedFiles) {
86+
if (globalEffectFiles.contains(f)) {
87+
logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)")
88+
rootProject.extra.set("useGitChanges", false)
89+
break
90+
}
91+
}
92+
93+
if (rootProject.extra.get("useGitChanges") as Boolean) {
94+
logger.warn("Git change tracking is enabled: $baseRef..$newRef")
95+
96+
val projects = subprojects.sortedByDescending { it.projectDir.path.length }
97+
val affectedProjects = mutableMapOf<Project, MutableSet<String>>()
98+
99+
// Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in
100+
// the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used.
101+
val matchers = listOf(
102+
mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"),
103+
mapOf("prefix" to "src/test/", "task" to "testClasses"),
104+
mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses")
105+
)
106+
107+
for (f in filteredChangedFiles) {
108+
val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") }
109+
if (p == null) {
110+
logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)")
111+
rootProject.extra.set("useGitChanges", false)
112+
break
113+
}
114+
115+
// Make sure path separator is /
116+
val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/")
117+
val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all"
118+
logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)")
119+
affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task)
120+
}
121+
122+
rootProject.extra.set("affectedProjects", affectedProjects)
123+
}
124+
}
125+
126+
tasks.register("runMuzzle") {
127+
val muzzleSubprojects = subprojects.filter { p ->
128+
val activePartition = p.extra.get("activePartition") as Boolean
129+
activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle")
130+
}
131+
dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" })
132+
}

0 commit comments

Comments
 (0)