Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bazel builds enabled along with sbt #893

Merged
merged 32 commits into from
Jan 23, 2025
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
d89ef49
first commit
knarukulla Dec 19, 2024
e357725
api build
knarukulla Dec 20, 2024
4608d2b
building sources tests and quick start completed
knarukulla Dec 31, 2024
36c89d4
Merge branch 'airbnb:main' into bazel-phase1
krisnaru Dec 31, 2024
0ebd1ce
cleanup
knarukulla Dec 31, 2024
44529e3
fixing online to work with scala 2.13
knarukulla Dec 31, 2024
513984e
spark 3.5 builds enabled, some refactor and cleanup
knarukulla Jan 1, 2025
0db49c4
some refactor
knarukulla Jan 1, 2025
9aa98f2
2.11 cleanup and spark 3.5 support
knarukulla Jan 1, 2025
11797c8
cleanup
knarukulla Jan 1, 2025
f0a0004
sbt support for 3_5
nikhil-zlai Jan 2, 2025
ceef637
thrift rule generation
knarukulla Jan 2, 2025
9bb179c
platform agnostic thrift lib
knarukulla Jan 2, 2025
ac96222
cr feedback
knarukulla Jan 2, 2025
bc58b6f
Future compat issue for scala_2.13
nikhil-zlai Jan 2, 2025
9107216
python builds enabled
knarukulla Jan 5, 2025
0c31cf2
import fix
nikhil-zlai Jan 6, 2025
27ad169
fixing imports
knarukulla Jan 7, 2025
efbffc4
fixing python tests
knarukulla Jan 7, 2025
4babcb4
test files update according to workspace path
knarukulla Jan 7, 2025
1b1b5d2
documentation
knarukulla Jan 9, 2025
00ee54a
fix comments
knarukulla Jan 10, 2025
deba1aa
Merge branch 'main' into bazel-phase1
krisnaru Jan 12, 2025
1bb4d33
python tests succeeds
knarukulla Jan 14, 2025
3c6cf7c
use bazel for python tests
knarukulla Jan 14, 2025
475ba7a
fix CI
knarukulla Jan 14, 2025
ddd9549
flink tests passed
knarukulla Jan 15, 2025
afd6805
Merge branch 'main' into bazel-phase1
krisnaru Jan 16, 2025
14ea69a
Merge branch 'main' into bazel-phase1
krisnaru Jan 19, 2025
f102de7
Merge branch 'main' into bazel-phase1
krisnaru Jan 23, 2025
6cc2d4c
code review feedback
knarukulla Jan 23, 2025
d6a36c7
build fix
knarukulla Jan 23, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .bazeliskrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
USE_BAZEL_VERSION=6.4.0
18 changes: 18 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
## Disable remote cache completely when --config=local is passed
build:local --remote_cache=

# Scala version config flags:
krisnaru marked this conversation as resolved.
Show resolved Hide resolved
# To build with Scala 2.12, pass "--config scala_2.12" to "bazel build"
# To set a different default Scala version, add the following to
# user.bazelrc:
# common --config scala_2.12
common:scala_2.12 --repo_env=SCALA_VERSION=2.12.18
common:scala_2.13 --repo_env=SCALA_VERSION=2.13.12
# Default scala version to 2.12
common --repo_env=SCALA_VERSION=2.12.18
# Default to Spark 3.2 if no value is specified
build --define=spark_version=3.2
build --javacopt=-Xep:DoubleBraceInitialization:OFF

# Don't implicitly create __init__.py files
build --incompatible_default_to_explicit_init_py
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,6 @@ mvn_settings.xml
# Release folder
releases

# bazel
/bazel-*
/user.bazelrc
16 changes: 16 additions & 0 deletions .ijwb/.bazelproject
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
directories:
# Add the directories you want added as source here
# By default, we've added your entire workspace ('.')
.

# Automatically includes all relevant targets under the 'directories' above
derive_targets_from_directories: true

targets:
# If source code isn't resolving, add additional targets that compile it here

additional_languages:
# Uncomment any additional languages you want supported
python
scala
java
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@ Daniel Kristjansson (Stripe)
Piyush Narang (Stripe)
Caio Camatta (Stripe)
Divya Manohar (Stripe)
Krish Narukulla (Roku)
17 changes: 17 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package(default_visibility = ["//visibility:public"])

load("@rules_python//python:pip.bzl", "compile_pip_requirements")

# To update py3_requirements_lock.txt, run:
compile_pip_requirements(
name = "pip",
extra_args = [
"--allow-unsafe",
"--resolver=backtracking",
],
requirements_in = "//:requirements.txt",
requirements_txt = "//:requirements_lock.txt",
# force this to mac-only, since that's where we expect it to run.
# remove + adopt requirements_[platform] arguments when we're on rules_python>=0.10.0
tags = ["manual"],
)
134 changes: 134 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
workspace(name = "chronon")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
name = "bazel_skylib",
sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
],
)

http_archive(
name = "rules_python",
sha256 = "ca77768989a7f311186a29747e3e95c936a41dffac779aff6b443db22290d913",
strip_prefix = "rules_python-0.36.0",
url = "https://github.com/bazelbuild/rules_python/releases/download/0.36.0/rules_python-0.36.0.tar.gz",
)

load("@rules_python//python:repositories.bzl", "py_repositories")

py_repositories()

load("@rules_python//python:repositories.bzl", "python_register_toolchains")

python_register_toolchains(
name = "python_3_8",
# Available versions are listed in @rules_python//python:versions.bzl.
# We recommend using the same version your team is already standardized on.
python_version = "3.8",
)

load("@rules_python//python:pip.bzl", "pip_parse")
load("@python_3_8//:defs.bzl", "interpreter")

pip_parse(
name = "pypi",
python_interpreter_target = interpreter,
requirements_lock = "//:requirements_lock.txt",
)

load("@pypi//:requirements.bzl", "install_deps")

install_deps()

# Remove all the remote_java_tools above when upgrading to rules_java 7.5.0 or greater
http_archive(
name = "rules_java",
sha256 = "e81e9deaae0d9d99ef3dd5f6c1b32338447fe16d5564155531ea4eb7ef38854b",
urls = [
"https://github.com/bazelbuild/rules_java/releases/download/7.0.6/rules_java-7.0.6.tar.gz",
],
)

load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains")

rules_java_dependencies()

rules_java_toolchains()

load("@rules_java//java:repositories.bzl", "remote_jdk8_repos")

remote_jdk8_repos()

#############################
# Protobuf #
#############################
http_archive(
name = "rules_proto",
krisnaru marked this conversation as resolved.
Show resolved Hide resolved
sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
strip_prefix = "rules_proto-5.3.0-21.7",
urls = [
"https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz",
],
)

load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")

rules_proto_dependencies()

rules_proto_toolchains()

# Maven rules
RULES_JVM_EXTERNAL_TAG = "4.5"

RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6"

http_archive(
name = "rules_jvm_external",
sha256 = RULES_JVM_EXTERNAL_SHA,
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
)

load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")

rules_jvm_external_deps()

load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")

rules_jvm_external_setup()

## Scala support

# See https://github.com/bazelbuild/rules_scala/releases for up to date version information.
http_archive(
name = "io_bazel_rules_scala",
sha256 = "e734eef95cf26c0171566bdc24d83bd82bdaf8ca7873bec6ce9b0d524bdaf05d",
strip_prefix = "rules_scala-6.6.0",
url = "https://github.com/bazelbuild/rules_scala/releases/download/v6.6.0/rules_scala-v6.6.0.tar.gz",
)

load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")

scala_config(scala_version = "2.12.12")

load("@io_bazel_rules_scala//scala:scala.bzl", "scala_repositories")

scala_repositories()

load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")

scala_register_toolchains()

load("@io_bazel_rules_scala//testing:scalatest.bzl", "scalatest_repositories", "scalatest_toolchain")

scalatest_repositories()

scalatest_toolchain()

load("//jvm:mirrors.bzl", "load_deps")

load_deps()
87 changes: 87 additions & 0 deletions aggregator/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
load("@io_bazel_rules_scala//scala:scala_cross_version_select.bzl", "select_for_scala_version")

scala_library(
name = "aggregator",
srcs = glob(["src/main/scala/ai/chronon/aggregator/**/*.scala"]),
visibility = ["//visibility:public"],
deps = [
"//api:api-lib",
"//api:api-models",
maven_artifact("com.fasterxml.jackson.core:jackson-core"),
maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
maven_artifact("com.google.code.gson:gson"),
maven_artifact("com.yahoo.datasketches:sketches-core"),
maven_artifact("com.yahoo.datasketches:memory"),
maven_artifact("org.apache.commons:commons-lang3"),
maven_artifact("org.apache.thrift:libthrift"),
maven_artifact("org.slf4j:slf4j-api"),
maven_artifact("org.slf4j:slf4j-log4j12"),
] + select_for_scala_version(
before_2_13 = [
scala_artifact("org.scala-lang.modules:scala-collection-compat"),
scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"),
],
since_2_13 = [
scala_artifact("org.scala-lang.modules:scala-parallel-collections"),
],
),
)

scala_library(
name = "test-lib",
srcs = glob(["src/test/scala/ai/chronon/aggregator/test/*.scala"]),
visibility = ["//visibility:public"],
deps = [
":aggregator",
"//api:api-lib",
"//api:api-models",
maven_artifact("junit:junit"),
maven_artifact("com.novocode:junit-interface"),
maven_artifact("org.slf4j:slf4j-api"),
maven_artifact("org.slf4j:slf4j-log4j12"),
maven_artifact("com.google.code.gson:gson"),
maven_artifact("org.apache.thrift:libthrift"),
maven_artifact("org.apache.commons:commons-lang3"),
maven_artifact("org.apache.commons:commons-math3"),
maven_artifact("com.yahoo.datasketches:sketches-core"),
maven_artifact("com.yahoo.datasketches:memory"),
] + select_for_scala_version(
before_2_12 = [],
) + select_for_scala_version(before_2_13 = [
maven_artifact("org.scala-lang.modules:scala-collection-compat_2.12"),
maven_artifact("com.fasterxml.jackson.module:jackson-module-scala_2.12"),
]) +
select_for_scala_version(after_2_12 = [
maven_artifact("org.scala-lang.modules:scala-parallel-collections_2.13"),
]),
)

scala_test_suite(
name = "test",
srcs = glob(["src/test/scala/ai/chronon/aggregator/test/*.scala"]),
visibility = ["//visibility:public"],
deps = [
":aggregator",
":test-lib",
"//api:api-lib",
"//api:api-models",
maven_artifact("junit:junit"),
maven_artifact("com.novocode:junit-interface"),
maven_artifact("org.slf4j:slf4j-api"),
maven_artifact("org.slf4j:slf4j-log4j12"),
maven_artifact("org.apache.thrift:libthrift"),
maven_artifact("com.yahoo.datasketches:sketches-core"),
maven_artifact("com.google.code.gson:gson"),
maven_artifact("com.yahoo.datasketches:memory"),
maven_artifact("org.apache.commons:commons-lang3"),
maven_artifact("org.apache.commons:commons-math3"),
] + select_for_scala_version(
before_2_12 = [],
) + select_for_scala_version(before_2_13 = [
maven_artifact("org.scala-lang.modules:scala-collection-compat_2.12"),
maven_artifact("com.fasterxml.jackson.module:jackson-module-scala_2.12"),
]) +
select_for_scala_version(after_2_12 = [
maven_artifact("org.scala-lang.modules:scala-parallel-collections_2.13"),
]),
)
68 changes: 68 additions & 0 deletions api/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
load("@io_bazel_rules_scala//scala:scala_cross_version_select.bzl", "select_for_scala_version")

scala_library(
name = "api-lib",
srcs = glob(["src/main/scala/ai/chronon/api/*.scala"]) +
select_for_scala_version(
between_2_12_and_2_13 = [
"//api/src/main/scala-2.12/scala/util:ScalaVersionSpecificCollectionsConverter.scala",
],
since_2_13 = [
"//api/src/main/scala-2.13/scala/util:ScalaVersionSpecificCollectionsConverter.scala",
],
),
visibility = ["//visibility:public"],
deps = [
":api-models",
"//third_party/java/spark:spark-exec",
maven_artifact("com.fasterxml.jackson.core:jackson-core"),
maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
maven_artifact("org.apache.thrift:libthrift"),
maven_artifact("org.slf4j:slf4j-api"),
maven_artifact("org.slf4j:slf4j-log4j12"),
] + select_for_scala_version(
before_2_13 = [
scala_artifact("org.scala-lang.modules:scala-collection-compat"),
],
since_2_13 = [
scala_artifact("org.scala-lang.modules:scala-parallel-collections"),
],
),
)

scala_test_suite(
name = "test",
srcs = glob(["src/test/scala/ai/chronon/api/test/*.scala"]),
visibility = ["//visibility:public"],
deps = [
":api-models",
":api-lib",
"//third_party/java/spark:spark-exec",
maven_artifact("com.fasterxml.jackson.core:jackson-core"),
maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
maven_artifact("org.apache.thrift:libthrift"),
maven_artifact("org.slf4j:slf4j-api"),
maven_artifact("org.slf4j:slf4j-log4j12"),
maven_artifact("junit:junit"),
maven_artifact("com.novocode:junit-interface"),
scala_artifact("org.scalatestplus:mockito-3-4"),
maven_artifact("org.mockito:mockito-core"),
scala_artifact(
"org.scala-lang.modules:scala-collection-compat",
),
] + select_for_scala_version(
since_2_13 = [
maven_artifact("org.scala-lang.modules:scala-parallel-collections_2.13"),
],
),
)

java_library(
name = "api-models",
srcs = ["//api/thrift:api-models-java"],
visibility = ["//visibility:public"],
deps = [
maven_artifact("javax.annotation:javax.annotation.api"),
maven_artifact("org.apache.thrift:libthrift"),
],
)
Loading