Skip to content

Commit

Permalink
bazel: Support maven_install
Browse files Browse the repository at this point in the history 
maven_install is strongly superior to previous forms of grabbing dependencies
from Maven as it computes the appropriate versions to use from the full
transitive dependencies of all libraries used by an application. It also has
much less boilerplate and includes dependencies with generated targets.

In the future we will drop the jvm_maven_import_external usages and require
maven_install, at which point we can swap to using the `@maven' repository and
no longer depend on compat_repositories.

Fixes #5359
  • Loading branch information
@ejona86
ejona86 committed Dec 30, 2019
1 parent 6a5ee19 commit c606519
Show file tree
Hide file tree
Showing 9 changed files with 121 additions and 33 deletions.
27 changes: 27 additions & 0 deletions WORKSPACE
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,35 @@
workspace(name = "io_grpc_grpc_java")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
name = "rules_jvm_external",
sha256 = "62133c125bf4109dfd9d2af64830208356ce4ef8b165a6ef15bbff7460b35c3a",
strip_prefix = "rules_jvm_external-3.0",
url = "https://github.com/bazelbuild/rules_jvm_external/archive/3.0.zip",
)

load("@rules_jvm_external//:defs.bzl", "maven_install")
load("//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS")
load("//:repositories.bzl", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS")

maven_install(
artifacts = IO_GRPC_GRPC_JAVA_ARTIFACTS,
generate_compat_repositories = True,
override_targets = IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS,
repositories = [
"https://repo.maven.apache.org/maven2/",
],
)

load("@maven//:compat.bzl", "compat_repositories")

compat_repositories()

load("//:repositories.bzl", "grpc_java_repositories")

grpc_java_repositories()

load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

protobuf_deps()
2 changes: 1 addition & 1 deletion buildscripts/kokoro/bazel.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
set -exu -o pipefail
cat /VERSION

use_bazel.sh 0.28.1
use_bazel.sh 1.0.1
bazel version

cd github/grpc-java
Expand Down
8 changes: 4 additions & 4 deletions examples/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,15 @@ java_library(
":helloworld_java_proto",
":route_guide_java_grpc",
":route_guide_java_proto",
"@com_google_api_grpc_proto_google_common_protos//jar",
"@com_google_code_findbugs_jsr305//jar",
"@com_google_code_gson_gson",
"@com_google_guava_guava//jar",
"@com_google_protobuf//:protobuf_java",
"@com_google_protobuf//:protobuf_java_util",
"@io_grpc_grpc_java//api",
"@io_grpc_grpc_java//protobuf",
"@io_grpc_grpc_java//stub",
"@maven//:com_google_api_grpc_proto_google_common_protos",
"@maven//:com_google_code_findbugs_jsr305",
"@maven//:com_google_code_gson_gson",
"@maven//:com_google_guava_guava",
],
)

Expand Down
38 changes: 27 additions & 11 deletions examples/WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,41 @@ local_repository(
path = "..",
)

load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories")


load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

maven_jar(
name = "com_google_api_grpc_cloud_pubsub_v1",
artifact = "com.google.api.grpc:grpc-google-cloud-pubsub-v1:0.1.24",
sha1 = "601d8be0fd0cc0e050b1af3b88f191ada9a2f4e5",
http_archive(
name = "rules_jvm_external",
sha256 = "62133c125bf4109dfd9d2af64830208356ce4ef8b165a6ef15bbff7460b35c3a",
strip_prefix = "rules_jvm_external-3.0",
url = "https://github.com/bazelbuild/rules_jvm_external/archive/3.0.zip",
)

maven_jar(
name = "com_google_api_grpc_proto_cloud_pubsub_v1",
artifact = "com.google.api.grpc:proto-google-cloud-pubsub-v1:0.1.24",
sha1 = "e6dd66635f674b4e380dfd3de252ae019a51a67e",
load("@rules_jvm_external//:defs.bzl", "maven_install")
load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS")
load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS")

maven_install(
artifacts = [
"com.google.api.grpc:grpc-google-cloud-pubsub-v1:0.1.24",
"com.google.api.grpc:proto-google-cloud-pubsub-v1:0.1.24",
] + IO_GRPC_GRPC_JAVA_ARTIFACTS,
generate_compat_repositories = True,
override_targets = IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS,
repositories = [
"https://repo.maven.apache.org/maven2/",
],
)

load("@maven//:compat.bzl", "compat_repositories")

compat_repositories()

load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories")

# Run grpc_java_repositories after compat_repositories to ensure the
# maven_install-selected dependencies are used.
grpc_java_repositories()

load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

protobuf_deps()
6 changes: 3 additions & 3 deletions examples/example-gauth/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ java_library(
["src/main/java/**/*.java"],
),
deps = [
"@com_google_api_grpc_cloud_pubsub_v1//jar",
"@com_google_api_grpc_proto_cloud_pubsub_v1//jar",
"@com_google_auth_google_auth_library_oauth2_http//jar",
"@io_grpc_grpc_java//api",
"@io_grpc_grpc_java//auth",
"@io_grpc_grpc_java//protobuf",
"@io_grpc_grpc_java//stub",
"@maven//:com_google_api_grpc_grpc_google_cloud_pubsub_v1",
"@maven//:com_google_api_grpc_proto_google_cloud_pubsub_v1",
"@maven//:com_google_auth_google_auth_library_oauth2_http",
],
)

Expand Down
4 changes: 2 additions & 2 deletions examples/example-tls/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ java_library(
["src/main/java/**/*.java"],
),
runtime_deps = [
"@io_netty_netty_tcnative_boringssl_static//jar",
"@maven//:io_netty_netty_tcnative_boringssl_static",
],
deps = [
":helloworld_java_grpc",
Expand All @@ -33,7 +33,7 @@ java_library(
"@io_grpc_grpc_java//netty",
"@io_grpc_grpc_java//protobuf",
"@io_grpc_grpc_java//stub",
"@io_netty_netty_handler//jar",
"@maven//:io_netty_netty_handler",
],
)

Expand Down
2 changes: 1 addition & 1 deletion netty/shaded/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ java_library(
runtime_deps = [
"//netty",
"@io_netty_netty_tcnative_boringssl_static//jar",
"@io_netty_netty_transport_native_epoll//jar",
"@io_netty_netty_transport_native_epoll_linux_x86_64//jar",
],
)
1 change: 0 additions & 1 deletion protobuf/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,5 @@ java_library(
"@com_google_guava_guava//jar",
"@com_google_j2objc_j2objc_annotations//jar",
"@com_google_protobuf//:protobuf_java",
"@com_google_protobuf//:protobuf_java_util",
],
)
66 changes: 56 additions & 10 deletions repositories.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,48 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:jvm.bzl", "jvm_maven_import_external")

# For use with maven_install's artifacts.
# maven_install(
# ...
# artifacts = [
# # Your own deps
# ] + IO_GRPC_GRPC_JAVA_ARTIFACTS,
# )
IO_GRPC_GRPC_JAVA_ARTIFACTS = [
"com.google.android:annotations:4.1.1.4",
"com.google.api.grpc:proto-google-common-protos:1.17.0",
"com.google.auth:google-auth-library-credentials:0.19.0",
"com.google.auth:google-auth-library-oauth2-http:0.19.0",
"com.google.code.findbugs:jsr305:3.0.2",
"com.google.code.gson:gson:jar:2.8.6",
"com.google.errorprone:error_prone_annotations:2.3.3",
"com.google.guava:failureaccess:1.0.1",
"com.google.guava:guava:28.1-android",
"com.google.j2objc:j2objc-annotations:1.3",
"com.google.truth:truth:1.0",
"com.squareup.okhttp:okhttp:2.5.0",
"com.squareup.okio:okio:1.13.0",
"io.netty:netty-buffer:4.1.42.Final",
"io.netty:netty-codec-http2:4.1.42.Final",
"io.netty:netty-codec-http:4.1.42.Final",
"io.netty:netty-codec-socks:4.1.42.Final",
"io.netty:netty-codec:4.1.42.Final",
"io.netty:netty-common:4.1.42.Final",
"io.netty:netty-handler-proxy:4.1.42.Final",
"io.netty:netty-handler:4.1.42.Final",
"io.netty:netty-resolver:4.1.42.Final",
"io.netty:netty-tcnative-boringssl-static:2.0.26.Final",
"io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.42.Final",
"io.netty:netty-transport:4.1.42.Final",
"io.opencensus:opencensus-api:0.24.0",
"io.opencensus:opencensus-contrib-grpc-metrics:0.24.0",
"io.perfmark:perfmark-api:0.19.0",
"javax.annotation:javax.annotation-api:1.2",
"junit:junit:4.12",
"org.apache.commons:commons-lang3:3.5",
"org.codehaus.mojo:animal-sniffer-annotations:1.18",
]

# For use with maven_install's override_targets.
# maven_install(
# ...
Expand All @@ -22,6 +64,9 @@ load("@bazel_tools//tools/build_defs/repo:jvm.bzl", "jvm_maven_import_external")
# "your.target:artifact": "@//third_party/artifact",
# )
IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS = {
"com.google.protobuf:protobuf-java": "@com_google_protobuf//:protobuf_java",
"com.google.protobuf:protobuf-java-util": "@com_google_protobuf//:protobuf_java_util",
"com.google.protobuf:protobuf-javalite": "@com_google_protobuf_javalite//:protobuf_java_lite",
"io.grpc:grpc-alts": "@io_grpc_grpc_java//alts",
"io.grpc:grpc-api": "@io_grpc_grpc_java//api",
"io.grpc:grpc-auth": "@io_grpc_grpc_java//auth",
Expand All @@ -39,6 +84,13 @@ IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS = {

def grpc_java_repositories():
"""Imports dependencies for grpc-java."""
if not native.existing_rule("com_google_protobuf"):
com_google_protobuf()
if not native.existing_rule("com_google_protobuf_javalite"):
com_google_protobuf_javalite()
if not native.existing_rule("io_grpc_grpc_proto"):
io_grpc_grpc_proto()

if not native.existing_rule("com_google_android_annotations"):
com_google_android_annotations()
if not native.existing_rule("com_google_api_grpc_proto_google_common_protos"):
Expand All @@ -59,14 +111,8 @@ def grpc_java_repositories():
com_google_guava_failureaccess()
if not native.existing_rule("com_google_j2objc_j2objc_annotations"):
com_google_j2objc_j2objc_annotations()
if not native.existing_rule("com_google_protobuf"):
com_google_protobuf()
if not native.existing_rule("com_google_protobuf_javalite"):
com_google_protobuf_javalite()
if not native.existing_rule("com_google_truth_truth"):
com_google_truth_truth()
if not native.existing_rule("io_grpc_grpc_proto"):
io_grpc_grpc_proto()
if not native.existing_rule("com_squareup_okhttp_okhttp"):
com_squareup_okhttp_okhttp()
if not native.existing_rule("com_squareup_okio_okio"):
Expand All @@ -93,8 +139,8 @@ def grpc_java_repositories():
io_netty_netty_tcnative_boringssl_static()
if not native.existing_rule("io_netty_netty_transport"):
io_netty_netty_transport()
if not native.existing_rule("io_netty_netty_transport_native_epoll"):
io_netty_netty_transport_native_epoll()
if not native.existing_rule("io_netty_netty_transport_native_epoll_linux_x86_64"):
io_netty_netty_transport_native_epoll_linux_x86_64()
if not native.existing_rule("io_opencensus_opencensus_api"):
io_opencensus_opencensus_api()
if not native.existing_rule("io_opencensus_opencensus_contrib_grpc_metrics"):
Expand Down Expand Up @@ -368,9 +414,9 @@ def io_netty_netty_transport():
licenses = ["notice"], # Apache 2.0
)

def io_netty_netty_transport_native_epoll():
def io_netty_netty_transport_native_epoll_linux_x86_64():
jvm_maven_import_external(
name = "io_netty_netty_transport_native_epoll",
name = "io_netty_netty_transport_native_epoll_linux_x86_64",
artifact = "io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.42.Final",
server_urls = ["https://repo.maven.apache.org/maven2/"],
artifact_sha256 = "7bdf3003d5b60b061b494e62d1bafc420caf800efb743b14ec01ceaef1d3fa3e",
Expand Down

3 comments on commit c606519

@Mythra
Copy link

@Mythra Mythra commented on c606519 Nov 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW for those having trouble integrating this patch, here's a patch file you can specify with patches to use only maven_install targets: https://gist.github.com/Mythra/c73c3accf416aeb2beda32d656d28651 . This will prevent you from needing to use maven_compat with the latest release. An example rule may look something like:

  http_archive(
    name = "io_grpc_grpc_java",
    sha256 =
        "6990bf48696f0f5ca2170b28a05893f7875449cb3734d0827d50f5c69f746e33",
    strip_prefix = "grpc-java-1.33.1",
    urls = [
        "https://github.com/grpc/grpc-java/archive/v1.33.1.tar.gz",
    ],
    patches = ["//folder/to/patches:io_grpc_grpc_java.patch"],
  )

We had the following problems with maven_compat for reference:

  • It required our docker images for building C/C++ code to install java even though they weren't building any java targets (they normally let bazel use it's embedded JRE), which bloated them.
  • It added an extra 30 seconds to our build times for each build to generate maven_compat rules.

@ejona86
Copy link
Member Author

@ejona86 ejona86 commented on c606519 Nov 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Mythra, interesting. If you use pinned versions via maven_install_json, I'd expect that wouldn't be necessary.

@Mythra
Copy link

@Mythra Mythra commented on c606519 Nov 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ejona86 is possible, I didn't dig into using maven_install_json as it didn't seem relevant. Wasn't mentioned as far as I can tell in any release docs/the example, other than in rules_jvm_external which mentioned "pinning" dependencies. I guess in hindsight it makes sense as to it also being a potential fix, but at the same time I'm not too concerned with a patch for my usecase. Good to know for future reference though.

Please sign in to comment.