Allow non-default targets (#187)

README.md

@@ -10,6 +10,8 @@
 The set of scripts in this repository provides a solution for:
  - Generating pom.xml files for jars built by Bazel (typically ```java_library``` or ```java_binary``` rules)
  - Uploading the pom.xmls and jars to a Maven Artifact Repository such as Nexus (or installing them into the local Maven Repository at ~/.m2/repository)
+  - Handling the grouping of related jars into a "library", similar to a multi-module Maven project: all jars that are part of the same library are uploaded together
+  - Crawling library references and uploading those that have changed since they were last uploaded to Nexus
 
 pomgen does not run as part of the Bazel build - therefore Bazel BUILD files can remain free of pom.xml related metadata.
 
@@ -54,6 +56,7 @@ See [this doc](docs/change_detection.md) for more information on change detectio
 
 Please see the [hello-world example](examples/hello-world/README.md) to see how pomgen works.
 
+
 ## External Dependencies
 
 - Bazel, ideally through [bazelisk](https://github.com/bazelbuild/bazelisk)
@@ -176,6 +179,7 @@ Please see [more information about transitives versioning](docs/ci.md#using-a-di
 
 See [this example](examples/dep-overrides).
 
+
 ## CI setup
 
 [This document](docs/ci.md) goes over the CI setup.

crawl/buildpom.py

@@ -21,6 +21,7 @@ class MavenArtifactDef(object):
     Represents an instance of a maven_artifact rule defined in BUILD.pom file.
     Information from the BUILD.pom.released file is added, if that file exists.
 
+
     ==== Read out of the BUILD.pom file ====
 
     group_id: the maven artifact groupId of the bazel package.
@@ -64,6 +65,7 @@ class MavenArtifactDef(object):
     version_increment_strategy_name: specifies how this artifacts version should
         be incremented.
 
+
     ==== Read out of the optional BUILD.pom.released file ====
 
     released_version: the previously released version to Nexus
@@ -72,19 +74,22 @@ class MavenArtifactDef(object):
         previously released to Nexus
 
 
+
     ===== Internal attributes (never specified by the user) ====
 
     deps: additional targets this package depends on; list of Bazel labels.
         For example: deps = ["//projects/libs/servicelibs/srpc/srpc-thrift-svc-runtime"]
 
-        Only used by tests.
+        The deps attribute is only used by tests.
+
+    bazel_package: the bazel package the BUILD (and MVN-INF/) files live in
 
-    bazel_package: the bazel package the BUILD.pom file lives in 
+    bazel_target: the bazel target that builds this artifact
 
-    library_path: the path to the root directory of the library this
-        monorepo package is part of
+    library_path: the path to the root directory of the library this artifact
+        is part of
 
-    requires_release: whether this monorepo package should be released (to Nexus
+    requires_release: whether this artifact should be released (to Nexus
         or local Maven repository)
 
     release_reason: the reason for releasing this artifact
@@ -93,6 +98,7 @@ class MavenArtifactDef(object):
         BUILD.pom file, the content of the pom.xml.released file
     =====
 
+
     Implementation notes:
         - properties are kept read-only whenever possible
         - the constructor provides default values for easier instantiation
@@ -115,6 +121,7 @@ def __init__(self,
                  released_version=None,
                  released_artifact_hash=None,
                  bazel_package=None,
+                 bazel_target=None,
                  library_path=None,
                  requires_release=None,
                  released_pom_content=None):
@@ -133,6 +140,7 @@ def __init__(self,
         self._released_version = released_version
         self._released_artifact_hash = released_artifact_hash
         self._bazel_package = bazel_package
+        self._bazel_target = bazel_target
         self._library_path = library_path
         self._requires_release = requires_release
         self._release_reason = None
@@ -206,6 +214,10 @@ def released_artifact_hash(self, value):
     def bazel_package(self):
         return self._bazel_package
 
+    @property
+    def bazel_target(self):
+        return self._bazel_target
+
     @property
     def library_path(self):
         return self._library_path
@@ -279,6 +291,7 @@ def parse_maven_artifact_def(root_path, package):
         additional_change_detected_packages=ma_attrs.get("additional_change_detected_packages", []),
         gen_dependency_management_pom=ma_attrs.get("generate_dependency_management_pom", False),
         jar_path=ma_attrs.get("jar_path", None),
+        bazel_target=ma_attrs.get("target_name", None),
         deps=ma_attrs.get("deps", []))
 
     template_path = ma_attrs.get("pom_template_file", None)
@@ -348,6 +361,7 @@ def _augment_art_def_values(user_art_def, rel_art_def, bazel_package,
         gen_dependency_management_pom=False if user_art_def.gen_dependency_management_pom is None else user_art_def.gen_dependency_management_pom,
         jar_path=None if user_art_def.jar_path is None else os.path.normpath(os.path.join(bazel_package, mdfiles.MD_DIR_NAME, user_art_def.jar_path)),
         deps=user_art_def.deps,
+        bazel_target=user_art_def.bazel_target if user_art_def.bazel_target is not None else os.path.basename(bazel_package),
         released_version=rel_art_def.version if rel_art_def is not None else None,
         released_artifact_hash=rel_art_def.artifact_hash if rel_art_def is not None else None,
         bazel_package=bazel_package,

crawl/crawler.py

@@ -220,12 +220,11 @@ def _get_deps_transitive_closure_for_library(self, library_path,
         # other, but these references are not guaranteed)
         artifacts = self.library_to_artifact[library_path]
         for art_def in artifacts:
-            all_deps.add(dependency.new_dep_from_maven_artifact_def(art_def, bazel_target=None))
-
+            all_deps.add(dependency.new_dep_from_maven_artifact_def(art_def))
         return all_deps
 
     def _get_crawled_packages_as_deps(self):
-        deps = [dependency.new_dep_from_maven_artifact_def(art_def, bazel_target=None) for art_def in self.package_to_artifact.values()]
+        deps = [dependency.new_dep_from_maven_artifact_def(art_def) for art_def in self.package_to_artifact.values()]
         deps = set(self._filter_non_artifact_referencing_deps(deps))
         return deps
 
@@ -510,7 +509,10 @@ def _crawl(self, package, dep, parent_node, follow_references):
 
         Returns a Node instance for the crawled package.
         """
-        target_key = self._get_target_key(package, dep)
+        artifact_def = self.workspace.parse_maven_artifact_def(package)
+        if artifact_def is None:
+            raise Exception("No artifact defined at package %s" % package)
+        target_key = self._get_target_key(package, dep, artifact_def)
         if target_key in self.target_to_node:
             # if we have already processed this target, we can re-use the
             # children we discovered previously
@@ -532,13 +534,7 @@ def _crawl(self, package, dep, parent_node, follow_references):
         else:
             if self.verbose:
                 logger.info("Processing [%s]" % target_key)
-            artifact_def = self.workspace.parse_maven_artifact_def(package)
-
-            if artifact_def is None:
-                raise Exception("No artifact defined at package %s" % package)
 
-            self._validate_default_target_dep(parent_node, dep, artifact_def)
-
             self.package_to_artifact[package] = artifact_def
             self.library_to_artifact[artifact_def.library_path].append(artifact_def)
             pomgen = self._get_pom_generator(artifact_def, dep)
@@ -563,20 +559,6 @@ def _crawl(self, package, dep, parent_node, follow_references):
             self._store_if_leafnode(node)
             return node
 
-    def _validate_default_target_dep(self, parent_node, dep, artifact_def):
-        if dep is not None:
-            if artifact_def.pom_generation_mode.produces_artifact:
-                # if the current bazel target produces an artifact 
-                # (pom/jar that goes to Nexus), validate that the BUILD 
-                # file pointing at this target uses the default bazel 
-                # package target 
-                # this is a current pomgen requirement: 
-                # 1 bazel package produces one artifact, named after the 
-                # bazel package
-                dflt_package_name = os.path.basename(artifact_def.bazel_package)
-                if dep.bazel_target != dflt_package_name:
-                    raise Exception("Non default-package references are only supported to non-artifact producing packages: [%s] can only reference [%s], [%s:%s] is not allowed" % (parent_node.artifact_def.bazel_package, artifact_def.bazel_package, artifact_def.bazel_package, dep.bazel_target))
-
     def _get_pom_generator(self, artifact_def, dep):
         if dep is None:
             # make a real dependency instance here so we can pass it along
@@ -587,12 +569,15 @@ def _get_pom_generator(self, artifact_def, dep):
                                      dep)
 
     @classmethod
-    def _get_target_key(clazz, package, dep):
+    def _get_target_key(clazz, package, dep, artifact_def=None):
         if dep is None:
-            target = os.path.basename(package)
+            # initial bootstrap - we start a bazel package and we don't
+            # have a dep pointing here
+            assert artifact_def is not None
+            target = artifact_def.bazel_target
         else:
             target = dep.bazel_target
-        assert target is not None, "Target is None for dep %s" % dep
+        assert target is not None, "Target is None for package %s" % package
         return "%s:%s" % (package, target)
 
     def _store_if_leafnode(self, node):

crawl/dependency.py

@@ -314,8 +314,9 @@ def new_dep_from_maven_art_str(maven_artifact_str, name):
 
 
 def new_dep_from_maven_artifact_def(artifact_def, bazel_target=None):
-    if bazel_target is not None:
-        assert len(bazel_target) > 0, "bazel target must not be empty for artifact def %s" % artifact_def.bazel_package
+    if bazel_target  is None:
+        bazel_target = artifact_def.bazel_target
+    assert bazel_target is not None
     return MonorepoDependency(artifact_def, bazel_target)
 
 

crawl/pomparser.py

@@ -213,5 +213,5 @@ def _import_lxml():
         return etree
     except ImportError as ex:
         print("Module lxml is not installed, please execute the following in your environment:")
-        print("pip install --user lxml")
+        print("pip3 install --user lxml")
         return None

crawl/workspace.py

@@ -126,7 +126,7 @@ def _parse_dep_label(self, dep_label):
                 raise Exception("Unknown external dependency - please make sure all maven install json files have been registered with pomgen (by setting maven_install_paths in the pomgen config file): [%s]" % dep_label)
             return self._label_to_ext_dep[dep_label]
         elif dep_label.startswith("//"):
-            # monorepo src ref:
+            # src ref:
             package_path = dep_label[2:] # remove leading "//"
             target_name = None
             i = package_path.rfind(":")

docs/mdfiles.md

@@ -22,6 +22,7 @@ maven_artifact_update(
 )
 ```
 
+
 #### Required Attributes
 
 ##### maven_artifact.group_id
@@ -50,8 +51,13 @@ Given a current version of `1.2.3`, the `major`, `minor`, and `patch` incrementi
 
 Given a current version of `20230605.1`, the `calver` incrementing strategy would produce `<todaydate>.1` (or `<todaydate>.2` if the current version is already `<todaydate>.1`).
 
+
 #### Optional Attributes
 
+##### maven_artifact.target_name
+
+The name of the bazel target that builds the jar artifact pomgen will be processing. Defaults to the [default target](https://bazel.build/concepts/labels) of the bazel package (aka the target that has the same name as the directory the BUILD file, that defines the target, lives in).
+
 ##### maven_artifact.change_detection
 
 Controls whether change detection should be enabled for this artifact. If set to `False`, this artifact will always be marked as needing to be released (and a new pom will always be generated).
@@ -85,6 +91,7 @@ Default value: `None`
 
 See the `java_import` [example](../examples/java-import).
 
+
 ### LIBRARY.root (required)
 
 The LIBRARY.root file is a marker file that is currently empty.  It groups together multiple artifacts, defined by BUILD.pom files, into a single "library". All artifacts that belong to a single library are processed (installed/uploaded) together. Change detection also operates at the library level. If a single artifact in a library has changed, then all artifacts in the library are marked as needing to be released.

examples/compile-only-dependencies/README.md

@@ -1,6 +1,5 @@
 # Compilation time only dependencies should not be evaluated by pomgen
 
-
 There are some dependencies that are needed only at compilation time, for example Lombok. To ensure compile-time only dependencies are not included at runtime, Bazel has the `neverlink` attribute, which can be added to `java_library` rules.
 Typically compile-time only dependencies won't have a `BUILD.pom` file, so they have to be ignored by `pomgen`.
 

examples/dependency-management/README.md

@@ -4,6 +4,7 @@ This examples shows how pomgen can optionally generate a dependencyManagement "c
 
 The dependency management pom contains a `<dependencyManagement>` section with the transitive closure of all dependencies of the artifact it was generated for. It uses the `artifact_id` specified in the BUILD.pom file, suffixed with `.depmanagement`.
 
+
 ### Try this example
 
 From the root of the repository:

examples/hello-world/README.md

@@ -5,13 +5,15 @@
 
 In the Maven world, a typical project setup consists of a top level pom.xml file with multiple modules in subdirectories. Each module produces a Maven Artifact (typically a jar or a pom.xml).
 
-The pomgen terminology for a top level project with modules is a "library". Subdirectories of the library (the modules) are Bazel Packages that each produce a Maven Artifact (jar or pom). pomgen processes all modules that are part of a library together.
+The pomgen terminology for a top level project with modules is a `library`. Subdirectories of the library (the modules) are Bazel Packages that each produce a Maven Artifact (jar or pom). pomgen processes all modules that are part of a library together.
 
 This example has 3 libraries. A library is defined by the presence of a [LIBRARY.root](healthyfoods/MVN-INF/LIBRARY.root) marker file.
 
-A Bazel Package that produces a Maven Artifact must have a [BUILD.pom](healthyfoods/fruit-api/MVN-INF/BUILD.pom) file that defines Maven specific metadata. Note that the `java_library` target that builds the jar Maven Artifact must be the default target, ie it must have the same name as the directory its BUILD file lives in.
+A Bazel Package that produces a Maven Artifact must have a [BUILD.pom](healthyfoods/fruit-api/MVN-INF/BUILD.pom) file that defines Maven specific metadata.
 
-The libraries in this example are, and reference each other in this order:
+The `java_library` target that builds the jar Maven Artifact is typically  the default target, ie it the target that the same name as the directory its BUILD file lives in. If the target is not the default target, its name must be explicitly specified in the BUILD.pom file using the `maven_artifact.target_name` attribute.
+
+The libraries in this example reference each other in this order:
 - [juicer](juicer)
 - [wintervegetables](wintervegetables)
 - [healthyfoods](healthyfoods)
@@ -67,7 +69,7 @@ The command above specifies:
 - The **library** to generate poms for: `examples/hello-world/juicer`
 
 
-pomgen follows refernces between libraries; since `juicer` depends on 2 other libraries `healthyfoods` and `wintervegerables`, pomgen generated pom.xml files for all 3 libraries, ie for all modules that are part of those libraries. Usually this is the right behavior, but if there a lot of upstream libraries, it may be desirable in some cases to not follow library references. This can be accomplished by setting `-i` (ignore references) flag:
+pomgen follows references between libraries; since `juicer` depends on 2 other libraries `healthyfoods` and `wintervegerables`, pomgen generated pom.xml files for all 3 libraries, ie for all modules that are part of those libraries. Usually this is the right behavior, but if there a lot of upstream libraries, it may be desirable in some cases to not follow library references. This can be accomplished by setting `-i` (ignore references) flag:
 
 ```
 bazel run @pomgen//maven -- -a pomgen -l examples/hello-world/juicer -i

examples/hello-world/juicer/BUILD

@@ -1,15 +1,15 @@
 java_library(
-    name = "juicer",
+    name = "juicer_lib",
     srcs = glob(["src/main/java/**/*.java"]),
     deps = ["//examples/hello-world/healthyfoods/fruit-api", 
             "//examples/hello-world/healthyfoods/vegetable-api",
-            "//examples/hello-world/wintervegetables",
+            "//examples/hello-world/wintervegetables:wintervegetables_lib2",
             "@maven//:com_google_guava_guava",
     ]
 )
 
 java_binary(
     name = "make-juice",
-    runtime_deps = [":juicer"],
+    runtime_deps = [":juicer_lib"],
     main_class = "com.pomgen.example.Main",
 )

examples/hello-world/juicer/MVN-INF/BUILD.pom

@@ -3,6 +3,7 @@ maven_artifact(
     artifact_id = "juicer",
     version = "10.0.0-qual1-SNAPSHOT",
     pom_generation_mode = "dynamic",
+    target_name = "juicer_lib",
 )
 
 maven_artifact_update(

examples/hello-world/wintervegetables/BUILD

@@ -1,5 +1,5 @@
 java_library(
-    name = "wintervegetables",
+    name = "wintervegetables_lib2",
     srcs = glob(["src/main/java/**/*.java"]),
     deps = ["//examples/hello-world/healthyfoods/vegetable-api"],
     visibility = ["//examples/hello-world/juicer:__subpackages__"],

examples/hello-world/wintervegetables/MVN-INF/BUILD.pom

@@ -3,6 +3,7 @@ maven_artifact(
     artifact_id = "wintervegetables",
     version = "20200416.1-SNAPSHOT",
     pom_generation_mode = "dynamic",
+    target_name = "wintervegetables_lib2",
 )
 
 maven_artifact_update(