Merge pull request #21 from menny/refactor-sources-locator

Refactor sources locator
menny · Feb 9, 2019 · 2d7dbf3 · 2d7dbf3
2 parents be64bb3 + 694ab94
commit 2d7dbf3
Show file tree

Hide file tree

Showing 12 changed files with 428 additions and 35 deletions.
diff --git a/examples/android/program/main_deps/BUILD.bazel b/examples/android/program/main_deps/BUILD.bazel
@@ -1,3 +1,3 @@
 # Auto-generated by https://github.com/menny/bazel-mvn-deps
 
-# Args: --graph_file=program/maven_dependency_graph_resolving_rule_androidx_fragment__fragment__1_1_0_alpha02-transitive-graph.json --graph_file=program/maven_dependency_graph_resolving_rule_androidx_appcompat__appcompat__1_0_2-transitive-graph.json --graph_file=program/maven_dependency_graph_resolving_rule_androidx_annotation__annotation__1_0_0-transitive-graph.json --graph_file=program/maven_dependency_graph_resolving_rule_com_github_menny_Chauffeur__permissions__90e703256785b7baf9cd37370b62b1d7d3830406-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/bazel-mvn-deps/examples/android/program/main_deps/ --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
+# Args: --graph_file=program/maven_dependency_graph_resolving_rule_androidx_fragment__fragment__1_1_0_alpha02-transitive-graph.json --graph_file=program/maven_dependency_graph_resolving_rule_androidx_appcompat__appcompat__1_0_2-transitive-graph.json --graph_file=program/maven_dependency_graph_resolving_rule_androidx_annotation__annotation__1_0_0-transitive-graph.json --graph_file=program/maven_dependency_graph_resolving_rule_com_github_menny_Chauffeur__permissions__90e703256785b7baf9cd37370b62b1d7d3830406-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/android/program/main_deps/ --fetch_srcjar=true --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
diff --git a/examples/android/program/main_deps/dependencies.bzl b/examples/android/program/main_deps/dependencies.bzl
diff --git a/examples/kotlin/program/main_deps/BUILD.bazel b/examples/kotlin/program/main_deps/BUILD.bazel
@@ -1,3 +1,3 @@
 # Auto-generated by https://github.com/menny/bazel-mvn-deps
 
-# Args: --graph_file=program/maven_dependency_graph_resolving_rule_com_github_salomonbrys_kotson__kotson__2_5_0-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/kotlin/program/main_deps/ --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
+# Args: --graph_file=program/maven_dependency_graph_resolving_rule_com_github_salomonbrys_kotson__kotson__2_5_0-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/kotlin/program/main_deps/ --fetch_srcjar=true --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
diff --git a/examples/kotlin/program/main_deps/dependencies.bzl b/examples/kotlin/program/main_deps/dependencies.bzl
@@ -111,7 +111,7 @@ def generate_transitive_dependency_targets(kt_jvm_import=None, kt_jvm_library=No
     # from com.google.code.gson:gson:2.8.0
     native.java_import(name = 'main_deps___com_google_code_gson__gson__2_8_0',
         jars = ['@main_deps___com_google_code_gson__gson__2_8_0//file'],
-        licenses = [],
+        licenses = ['notice'],
         deps = [],
         exports = [],
         runtime_deps = [],

diff --git a/examples/kotlin_with_kt_rules/program/main_deps/BUILD.bazel b/examples/kotlin_with_kt_rules/program/main_deps/BUILD.bazel
@@ -1,3 +1,3 @@
 # Auto-generated by https://github.com/menny/bazel-mvn-deps
 
-# Args: --graph_file=program/maven_dependency_graph_resolving_rule_com_github_salomonbrys_kotson__kotson__2_5_0-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/kotlin_with_kt_rules/program/main_deps/ --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
+# Args: --graph_file=program/maven_dependency_graph_resolving_rule_com_github_salomonbrys_kotson__kotson__2_5_0-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/kotlin_with_kt_rules/program/main_deps/ --fetch_srcjar=true --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
diff --git a/examples/kotlin_with_kt_rules/program/main_deps/dependencies.bzl b/examples/kotlin_with_kt_rules/program/main_deps/dependencies.bzl
@@ -111,7 +111,7 @@ def generate_transitive_dependency_targets(kt_jvm_import=None, kt_jvm_library=No
     # from com.google.code.gson:gson:2.8.0
     native.java_import(name = 'main_deps___com_google_code_gson__gson__2_8_0',
         jars = ['@main_deps___com_google_code_gson__gson__2_8_0//file'],
-        licenses = [],
+        licenses = ['notice'],
         deps = [],
         exports = [],
         runtime_deps = [],

diff --git a/examples/plain_java/program/main_deps/BUILD.bazel b/examples/plain_java/program/main_deps/BUILD.bazel
@@ -1,3 +1,3 @@
 # Auto-generated by https://github.com/menny/bazel-mvn-deps
 
-# Args: --graph_file=program/maven_dependency_graph_resolving_rule_com_google_guava__guava__20_0-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/plain_java/program/main_deps/ --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
+# Args: --graph_file=program/maven_dependency_graph_resolving_rule_com_google_guava__guava__20_0-transitive-graph.json --output_macro_file_path=dependencies.bzl --output_target_build_files_base_path=/Users/menny/dev/menny/mabel/examples/plain_java/program/main_deps/ --fetch_srcjar=true --package_path=program --rule_prefix=main_deps___ --create_deps_sub_folders=true
diff --git a/resolver/BUILD.bazel b/resolver/BUILD.bazel
@@ -340,3 +340,17 @@ java_test(name = "graph_resolver_lib_test",
   size="small",
   test_class = "com.google.devtools.bazel.workspace.maven.ArtifactBuilderTest",
 )
+
+java_test(name = "sources_locator_test",
+  deps=[
+       ":merger",
+       ':api_lib',
+
+       "//resolver/main_deps/junit/junit",
+       '//resolver/main_deps/org/mockito/mockito-core',
+       '//resolver/main_deps/com/google/guava/guava',
+  ],
+  srcs=["src/main/javatest/net/evendanan/bazel/mvn/merger/SourcesLocatorTest.java"],
+  size="small",
+  test_class = "net.evendanan.bazel.mvn.merger.SourcesLocatorTest",
+)
diff --git a/...ver/src/main/java/com/google/devtools/bazel/workspace/maven/adapter/RuleToDependency.java b/...ver/src/main/java/com/google/devtools/bazel/workspace/maven/adapter/RuleToDependency.java
@@ -1,10 +1,7 @@
 package com.google.devtools.bazel.workspace.maven.adapter;
 
 import com.google.devtools.bazel.workspace.maven.Rule;
-import java.net.HttpURLConnection;
 import java.net.URI;
-import java.net.URL;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.stream.Collectors;
 import net.evendanan.bazel.mvn.api.Dependency;
@@ -13,39 +10,17 @@
 public class RuleToDependency {
 
     public static Dependency from(Rule rule) {
-        URI sources = uriWithClassifier(rule.getUrl(), "sources");
         return new Dependency(rule.groupId(), rule.artifactId(), rule.version(), rule.packaging(),
                 rule.getDeps().stream().map(RuleToDependency::from).collect(Collectors.toList()),
                 rule.getExportDeps().stream().map(RuleToDependency::from).collect(Collectors.toList()),
                 rule.getRuntimeDeps().stream().map(RuleToDependency::from).collect(Collectors.toList()),
                 URI.create(rule.getUrl()),
-                sources,
+                URI.create(""),
                 URI.create(""),
                 rule.getLicenses().stream()
                         .map(License::getName)
                         .map(net.evendanan.bazel.mvn.api.Dependency.License::fromLicenseName)
                         .filter(Objects::nonNull)
                         .collect(Collectors.toList()));
     }
-
-    private static URI uriWithClassifier(final String url, final String classifier) {
-        final int extStartIndex = url.lastIndexOf(".");
-        if (extStartIndex > 0) {
-            try {
-                URL classifiedUrl = new URL(String.format(Locale.US, "%s-%s%s", url.substring(0, extStartIndex), classifier, url.substring(extStartIndex)));
-                HttpURLConnection con = (HttpURLConnection) classifiedUrl.openConnection();
-                con.setRequestMethod("HEAD");
-                final int responseCode = con.getResponseCode();
-                if (responseCode >= 200 && responseCode < 300) {
-                    return classifiedUrl.toURI();
-                } else {
-                    return URI.create("");
-                }
-            } catch (Exception e) {
-                return URI.create("");
-            }
-        } else {
-            return URI.create("");
-        }
-    }
 }
diff --git a/resolver/src/main/java/net/evendanan/bazel/mvn/Merger.java b/resolver/src/main/java/net/evendanan/bazel/mvn/Merger.java
@@ -31,6 +31,7 @@
 import net.evendanan.bazel.mvn.merger.ClearSrcJarAttribute;
 import net.evendanan.bazel.mvn.merger.DefaultMerger;
 import net.evendanan.bazel.mvn.merger.DependencyTreeFlatter;
+import net.evendanan.bazel.mvn.merger.SourcesJarLocator;
 import net.evendanan.bazel.mvn.serialization.Serialization;
 import net.evendanan.timing.TaskTiming;
 import net.evendanan.timing.TimingData;
@@ -82,10 +83,14 @@ public static void main(String[] args) throws Exception {
         Merger driver = new Merger(options);
         Collection<Dependency> dependencies = driver.generateFromInputs(options);
 
-        if (!options.fetch_srcjar) {
+        if (options.fetch_srcjar) {
+            System.out.print("Locating sources JARs for resolved dependencies...");
+            dependencies = new SourcesJarLocator().fillSourcesAttribute(dependencies);
+            System.out.println("✓");
+        } else {
             dependencies = ClearSrcJarAttribute.clearSrcJar(dependencies);
         }
-        
+
         Function<Dependency, Dependency> prefixer = dependency -> DependencyWithPrefix.wrap(options.rule_prefix, dependency);
 
         driver.writeResults(dependencies.stream().map(prefixer).collect(Collectors.toList()), args);

diff --git a/resolver/src/main/java/net/evendanan/bazel/mvn/merger/SourcesJarLocator.java b/resolver/src/main/java/net/evendanan/bazel/mvn/merger/SourcesJarLocator.java
@@ -0,0 +1,76 @@
+package net.evendanan.bazel.mvn.merger;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+import net.evendanan.bazel.mvn.api.Dependency;
+
+public class SourcesJarLocator {
+
+    private static final String SOURCES_CLASSIFIER = "sources";
+
+    private final ConnectionOpener mConnectionOpenner;
+    private final Map<URI, URI> mURLCache = new HashMap<>();
+
+    public SourcesJarLocator() {
+        this(url -> (HttpURLConnection) url.openConnection());
+    }
+
+    @VisibleForTesting
+    SourcesJarLocator(final ConnectionOpener opener) {
+        mConnectionOpenner = opener;
+    }
+
+    public Collection<Dependency> fillSourcesAttribute(Collection<Dependency> dependencies) {
+        return dependencies.stream()
+                .map(dependency -> new Dependency(dependency.groupId(), dependency.artifactId(), dependency.version(), dependency.packaging(),
+                        fillSourcesAttribute(dependency.dependencies()),
+                        fillSourcesAttribute(dependency.exports()),
+                        fillSourcesAttribute(dependency.runtimeDependencies()),
+                        dependency.url(), uriWithClassifier(dependency.url()), dependency.javadocUrl(), dependency.licenses()))
+                .collect(Collectors.toList());
+    }
+
+    private URI uriWithClassifier(final URI uri) {
+        return mURLCache.computeIfAbsent(uri, this::hotFetch);
+    }
+
+    private URI hotFetch(final URI uri) {
+        System.out.print('.');
+        final String url = uri.toASCIIString();
+        final int extStartIndex = url.lastIndexOf(".");
+        if (extStartIndex > 0) {
+            try {
+                //for example
+                // https://repo1.maven.org/maven2/com/google/guava/guava/20.0/guava-20.0.jar
+                // will become
+                // https://repo1.maven.org/maven2/com/google/guava/guava/20.0/guava-20.0-sources.jar (always jar ext)
+                final String urlWithClassifier = String.format(Locale.US, "%s-%s.jar", url.substring(0, extStartIndex), SOURCES_CLASSIFIER);
+                final URL classifiedUrl = new URL(urlWithClassifier);
+                HttpURLConnection con = mConnectionOpenner.openUrlConnection(classifiedUrl);
+                con.setRequestMethod("HEAD");
+                final int responseCode = con.getResponseCode();
+                if (responseCode >= 200 && responseCode < 300) {
+                    return classifiedUrl.toURI();
+                } else {
+                    return URI.create("");
+                }
+            } catch (Exception e) {
+                return URI.create("");
+            }
+        } else {
+            return URI.create("");
+        }
+    }
+
+    interface ConnectionOpener {
+        HttpURLConnection openUrlConnection(URL url) throws IOException;
+    }
+}