From c87fc5717572f894f2fa4fac8c637c7ac0fb265f Mon Sep 17 00:00:00 2001 From: cmhulbert Date: Mon, 25 Jan 2021 16:13:10 -0500 Subject: [PATCH 1/3] allow specifying certain modules for module-path instead of classpath --- jgo/jgo.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/jgo/jgo.py b/jgo/jgo.py index 642ba74..2a5c4ca 100644 --- a/jgo/jgo.py +++ b/jgo/jgo.py @@ -256,7 +256,7 @@ def jgo_parser(log_levels = _default_log_levels): parser = argparse.ArgumentParser( description = 'Run Java main class from maven coordinates.', - usage = '%(prog)s [-v] [-u] [-U] [-m] [-q] [--log-level] [--ignore-jgorc] [--link-type type] [--additional-jars jar [jar ...]] [--additional-endpoints endpoint [endpoint ...]] [JVM_OPTIONS [JVM_OPTIONS ...]] [main-args]', + usage = '%(prog)s [-v] [-u] [-U] [-m] [-q] [--log-level] [--ignore-jgorc] [--link-type type] [--modules-dependencies dep [dep ..]] [--additional-jars jar [jar ...]] [--additional-endpoints endpoint [endpoint ...]] [JVM_OPTIONS [JVM_OPTIONS ...]] [main-args]', epilog = epilog, formatter_class = argparse.RawTextHelpFormatter ) @@ -267,6 +267,7 @@ def jgo_parser(log_levels = _default_log_levels): parser.add_argument('-r', '--repository', nargs='+', help='Add additional maven repository (key=url format)', default=[], required=False) parser.add_argument('-a', '--additional-jars', nargs='+', help='Add additional jars to classpath', default=[], required=False) parser.add_argument('-q', '--quiet', action='store_true', required=False, help='Suppress jgo output, including logging') + parser.add_argument('--module-dependencies', nargs='+', help='Specify which dependencies should be treated as modules (of the form "group:artifact:version[:classifier]', default=[], required=False) parser.add_argument( '--additional-endpoints', nargs='+', help='Add additional endpoints', default=[], required=False) parser.add_argument('--ignore-jgorc', action='store_true', help='Ignore ~/.jgorc') parser.add_argument('--link-type', default=None, type=str, help='How to link from local maven repository into jgo cache. Defaults to the `links\' setting in ~/.jgorc or \'auto\' if not specified.', choices=('hard', 'soft', 'copy', 'auto')) @@ -396,6 +397,7 @@ def resolve_dependencies( manage_dependencies=False, repositories={}, shortcuts={}, + modules_deps=[], verbose=0): @@ -406,6 +408,7 @@ def resolve_dependencies( repo_str = ''.join('{rid}{url}'.format(rid=k, url=v) for (k, v) in repositories.items()) coordinates = coordinates_from_endpoints(endpoints) workspace = workspace_dir_from_coordinates(coordinates, cache_dir=cache_dir) + module_dir = os.path.join(workspace, "modules") build_success_file = os.path.join(workspace, 'buildSuccess') update_cache = True if force_update else update_cache @@ -424,6 +427,7 @@ def resolve_dependencies( shutil.rmtree(workspace, True) os.makedirs(workspace, exist_ok=True) + os.makedirs(module_dir, exist_ok=True) # TODO should this be for all endpoints or only the primary endpoint? if manage_dependencies: @@ -480,7 +484,7 @@ def resolve_dependencies( info_regex = re.compile('^.*\\[[A-Z]+\\] *') - relevant_jars = [] + modules_discovered = set() for l in str(mvn_out).split('\\n'): # TODO: the compile|runtime|provided matches might fail if an artifactId starts with accordingly # TODO: If that ever turns out to be an issue, it is going to be necessary to update these checks @@ -507,13 +511,23 @@ def resolve_dependencies( jar_file = '{}.{}'.format(artifact_name, extension) jar_file_in_workspace = os.path.join(workspace, jar_file) - relevant_jars.append(jar_file_in_workspace) try: + dependency_identifier = ':'.join((g, a, version, c) if c else (g, a, version)) + if dependency_identifier in modules_deps: + jar_file_in_module_dir = os.path.join(module_dir, jar_file) + link(os.path.join(m2_repo, *g.split('.'), a, version, jar_file), jar_file_in_module_dir, link_type=link_type) + modules_discovered.add(dependency_identifier) + else: + jar_file_in_workspace = os.path.join(workspace, jar_file) link(os.path.join(m2_repo, *g.split('.'), a, version, jar_file), jar_file_in_workspace, link_type=link_type) except FileExistsError as e: # Do not throw exceptionif target file exists. pass + if len(set(modules_deps)) != len(modules_discovered): + missing_mods = '\n'.join(x for x in modules_deps - modules_discovered) + _logger.info(f"Missing Modules:\n{missing_mods}") + raise Exception("Could not discover all modules dependencies") pathlib.Path(build_success_file).touch(exist_ok=True) return primary_endpoint, workspace @@ -579,7 +593,13 @@ def run(parser, argv=sys.argv[1:], stdout=None, stderr=None): repositories = repositories, shortcuts = shortcuts, verbose = args.verbose, - link_type = link_type) + link_type = link_type, + modules_deps=args.module_dependencies) + + if len(args.module_dependencies) > 0: + jvm_args.append("--module-path") + jvm_args.append(os.path.join(workspace, "modules")) + main_class_file = os.path.join(workspace, primary_endpoint.main_class) if primary_endpoint.main_class else os.path.join(workspace, 'mainClass') try: From 2601284c2a8a8d2139e003e067d3862134a4104d Mon Sep 17 00:00:00 2001 From: cmhulbert Date: Mon, 25 Jan 2021 16:49:44 -0500 Subject: [PATCH 2/3] add test for module path dependencies --- jgo/jgo.py | 2 +- tests/test_parsington.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/jgo/jgo.py b/jgo/jgo.py index 2a5c4ca..30691bb 100644 --- a/jgo/jgo.py +++ b/jgo/jgo.py @@ -520,7 +520,7 @@ def resolve_dependencies( modules_discovered.add(dependency_identifier) else: jar_file_in_workspace = os.path.join(workspace, jar_file) - link(os.path.join(m2_repo, *g.split('.'), a, version, jar_file), jar_file_in_workspace, link_type=link_type) + link(os.path.join(m2_repo, *g.split('.'), a, version, jar_file), jar_file_in_workspace, link_type=link_type) except FileExistsError as e: # Do not throw exceptionif target file exists. pass diff --git a/tests/test_parsington.py b/tests/test_parsington.py index 5cfb590..b380077 100644 --- a/tests/test_parsington.py +++ b/tests/test_parsington.py @@ -88,6 +88,25 @@ def test_resolve_auto(self): finally: shutil.rmtree(tmp_dir) + def test_resolve_as_module(self): + tmp_dir = tempfile.mkdtemp(prefix='jgo-test-cache-dir') + m2_repo = os.path.join(str(pathlib.Path.home()), '.m2', 'repository') + try: + _, workspace = jgo.resolve_dependencies( + PARSINGTON_ENDPOINT, + m2_repo=m2_repo, + cache_dir=tmp_dir, + link_type='auto', + modules_deps=[PARSINGTON_ENDPOINT] + ) + jars = glob.glob(os.path.join(workspace, '*jar')) + self.assertEqual(len(jars), 0, 'Expected zero jars in workspace') + modules = glob.glob(os.path.join(workspace, "modules", '*jar')) + self.assertEqual(len(modules), 1, 'Expected one jar in the module directory') + self.assertEqual(modules[0], os.path.join(workspace, "modules", 'parsington-%s.jar' % PARSINGTON_VERSION), 'Expected parsington jar') + finally: + shutil.rmtree(tmp_dir) + def test_run_jgo(self): tmp_dir = tempfile.mkdtemp(prefix='jgo-test-cache-dir') From 4f9ac7901b7ff1fd17c3edeadcaa999a20a8c50c Mon Sep 17 00:00:00 2001 From: cmhulbert Date: Mon, 25 Jan 2021 16:53:24 -0500 Subject: [PATCH 3/3] cleanup error reportiong --- jgo/jgo.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jgo/jgo.py b/jgo/jgo.py index 30691bb..59a166d 100644 --- a/jgo/jgo.py +++ b/jgo/jgo.py @@ -524,10 +524,11 @@ def resolve_dependencies( except FileExistsError as e: # Do not throw exceptionif target file exists. pass - if len(set(modules_deps)) != len(modules_discovered): - missing_mods = '\n'.join(x for x in modules_deps - modules_discovered) - _logger.info(f"Missing Modules:\n{missing_mods}") - raise Exception("Could not discover all modules dependencies") + module_set = set(modules_deps) + if len(module_set) != len(modules_discovered): + missing_mods = '\n'.join(x for x in module_set - modules_discovered) + _logger.info("Missing Modules:\n{}".format(missing_mods)) + raise Exception("Could not discover all module dependencies") pathlib.Path(build_success_file).touch(exist_ok=True) return primary_endpoint, workspace