From 00b1b5e5dc6df96631c9cd0026953b3ccaf6df81 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 12 Jan 2025 15:54:41 +0100 Subject: [PATCH] composer meta analyzer: add DEBUG logging --- .../persistence/DefaultObjectGenerator.java | 21 +++++++++++++++++++ .../repositories/ComposerMetaAnalyzer.java | 8 +++++++ .../resources/v1/RepositoryResourceTest.java | 5 +++-- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java b/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java index 1cc6729b57..305204d897 100644 --- a/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java +++ b/src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java @@ -214,6 +214,7 @@ private List getBadgesPermissions(final List fullList) { public void loadDefaultRepositories() { try (QueryManager qm = new QueryManager()) { LOGGER.info("Synchronizing default repositories to datastore"); +<<<<<<< Updated upstream qm.createRepository(RepositoryType.CPAN, "cpan-public-registry", "https://fastapi.metacpan.org/v1/", true, false, false, null, null); qm.createRepository(RepositoryType.GEM, "rubygems.org", "https://rubygems.org/", true, false, false, null, null); qm.createRepository(RepositoryType.HEX, "hex.pm", "https://hex.pm/", true, false, false, null, null); @@ -231,6 +232,26 @@ public void loadDefaultRepositories() { qm.createRepository(RepositoryType.CARGO, "crates.io", "https://crates.io", true, false, false, null, null); qm.createRepository(RepositoryType.GO_MODULES, "proxy.golang.org", "https://proxy.golang.org", true, false, false, null, null); qm.createRepository(RepositoryType.GITHUB, "github.com", "https://github.com", true, false, false, null, null); +======= + qm.createRepository(RepositoryType.CPAN, "cpan-public-registry", "https://fastapi.metacpan.org/v1/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.GEM, "rubygems.org", "https://rubygems.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.HEX, "hex.pm", "https://hex.pm/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.HACKAGE, "hackage.haskell.org", "https://hackage.haskell.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.MAVEN, "central", "https://repo1.maven.org/maven2/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.MAVEN, "atlassian-public", "https://packages.atlassian.com/content/repositories/atlassian-public/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.MAVEN, "jboss-releases", "https://repository.jboss.org/nexus/content/repositories/releases/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.MAVEN, "clojars", "https://repo.clojars.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.MAVEN, "google-android", "https://maven.google.com/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.NIXPKGS, "nixpkgs-unstable", "https://channels.nixos.org/nixpkgs-unstable/packages.json.br", true, false, false, null, null, null); + qm.createRepository(RepositoryType.NPM, "npm-public-registry", "https://registry.npmjs.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.PYPI, "pypi.org", "https://pypi.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.NUGET, "nuget-gallery", "https://api.nuget.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.COMPOSER, "packagist", "https://repo.packagist.org/", true, false, false, null, null, null); + qm.createRepository(RepositoryType.COMPOSER, "drupal8", "https://packages.drupal.org/8", false, false, false, null, null, null); + qm.createRepository(RepositoryType.CARGO, "crates.io", "https://crates.io", true, false, false, null, null, null); + qm.createRepository(RepositoryType.GO_MODULES, "proxy.golang.org", "https://proxy.golang.org", true, false, false, null, null, null); + qm.createRepository(RepositoryType.GITHUB, "github.com", "https://github.com", true, false, false, null, null, null); +>>>>>>> Stashed changes } } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java index a144360249..02f6bb9b1b 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java @@ -114,6 +114,7 @@ public MetaModel analyze(final Component component) { final JSONObject repoRoot = getRepoRoot(); if (repoRoot == null) { + LOGGER.debug("repoRoot is null, fallback to v1 metadata url %s".formatted(PACKAGE_META_DATA_PATH_PATTERN_V1)); // absence of packages.json shouldn't happen, but let's try to get metadata as // we did in <=4.12.2 return analyzeFromMetadataUrl(meta, component, PACKAGE_META_DATA_PATH_PATTERN_V1); @@ -127,6 +128,7 @@ public MetaModel analyze(final Component component) { // According to https://github.com/composer/composer/blob/fb397acaa0648ba2668893e4b786af6465a41696/doc/05-repositories.md?plain=1#L197 // available-packages should contain ALL the packages in the repo. // But in the Composer implementation the patterns are consulted even if available-packages is present and doesn't contain the package + LOGGER.debug("package not present in available-packages nor available-package-patterns"); return meta; } } @@ -140,6 +142,7 @@ public MetaModel analyze(final Component component) { .anyMatch(pattern -> composerPackageName.matches(pattern)); if (!found) { + LOGGER.debug("package doesn't match available-package-patterns"); return meta; } } @@ -148,6 +151,7 @@ public MetaModel analyze(final Component component) { // presence of metadata-url implies V2 repository, and takes precedence over // included packages and other V1 features final String packageMetaDataPathPattern = repoRoot.getString("metadata-url"); + LOGGER.debug("using metadata-url pattern from packages.json: " + packageMetaDataPathPattern); return analyzeFromMetadataUrl(meta, component, packageMetaDataPathPattern); } @@ -163,6 +167,7 @@ public MetaModel analyze(final Component component) { JSONObject packages = repoRoot.getJSONObject("packages"); if (!packages.isEmpty()) { if (!packages.has(getComposerPackageName(component))) { + LOGGER.debug("package %s not found in this repository.".formatted(component.getPurl())); return meta; } JSONObject packageVersions = packages.getJSONObject(getComposerPackageName(component)); @@ -172,6 +177,7 @@ public MetaModel analyze(final Component component) { // V1 and no included packages, so we have to retrieve the package specific // metadata + LOGGER.debug("no metadata-url pattern and package %s not found in included packages, analyzing using v1 url pattern: %s".formatted(component.getPurl(), PACKAGE_META_DATA_PATH_PATTERN_V1)); return analyzeFromMetadataUrl(meta, component, PACKAGE_META_DATA_PATH_PATTERN_V1); } @@ -297,6 +303,7 @@ private MetaModel analyzeFromMetadataUrl(final MetaModel meta, final Component c if (!responsePackages.has(expectedResponsePackage)) { // the package no longer exists - for v2 there's no example (yet), v1 example // https://repo.packagist.org/p/magento/adobe-ims.json + LOGGER.debug("Package %s no longer exists in this repository.". formatted(component.getPurl())); return meta; } @@ -345,6 +352,7 @@ private MetaModel analyzePackageVersions(final MetaModel meta, Component compone final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion())); final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); + LOGGER.debug("analyzing package versions for: " + component.getPurl()); packageVersions.names().forEach(item -> { JSONObject packageVersion = packageVersions.getJSONObject((String) item); // Sometimes the JSON key differs from the the version inside the JSON value. The latter is leading. diff --git a/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java index bf62d0bc44..53eb6d6727 100644 --- a/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java +++ b/src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java @@ -171,7 +171,10 @@ public void getRepositoryMetaUntrackedComponentTest() { Assert.assertEquals("The repository metadata for the specified component cannot be found.", body); } +<<<<<<< Updated upstream +======= +>>>>>>> Stashed changes @Test public void createRepositoryTest() { Repository repository = new Repository(); @@ -187,7 +190,6 @@ public void createRepositoryTest() { .put(Entity.entity(repository, MediaType.APPLICATION_JSON)); Assert.assertEquals(201, response.getStatus()); - response = jersey.target(V1_REPOSITORY).request().header(X_API_KEY, apiKey).get(Response.class); Assert.assertEquals(200, response.getStatus(), 0); Assert.assertEquals(String.valueOf(18), response.getHeaderString(TOTAL_COUNT_HEADER)); @@ -219,7 +221,6 @@ public void createNonInternalRepositoryTest() { .put(Entity.entity(repository, MediaType.APPLICATION_JSON)); Assert.assertEquals(201, response.getStatus()); - response = jersey.target(V1_REPOSITORY).request().header(X_API_KEY, apiKey).get(Response.class); Assert.assertEquals(200, response.getStatus(), 0); Assert.assertEquals(String.valueOf(18), response.getHeaderString(TOTAL_COUNT_HEADER));