Skip to content

Commit

Permalink
composer meta analyzer: add DEBUG logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@valentijnscholten
valentijnscholten committed Jan 12, 2025
1 parent bcd2651 commit 00b1b5e
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 2 deletions.
21 changes: 21 additions & 0 deletions src/main/java/org/dependencytrack/persistence/DefaultObjectGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ private List<Permission> getBadgesPermissions(final List<Permission> fullList) {
public void loadDefaultRepositories() {
try (QueryManager qm = new QueryManager()) {
LOGGER.info("Synchronizing default repositories to datastore");
<<<<<<< Updated upstream
qm.createRepository(RepositoryType.CPAN, "cpan-public-registry", "https://fastapi.metacpan.org/v1/", true, false, false, null, null);
qm.createRepository(RepositoryType.GEM, "rubygems.org", "https://rubygems.org/", true, false, false, null, null);
qm.createRepository(RepositoryType.HEX, "hex.pm", "https://hex.pm/", true, false, false, null, null);
Expand All @@ -231,6 +232,26 @@ public void loadDefaultRepositories() {
qm.createRepository(RepositoryType.CARGO, "crates.io", "https://crates.io", true, false, false, null, null);
qm.createRepository(RepositoryType.GO_MODULES, "proxy.golang.org", "https://proxy.golang.org", true, false, false, null, null);
qm.createRepository(RepositoryType.GITHUB, "github.com", "https://github.com", true, false, false, null, null);
=======
qm.createRepository(RepositoryType.CPAN, "cpan-public-registry", "https://fastapi.metacpan.org/v1/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.GEM, "rubygems.org", "https://rubygems.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.HEX, "hex.pm", "https://hex.pm/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.HACKAGE, "hackage.haskell.org", "https://hackage.haskell.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.MAVEN, "central", "https://repo1.maven.org/maven2/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.MAVEN, "atlassian-public", "https://packages.atlassian.com/content/repositories/atlassian-public/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.MAVEN, "jboss-releases", "https://repository.jboss.org/nexus/content/repositories/releases/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.MAVEN, "clojars", "https://repo.clojars.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.MAVEN, "google-android", "https://maven.google.com/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.NIXPKGS, "nixpkgs-unstable", "https://channels.nixos.org/nixpkgs-unstable/packages.json.br", true, false, false, null, null, null);
qm.createRepository(RepositoryType.NPM, "npm-public-registry", "https://registry.npmjs.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.PYPI, "pypi.org", "https://pypi.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.NUGET, "nuget-gallery", "https://api.nuget.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.COMPOSER, "packagist", "https://repo.packagist.org/", true, false, false, null, null, null);
qm.createRepository(RepositoryType.COMPOSER, "drupal8", "https://packages.drupal.org/8", false, false, false, null, null, null);
qm.createRepository(RepositoryType.CARGO, "crates.io", "https://crates.io", true, false, false, null, null, null);
qm.createRepository(RepositoryType.GO_MODULES, "proxy.golang.org", "https://proxy.golang.org", true, false, false, null, null, null);
qm.createRepository(RepositoryType.GITHUB, "github.com", "https://github.com", true, false, false, null, null, null);
>>>>>>> Stashed changes
}
}

Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ public MetaModel analyze(final Component component) {

final JSONObject repoRoot = getRepoRoot();
if (repoRoot == null) {
LOGGER.debug("repoRoot is null, fallback to v1 metadata url %s".formatted(PACKAGE_META_DATA_PATH_PATTERN_V1));
// absence of packages.json shouldn't happen, but let's try to get metadata as
// we did in <=4.12.2
return analyzeFromMetadataUrl(meta, component, PACKAGE_META_DATA_PATH_PATTERN_V1);
Expand All @@ -127,6 +128,7 @@ public MetaModel analyze(final Component component) {
// According to https://github.com/composer/composer/blob/fb397acaa0648ba2668893e4b786af6465a41696/doc/05-repositories.md?plain=1#L197
// available-packages should contain ALL the packages in the repo.
// But in the Composer implementation the patterns are consulted even if available-packages is present and doesn't contain the package
LOGGER.debug("package not present in available-packages nor available-package-patterns");
return meta;
}
}
Expand All @@ -140,6 +142,7 @@ public MetaModel analyze(final Component component) {
.anyMatch(pattern -> composerPackageName.matches(pattern));

if (!found) {
LOGGER.debug("package doesn't match available-package-patterns");
return meta;
}
}
Expand All @@ -148,6 +151,7 @@ public MetaModel analyze(final Component component) {
// presence of metadata-url implies V2 repository, and takes precedence over
// included packages and other V1 features
final String packageMetaDataPathPattern = repoRoot.getString("metadata-url");
LOGGER.debug("using metadata-url pattern from packages.json: " + packageMetaDataPathPattern);
return analyzeFromMetadataUrl(meta, component, packageMetaDataPathPattern);
}

Expand All @@ -163,6 +167,7 @@ public MetaModel analyze(final Component component) {
JSONObject packages = repoRoot.getJSONObject("packages");
if (!packages.isEmpty()) {
if (!packages.has(getComposerPackageName(component))) {
LOGGER.debug("package %s not found in this repository.".formatted(component.getPurl()));
return meta;
}
JSONObject packageVersions = packages.getJSONObject(getComposerPackageName(component));
Expand All @@ -172,6 +177,7 @@ public MetaModel analyze(final Component component) {

// V1 and no included packages, so we have to retrieve the package specific
// metadata
LOGGER.debug("no metadata-url pattern and package %s not found in included packages, analyzing using v1 url pattern: %s".formatted(component.getPurl(), PACKAGE_META_DATA_PATH_PATTERN_V1));
return analyzeFromMetadataUrl(meta, component, PACKAGE_META_DATA_PATH_PATTERN_V1);
}

Expand Down Expand Up @@ -297,6 +303,7 @@ private MetaModel analyzeFromMetadataUrl(final MetaModel meta, final Component c
if (!responsePackages.has(expectedResponsePackage)) {
// the package no longer exists - for v2 there's no example (yet), v1 example
// https://repo.packagist.org/p/magento/adobe-ims.json
LOGGER.debug("Package %s no longer exists in this repository.". formatted(component.getPurl()));
return meta;
}

Expand Down Expand Up @@ -345,6 +352,7 @@ private MetaModel analyzePackageVersions(final MetaModel meta, Component compone
final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion()));
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");

LOGGER.debug("analyzing package versions for: " + component.getPurl());
packageVersions.names().forEach(item -> {
JSONObject packageVersion = packageVersions.getJSONObject((String) item);
// Sometimes the JSON key differs from the the version inside the JSON value. The latter is leading.
Expand Down
5 changes: 3 additions & 2 deletions src/test/java/org/dependencytrack/resources/v1/RepositoryResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,10 @@ public void getRepositoryMetaUntrackedComponentTest() {
Assert.assertEquals("The repository metadata for the specified component cannot be found.", body);
}

<<<<<<< Updated upstream

=======
>>>>>>> Stashed changes
@Test
public void createRepositoryTest() {
Repository repository = new Repository();
Expand All @@ -187,7 +190,6 @@ public void createRepositoryTest() {
.put(Entity.entity(repository, MediaType.APPLICATION_JSON));
Assert.assertEquals(201, response.getStatus());


response = jersey.target(V1_REPOSITORY).request().header(X_API_KEY, apiKey).get(Response.class);
Assert.assertEquals(200, response.getStatus(), 0);
Assert.assertEquals(String.valueOf(18), response.getHeaderString(TOTAL_COUNT_HEADER));
Expand Down Expand Up @@ -219,7 +221,6 @@ public void createNonInternalRepositoryTest() {
.put(Entity.entity(repository, MediaType.APPLICATION_JSON));
Assert.assertEquals(201, response.getStatus());


response = jersey.target(V1_REPOSITORY).request().header(X_API_KEY, apiKey).get(Response.class);
Assert.assertEquals(200, response.getStatus(), 0);
Assert.assertEquals(String.valueOf(18), response.getHeaderString(TOTAL_COUNT_HEADER));
Expand Down

0 comments on commit 00b1b5e

Please sign in to comment.