Skip to content

Commit

Permalink
refactor: use href instead of title
Browse files Browse the repository at this point in the history
  • Loading branch information
@DmitriyLewen
DmitriyLewen committed Mar 6, 2024
1 parent 3ed9e32 commit 37e4c01
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 50 deletions.
4 changes: 2 additions & 2 deletions pkg/crawler/crawler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,8 +258,8 @@ func (c *Crawler) sha1Urls(ctx context.Context, url string) ([]string, error) {
d.Find("a").Each(func(i int, selection *goquery.Selection) {
// There are times when the file name is very long.
// e.g. https://repo.maven.apache.org/maven2/africa/absa/inception-oauth2-resource-server/1.0.0/
// We need to use `title` to make sure we use the correct filename
if fileName, ok := selection.Attr("title"); ok {
// We need to use `href` to make sure we use the correct filename
if fileName, ok := selection.Attr("href"); ok {
if strings.HasSuffix(fileName, ".jar.sha1") && !strings.HasSuffix(fileName, "sources.jar.sha1") &&
!strings.HasSuffix(fileName, "test.jar.sha1") && !strings.HasSuffix(fileName, "javadoc.jar.sha1") {
sha1URLs = append(sha1URLs, url+fileName)
Expand Down
5 changes: 3 additions & 2 deletions pkg/crawler/testdata/abbot.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ <h1>abbot</h1>
</header>
<hr>
<main>
<pre id="contents"><a href="https://repo.maven.apache.org/maven2/">../</a>
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/" title="abbot/">abbot/</a> - - - -
<pre id="contents">
<a href="../">../</a>
<a href="abbot/" title="abbot/">abbot/</a> - -
</pre>
</main>
<hr>
Expand Down
13 changes: 7 additions & 6 deletions pkg/crawler/testdata/abbot_abbot.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@ <h1>abbot/abbot</h1>
<hr>
<main>
<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/">../</a>
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/" title="0.12.3/">0.12.3/</a> 2005-09-20 05:44 -
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/" title="0.13.0/">0.13.0/</a> 2005-09-20 05:44 -
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/" title="1.4.0/">1.4.0/</a> 2015-09-22 16:03 -
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a> 2015-09-24 14:18 402
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/maven-metadata.xml.md5" title="maven-metadata.xml.md5">maven-metadata.xml.md5</a> 2015-09-24 14:18 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/maven-metadata.xml.sha1" title="maven-metadata.xml.sha1">maven-metadata.xml.sha1</a> 2015-09-24 14:18 40
<a href="../">../</a>
<a href="0.12.3/" title="0.12.3/">0.12.3/</a> 2005-09-20 05:44 -
<a href="0.13.0/" title="0.13.0/">0.13.0/</a> 2005-09-20 05:44 -
<a href="1.4.0/" title="1.4.0/">1.4.0/</a> 2015-09-22 16:03 -
<a href="maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a> 2015-09-24 14:18 402
<a href="maven-metadata.xml.md5" title="maven-metadata.xml.md5">maven-metadata.xml.md5</a> 2015-09-24 14:18 32
<a href="maven-metadata.xml.sha1" title="maven-metadata.xml.sha1">maven-metadata.xml.sha1</a> 2015-09-24 14:18 40
</pre>
</main>
<hr>
Expand Down
18 changes: 9 additions & 9 deletions pkg/crawler/testdata/abbot_abbot_0.12.3.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ <h1>abbot/abbot/0.12.3</h1>
<hr>
<main>
<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/abbot/">../</a>
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar" title="abbot-0.12.3.jar">abbot-0.12.3.jar</a> 2005-09-20 05:44 689791
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.md5" title="abbot-0.12.3.jar.md5">abbot-0.12.3.jar.md5</a> 2005-09-20 05:44 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1" title="abbot-0.12.3.jar.sha1">abbot-0.12.3.j...</a> 2005-09-20 05:44 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/abbot-0.12.3.pom" title="abbot-0.12.3.pom">abbot-0.12.3.pom</a> 2005-09-20 05:44 166
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/abbot-0.12.3.pom.md5" title="abbot-0.12.3.pom.md5">abbot-0.12.3.pom.md5</a> 2005-09-20 05:44 128
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/abbot-0.12.3.pom.sha1" title="abbot-0.12.3.pom.sha1">abbot-0.12.3.pom.sha1</a> 2005-09-20 05:44 136
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a> 2005-09-20 05:44 110
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/maven-metadata.xml.md5" title="maven-metadata.xml.md5">maven-metadata.xml.md5</a> 2005-09-20 05:44 74
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.12.3/maven-metadata.xml.sha1" title="maven-metadata.xml.sha1">maven-metadata.xml.sha1</a> 2005-09-20 05:44 129
<a href="abbot-0.12.3.jar" title="abbot-0.12.3.jar">abbot-0.12.3.jar</a> 2005-09-20 05:44 689791
<a href="abbot-0.12.3.jar.md5" title="abbot-0.12.3.jar.md5">abbot-0.12.3.jar.md5</a> 2005-09-20 05:44 32
<a href="abbot-0.12.3.jar.sha1" title="abbot-0.12.3.jar.sha1">abbot-0.12.3.j...</a> 2005-09-20 05:44 40
<a href="abbot-0.12.3.pom" title="abbot-0.12.3.pom">abbot-0.12.3.pom</a> 2005-09-20 05:44 166
<a href="abbot-0.12.3.pom.md5" title="abbot-0.12.3.pom.md5">abbot-0.12.3.pom.md5</a> 2005-09-20 05:44 128
<a href="abbot-0.12.3.pom.sha1" title="abbot-0.12.3.pom.sha1">abbot-0.12.3.pom.sha1</a> 2005-09-20 05:44 136
<a href="maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a> 2005-09-20 05:44 110
<a href="maven-metadata.xml.md5" title="maven-metadata.xml.md5">maven-metadata.xml.md5</a> 2005-09-20 05:44 74
<a href="maven-metadata.xml.sha1" title="maven-metadata.xml.sha1">maven-metadata.xml.sha1</a> 2005-09-20 05:44 129
</pre>
</main>
<hr>
Expand Down
18 changes: 9 additions & 9 deletions pkg/crawler/testdata/abbot_abbot_0.13.0.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ <h1>abbot/abbot/0.13.0</h1>
<hr>
<main>
<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/abbot/">../</a>
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar" title="abbot-0.13.0.jar">abbot-0.13.0.jar</a> 2005-09-20 05:44 779426
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.md5" title="abbot-0.13.0.jar.md5">abbot-0.13.0.jar.md5</a> 2005-09-20 05:44 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1" title="abbot-0.13.0.jar.sha1">abbot-0.13.0.jar.sha1</a> 2005-09-20 05:44 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/abbot-0.13.0.pom" title="abbot-0.13.0.pom">abbot-0.13.0.pom</a> 2005-09-20 05:44 166
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/abbot-0.13.0.pom.md5" title="abbot-0.13.0.pom.md5">abbot-0.13.0.pom.md5</a> 2005-09-20 05:44 128
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/abbot-0.13.0.pom.sha1" title="abbot-0.13.0.pom.sha1">abbot-0.13.0.pom.sha1</a> 2005-09-20 05:44 136
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a> 2005-09-20 05:44 110
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/maven-metadata.xml.md5" title="maven-metadata.xml.md5">maven-metadata.xml.md5</a> 2005-09-20 05:44 74
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/0.13.0/maven-metadata.xml.sha1" title="maven-metadata.xml.sha1">maven-metadata.xml.sha1</a> 2005-09-20 05:44 129
<a href="abbot-0.13.0.jar" title="abbot-0.13.0.jar">abbot-0.13.0.jar</a> 2005-09-20 05:44 779426
<a href="abbot-0.13.0.jar.md5" title="abbot-0.13.0.jar.md5">abbot-0.13.0.jar.md5</a> 2005-09-20 05:44 32
<a href="abbot-0.13.0.jar.sha1" title="abbot-0.13.0.jar.sha1">abbot-0.13.0.jar.sha1</a> 2005-09-20 05:44 40
<a href="abbot-0.13.0.pom" title="abbot-0.13.0.pom">abbot-0.13.0.pom</a> 2005-09-20 05:44 166
<a href="abbot-0.13.0.pom.md5" title="abbot-0.13.0.pom.md5">abbot-0.13.0.pom.md5</a> 2005-09-20 05:44 128
<a href="abbot-0.13.0.pom.sha1" title="abbot-0.13.0.pom.sha1">abbot-0.13.0.pom.sha1</a> 2005-09-20 05:44 136
<a href="maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a> 2005-09-20 05:44 110
<a href="maven-metadata.xml.md5" title="maven-metadata.xml.md5">maven-metadata.xml.md5</a> 2005-09-20 05:44 74
<a href="maven-metadata.xml.sha1" title="maven-metadata.xml.sha1">maven-metadata.xml.sha1</a> 2005-09-20 05:44 129
</pre>
</main>
<hr>
Expand Down
44 changes: 22 additions & 22 deletions pkg/crawler/testdata/abbot_abbot_1.4.0.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,28 @@ <h1>abbot/abbot/1.4.0</h1>
<main>
<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/abbot/">../</a>
<a href="jasypt-1.9.3-javadoc.jar" title="jasypt-1.9.3-javadoc.jar">jasypt-1.9.3-javadoc.jar</a> 2019-05-25 16:34 748409
<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar" title="abbot-1.4.0-lite.jar">abbot-1.4.0-lite.jar</a> 2019-05-25 16:34 74953
<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.asc" title="abbot-1.4.0-lite.jar.asc">abbot-1.4.0-lite.jar.asc</a> 2019-05-25 16:34 516
<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.md5" title="abbot-1.4.0-lite.jar.md5">abbot-1.4.0-lite.jar.md5</a> 2019-05-25 16:34 32
<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1" title="abbot-1.4.0-lite.jar.sha1">abbot-1.4.0-lite.jar.sha1</a> 2019-05-25 16:34 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar" title="abbot-1.4.0-sources.jar">abbot-1.4.0-sources.jar</a> 2015-09-22 16:03 310023
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.asc" title="abbot-1.4.0-sources.jar.asc">abbot-1.4.0-sources.jar.asc</a> 2015-09-22 16:03 490
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.asc.md5" title="abbot-1.4.0-sources.jar.asc.md5">abbot-1.4.0-sources.jar.asc.md5</a> 2015-09-22 16:03 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.asc.sha1" title="abbot-1.4.0-sources.jar.asc.sha1">abbot-1.4.0-sources.jar.asc.sha1</a> 2015-09-22 16:03 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.md5" title="abbot-1.4.0-sources.jar.md5">abbot-1.4.0-sources.jar.md5</a> 2015-09-22 16:03 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.sha1" title="abbot-1.4.0-sources.jar.sha1">abbot-1.4.0-sources.jar.sha1</a> 2015-09-22 16:03 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar" title="abbot-1.4.0.jar">abbot-1.4.0.jar</a> 2015-09-22 16:03 687192
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.asc" title="abbot-1.4.0.jar.asc">abbot-1.4.0.jar.asc</a> 2015-09-22 16:03 490
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.asc.md5" title="abbot-1.4.0.jar.asc.md5">abbot-1.4.0.jar.asc.md5</a> 2015-09-22 16:03 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.asc.sha1" title="abbot-1.4.0.jar.asc.sha1">abbot-1.4.0.jar.asc.sha1</a> 2015-09-22 16:03 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.md5" title="abbot-1.4.0.jar.md5">abbot-1.4.0.jar.md5</a> 2015-09-22 16:03 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1" title="abbot-1.4.0.jar.sha1">abbot-1.4.0.jar.sha1</a> 2015-09-22 16:03 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.pom" title="abbot-1.4.0.pom">abbot-1.4.0.pom</a> 2015-09-22 16:03 1292
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.pom.asc" title="abbot-1.4.0.pom.asc">abbot-1.4.0.pom.asc</a> 2015-09-22 16:03 490
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.pom.asc.md5" title="abbot-1.4.0.pom.asc.md5">abbot-1.4.0.pom.asc.md5</a> 2015-09-22 16:03 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.pom.asc.sha1" title="abbot-1.4.0.pom.asc.sha1">abbot-1.4.0.pom.asc.sha1</a> 2015-09-22 16:03 40
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.pom.md5" title="abbot-1.4.0.pom.md5">abbot-1.4.0.pom.md5</a> 2015-09-22 16:03 32
<a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.pom.sha1" title="abbot-1.4.0.pom.sha1">abbot-1.4.0.pom.sha1</a> 2015-09-22 16:03 40
<a href="abbot-1.4.0-lite.jar" title="abbot-1.4.0-lite.jar">abbot-1.4.0-lite.jar</a> 2019-05-25 16:34 74953
<a href="abbot-1.4.0-lite.jar.asc" title="abbot-1.4.0-lite.jar.asc">abbot-1.4.0-lite.jar.asc</a> 2019-05-25 16:34 516
<a href="abbot-1.4.0-lite.jar.md5" title="abbot-1.4.0-lite.jar.md5">abbot-1.4.0-lite.jar.md5</a> 2019-05-25 16:34 32
<a href="abbot-1.4.0-lite.jar.sha1" title="abbot-1.4.0-lite.jar.sha1">abbot-1.4.0-lite.jar.sha1</a> 2019-05-25 16:34 40
<a href="abbot-1.4.0-sources.jar" title="abbot-1.4.0-sources.jar">abbot-1.4.0-sources.jar</a> 2015-09-22 16:03 310023
<a href="abbot-1.4.0-sources.jar.asc" title="abbot-1.4.0-sources.jar.asc">abbot-1.4.0-sources.jar.asc</a> 2015-09-22 16:03 490
<a href="abbot-1.4.0-sources.jar.asc.md5" title="abbot-1.4.0-sources.jar.asc.md5">abbot-1.4.0-sources.jar.asc.md5</a> 2015-09-22 16:03 32
<a href="abbot-1.4.0-sources.jar.asc.sha1" title="abbot-1.4.0-sources.jar.asc.sha1">abbot-1.4.0-sources.jar.asc.sha1</a> 2015-09-22 16:03 40
<a href="abbot-1.4.0-sources.jar.md5" title="abbot-1.4.0-sources.jar.md5">abbot-1.4.0-sources.jar.md5</a> 2015-09-22 16:03 32
<a href="abbot-1.4.0-sources.jar.sha1" title="abbot-1.4.0-sources.jar.sha1">abbot-1.4.0-sources.jar.sha1</a> 2015-09-22 16:03 40
<a href="abbot-1.4.0.jar" title="abbot-1.4.0.jar">abbot-1.4.0.jar</a> 2015-09-22 16:03 687192
<a href="abbot-1.4.0.jar.asc" title="abbot-1.4.0.jar.asc">abbot-1.4.0.jar.asc</a> 2015-09-22 16:03 490
<a href="abbot-1.4.0.jar.asc.md5" title="abbot-1.4.0.jar.asc.md5">abbot-1.4.0.jar.asc.md5</a> 2015-09-22 16:03 32
<a href="abbot-1.4.0.jar.asc.sha1" title="abbot-1.4.0.jar.asc.sha1">abbot-1.4.0.jar.asc.sha1</a> 2015-09-22 16:03 40
<a href="abbot-1.4.0.jar.md5" title="abbot-1.4.0.jar.md5">abbot-1.4.0.jar.md5</a> 2015-09-22 16:03 32
<a href="abbot-1.4.0.jar.sha1" title="abbot-1.4.0.jar.sha1">abbot-1.4.0.jar.sha1</a> 2015-09-22 16:03 40
<a href="abbot-1.4.0.pom" title="abbot-1.4.0.pom">abbot-1.4.0.pom</a> 2015-09-22 16:03 1292
<a href="abbot-1.4.0.pom.asc" title="abbot-1.4.0.pom.asc">abbot-1.4.0.pom.asc</a> 2015-09-22 16:03 490
<a href="abbot-1.4.0.pom.asc.md5" title="abbot-1.4.0.pom.asc.md5">abbot-1.4.0.pom.asc.md5</a> 2015-09-22 16:03 32
<a href="abbot-1.4.0.pom.asc.sha1" title="abbot-1.4.0.pom.asc.sha1">abbot-1.4.0.pom.asc.sha1</a> 2015-09-22 16:03 40
<a href="abbot-1.4.0.pom.md5" title="abbot-1.4.0.pom.md5">abbot-1.4.0.pom.md5</a> 2015-09-22 16:03 32
<a href="abbot-1.4.0.pom.sha1" title="abbot-1.4.0.pom.sha1">abbot-1.4.0.pom.sha1</a> 2015-09-22 16:03 40
</pre>
</main>
<hr>
Expand Down

0 comments on commit 37e4c01

Please sign in to comment.