From b6c2436228ccb93096a68f72da52ad847a739ce0 Mon Sep 17 00:00:00 2001
From: DmitriyLewen <dmitriy.lewen@smartforce.io>
Date: Tue, 12 Mar 2024 12:18:14 +0600
Subject: [PATCH] refactor + added logic to avoid overwriting dir version

---
 go.mod                                        |  10 +-
 go.sum                                        |  34 ++---
 pkg/crawler/crawler.go                        | 119 +++++++++++-------
 pkg/crawler/crawler_test.go                   |  23 ++--
 .../testdata/abbot-0.13.0-copy.jar.sha1       |   1 +
 pkg/crawler/testdata/abbot_abbot.html         |   2 +-
 pkg/crawler/testdata/abbot_abbot_0.13.0.html  |   3 +
 7 files changed, 111 insertions(+), 81 deletions(-)
 create mode 100644 pkg/crawler/testdata/abbot-0.13.0-copy.jar.sha1

diff --git a/go.mod b/go.mod
index 1560382..6bcc437 100644
--- a/go.mod
+++ b/go.mod
@@ -6,9 +6,10 @@ require (
 	github.com/PuerkitoBio/goquery v1.5.1
 	github.com/cheggaaa/pb/v3 v3.1.0
 	github.com/hashicorp/go-retryablehttp v0.7.2
+	github.com/samber/lo v1.39.0
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.1
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
 	k8s.io/utils v0.0.0-20230115233650-391b47cb4029
 	modernc.org/sqlite v1.20.3
@@ -31,10 +32,11 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/mod v0.3.0 // indirect
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
 	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
-	golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect
+	golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
diff --git a/go.sum b/go.sum
index 91ffddd..c8600cb 100644
--- a/go.sum
+++ b/go.sum
@@ -45,6 +45,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
+github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -57,38 +59,26 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
+golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4=
+golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs=
-golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI=
+golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
index 00cf69a..3a0afca 100644
--- a/pkg/crawler/crawler.go
+++ b/pkg/crawler/crawler.go
@@ -1,12 +1,11 @@
 package crawler
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"encoding/xml"
 	"fmt"
-	"github.com/aquasecurity/trivy-java-db/pkg/fileutil"
-	"github.com/aquasecurity/trivy-java-db/pkg/types"
 	"io"
 	"log"
 	"net/http"
@@ -16,8 +15,12 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/hashicorp/go-retryablehttp"
+	"github.com/samber/lo"
 	"golang.org/x/sync/semaphore"
 	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/trivy-java-db/pkg/fileutil"
+	"github.com/aquasecurity/trivy-java-db/pkg/types"
 )
 
 const mavenRepoURL = "https://repo.maven.apache.org/maven2/"
@@ -146,17 +149,15 @@ func (c *Crawler) Visit(ctx context.Context, url string) error {
 	var children []string
 	var foundMetadata bool
 	d.Find("a").Each(func(i int, selection *goquery.Selection) {
-		if link, ok := selection.Attr("href"); ok {
-			if link == "maven-metadata.xml" {
-				foundMetadata = true
-				return
-			} else if link == "../" || !strings.HasSuffix(link, "/") {
-				// only `../` and dirs have `/` suffix. We don't need to check other files.
-				return
-			}
-			children = append(children, link)
+		link := linkFromSelection(selection)
+		if link == "maven-metadata.xml" {
+			foundMetadata = true
+			return
+		} else if link == "../" || !strings.HasSuffix(link, "/") {
+			// only `../` and dirs have `/` suffix. We don't need to check other files.
+			return
 		}
-
+		children = append(children, link)
 	})
 
 	if foundMetadata {
@@ -190,41 +191,61 @@ func (c *Crawler) Visit(ctx context.Context, url string) error {
 	return nil
 }
 
-func (c *Crawler) crawlSHA1(ctx context.Context, baseURL string, meta *Metadata, versionDirs []string) error {
-	var sha1URLs []string
+func (c *Crawler) crawlSHA1(ctx context.Context, baseURL string, meta *Metadata, dirs []string) error {
+	var foundVersions []Version
 	// Check each version dir to find links to `*.jar.sha1` files.
-	for _, versionDir := range versionDirs {
-		versionDirURL := baseURL + versionDir
-		urls, err := c.sha1Urls(ctx, versionDirURL)
+	for _, dir := range dirs {
+		dirURL := baseURL + dir
+		sha1Urls, err := c.sha1Urls(ctx, dirURL)
 		if err != nil {
-			return xerrors.Errorf("unable to get list of sha1 files from %q: %s", versionDirURL, err)
+			return xerrors.Errorf("unable to get list of sha1 files from %q: %s", dirURL, err)
 		}
-		sha1URLs = append(sha1URLs, urls...)
-	}
 
-	var versions []Version
-	for _, sha1URL := range sha1URLs {
-		sha1, err := c.fetchSHA1(ctx, sha1URL)
-		if err != nil {
-			return xerrors.Errorf("unable to fetch sha1: %s", err)
-		}
-		if ver := versionFromSha1URL(meta.ArtifactID, sha1URL); ver != "" && len(sha1) != 0 {
-			v := Version{
-				Version: ver,
-				SHA1:    sha1,
+		// Remove the `/` suffix to correctly compare file versions with version from directory name.
+		dirVersion := strings.TrimSuffix(dir, "/")
+		var dirVersionSha1 []byte
+		var versions []Version
+		for _, sha1Url := range sha1Urls {
+			sha1, err := c.fetchSHA1(ctx, sha1Url)
+			if err != nil {
+				return xerrors.Errorf("unable to fetch sha1: %s", err)
+			}
+			if ver := versionFromSha1URL(meta.ArtifactID, sha1Url); ver != "" && len(sha1) != 0 {
+				// Save sha1 for the file where the version is equal to the version from the directory name in order to remove duplicates later
+				// Avoid overwriting dirVersion when inserting versions into the database (sha1 is uniq blob)
+				// e.g. `cudf-0.14-cuda10-1.jar.sha1` should not overwrite `cudf-0.14.jar.sha1`
+				// https://repo.maven.apache.org/maven2/ai/rapids/cudf/0.14/
+				if ver == dirVersion {
+					dirVersionSha1 = sha1
+				} else {
+					versions = append(versions, Version{
+						Version: ver,
+						SHA1:    sha1,
+					})
+				}
 			}
-			versions = append(versions, v)
 		}
+		// Remove duplicates of dirVersionSha1
+		versions = lo.Filter(versions, func(v Version, _ int) bool {
+			return !bytes.Equal(v.SHA1, dirVersionSha1)
+		})
+
+		versions = append(versions, Version{
+			Version: dirVersion,
+			SHA1:    dirVersionSha1,
+		})
+
+		foundVersions = append(foundVersions, versions...)
 	}
 
-	if len(versions) == 0 {
+	if len(foundVersions) == 0 {
 		return nil
 	}
 
 	index := &Index{
 		GroupID:     meta.GroupID,
 		ArtifactID:  meta.ArtifactID,
-		Versions:    versions,
+		Versions:    foundVersions,
 		ArchiveType: types.JarType,
 	}
 	fileName := fmt.Sprintf("%s.json", index.ArtifactID)
@@ -256,16 +277,12 @@ func (c *Crawler) sha1Urls(ctx context.Context, url string) ([]string, error) {
 	// We need to take all links.
 	var sha1URLs []string
 	d.Find("a").Each(func(i int, selection *goquery.Selection) {
-		// There are times when the file name is very long.
-		// e.g. https://repo.maven.apache.org/maven2/africa/absa/inception-oauth2-resource-server/1.0.0/
-		// We need to use `href` to make sure we use the correct filename
-		if fileName, ok := selection.Attr("href"); ok {
-			// don't include sources, test, javadocs, scaladoc files
-			if strings.HasSuffix(fileName, ".jar.sha1") && !strings.HasSuffix(fileName, "sources.jar.sha1") &&
-				!strings.HasSuffix(fileName, "test.jar.sha1") && !strings.HasSuffix(fileName, "tests.jar.sha1") &&
-				!strings.HasSuffix(fileName, "javadoc.jar.sha1") && !strings.HasSuffix(fileName, "scaladoc.jar.sha1") {
-				sha1URLs = append(sha1URLs, url+fileName)
-			}
+		link := linkFromSelection(selection)
+		// Don't include sources, test, javadocs, scaladoc files
+		if strings.HasSuffix(link, ".jar.sha1") && !strings.HasSuffix(link, "sources.jar.sha1") &&
+			!strings.HasSuffix(link, "test.jar.sha1") && !strings.HasSuffix(link, "tests.jar.sha1") &&
+			!strings.HasSuffix(link, "javadoc.jar.sha1") && !strings.HasSuffix(link, "scaladoc.jar.sha1") {
+			sha1URLs = append(sha1URLs, url+link)
 		}
 	})
 	return sha1URLs, nil
@@ -360,3 +377,19 @@ func versionFromSha1URL(artifactId, sha1URL string) string {
 	}
 	return strings.TrimSuffix(strings.TrimPrefix(fileName, artifactId+"-"), ".jar.sha1")
 }
+
+// linkFromSelection returns the link from goquery.Selection.
+// There are times when maven breaks `text` - it removes part of the `text` and adds the suffix `...` (`.../` for dirs).
+// e.g. `<a href="v1.1.0-226-g847ecff2d8e26f249422247d7665fe15f07b1744/">v1.1.0-226-g847ecff2d8e26f249422247d7665fe15.../</a>`
+// In this case we should take `href`.
+// But we don't need to get `href` if the text isn't broken.
+// To avoid checking unnecessary links.
+// e.g. `<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/">../</a>`
+func linkFromSelection(selection *goquery.Selection) string {
+	link := selection.Text()
+	// maven uses `.../` suffix for dirs and `...` suffix for files.
+	if href, ok := selection.Attr("href"); ok && (strings.HasSuffix(link, ".../") || (strings.HasSuffix(link, "..."))) {
+		link = href
+	}
+	return link
+}
diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go
index a04d707..5cd9fee 100644
--- a/pkg/crawler/crawler_test.go
+++ b/pkg/crawler/crawler_test.go
@@ -22,17 +22,18 @@ func TestCrawl(t *testing.T) {
 		{
 			name: "happy path",
 			fileNames: map[string]string{
-				"/maven2/":                                            "testdata/index.html",
-				"/maven2/abbot/":                                      "testdata/abbot.html",
-				"/maven2/abbot/abbot/":                                "testdata/abbot_abbot.html",
-				"/maven2/abbot/abbot/maven-metadata.xml":              "testdata/maven-metadata.xml",
-				"/maven2/abbot/abbot/0.12.3/":                         "testdata/abbot_abbot_0.12.3.html",
-				"/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1":    "testdata/abbot-0.12.3.jar.sha1",
-				"/maven2/abbot/abbot/0.13.0/":                         "testdata/abbot_abbot_0.13.0.html",
-				"/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1":    "testdata/abbot-0.13.0.jar.sha1",
-				"/maven2/abbot/abbot/1.4.0/":                          "testdata/abbot_abbot_1.4.0.html",
-				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1":      "testdata/abbot-1.4.0.jar.sha1",
-				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1": "testdata/abbot-1.4.0-lite.jar.sha1",
+				"/maven2/":                                              "testdata/index.html",
+				"/maven2/abbot/":                                        "testdata/abbot.html",
+				"/maven2/abbot/abbot/":                                  "testdata/abbot_abbot.html",
+				"/maven2/abbot/abbot/maven-metadata.xml":                "testdata/maven-metadata.xml",
+				"/maven2/abbot/abbot/0.12.3/":                           "testdata/abbot_abbot_0.12.3.html",
+				"/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1":      "testdata/abbot-0.12.3.jar.sha1",
+				"/maven2/abbot/abbot/0.13.0/":                           "testdata/abbot_abbot_0.13.0.html",
+				"/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1":      "testdata/abbot-0.13.0.jar.sha1",
+				"/maven2/abbot/abbot/0.13.0/abbot-0.13.0-copy.jar.sha1": "testdata/abbot-0.13.0-copy.jar.sha1",
+				"/maven2/abbot/abbot/1.4.0/":                            "testdata/abbot_abbot_1.4.0.html",
+				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1":        "testdata/abbot-1.4.0.jar.sha1",
+				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1":   "testdata/abbot-1.4.0-lite.jar.sha1",
 			},
 			goldenPath: "testdata/golden/abbot.json",
 			filePath:   "indexes/abbot/abbot.json",
diff --git a/pkg/crawler/testdata/abbot-0.13.0-copy.jar.sha1 b/pkg/crawler/testdata/abbot-0.13.0-copy.jar.sha1
new file mode 100644
index 0000000..ad11b66
--- /dev/null
+++ b/pkg/crawler/testdata/abbot-0.13.0-copy.jar.sha1
@@ -0,0 +1 @@
+596d91e67631b0deb05fb685d8d1b6735f3e4f60
\ No newline at end of file
diff --git a/pkg/crawler/testdata/abbot_abbot.html b/pkg/crawler/testdata/abbot_abbot.html
index 76d8551..c12f4bf 100644
--- a/pkg/crawler/testdata/abbot_abbot.html
+++ b/pkg/crawler/testdata/abbot_abbot.html
@@ -18,7 +18,7 @@ <h1>abbot/abbot</h1>
 	<main>
 		<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/">../</a>
 <a href="../">../</a>
-<a href="0.12.3/" title="0.12.3/">0.12.3/</a>                                           2005-09-20 05:44         -
+<a href="0.12.3/" title="0.12.3/">0.12..../</a>                                           2005-09-20 05:44         -
 <a href="0.13.0/" title="0.13.0/">0.13.0/</a>                                           2005-09-20 05:44         -
 <a href="1.4.0/" title="1.4.0/">1.4.0/</a>                                            2015-09-22 16:03         -
 <a href="maven-metadata.xml" title="maven-metadata.xml">maven-metadata.xml</a>                                2015-09-24 14:18       402
diff --git a/pkg/crawler/testdata/abbot_abbot_0.13.0.html b/pkg/crawler/testdata/abbot_abbot_0.13.0.html
index 7e57d66..b5661f0 100644
--- a/pkg/crawler/testdata/abbot_abbot_0.13.0.html
+++ b/pkg/crawler/testdata/abbot_abbot_0.13.0.html
@@ -17,6 +17,9 @@ <h1>abbot/abbot/0.13.0</h1>
 	<hr>
 	<main>
 		<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/abbot/">../</a>
+<a href="abbot-0.13.0-copy.jar" title="abbot-0.13.0.jar">abbot-0.13.0.jar</a>                                  2005-09-20 05:44    779426
+<a href="abbot-0.13.0-copy.jar.md5" title="abbot-0.13.0.jar.md5">abbot-0.13.0.jar.md5</a>                              2005-09-20 05:44        32
+<a href="abbot-0.13.0-copy.jar.sha1" title="abbot-0.13.0.jar.sha1">abbot-0.13.0.jar.sha1</a>                             2005-09-20 05:44        40
 <a href="abbot-0.13.0.jar" title="abbot-0.13.0.jar">abbot-0.13.0.jar</a>                                  2005-09-20 05:44    779426      
 <a href="abbot-0.13.0.jar.md5" title="abbot-0.13.0.jar.md5">abbot-0.13.0.jar.md5</a>                              2005-09-20 05:44        32      
 <a href="abbot-0.13.0.jar.sha1" title="abbot-0.13.0.jar.sha1">abbot-0.13.0.jar.sha1</a>                             2005-09-20 05:44        40