diff --git a/README.md b/README.md index fbf0862..0f9c871 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,14 @@ ## Support dependencies files -| Language | file | status | -| -------- | --------------- | :----: | -| Ruby | Gemfile.lock | YES | -| Ruby | gemspec | NO (but soon) | -| JavaScript | yarn.lock | YES | -| JavaScript | package.json | NO (but soon) | -| Go | go.sum | NO (but soon) | +| Language | package manager | file (e.g.) | status | +| -------- | ------------- | -- | :----: | +| Ruby | bundler | Gemfile.lock | :heavy_check_mark: | +| Ruby | bundler | gemspec | (soon) | +| JavaScript | yarn | yarn.lock | :heavy_check_mark: | +| JavaScript | npm | package.json | (soon) | +| Python | pip | requirements.txt | :heavy_check_mark: | +| Go | | go.sum | (soon) | ## Install diff --git a/cmd/diagnose.go b/cmd/diagnose.go index 28a4049..8290420 100644 --- a/cmd/diagnose.go +++ b/cmd/diagnose.go @@ -57,6 +57,7 @@ var ( var doctors = map[string]Doctor{ "bundler": NewBundlerDoctor(), "yarn": NewYarnDoctor(), + "pip": NewPipDoctor(), } var diagnoseCmd = &cobra.Command{ diff --git a/cmd/pip.go b/cmd/pip.go new file mode 100644 index 0000000..473c5fd --- /dev/null +++ b/cmd/pip.go @@ -0,0 +1,135 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + parser_io "github.com/aquasecurity/go-dep-parser/pkg/io" + "github.com/aquasecurity/go-dep-parser/pkg/python/pip" + "github.com/kyoshidajp/dep-doctor/cmd/github" +) + +// https://warehouse.pypa.io/api-reference/json.html +const PYPI_REGISTRY_API = "https://pypi.org/pypi/%s/json" + +type PipRegistryResponse struct { + Info struct { + ProjectUrls struct { + SourceCode string `json:"Source Code"` + Source string `json:"Source"` + } `json:"project_urls"` + } `json:"info"` +} + +type PipDoctor struct { +} + +func NewPipDoctor() *PipDoctor { + return &PipDoctor{} +} + +func (d *PipDoctor) fetchURLFromRepository(name string) (string, error) { + url := fmt.Sprintf(PYPI_REGISTRY_API, name) + req, _ := http.NewRequest(http.MethodGet, url, nil) + client := new(http.Client) + resp, _ := client.Do(req) + body, _ := io.ReadAll(resp.Body) + + var PipRegistryResponse PipRegistryResponse + err := json.Unmarshal(body, &PipRegistryResponse) + if err != nil { + return "", nil + } + + if PipRegistryResponse.Info.ProjectUrls.SourceCode != "" { + return PipRegistryResponse.Info.ProjectUrls.SourceCode, nil + } + + return PipRegistryResponse.Info.ProjectUrls.Source, nil +} + +func (d *PipDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string]Diagnosis { + diagnoses := make(map[string]Diagnosis) + slicedNameWithOwners := [][]github.NameWithOwner{} + nameWithOwners := d.NameWithOwners(r) + sliceSize := len(nameWithOwners) + + for i := 0; i < sliceSize; i += GITHUB_SEARCH_REPO_COUNT_PER_ONCE { + end := i + GITHUB_SEARCH_REPO_COUNT_PER_ONCE + if sliceSize < end { + end = sliceSize + } + slicedNameWithOwners = append(slicedNameWithOwners, nameWithOwners[i:end]) + } + + for _, nameWithOwners := range slicedNameWithOwners { + repos := github.FetchFromGitHub(nameWithOwners) + for _, r := range repos { + diagnosis := Diagnosis{ + Name: r.Name, + Url: r.Url, + Archived: r.Archived, + Diagnosed: true, + IsActive: r.IsActive(year), + } + diagnoses[r.Name] = diagnosis + } + } + + for _, nameWithOwner := range nameWithOwners { + if nameWithOwner.CanSearch { + continue + } + + diagnosis := Diagnosis{ + Name: nameWithOwner.PackageName, + Diagnosed: false, + } + diagnoses[nameWithOwner.PackageName] = diagnosis + } + return diagnoses +} + +func (d *PipDoctor) NameWithOwners(r parser_io.ReadSeekerAt) []github.NameWithOwner { + var nameWithOwners []github.NameWithOwner + libs, _, _ := pip.NewParser().Parse(r) + + for _, lib := range libs { + fmt.Printf("%s\n", lib.Name) + + githubUrl, err := d.fetchURLFromRepository(lib.Name) + if err != nil { + nameWithOwners = append(nameWithOwners, + github.NameWithOwner{ + PackageName: lib.Name, + CanSearch: false, + }, + ) + continue + } + + repo, err := github.ParseGitHubUrl(githubUrl) + if err != nil { + nameWithOwners = append(nameWithOwners, + github.NameWithOwner{ + PackageName: lib.Name, + CanSearch: false, + }, + ) + continue + } + + nameWithOwners = append(nameWithOwners, + github.NameWithOwner{ + Repo: repo.Repo, + Owner: repo.Owner, + PackageName: lib.Name, + CanSearch: true, + }, + ) + } + + return nameWithOwners +} diff --git a/cmd/pip_test.go b/cmd/pip_test.go new file mode 100644 index 0000000..feccbf2 --- /dev/null +++ b/cmd/pip_test.go @@ -0,0 +1,37 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPyPiDoctor_fetchURLFromRepository(t *testing.T) { + tests := []struct { + name string + gem_name string + }{ + { + name: "source_code_uri exists", + gem_name: "pip", + }, + } + expects := []struct { + name string + url string + }{ + { + name: "source_code_uri exists", + url: "https://github.com/pypa/pip", + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := PipDoctor{} + r, _ := s.fetchURLFromRepository(tt.gem_name) + expect := expects[i] + assert.Equal(t, expect.url, r) + }) + } +} diff --git a/cmd/python/pip/testdata/requirements.txt b/cmd/python/pip/testdata/requirements.txt new file mode 100644 index 0000000..9072f44 --- /dev/null +++ b/cmd/python/pip/testdata/requirements.txt @@ -0,0 +1,6 @@ +click==8.0.0 +Flask==2.0.0 +itsdangerous==2.0.0 +Jinja2==3.0.0 +MarkupSafe==2.0.0 +Werkzeug==2.0.0 \ No newline at end of file diff --git a/go.mod b/go.mod index fd6c9b5..97fa933 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index c981341..c2b84fc 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=