Merge pull request #11 from kyoshidajp/support_pip

Support pip

README.md

@@ -10,13 +10,14 @@
 
 ## Support dependencies files
 
-| Language | file | status |
-| -------- | --------------- | :----: |
-| Ruby | Gemfile.lock | YES |
-| Ruby | gemspec | NO (but soon) |
-| JavaScript | yarn.lock | YES |
-| JavaScript | package.json | NO (but soon) |
-| Go | go.sum | NO (but soon) |
+| Language | package manager | file (e.g.) | status |
+| -------- | ------------- | -- | :----: |
+| Ruby | bundler | Gemfile.lock | :heavy_check_mark: |
+| Ruby | bundler | gemspec | (soon) |
+| JavaScript | yarn | yarn.lock | :heavy_check_mark: |
+| JavaScript | npm | package.json | (soon) |
+| Python | pip | requirements.txt | :heavy_check_mark: |
+| Go | | go.sum | (soon) |
 
 ## Install
 

cmd/diagnose.go

@@ -57,6 +57,7 @@ var (
 var doctors = map[string]Doctor{
 	"bundler": NewBundlerDoctor(),
 	"yarn":    NewYarnDoctor(),
+	"pip":     NewPipDoctor(),
 }
 
 var diagnoseCmd = &cobra.Command{

cmd/pip.go

@@ -0,0 +1,135 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	parser_io "github.com/aquasecurity/go-dep-parser/pkg/io"
+	"github.com/aquasecurity/go-dep-parser/pkg/python/pip"
+	"github.com/kyoshidajp/dep-doctor/cmd/github"
+)
+
+// https://warehouse.pypa.io/api-reference/json.html
+const PYPI_REGISTRY_API = "https://pypi.org/pypi/%s/json"
+
+type PipRegistryResponse struct {
+	Info struct {
+		ProjectUrls struct {
+			SourceCode string `json:"Source Code"`
+			Source     string `json:"Source"`
+		} `json:"project_urls"`
+	} `json:"info"`
+}
+
+type PipDoctor struct {
+}
+
+func NewPipDoctor() *PipDoctor {
+	return &PipDoctor{}
+}
+
+func (d *PipDoctor) fetchURLFromRepository(name string) (string, error) {
+	url := fmt.Sprintf(PYPI_REGISTRY_API, name)
+	req, _ := http.NewRequest(http.MethodGet, url, nil)
+	client := new(http.Client)
+	resp, _ := client.Do(req)
+	body, _ := io.ReadAll(resp.Body)
+
+	var PipRegistryResponse PipRegistryResponse
+	err := json.Unmarshal(body, &PipRegistryResponse)
+	if err != nil {
+		return "", nil
+	}
+
+	if PipRegistryResponse.Info.ProjectUrls.SourceCode != "" {
+		return PipRegistryResponse.Info.ProjectUrls.SourceCode, nil
+	}
+
+	return PipRegistryResponse.Info.ProjectUrls.Source, nil
+}
+
+func (d *PipDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string]Diagnosis {
+	diagnoses := make(map[string]Diagnosis)
+	slicedNameWithOwners := [][]github.NameWithOwner{}
+	nameWithOwners := d.NameWithOwners(r)
+	sliceSize := len(nameWithOwners)
+
+	for i := 0; i < sliceSize; i += GITHUB_SEARCH_REPO_COUNT_PER_ONCE {
+		end := i + GITHUB_SEARCH_REPO_COUNT_PER_ONCE
+		if sliceSize < end {
+			end = sliceSize
+		}
+		slicedNameWithOwners = append(slicedNameWithOwners, nameWithOwners[i:end])
+	}
+
+	for _, nameWithOwners := range slicedNameWithOwners {
+		repos := github.FetchFromGitHub(nameWithOwners)
+		for _, r := range repos {
+			diagnosis := Diagnosis{
+				Name:      r.Name,
+				Url:       r.Url,
+				Archived:  r.Archived,
+				Diagnosed: true,
+				IsActive:  r.IsActive(year),
+			}
+			diagnoses[r.Name] = diagnosis
+		}
+	}
+
+	for _, nameWithOwner := range nameWithOwners {
+		if nameWithOwner.CanSearch {
+			continue
+		}
+
+		diagnosis := Diagnosis{
+			Name:      nameWithOwner.PackageName,
+			Diagnosed: false,
+		}
+		diagnoses[nameWithOwner.PackageName] = diagnosis
+	}
+	return diagnoses
+}
+
+func (d *PipDoctor) NameWithOwners(r parser_io.ReadSeekerAt) []github.NameWithOwner {
+	var nameWithOwners []github.NameWithOwner
+	libs, _, _ := pip.NewParser().Parse(r)
+
+	for _, lib := range libs {
+		fmt.Printf("%s\n", lib.Name)
+
+		githubUrl, err := d.fetchURLFromRepository(lib.Name)
+		if err != nil {
+			nameWithOwners = append(nameWithOwners,
+				github.NameWithOwner{
+					PackageName: lib.Name,
+					CanSearch:   false,
+				},
+			)
+			continue
+		}
+
+		repo, err := github.ParseGitHubUrl(githubUrl)
+		if err != nil {
+			nameWithOwners = append(nameWithOwners,
+				github.NameWithOwner{
+					PackageName: lib.Name,
+					CanSearch:   false,
+				},
+			)
+			continue
+		}
+
+		nameWithOwners = append(nameWithOwners,
+			github.NameWithOwner{
+				Repo:        repo.Repo,
+				Owner:       repo.Owner,
+				PackageName: lib.Name,
+				CanSearch:   true,
+			},
+		)
+	}
+
+	return nameWithOwners
+}

cmd/pip_test.go

@@ -0,0 +1,37 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPyPiDoctor_fetchURLFromRepository(t *testing.T) {
+	tests := []struct {
+		name     string
+		gem_name string
+	}{
+		{
+			name:     "source_code_uri exists",
+			gem_name: "pip",
+		},
+	}
+	expects := []struct {
+		name string
+		url  string
+	}{
+		{
+			name: "source_code_uri exists",
+			url:  "https://github.com/pypa/pip",
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := PipDoctor{}
+			r, _ := s.fetchURLFromRepository(tt.gem_name)
+			expect := expects[i]
+			assert.Equal(t, expect.url, r)
+		})
+	}
+}

cmd/python/pip/testdata/requirements.txt

@@ -0,0 +1,6 @@
+click==8.0.0
+Flask==2.0.0
+itsdangerous==2.0.0
+Jinja2==3.0.0
+MarkupSafe==2.0.0
+Werkzeug==2.0.0

go.mod

@@ -20,6 +20,7 @@ require (
 	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -76,6 +76,8 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=