From 544936b34524705c4bc851a8c623c1a26983ec0e Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Fri, 27 Oct 2023 09:02:36 +0900 Subject: [PATCH 1/3] Delete unused code --- cmd/diagnose.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/diagnose.go b/cmd/diagnose.go index 00d14dc..4cb13fe 100644 --- a/cmd/diagnose.go +++ b/cmd/diagnose.go @@ -80,7 +80,6 @@ var diagnoseCmd = &cobra.Command{ _ = f.Close() }() if err != nil { - fmt.Fprintln(os.Stderr, err) m := fmt.Sprintf("Can't open: %s.", o.lockFilePath) log.Fatal(m) } From 2081ea32440ce86a8f2040d6fb7be74a186d272d Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Fri, 27 Oct 2023 09:06:38 +0900 Subject: [PATCH 2/3] Refactor search repository count in GitHub --- cmd/bundler.go | 6 ++---- cmd/github/github.go | 1 + cmd/pip.go | 4 ++-- cmd/yarn.go | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cmd/bundler.go b/cmd/bundler.go index c89479a..220c3fe 100644 --- a/cmd/bundler.go +++ b/cmd/bundler.go @@ -12,8 +12,6 @@ import ( "github.com/kyoshidajp/dep-doctor/cmd/github" ) -const GITHUB_SEARCH_REPO_COUNT_PER_ONCE = 20 - // https://guides.rubygems.org/rubygems-org-api/ const RUBYGEMS_ORG_API = "https://rubygems.org/api/v1/gems/%s.json" @@ -36,8 +34,8 @@ func (b *BundlerDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string] nameWithOwners := b.NameWithOwners(r) sliceSize := len(nameWithOwners) - for i := 0; i < sliceSize; i += GITHUB_SEARCH_REPO_COUNT_PER_ONCE { - end := i + GITHUB_SEARCH_REPO_COUNT_PER_ONCE + for i := 0; i < sliceSize; i += github.SEARCH_REPOS_PER_ONCE { + end := i + github.SEARCH_REPOS_PER_ONCE if sliceSize < end { end = sliceSize } diff --git a/cmd/github/github.go b/cmd/github/github.go index 01c4646..4622135 100644 --- a/cmd/github/github.go +++ b/cmd/github/github.go @@ -16,6 +16,7 @@ import ( ) const QUERY_SEPARATOR = " " +const SEARCH_REPOS_PER_ONCE = 20 type GitHubRepository struct { Name string diff --git a/cmd/pip.go b/cmd/pip.go index 473c5fd..89d018f 100644 --- a/cmd/pip.go +++ b/cmd/pip.go @@ -56,8 +56,8 @@ func (d *PipDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string]Diag nameWithOwners := d.NameWithOwners(r) sliceSize := len(nameWithOwners) - for i := 0; i < sliceSize; i += GITHUB_SEARCH_REPO_COUNT_PER_ONCE { - end := i + GITHUB_SEARCH_REPO_COUNT_PER_ONCE + for i := 0; i < sliceSize; i += github.SEARCH_REPOS_PER_ONCE { + end := i + github.SEARCH_REPOS_PER_ONCE if sliceSize < end { end = sliceSize } diff --git a/cmd/yarn.go b/cmd/yarn.go index f5da87d..75ffa63 100644 --- a/cmd/yarn.go +++ b/cmd/yarn.go @@ -48,8 +48,8 @@ func (d *YarnDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string]Dia nameWithOwners := d.NameWithOwners(r) sliceSize := len(nameWithOwners) - for i := 0; i < sliceSize; i += GITHUB_SEARCH_REPO_COUNT_PER_ONCE { - end := i + GITHUB_SEARCH_REPO_COUNT_PER_ONCE + for i := 0; i < sliceSize; i += github.SEARCH_REPOS_PER_ONCE { + end := i + github.SEARCH_REPOS_PER_ONCE if sliceSize < end { end = sliceSize } From d8cc65217a8a39bfa2bf8e66c996d5ef5576170e Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Fri, 27 Oct 2023 16:55:11 +0900 Subject: [PATCH 3/3] Refactor pick up registry --- cmd/bundler.go | 38 +------------------ cmd/diagnose.go | 1 - cmd/nodejs.go | 36 ++++++++++++++++++ cmd/{yarn_test.go => nodejs_test.go} | 6 +-- cmd/pip.go | 38 +------------------ cmd/pypi.go | 43 ++++++++++++++++++++++ cmd/{pip_test.go => pypi_test.go} | 6 +-- cmd/ruby_gems.go | 43 ++++++++++++++++++++++ cmd/{bundler_test.go => ruby_gems_test.go} | 6 +-- cmd/yarn.go | 30 +-------------- 10 files changed, 137 insertions(+), 110 deletions(-) create mode 100644 cmd/nodejs.go rename cmd/{yarn_test.go => nodejs_test.go} (80%) create mode 100644 cmd/pypi.go rename cmd/{pip_test.go => pypi_test.go} (79%) create mode 100644 cmd/ruby_gems.go rename cmd/{bundler_test.go => ruby_gems_test.go} (84%) diff --git a/cmd/bundler.go b/cmd/bundler.go index 220c3fe..0049d64 100644 --- a/cmd/bundler.go +++ b/cmd/bundler.go @@ -1,26 +1,13 @@ package cmd import ( - "encoding/json" - "errors" "fmt" - "io" - "net/http" parser_io "github.com/aquasecurity/go-dep-parser/pkg/io" "github.com/aquasecurity/go-dep-parser/pkg/ruby/bundler" "github.com/kyoshidajp/dep-doctor/cmd/github" ) -// https://guides.rubygems.org/rubygems-org-api/ -const RUBYGEMS_ORG_API = "https://rubygems.org/api/v1/gems/%s.json" - -type GemResponse struct { - Name string `json:"name"` - SourceCodeUri string `json:"source_code_uri"` - HomepageUri string `json:"homepage_uri"` -} - type BundlerDoctor struct { } @@ -70,37 +57,16 @@ func (b *BundlerDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string] return diagnoses } -func (d *BundlerDoctor) fetchURLFromRepository(name string) (string, error) { - url := fmt.Sprintf(RUBYGEMS_ORG_API, name) - req, _ := http.NewRequest(http.MethodGet, url, nil) - client := new(http.Client) - resp, _ := client.Do(req) - body, _ := io.ReadAll(resp.Body) - - var Gem GemResponse - err := json.Unmarshal(body, &Gem) - if err != nil { - return "", errors.New("error: Unknown response") - } - - if Gem.SourceCodeUri != "" { - return Gem.SourceCodeUri, nil - } else if Gem.HomepageUri != "" { - return Gem.HomepageUri, nil - } - - return "", nil -} - func (d *BundlerDoctor) NameWithOwners(r parser_io.ReadSeekerAt) []github.NameWithOwner { var nameWithOwners []github.NameWithOwner p := &bundler.Parser{} libs, _, _ := p.Parse(r) + rubyGems := RubyGems{} for _, lib := range libs { fmt.Printf("%s\n", lib.Name) - githubUrl, err := d.fetchURLFromRepository(lib.Name) + githubUrl, err := rubyGems.fetchURLFromRegistry(lib.Name) if err != nil { continue } diff --git a/cmd/diagnose.go b/cmd/diagnose.go index 4cb13fe..5b24cc3 100644 --- a/cmd/diagnose.go +++ b/cmd/diagnose.go @@ -19,7 +19,6 @@ const MAX_YEAR_TO_BE_BLANK = 5 type Doctor interface { Diagnose(r io.ReadSeekerAt, year int) map[string]Diagnosis - fetchURLFromRepository(name string) (string, error) NameWithOwners(r parser_io.ReadSeekerAt) []github.NameWithOwner } diff --git a/cmd/nodejs.go b/cmd/nodejs.go new file mode 100644 index 0000000..a386b0b --- /dev/null +++ b/cmd/nodejs.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" +) + +// https://docs.npmjs.com/cli/v8/using-npm/registry +const NODEJS_REGISTRY_API = "https://registry.npmjs.org/%s" + +type NodejsRegistryResponse struct { + Repository struct { + Url string `json:"url"` + } +} + +type Nodejs struct { +} + +func (n *Nodejs) fetchURLFromRegistry(name string) (string, error) { + url := fmt.Sprintf(NODEJS_REGISTRY_API, name) + req, _ := http.NewRequest(http.MethodGet, url, nil) + client := new(http.Client) + resp, _ := client.Do(req) + body, _ := io.ReadAll(resp.Body) + + var NodejsRegistryResponse NodejsRegistryResponse + err := json.Unmarshal(body, &NodejsRegistryResponse) + if err != nil { + return "", nil + } + + return NodejsRegistryResponse.Repository.Url, nil +} diff --git a/cmd/yarn_test.go b/cmd/nodejs_test.go similarity index 80% rename from cmd/yarn_test.go rename to cmd/nodejs_test.go index bbdacbb..9bd7cf6 100644 --- a/cmd/yarn_test.go +++ b/cmd/nodejs_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestYarnDoctor_fetchURLFromRepository(t *testing.T) { +func TestNodejs_fetchURLFromRegistry(t *testing.T) { tests := []struct { name string gem_name string @@ -29,8 +29,8 @@ func TestYarnDoctor_fetchURLFromRepository(t *testing.T) { for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := YarnDoctor{} - r, _ := s.fetchURLFromRepository(tt.gem_name) + n := Nodejs{} + r, _ := n.fetchURLFromRegistry(tt.gem_name) expect := expects[i] assert.Equal(t, true, strings.HasPrefix(r, expect.url)) }) diff --git a/cmd/pip.go b/cmd/pip.go index 89d018f..e80ff96 100644 --- a/cmd/pip.go +++ b/cmd/pip.go @@ -1,28 +1,13 @@ package cmd import ( - "encoding/json" "fmt" - "io" - "net/http" parser_io "github.com/aquasecurity/go-dep-parser/pkg/io" "github.com/aquasecurity/go-dep-parser/pkg/python/pip" "github.com/kyoshidajp/dep-doctor/cmd/github" ) -// https://warehouse.pypa.io/api-reference/json.html -const PYPI_REGISTRY_API = "https://pypi.org/pypi/%s/json" - -type PipRegistryResponse struct { - Info struct { - ProjectUrls struct { - SourceCode string `json:"Source Code"` - Source string `json:"Source"` - } `json:"project_urls"` - } `json:"info"` -} - type PipDoctor struct { } @@ -30,26 +15,6 @@ func NewPipDoctor() *PipDoctor { return &PipDoctor{} } -func (d *PipDoctor) fetchURLFromRepository(name string) (string, error) { - url := fmt.Sprintf(PYPI_REGISTRY_API, name) - req, _ := http.NewRequest(http.MethodGet, url, nil) - client := new(http.Client) - resp, _ := client.Do(req) - body, _ := io.ReadAll(resp.Body) - - var PipRegistryResponse PipRegistryResponse - err := json.Unmarshal(body, &PipRegistryResponse) - if err != nil { - return "", nil - } - - if PipRegistryResponse.Info.ProjectUrls.SourceCode != "" { - return PipRegistryResponse.Info.ProjectUrls.SourceCode, nil - } - - return PipRegistryResponse.Info.ProjectUrls.Source, nil -} - func (d *PipDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string]Diagnosis { diagnoses := make(map[string]Diagnosis) slicedNameWithOwners := [][]github.NameWithOwner{} @@ -96,10 +61,11 @@ func (d *PipDoctor) NameWithOwners(r parser_io.ReadSeekerAt) []github.NameWithOw var nameWithOwners []github.NameWithOwner libs, _, _ := pip.NewParser().Parse(r) + pypi := Pypi{} for _, lib := range libs { fmt.Printf("%s\n", lib.Name) - githubUrl, err := d.fetchURLFromRepository(lib.Name) + githubUrl, err := pypi.fetchURLFromRepository(lib.Name) if err != nil { nameWithOwners = append(nameWithOwners, github.NameWithOwner{ diff --git a/cmd/pypi.go b/cmd/pypi.go new file mode 100644 index 0000000..6149ed3 --- /dev/null +++ b/cmd/pypi.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" +) + +// https://warehouse.pypa.io/api-reference/json.html +const PYPI_REGISTRY_API = "https://pypi.org/pypi/%s/json" + +type PypiRegistryResponse struct { + Info struct { + ProjectUrls struct { + SourceCode string `json:"Source Code"` + Source string `json:"Source"` + } `json:"project_urls"` + } `json:"info"` +} + +type Pypi struct { +} + +func (p *Pypi) fetchURLFromRepository(name string) (string, error) { + url := fmt.Sprintf(PYPI_REGISTRY_API, name) + req, _ := http.NewRequest(http.MethodGet, url, nil) + client := new(http.Client) + resp, _ := client.Do(req) + body, _ := io.ReadAll(resp.Body) + + var PypiRegistryResponse PypiRegistryResponse + err := json.Unmarshal(body, &PypiRegistryResponse) + if err != nil { + return "", nil + } + + if PypiRegistryResponse.Info.ProjectUrls.SourceCode != "" { + return PypiRegistryResponse.Info.ProjectUrls.SourceCode, nil + } + + return PypiRegistryResponse.Info.ProjectUrls.Source, nil +} diff --git a/cmd/pip_test.go b/cmd/pypi_test.go similarity index 79% rename from cmd/pip_test.go rename to cmd/pypi_test.go index feccbf2..e0add85 100644 --- a/cmd/pip_test.go +++ b/cmd/pypi_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPyPiDoctor_fetchURLFromRepository(t *testing.T) { +func TestPyPi_fetchURLFromRegistry(t *testing.T) { tests := []struct { name string gem_name string @@ -28,8 +28,8 @@ func TestPyPiDoctor_fetchURLFromRepository(t *testing.T) { for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := PipDoctor{} - r, _ := s.fetchURLFromRepository(tt.gem_name) + p := Pypi{} + r, _ := p.fetchURLFromRepository(tt.gem_name) expect := expects[i] assert.Equal(t, expect.url, r) }) diff --git a/cmd/ruby_gems.go b/cmd/ruby_gems.go new file mode 100644 index 0000000..dc06f00 --- /dev/null +++ b/cmd/ruby_gems.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" +) + +// https://guides.rubygems.org/rubygems-org-api/ +const RUBY_GEMS_REGISTRY_API = "https://rubygems.org/api/v1/gems/%s.json" + +type RubyGemsRegistryResponse struct { + Name string `json:"name"` + SourceCodeUri string `json:"source_code_uri"` + HomepageUri string `json:"homepage_uri"` +} + +type RubyGems struct { +} + +func (g *RubyGems) fetchURLFromRegistry(name string) (string, error) { + url := fmt.Sprintf(RUBY_GEMS_REGISTRY_API, name) + req, _ := http.NewRequest(http.MethodGet, url, nil) + client := new(http.Client) + resp, _ := client.Do(req) + body, _ := io.ReadAll(resp.Body) + + var Gem RubyGemsRegistryResponse + err := json.Unmarshal(body, &Gem) + if err != nil { + return "", errors.New("error: Unknown response") + } + + if Gem.SourceCodeUri != "" { + return Gem.SourceCodeUri, nil + } else if Gem.HomepageUri != "" { + return Gem.HomepageUri, nil + } + + return "", nil +} diff --git a/cmd/bundler_test.go b/cmd/ruby_gems_test.go similarity index 84% rename from cmd/bundler_test.go rename to cmd/ruby_gems_test.go index 8ea7d5e..d98f6e7 100644 --- a/cmd/bundler_test.go +++ b/cmd/ruby_gems_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBundlerDoctor_fetchURLFromRepository(t *testing.T) { +func TestRubyGems_fetchURLFromRepository(t *testing.T) { tests := []struct { name string gem_name string @@ -37,8 +37,8 @@ func TestBundlerDoctor_fetchURLFromRepository(t *testing.T) { for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := BundlerDoctor{} - r, _ := s.fetchURLFromRepository(tt.gem_name) + g := RubyGems{} + r, _ := g.fetchURLFromRegistry(tt.gem_name) expect := expects[i] assert.Equal(t, true, strings.HasPrefix(r, expect.url)) }) diff --git a/cmd/yarn.go b/cmd/yarn.go index 75ffa63..d2f9caa 100644 --- a/cmd/yarn.go +++ b/cmd/yarn.go @@ -1,24 +1,13 @@ package cmd import ( - "encoding/json" "fmt" - "io" - "net/http" parser_io "github.com/aquasecurity/go-dep-parser/pkg/io" "github.com/aquasecurity/go-dep-parser/pkg/nodejs/yarn" "github.com/kyoshidajp/dep-doctor/cmd/github" ) -const NODEJS_REGISTRY_API = "https://registry.npmjs.org/%s" - -type NodejsRegistryResponse struct { - Repository struct { - Url string `json:"url"` - } -} - type YarnDoctor struct { } @@ -26,22 +15,6 @@ func NewYarnDoctor() *YarnDoctor { return &YarnDoctor{} } -func (d *YarnDoctor) fetchURLFromRepository(name string) (string, error) { - url := fmt.Sprintf(NODEJS_REGISTRY_API, name) - req, _ := http.NewRequest(http.MethodGet, url, nil) - client := new(http.Client) - resp, _ := client.Do(req) - body, _ := io.ReadAll(resp.Body) - - var NodejsRegistryResponse NodejsRegistryResponse - err := json.Unmarshal(body, &NodejsRegistryResponse) - if err != nil { - return "", nil - } - - return NodejsRegistryResponse.Repository.Url, nil -} - func (d *YarnDoctor) Diagnose(r parser_io.ReadSeekerAt, year int) map[string]Diagnosis { diagnoses := make(map[string]Diagnosis) slicedNameWithOwners := [][]github.NameWithOwner{} @@ -88,10 +61,11 @@ func (d *YarnDoctor) NameWithOwners(r parser_io.ReadSeekerAt) []github.NameWithO var nameWithOwners []github.NameWithOwner libs, _, _ := yarn.NewParser().Parse(r) + nodejs := Nodejs{} for _, lib := range libs { fmt.Printf("%s\n", lib.Name) - githubUrl, err := d.fetchURLFromRepository(lib.Name) + githubUrl, err := nodejs.fetchURLFromRegistry(lib.Name) if err != nil { nameWithOwners = append(nameWithOwners, github.NameWithOwner{