diff --git a/.gitignore b/.gitignore index d9cde86..630d088 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ #* *~ stargazers +stargazer_cache/ \ No newline at end of file diff --git a/analyze/analyze.go b/analyze/analyze.go index 6d4b584..af72cd3 100644 --- a/analyze/analyze.go +++ b/analyze/analyze.go @@ -19,14 +19,13 @@ package analyze import ( "encoding/csv" "fmt" + "log" "os" "path/filepath" "sort" "strconv" "time" - "github.com/cockroachdb/cockroach/util" - "github.com/cockroachdb/cockroach/util/log" "github.com/spencerkimball/stargazers/fetch" ) @@ -111,17 +110,17 @@ func RunAll(c *fetch.Context, sg []*fetch.Stargazer, rs map[string]*fetch.Repo) // RunCumulativeStars creates a table of date and cumulative // star count for the provided stargazers. func RunCumulativeStars(c *fetch.Context, sg []*fetch.Stargazer) error { - log.Infof("running cumulative stars analysis") + log.Printf("running cumulative stars analysis") // Open file and prepare. f, err := createFile(c, "cumulative_stars.csv") if err != nil { - return util.Errorf("failed to create file: %s", err) + return fmt.Errorf("failed to create file: %s", err) } defer f.Close() w := csv.NewWriter(f) if err := w.Write([]string{"Date", "New", "Cumulative"}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } // Sort the stargazers. @@ -142,7 +141,7 @@ func RunCumulativeStars(c *fetch.Context, sg []*fetch.Stargazer) error { if count > 0 { t := time.Unix(lastDay*60*60*24, 0) if err := w.Write([]string{t.Format("01/02/2006"), strconv.Itoa(count), strconv.Itoa(total)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } lastDay = day @@ -155,11 +154,11 @@ func RunCumulativeStars(c *fetch.Context, sg []*fetch.Stargazer) error { if count > 0 { t := time.Unix(lastDay*60*60*24, 0) if err := w.Write([]string{t.Format("01/02/2006"), strconv.Itoa(count), strconv.Itoa(total)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } w.Flush() - log.Infof("wrote cumulative stars analysis to %s", f.Name()) + log.Printf("wrote cumulative stars analysis to %s", f.Name()) return nil } @@ -167,17 +166,17 @@ func RunCumulativeStars(c *fetch.Context, sg []*fetch.Stargazer) error { // RunCorrelatedRepos creates a map from repo name to count of // repos for repo lists of each stargazer. func RunCorrelatedRepos(c *fetch.Context, listType string, sg []*fetch.Stargazer, rs map[string]*fetch.Repo) error { - log.Infof("running correlated starred repos analysis") + log.Printf("running correlated starred repos analysis") // Open file and prepare. f, err := createFile(c, fmt.Sprintf("correlated_%s_repos.csv", listType)) if err != nil { - return util.Errorf("failed to create file: %s", err) + return fmt.Errorf("failed to create file: %s", err) } defer f.Close() w := csv.NewWriter(f) if err := w.Write([]string{"Repository", "URL", "Count", "Committers", "Commits", "Additions", "Deletions"}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } // Compute counts. counts := map[string]int{} @@ -205,21 +204,21 @@ func RunCorrelatedRepos(c *fetch.Context, listType string, sg []*fetch.Stargazer url := fmt.Sprintf("https://github.com/%s", rs[r.name].FullName) if err := w.Write([]string{r.name, url, strconv.Itoa(r.count), strconv.Itoa(len(rs[r.name].Statistics)), strconv.Itoa(c), strconv.Itoa(a), strconv.Itoa(d)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } w.Flush() - log.Infof("wrote correlated %s repos analysis to %s", listType, f.Name()) + log.Printf("wrote correlated %s repos analysis to %s", listType, f.Name()) // Open histogram file. fHist, err := createFile(c, fmt.Sprintf("correlated_%s_repos_hist.csv", listType)) if err != nil { - return util.Errorf("failed to create file: %s", err) + return fmt.Errorf("failed to create file: %s", err) } defer fHist.Close() wHist := csv.NewWriter(fHist) if err := wHist.Write([]string{"Correlation", "Count"}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } lastCorrelation := 0 count := 0 @@ -227,7 +226,7 @@ func RunCorrelatedRepos(c *fetch.Context, listType string, sg []*fetch.Stargazer if lastCorrelation != r.count { if count > 0 { if err := wHist.Write([]string{strconv.Itoa(lastCorrelation), strconv.Itoa(count)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } lastCorrelation = r.count @@ -238,11 +237,11 @@ func RunCorrelatedRepos(c *fetch.Context, listType string, sg []*fetch.Stargazer } if count > 0 { if err := wHist.Write([]string{strconv.Itoa(lastCorrelation), strconv.Itoa(count)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } wHist.Flush() - log.Infof("wrote correlated %s repos histogram to %s", listType, fHist.Name()) + log.Printf("wrote correlated %s repos histogram to %s", listType, fHist.Name()) return nil } @@ -250,17 +249,17 @@ func RunCorrelatedRepos(c *fetch.Context, listType string, sg []*fetch.Stargazer // RunFollowers computes the size of follower networks, as well as // the count of shared followers. func RunFollowers(c *fetch.Context, sg []*fetch.Stargazer) error { - log.Infof("running followers analysis") + log.Printf("running followers analysis") // Open file and prepare. f, err := createFile(c, "followers.csv") if err != nil { - return util.Errorf("failed to create file: %s", err) + return fmt.Errorf("failed to create file: %s", err) } defer f.Close() w := csv.NewWriter(f) if err := w.Write([]string{"Name", "Login", "URL", "Avatar URL", "Company", "Location", "Followers", "Shared Followers"}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } shared := map[string]int{} @@ -281,11 +280,11 @@ func RunFollowers(c *fetch.Context, sg []*fetch.Stargazer) error { } url := fmt.Sprintf("https://github.com/%s", s.Login) if err := w.Write([]string{s.Name, s.Login, url, s.AvatarURL, s.Company, s.Location, strconv.Itoa(s.User.Followers), strconv.Itoa(sharedCount)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } w.Flush() - log.Infof("wrote followers analysis to %s", f.Name()) + log.Printf("wrote followers analysis to %s", f.Name()) return nil } @@ -293,17 +292,17 @@ func RunFollowers(c *fetch.Context, sg []*fetch.Stargazer) error { // RunCommitters lists stargazers by commits to subscribed repos, from // most prolific committer to least. func RunCommitters(c *fetch.Context, sg []*fetch.Stargazer, rs map[string]*fetch.Repo) error { - log.Infof("running committers analysis") + log.Printf("running committers analysis") // Open file and prepare. f, err := createFile(c, "committers.csv") if err != nil { - return util.Errorf("failed to create file: %s", err) + return fmt.Errorf("failed to create file: %s", err) } defer f.Close() w := csv.NewWriter(f) if err := w.Write([]string{"Login", "Email", "Commits", "Additions", "Deletions"}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } // Sort the stargazers. @@ -317,11 +316,11 @@ func RunCommitters(c *fetch.Context, sg []*fetch.Stargazer, rs map[string]*fetch break } if err := w.Write([]string{s.Login, s.Email, strconv.Itoa(c), strconv.Itoa(a), strconv.Itoa(d)}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } } w.Flush() - log.Infof("wrote committers analysis to %s", f.Name()) + log.Printf("wrote committers analysis to %s", f.Name()) return nil } @@ -329,17 +328,17 @@ func RunCommitters(c *fetch.Context, sg []*fetch.Stargazer, rs map[string]*fetch // RunCumulativeStars creates a table of date and cumulative // star count for the provided stargazers. func RunAttributesByTime(c *fetch.Context, sg []*fetch.Stargazer, rs map[string]*fetch.Repo) error { - log.Infof("running stargazer attributes by time analysis") + log.Printf("running stargazer attributes by time analysis") // Open file and prepare. f, err := createFile(c, "attributes_by_time.csv") if err != nil { - return util.Errorf("failed to create file: %s", err) + return fmt.Errorf("failed to create file: %s", err) } defer f.Close() w := csv.NewWriter(f) if err := w.Write([]string{"Date", "New Stars", "Avg Age", "Avg Followers", "Avg Commits"}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } output := func(day int64, count, age, followers, commits int) error { @@ -348,7 +347,7 @@ func RunAttributesByTime(c *fetch.Context, sg []*fetch.Stargazer, rs map[string] avgFollowers := fmt.Sprintf("%.2f", float64(followers)/float64(count)) avgCommits := fmt.Sprintf("%.2f", float64(commits)/float64(count)) if err := w.Write([]string{t.Format("01/02/2006"), strconv.Itoa(count), avgAge, avgFollowers, avgCommits}); err != nil { - return util.Errorf("failed to write to CSV: %s", err) + return fmt.Errorf("failed to write to CSV: %s", err) } return nil } @@ -400,7 +399,7 @@ func RunAttributesByTime(c *fetch.Context, sg []*fetch.Stargazer, rs map[string] } } w.Flush() - log.Infof("wrote stargazer attributes by time analysis to %s", f.Name()) + log.Printf("wrote stargazer attributes by time analysis to %s", f.Name()) return nil } diff --git a/cmd/analyze.go b/cmd/analyze.go index 2ceeed4..b69fd52 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -18,8 +18,8 @@ package cmd import ( "errors" + "log" - "github.com/cockroachdb/cockroach/util/log" "github.com/spencerkimball/stargazers/analyze" "github.com/spencerkimball/stargazers/fetch" "github.com/spf13/cobra" @@ -51,19 +51,19 @@ func RunAnalyze(cmd *cobra.Command, args []string) error { if len(Repo) == 0 { return errors.New("repository not specified; use --repo=:owner/:repo") } - log.Infof("fetching saved GitHub stargazer data for repository %s", Repo) + log.Printf("fetching saved GitHub stargazer data for repository %s", Repo) fetchCtx := &fetch.Context{ Repo: Repo, CacheDir: CacheDir, } sg, rs, err := fetch.LoadState(fetchCtx) if err != nil { - log.Errorf("failed to load saved stargazer data: %s", err) + log.Printf("failed to load saved stargazer data: %s", err) return nil } - log.Infof("analyzing GitHub data for repository %s", Repo) + log.Printf("analyzing GitHub data for repository %s", Repo) if err := analyze.RunAll(fetchCtx, sg, rs); err != nil { - log.Errorf("failed to query stargazer data: %s", err) + log.Printf("failed to query stargazer data: %s", err) return nil } return nil diff --git a/cmd/clear.go b/cmd/clear.go index e972136..9bcdcaf 100644 --- a/cmd/clear.go +++ b/cmd/clear.go @@ -18,8 +18,8 @@ package cmd import ( "errors" + "log" - "github.com/cockroachdb/cockroach/util/log" "github.com/spencerkimball/stargazers/fetch" "github.com/spf13/cobra" ) @@ -41,13 +41,13 @@ func RunClear(cmd *cobra.Command, args []string) error { if len(Repo) == 0 { return errors.New("repository not specified; use --repo=:owner/:repo") } - log.Infof("clearing GitHub API response cache for repository %s", Repo) + log.Printf("clearing GitHub API response cache for repository %s", Repo) fetchCtx := &fetch.Context{ Repo: Repo, CacheDir: CacheDir, } if err := fetch.Clear(fetchCtx); err != nil { - log.Errorf("failed to clear cached responses: %s", err) + log.Printf("failed to clear cached responses: %s", err) return nil } return nil diff --git a/cmd/fetch.go b/cmd/fetch.go index 377c3e6..1ac3120 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -18,8 +18,8 @@ package cmd import ( "errors" + "log" - "github.com/cockroachdb/cockroach/util/log" "github.com/spencerkimball/stargazers/fetch" "github.com/spf13/cobra" ) @@ -50,14 +50,14 @@ func RunFetch(cmd *cobra.Command, args []string) error { if err != nil { return err } - log.Infof("fetching GitHub data for repository %s", Repo) + log.Printf("fetching GitHub data for repository %s", Repo) fetchCtx := &fetch.Context{ Repo: Repo, Token: token, CacheDir: CacheDir, } if err := fetch.QueryAll(fetchCtx); err != nil { - log.Errorf("failed to query stargazer data: %s", err) + log.Printf("failed to query stargazer data: %s", err) return nil } return nil diff --git a/fetch/cache.go b/fetch/cache.go index 1c8d62d..7f72771 100644 --- a/fetch/cache.go +++ b/fetch/cache.go @@ -21,13 +21,13 @@ import ( "bytes" "fmt" "io/ioutil" + "log" "net/http" "os" "path" "path/filepath" "strings" - "github.com/cockroachdb/cockroach/util/log" "github.com/kennygrant/sanitize" ) @@ -47,9 +47,8 @@ func getCache(c *Context, req *http.Request) (*http.Response, error) { } return nil, err } - if log.V(1) { - log.Infof("found %q in response cache", req.URL.String()) - } + log.Printf("found %q in response cache", req.URL.String()) + return resp, err } @@ -74,14 +73,12 @@ func putCache(c *Context, req *http.Request, resp *http.Response) error { return err } f.Close() - if log.V(1) { - log.Infof("wrote %q to response cache", req.URL.String()) - } + log.Printf("wrote %q to response cache", req.URL.String()) // TODO(spencer): this sucks, but we must re-read the response as // the body is closed during the call to resp.Write(). if readResp, err := readCachedResponse(filename, req); err != nil { - log.Errorf("failed reading cached response: %s", err) + log.Printf("failed reading cached response: %s", err) return err } else { resp.Body = readResp.Body diff --git a/fetch/fetch.go b/fetch/fetch.go index 904a887..e1c2b10 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -21,12 +21,11 @@ import ( "errors" "fmt" "io/ioutil" + "log" "net/http" "regexp" "strconv" "time" - - "github.com/cockroachdb/cockroach/util/log" ) // A rateLimitError is returned when the requestor's rate limit has @@ -106,15 +105,15 @@ func fetchURL(c *Context, url string, value interface{}, refresh bool) (string, switch t := err.(type) { case *rateLimitError: // Sleep until the expiration of the rate limit regime (+ 1s for clock offsets). - log.Error(t) + log.Printf("%s", t) time.Sleep(t.expiration()) case *httpError: // For now, regard HTTP errors as permanent. - log.Errorf("unable to fetch %q: %s", url, err) + log.Printf("unable to fetch %q: %s", url, err) return "", nil default: // Retry with exponential backoff on random connection and networking errors. - log.Error(t) + log.Printf("%s", t) backoff := int64((1 << i)) * 50000000 // nanoseconds, starting at 50ms if backoff > 1000000000 { backoff = 1000000000 @@ -124,7 +123,7 @@ func fetchURL(c *Context, url string, value interface{}, refresh bool) (string, } } if resp == nil { - log.Errorf("unable to fetch %q", url) + log.Printf("unable to fetch %q", url) return "", nil } @@ -151,7 +150,7 @@ func fetchURL(c *Context, url string, value interface{}, refresh bool) (string, if err != nil { // In the event corruption of the cached entry was encountered, clear // the entry and try again. - log.Errorf("cache entry %q corrupted; removing and refetching", url) + log.Printf("cache entry %q corrupted; removing and refetching", url) clearEntry(c, url) return fetchURL(c, url, value, refresh) } @@ -165,9 +164,7 @@ func fetchURL(c *Context, url string, value interface{}, refresh bool) (string, // cache on success. A rateLimitError is returned in the event that // the access token has exceeded its hourly limit. func doFetch(c *Context, url string, req *http.Request) (*http.Response, error) { - if log.V(1) { - log.Infof("fetching %q...", url) - } + log.Printf("fetching %q...", url) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err diff --git a/fetch/query.go b/fetch/query.go index d206e34..7016dfa 100644 --- a/fetch/query.go +++ b/fetch/query.go @@ -20,13 +20,12 @@ import ( "encoding/json" "errors" "fmt" + "log" "os" "path/filepath" "strconv" "strings" "time" - - "github.com/cockroachdb/cockroach/util/log" ) // TODO(spencer): this would all benefit from using a GitHub API @@ -197,7 +196,7 @@ func (s *Stargazer) Age() int64 { curDay := time.Now().Unix() createT, err := time.Parse(time.RFC3339, s.CreatedAt) if err != nil { - log.Errorf("failed to parse created at timestamp (%s): %s", s.CreatedAt, err) + log.Printf("failed to parse created at timestamp (%s): %s", s.CreatedAt, err) return 0 } return curDay - createT.Unix() @@ -255,7 +254,7 @@ func QueryAll(c *Context) error { func QueryStargazers(c *Context) ([]*Stargazer, error) { cCopy := *c cCopy.acceptHeader = "application/vnd.github.v3.star+json" - log.Infof("querying stargazers of repository %s", c.Repo) + log.Printf("querying stargazers of repository %s", c.Repo) url := fmt.Sprintf("%srepos/%s/stargazers", githubAPI, c.Repo) stargazers := []*Stargazer{} var err error @@ -275,7 +274,7 @@ func QueryStargazers(c *Context) ([]*Stargazer, error) { // QueryUserInfo queries user info for each stargazer. func QueryUserInfo(c *Context, sg []*Stargazer) error { - log.Infof("querying user info for each of %s stargazers...", format(len(sg))) + log.Printf("querying user info for each of %s stargazers...", format(len(sg))) fmt.Printf("*** user info for 0 stargazers") for i, s := range sg { if _, err := fetchURL(c, s.URL, &s.User, false); err != nil { @@ -289,7 +288,7 @@ func QueryUserInfo(c *Context, sg []*Stargazer) error { // QueryFollowers queries each stargazers list of followers. func QueryFollowers(c *Context, sg []*Stargazer) error { - log.Infof("querying followers for each of %s stargazers...", format(len(sg))) + log.Printf("querying followers for each of %s stargazers...", format(len(sg))) total := 0 fmt.Printf("*** 0 followers for 0 stargazers") uniqueFollowers := map[int]struct{}{} @@ -317,7 +316,7 @@ func QueryFollowers(c *Context, sg []*Stargazer) error { // QueryStarred queries all starred repos for each stargazer. func QueryStarred(c *Context, sg []*Stargazer, rs map[string]*Repo) error { - log.Infof("querying starred repos for each of %s stargazers...", format(len(sg))) + log.Printf("querying starred repos for each of %s stargazers...", format(len(sg))) starred := 0 fmt.Printf("*** 0 starred repos for 0 stargazers") uniqueStarred := map[int]struct{}{} @@ -349,7 +348,7 @@ func QueryStarred(c *Context, sg []*Stargazer, rs map[string]*Repo) error { // QuerySubscribed queries all subscribed repos for each stargazer. func QuerySubscribed(c *Context, sg []*Stargazer, rs map[string]*Repo) error { - log.Infof("querying subscribed repos for each of %s stargazers...", format(len(sg))) + log.Printf("querying subscribed repos for each of %s stargazers...", format(len(sg))) subscribed := 0 fmt.Printf("*** 0 subscribed repos for 0 stargazers") uniqueSubscribed := map[int]struct{}{} @@ -381,7 +380,7 @@ func QuerySubscribed(c *Context, sg []*Stargazer, rs map[string]*Repo) error { // QueryContributions queries all contributions to subscribed repos // for each stargazer. func QueryContributions(c *Context, sg []*Stargazer, rs map[string]*Repo) error { - log.Infof("querying contributions to subscribed repos for each of %s stargazers...", format(len(sg))) + log.Printf("querying contributions to subscribed repos for each of %s stargazers...", format(len(sg))) authors := map[string]struct{}{} for _, s := range sg { authors[s.Login] = struct{}{} @@ -447,7 +446,7 @@ func QueryStatistics(c *Context, r *Repo, authors map[string]struct{}) error { // SaveState writes all queried stargazer and repo data. func SaveState(c *Context, sg []*Stargazer, rs map[string]*Repo) error { - log.Infof("saving state") + log.Printf("saving state") filename := filepath.Join(c.CacheDir, c.Repo, "saved_state") f, err := os.Create(filename) if err != nil { @@ -455,11 +454,11 @@ func SaveState(c *Context, sg []*Stargazer, rs map[string]*Repo) error { } defer f.Close() enc := json.NewEncoder(f) - log.Infof("encoding stargazers data") + log.Printf("encoding stargazers data") if err := enc.Encode(sg); err != nil { return errors.New(fmt.Sprintf("failed to encode stargazer data: %s", err)) } - log.Infof("encoding repository data") + log.Printf("encoding repository data") if err := enc.Encode(rs); err != nil { return errors.New(fmt.Sprintf("failed to encode repo data: %s", err)) } @@ -468,7 +467,7 @@ func SaveState(c *Context, sg []*Stargazer, rs map[string]*Repo) error { // LoadState reads previously saved queried stargazer and repo data. func LoadState(c *Context) ([]*Stargazer, map[string]*Repo, error) { - log.Infof("loading state") + log.Printf("loading state") filename := filepath.Join(c.CacheDir, c.Repo, "saved_state") f, err := os.Open(filename) if err != nil { @@ -477,12 +476,12 @@ func LoadState(c *Context) ([]*Stargazer, map[string]*Repo, error) { defer f.Close() dec := json.NewDecoder(f) sg := []*Stargazer{} - log.Infof("decoding stargazers data") + log.Printf("decoding stargazers data") if err := dec.Decode(&sg); err != nil { return nil, nil, errors.New(fmt.Sprintf("failed to decode stargazer data: %s", err)) } rs := map[string]*Repo{} - log.Infof("decoding repository data") + log.Printf("decoding repository data") if err := dec.Decode(&rs); err != nil { return nil, nil, errors.New(fmt.Sprintf("failed to decode repo data: %s", err)) }