Skip to content
This repository has been archived by the owner on Nov 6, 2020. It is now read-only.

Commit

Permalink
Switch to pflag
Browse files Browse the repository at this point in the history
  • Loading branch information
Richard Patel committed Dec 12, 2018
1 parent d7bf3fd commit 0fe59cb
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 55 deletions.
79 changes: 51 additions & 28 deletions args.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,45 +2,68 @@ package main

import (
"fmt"
"github.com/akamensky/argparse"
"github.com/spf13/pflag"
"os"
)

var parser = argparse.NewParser("free-music-archive-scraper",
"Scraper for https://freemusicarchive.org/")
var genre = pflag.StringP("genre", "g", "", "Genre to archive")
var listGenres = pflag.Bool("list-genres", false, "Print a list of genres")
var concurrency = pflag.UintP("concurrency", "c", 4, "Number of concurrent downloads")
var minPage = pflag.Uint("min-page", 1, "Starting page")
var pageSize = pflag.Uint("page-size", 500, "Page size")
var dir = pflag.StringP("out-dir", "o", "Downloads", "Output directory")
var verbose = pflag.BoolP("verbose", "v", false, "More output")

var genre = parser.Selector("g", "genre", availableGenres[:], &argparse.Options{
Required: true,
Help: "Genre to scrape",
})

var concurrency = parser.Int("c", "concurrency", &argparse.Options{
Help: "Number of connections",
Default: 4,
})
var availableGenres = [...]string{
"Blues",
"Classical",
"Country",
"Electronic",
"Experimental",
"Folk",
"Hip-Hop",
"Instrumental",
"International",
"Jazz",
"Novelty",
"Old-Time__Historic",
"Pop",
"Rock",
"Soul-RB",
"Spoken",
}

var minPage = parser.Int("", "min-page", &argparse.Options{
Help: "Starting page",
Default: 1,
})
func parseArgs() error {
pflag.Usage = func() {
fmt.Fprintln(os.Stderr,
`Free Music Archive Scraper by terorie 2018
<< https://github.com/terorie/fma-scraper >>
var dir = parser.String("o", "out-dir", &argparse.Options{
Help: "Output directory",
Default: "Downloads",
})
Usage:`)
pflag.PrintDefaults()
}

var verbose = parser.Flag("v", "verbose", &argparse.Options{
Help: "More output",
})
pflag.Parse()

func parseArgs() {
if err := parser.Parse(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
if *listGenres {
fmt.Println("Available Genres:")
for _, g := range availableGenres {
fmt.Println(g)
}
os.Exit(1)
}

if *genre == "" {
return fmt.Errorf("-g/--genre flag required")
}

if *concurrency <= 0 {
return fmt.Errorf("invalid value for --concurrency")
}

if err := os.MkdirAll(*dir, 0777); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
return err
}

return nil
}
39 changes: 12 additions & 27 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,34 +12,13 @@ import (
"time"
)

var availableGenres = [...]string{
"Blues",
"Classical",
"Country",
"Electronic",
"Experimental",
"Folk",
"Hip-Hop",
"Instrumental",
"International",
"Jazz",
"Novelty",
"Old-Time__Historic",
"Pop",
"Rock",
"Soul-RB",
"Spoken",
}

var startTime = time.Now()
var totalBytes int64
var numDownloaded int64
var downloadGroup sync.WaitGroup
var helperGroup sync.WaitGroup
var exitRequested int32

const pageSize = 500

type Track struct {
Artist string `json:"artist"`
Title string `json:"title"`
Expand All @@ -49,10 +28,16 @@ type Track struct {
}

func main() {
parseArgs()
if err := parseArgs(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}

logrus.Info("Starting Free Music Archive Scraper")
logrus.Info(" https://github.com/terorie/fma-scraper/")

jobs := make(chan Track, 2 * pageSize)
results := make(chan Track, 2 * pageSize)
jobs := make(chan Track, 2 * *pageSize)
results := make(chan Track, 2 * *pageSize)

c, cancel := context.WithCancel(context.Background())

Expand All @@ -64,8 +49,8 @@ func main() {
go logger(results)

// Start downloaderss
downloadGroup.Add(*concurrency)
for i := 0; i < *concurrency; i++ {
downloadGroup.Add(int(*concurrency))
for i := 0; i < int(*concurrency); i++ {
go downloader(jobs, results)
}

Expand All @@ -77,7 +62,7 @@ func main() {
}

err := backoff.Retry(func() error {
err := list(c, jobs, *genre, page)
err := list(c, jobs, *genre, int(page))
if err != nil {
logrus.WithError(err).
Errorf("Failed visiting page %d", page)
Expand Down

0 comments on commit 0fe59cb

Please sign in to comment.