diff --git a/kadai3_2/asuke-yasukuni/go.mod b/kadai3_2/asuke-yasukuni/go.mod new file mode 100644 index 0000000..f6f988f --- /dev/null +++ b/kadai3_2/asuke-yasukuni/go.mod @@ -0,0 +1,8 @@ +module github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni + +go 1.12 + +require ( + golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +) diff --git a/kadai3_2/asuke-yasukuni/go.sum b/kadai3_2/asuke-yasukuni/go.sum new file mode 100644 index 0000000..dd99482 --- /dev/null +++ b/kadai3_2/asuke-yasukuni/go.sum @@ -0,0 +1,7 @@ +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 h1:PdU68SuVQNpTFEyGl0zoQOMysY+E0innv/QbAqV853w= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/kadai3_2/asuke-yasukuni/main.go b/kadai3_2/asuke-yasukuni/main.go new file mode 100644 index 0000000..9e586f9 --- /dev/null +++ b/kadai3_2/asuke-yasukuni/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "log" + "os" + + "github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni/muget" +) + +var get = flag.String("get", "", "ダウンロードパス") +var out = flag.String("out", "", "ファイルパス") + +func main() { + flag.Parse() + + log.Println("Download Start") + log.Println(*get) + log.Println(*out) + + if err := muget.Run(*get, *out); err != nil { + log.Fatal(err) + } + + os.Exit(0) +} diff --git a/kadai3_2/asuke-yasukuni/muget/checker.go b/kadai3_2/asuke-yasukuni/muget/checker.go new file mode 100644 index 0000000..d3a6720 --- /dev/null +++ b/kadai3_2/asuke-yasukuni/muget/checker.go @@ -0,0 +1,27 @@ +package muget + +import ( + "context" + "errors" + "fmt" + "net/http" + + "golang.org/x/net/context/ctxhttp" +) + +func CheckRanges(ctx context.Context, url string) (int, error) { + res, err := ctxhttp.Head(ctx, http.DefaultClient, url) + if err != nil { + return 0, err + } + + if res.Header.Get("Accept-Ranges") != "bytes" { + return 0, fmt.Errorf("not supported range access: %s", url) + } + + if res.ContentLength <= 0 { + return 0, errors.New("invalid content length") + } + + return int(res.ContentLength), nil +} diff --git a/kadai3_2/asuke-yasukuni/muget/download.go b/kadai3_2/asuke-yasukuni/muget/download.go new file mode 100644 index 0000000..06ce920 --- /dev/null +++ b/kadai3_2/asuke-yasukuni/muget/download.go @@ -0,0 +1,57 @@ +package muget + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +func DownloadFile(url string, path string, downloadStartSize, downloadEndSize, downloadCount int) (err error) { + // Create the file + out, err := os.Create(path + fmt.Sprint(downloadCount) + filepath.Ext(url)) + if err != nil { + return err + } + defer func() { + err = out.Close() + if err != nil { + return + } + }() + + // Get the data + resp, err := RangeRequest(url, downloadStartSize, downloadEndSize) + if err != nil { + return err + } + defer func() { + err = resp.Body.Close() + if err != nil { + return + } + }() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} + +// RangeRequest return *http.Response include context and range header +func RangeRequest(url string, low, high int) (*http.Response, error) { + // create get request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + // set download ranges + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) + + return http.DefaultClient.Do(req) +} diff --git a/kadai3_2/asuke-yasukuni/muget/merge.go b/kadai3_2/asuke-yasukuni/muget/merge.go new file mode 100644 index 0000000..eab6119 --- /dev/null +++ b/kadai3_2/asuke-yasukuni/muget/merge.go @@ -0,0 +1,48 @@ +package muget + +import ( + "fmt" + "io" + "os" +) + +func MergeFiles(count int, fileName, ext string) (err error) { + + fh, err := os.Create(fileName) + if err != nil { + return err + } + defer func() { + err = fh.Close() + if err != nil { + return + } + }() + + var f string + for i := 0; i < count; i++ { + f = fmt.Sprintf("./%d%s", i, ext) + openFile, err := os.Open(f) + if err != nil { + return err + } + + _, err = io.Copy(fh, openFile) + if err != nil { + return err + } + + err = openFile.Close() + if err != nil { + return err + } + + // remove a file in download location for join + err = os.Remove(f) + if err != nil { + return err + } + } + + return +} diff --git a/kadai3_2/asuke-yasukuni/muget/run.go b/kadai3_2/asuke-yasukuni/muget/run.go new file mode 100644 index 0000000..c12ea0e --- /dev/null +++ b/kadai3_2/asuke-yasukuni/muget/run.go @@ -0,0 +1,59 @@ +package muget + +import ( + "context" + "fmt" + "path/filepath" + + "golang.org/x/sync/errgroup" +) + +type Range struct { + Start int + End int +} + +func Run(url, outPutPath string) error { + size, err := CheckRanges(context.Background(), url) + if err != nil { + return err + } + + var ( + start, end int + ranges []Range + ) + + for start <= size { + end = start + (size / 10) + ranges = append(ranges, Range{ + Start: start, + End: end, + }) + start = end + } + + // TODO: contextとかつかってうまくキャンセルしてあげる + eg := errgroup.Group{} + for i, r := range ranges { + i := i + r := r + eg.Go(func() error { + return DownloadFile(url, outPutPath, r.Start, r.End, i) + }) + } + + //ダウンロード完了まで待つ + if err := eg.Wait(); err != nil { + return err + } + + fmt.Println("\nbinding with files...") + + //ダウンロードファイルをマージ + if err := MergeFiles(len(ranges), filepath.Base(url), filepath.Ext(url)); err != nil { + return err + } + + return nil +}