dojo7 [課題3-2] 分割ダウンローダを作ろう

kadai3_2/asuke-yasukuni/go.mod

@@ -0,0 +1,8 @@
+module github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni
+
+go 1.12
+
+require (
+	golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+)

kadai3_2/asuke-yasukuni/go.sum

@@ -0,0 +1,7 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 h1:PdU68SuVQNpTFEyGl0zoQOMysY+E0innv/QbAqV853w=
+golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

kadai3_2/asuke-yasukuni/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni/muget"
+)
+
+var get = flag.String("get", "", "ダウンロードパス")
+var out = flag.String("out", "", "ファイルパス")
+
+func main() {
+	flag.Parse()
+
+	log.Println("Download Start")
+	log.Println(*get)
+	log.Println(*out)
+
+	if err := muget.Run(*get, *out); err != nil {
+		log.Fatal(err)
+	}
+
+	os.Exit(0)
+}

kadai3_2/asuke-yasukuni/muget/checker.go

@@ -0,0 +1,27 @@
+package muget
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"golang.org/x/net/context/ctxhttp"
+)
+
+func CheckRanges(ctx context.Context, url string) (int, error) {
+	res, err := ctxhttp.Head(ctx, http.DefaultClient, url)
+	if err != nil {
+		return 0, err
+	}
+
+	if res.Header.Get("Accept-Ranges") != "bytes" {
+		return 0, fmt.Errorf("not supported range access: %s", url)
+	}
+
+	if res.ContentLength <= 0 {
+		return 0, errors.New("invalid content length")
+	}
+
+	return int(res.ContentLength), nil
+}

kadai3_2/asuke-yasukuni/muget/download.go

@@ -0,0 +1,57 @@
+package muget
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+func DownloadFile(url string, path string, downloadStartSize, downloadEndSize, downloadCount int) (err error) {
+	// Create the file
+	out, err := os.Create(path + fmt.Sprint(downloadCount) + filepath.Ext(url))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = out.Close()
+		if err != nil {
+			return
+		}
+	}()
+
+	// Get the data
+	resp, err := RangeRequest(url, downloadStartSize, downloadEndSize)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = resp.Body.Close()
+		if err != nil {
+			return
+		}
+	}()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RangeRequest return *http.Response include context and range header
+func RangeRequest(url string, low, high int) (*http.Response, error) {
+	// create get request
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// set download ranges
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high))
+
+	return http.DefaultClient.Do(req)
+}

kadai3_2/asuke-yasukuni/muget/merge.go

@@ -0,0 +1,48 @@
+package muget
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+func MergeFiles(count int, fileName, ext string) (err error) {
+
+	fh, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = fh.Close()
+		if err != nil {
+			return
+		}
+	}()
+
+	var f string
+	for i := 0; i < count; i++ {
+		f = fmt.Sprintf("./%d%s", i, ext)
+		openFile, err := os.Open(f)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(fh, openFile)
+		if err != nil {
+			return err
+		}
+
+		err = openFile.Close()
+		if err != nil {
+			return err
+		}
+
+		// remove a file in download location for join
+		err = os.Remove(f)
+		if err != nil {
+			return err
+		}
+	}
+
+	return
+}

kadai3_2/asuke-yasukuni/muget/run.go

@@ -0,0 +1,59 @@
+package muget
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"golang.org/x/sync/errgroup"
+)
+
+type Range struct {
+	Start int
+	End   int
+}
+
+func Run(url, outPutPath string) error {
+	size, err := CheckRanges(context.Background(), url)
+	if err != nil {
+		return err
+	}
+
+	var (
+		start, end int
+		ranges     []Range
+	)
+
+	for start <= size {
+		end = start + (size / 10)
+		ranges = append(ranges, Range{
+			Start: start,
+			End:   end,
+		})
+		start = end
+	}
+
+	// TODO: contextとかつかってうまくキャンセルしてあげる
+	eg := errgroup.Group{}
+	for i, r := range ranges {
+		i := i
+		r := r
+		eg.Go(func() error {
+			return DownloadFile(url, outPutPath, r.Start, r.End, i)
+		})
+	}
+
+	//ダウンロード完了まで待つ
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	fmt.Println("\nbinding with files...")
+
+	//ダウンロードファイルをマージ
+	if err := MergeFiles(len(ranges), filepath.Base(url), filepath.Ext(url)); err != nil {
+		return err
+	}
+
+	return nil
+}