hokita / 課題3-2

kadai3-2/hokita/README.md

@@ -0,0 +1,45 @@
+# 課題3-2
+## 分割ダウンローダを作ろう
+- 分割ダウンロードを行う
+  - Rangeアクセスを用いる
+  - いくつかのゴルーチンでダウンロードしてマージする
+  - エラー処理を工夫する
+  - golang.org/x/sync/errgourpパッケージなどを使ってみる
+  - キャンセルが発生した場合の実装を行う
+
+### 動作
+```shell
+$ go build -o pdl cmd/pdl/main.go
+
+$ ./pdl -proc 10 https://blog.golang.org/gopher/header.jpg
+start download worker: 1
+start download worker: 10
+start download worker: 4
+start download worker: 2
+start download worker: 5
+start download worker: 7
+start download worker: 9
+start download worker: 6
+start download worker: 3
+start download worker: 8
+finish download worker: 4
+finish download worker: 8
+finish download worker: 5
+finish download worker: 3
+finish download worker: 7
+finish download worker: 9
+finish download worker: 1
+finish download worker: 10
+finish download worker: 2
+finish download worker: 6
+finished
+
+$ ls testdata/header.jpg
+testdata/header.jpg
+```
+
+## わからなかったこと、むずかしかったこと
+- そもそもurlからダウンロードをどう実現するのかを考えるのに時間がかかった。
+  - `pget`を参考にした。
+    - cf. https://github.com/Code-Hex/pget
+  - 結局はurlでアクセスして読み込んだ情報(`io.Reader`)をファイルに書き込む(`io.writer`)だけだった。

kadai3-2/hokita/cmd/pdl/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/gopherdojo/dojo8/kadai3-2/hokita/pdl"
+)
+
+const (
+	ExitCode    = 0
+	ExitCodeErr = 1
+)
+
+var proc int
+var dir string
+
+func init() {
+	flag.IntVar(&proc, "proc", 10, "split ratio to download file")
+	flag.StringVar(&dir, "dir", "testdata", "output file")
+}
+
+func main() {
+	flag.Parse()
+	exitCode := run(flag.Arg(0))
+	os.Exit(exitCode)
+}
+
+func run(url string) int {
+	cli, err := pdl.New(proc, url, dir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error:%v\n", err)
+		return ExitCodeErr
+	}
+
+	if err := cli.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error:%v\n", err)
+		return ExitCodeErr
+	}
+
+	return ExitCode
+}

kadai3-2/hokita/go.mod

@@ -0,0 +1,8 @@
+module github.com/gopherdojo/dojo8/kadai3-2/hokita/pdl
+
+go 1.14
+
+require (
+	github.com/pkg/errors v0.9.1
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+)

kadai3-2/hokita/go.sum

@@ -0,0 +1,4 @@
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

kadai3-2/hokita/pdl.go

@@ -0,0 +1,216 @@
+package pdl
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+type PDL struct {
+	url      string
+	proc     uint
+	fileSize uint
+	split    uint
+	dir      string
+	filename string
+}
+
+type Range struct {
+	start uint
+	end   uint
+}
+
+func New(proc int, url, dir string) (*PDL, error) {
+	if url == "" {
+		return nil, errors.New("no url specified")
+	}
+
+	pdl := &PDL{
+		proc: uint(proc),
+		url:  url,
+		dir:  dir,
+	}
+
+	pdl.setSize()
+	pdl.setFilename()
+
+	return pdl, nil
+}
+
+func (p *PDL) Run() error {
+	if err := p.download(); err != nil {
+		return err
+	}
+
+	if err := p.merge(); err != nil {
+		return err
+	}
+
+	fmt.Println("finished")
+	return nil
+}
+
+func (p *PDL) download() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	eg, egctx := errgroup.WithContext(ctx)
+
+	for i := 0; i < int(p.proc); i++ {
+		i := i
+		eg.Go(func() error {
+			return p.partialDownload(egctx, i)
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (p *PDL) merge() (rerr error) {
+	out, err := os.Create(p.filePath())
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to Create %s in Merge", p.filePath()))
+	}
+	defer func() {
+		if err := out.Close(); err != nil {
+			rerr = err
+		}
+	}()
+
+	for i := 0; i < int(p.proc); i++ {
+		worker := i + 1
+
+		tmpfile, err := os.Open(p.tmpFilePath(worker))
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to Open %s in Merge", p.tmpFilePath(worker)))
+		}
+
+		_, err = io.Copy(out, tmpfile)
+
+		// Not use defer
+		tmpfile.Close()
+
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to Copy %s in Merge", p.tmpFilePath(worker)))
+		}
+
+		// delete
+		if err := os.Remove(p.tmpFilePath(worker)); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to Remove %s in Merge", p.tmpFilePath(worker)))
+		}
+	}
+
+	return nil
+}
+
+func (p *PDL) makeRange(i, proc uint) Range {
+	start := p.split * i
+	end := start + p.split - 1
+	if i == proc-1 {
+		end = p.fileSize
+	}
+
+	return Range{
+		start: start,
+		end:   end,
+	}
+}
+
+func (p *PDL) setSize() error {
+	resp, err := http.Head(p.url)
+	if err != nil {
+		return errors.Wrap(err, "failed to get Head")
+	}
+
+	p.fileSize = uint(resp.ContentLength)
+	p.split = p.fileSize / p.proc
+
+	return nil
+}
+
+func (p *PDL) setFilename() {
+	token := strings.Split(p.url, "/")
+
+	var original string
+	for i := 1; original == ""; i++ {
+		original = token[len(token)-i]
+	}
+
+	p.filename = original
+}
+
+func (p *PDL) tmpFilename(worker int) string {
+	return fmt.Sprintf("%s.%d", p.filename, worker)
+}
+
+func (p *PDL) partialDownload(ctx context.Context, index int) error {
+	worker := index + 1
+
+	fmt.Printf("start download worker: %d\n", worker)
+
+	// request
+	req, err := http.NewRequest("GET", p.url, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to create NewRequest for GET")
+	}
+
+	r := p.makeRange(uint(index), p.proc)
+
+	// set header
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, r.end))
+
+	// do
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "failed to access")
+	}
+	defer resp.Body.Close()
+
+	// write
+	if err := p.writeTmpfile(resp.Body, int(worker)); err != nil {
+		return err
+	}
+
+	select {
+	case <-ctx.Done():
+		fmt.Printf("cancelled worker: %d\n", worker)
+		return ctx.Err()
+	default:
+		fmt.Printf("finish download worker: %d\n", worker)
+		return nil
+	}
+}
+
+func (p *PDL) filePath() string {
+	return filepath.Join(p.dir, p.filename)
+}
+
+func (p *PDL) tmpFilePath(worker int) string {
+	return filepath.Join(p.dir, p.tmpFilename(worker))
+}
+
+func (p *PDL) writeTmpfile(body io.Reader, worker int) (rerr error) {
+	out, err := os.Create(p.tmpFilePath(worker))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to create file, worker: %d", worker))
+	}
+	defer func() {
+		if err := out.Close(); err != nil {
+			rerr = errors.Wrap(err, fmt.Sprintf("failed to close file, worker: %d", worker))
+		}
+	}()
+
+	if _, err := io.Copy(out, body); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to write file, worker: %d", worker))
+	}
+
+	return nil
+}

kadai3-2/hokita/pdl_test.go

@@ -0,0 +1,42 @@
+package pdl
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestRun(t *testing.T) {
+	tests := map[string]struct {
+		proc int
+		url  string
+		dir  string
+		want string
+	}{
+		"download": {
+			proc: 3,
+			url:  "https://blog.golang.org/gopher/header.jpg",
+			dir:  "testdata",
+			want: filepath.Join("testdata", "header.jpg"),
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			pdl := New(test.proc, test.url, test.dir)
+			err := pdl.Run()
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err := os.Stat(test.want); err != nil {
+				t.Errorf(`"%v" was not found`, test.want)
+			}
+
+			if err := os.Remove(test.want); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}