From e3af4bc2f72252ab342c8e9eb8cdc832a269c9a0 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 14:53:53 +0430 Subject: [PATCH 01/37] Update http.go --- http.go | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/http.go b/http.go index 642b027..c3de4e3 100644 --- a/http.go +++ b/http.go @@ -3,8 +3,6 @@ package main import ( "crypto/tls" "fmt" - "github.com/fatih/color" - pb "gopkg.in/cheggaaa/pb.v1" "io" "net" "net/http" @@ -14,6 +12,10 @@ import ( "strconv" "strings" "sync" + + "github.com/fatih/color" + "golang.org/x/net/proxy" + pb "gopkg.in/cheggaaa/pb.v1" ) var ( @@ -29,6 +31,7 @@ var ( ) type HttpDownloader struct { + proxy string url string file string par int64 @@ -39,9 +42,24 @@ type HttpDownloader struct { resumable bool } -func NewHttpDownloader(url string, par int, skipTls bool) *HttpDownloader { +func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) *HttpDownloader { var resumable = true + // setup a http client + httpTransport := &http.Transport{} + httpClient := &http.Client{Transport: httpTransport} + + // set our socks5 as the dialer + if len(socks5_proxy) > 0 { + // create a socks5 dialer + dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) + if err != nil { + fmt.Fprintln(os.Stderr, "can't connect to the proxy:", err) + os.Exit(1) + } + httpTransport.Dial = dialer.Dial + } + parsed, err := stdurl.Parse(url) FatalCheck(err) @@ -54,7 +72,7 @@ func NewHttpDownloader(url string, par int, skipTls bool) *HttpDownloader { req, err := http.NewRequest("GET", url, nil) FatalCheck(err) - resp, err := client.Do(req) + resp, err := httpClient.Do(req) FatalCheck(err) if resp.Header.Get(acceptRangeHeader) == "" { @@ -97,6 +115,7 @@ func NewHttpDownloader(url string, par int, skipTls bool) *HttpDownloader { ret.skipTls = skipTls ret.parts = partCalculate(int64(par), len, url) ret.resumable = resumable + ret.proxy = socks5_proxy return ret } @@ -145,6 +164,20 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan for i, p := range d.parts { ws.Add(1) go func(d *HttpDownloader, loop int64, part Part) { + // setup a http client + httpTransport := &http.Transport{} + httpClient := &http.Client{Transport: httpTransport} + + // set our socks5 as the dialer + if len(d.proxy) > 0 { + // create a socks5 dialer + dialer, err := proxy.SOCKS5("tcp", d.proxy, nil, proxy.Direct) + if err != nil { + fmt.Fprintln(os.Stderr, "can't connect to the proxy:", err) + os.Exit(1) + } + httpTransport.Dial = dialer.Dial + } defer ws.Done() var bar *pb.ProgressBar @@ -175,7 +208,7 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan } //write to file - resp, err := client.Do(req) + resp, err := httpClient.Do(req) if err != nil { errorChan <- err return From 144e418981401adeedf5efb248bf7e960f3101c1 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 14:55:26 +0430 Subject: [PATCH 02/37] Update main.go --- main.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 50f8059..129510d 100644 --- a/main.go +++ b/main.go @@ -13,9 +13,11 @@ var displayProgress = true func main() { var err error + var proxy string - conn := flag.Int("n", runtime.NumCPU(), "connection") + conn := flag.Int("n", runtime.NumCPU(), "connection") skiptls := flag.Bool("skip-tls", true, "skip verify certificate for https") + flag.StringVar(&proxy, "proxy", "", "socks5 proxy for downloading") flag.Parse() args := flag.Args() @@ -47,7 +49,7 @@ func main() { state, err := Resume(task) FatalCheck(err) - Execute(state.Url, state, *conn, *skiptls) + Execute(state.Url, state, *conn, *skiptls, proxy) return } else { if ExistDir(FolderOf(command)) { @@ -55,11 +57,11 @@ func main() { err := os.RemoveAll(FolderOf(command)) FatalCheck(err) } - Execute(command, nil, *conn, *skiptls) + Execute(command, nil, *conn, *skiptls, proxy) } } -func Execute(url string, state *State, conn int, skiptls bool) { +func Execute(url string, state *State, conn int, skiptls bool, proxy string) { //otherwise is hget command var err error @@ -84,7 +86,7 @@ func Execute(url string, state *State, conn int, skiptls bool) { var downloader *HttpDownloader if state == nil { - downloader = NewHttpDownloader(url, conn, skiptls) + downloader = NewHttpDownloader(url, conn, skiptls, proxy) } else { downloader = &HttpDownloader{url: state.Url, file: filepath.Base(state.Url), par: int64(len(state.Parts)), parts: state.Parts, resumable: true} } @@ -132,7 +134,7 @@ func Execute(url string, state *State, conn int, skiptls bool) { func usage() { Printf(`Usage: -hget [URL] [-n connection] [-skip-tls true] +hget [URL] [-n connection] [-skip-tls true] [-proxy socks5_proxy] hget tasks hget resume [TaskName] `) From 8575ef895203564422447ff74dc137ba44a4c455 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:01:44 +0430 Subject: [PATCH 03/37] Update http.go --- http.go | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/http.go b/http.go index c3de4e3..a18ba74 100644 --- a/http.go +++ b/http.go @@ -45,20 +45,7 @@ type HttpDownloader struct { func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) *HttpDownloader { var resumable = true - // setup a http client - httpTransport := &http.Transport{} - httpClient := &http.Client{Transport: httpTransport} - - // set our socks5 as the dialer - if len(socks5_proxy) > 0 { - // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) - if err != nil { - fmt.Fprintln(os.Stderr, "can't connect to the proxy:", err) - os.Exit(1) - } - httpTransport.Dial = dialer.Dial - } + client := ProxyAwareHttpClient(socks5_proxy) parsed, err := stdurl.Parse(url) FatalCheck(err) @@ -72,7 +59,7 @@ func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) * req, err := http.NewRequest("GET", url, nil) FatalCheck(err) - resp, err := httpClient.Do(req) + resp, err := client.Do(req) FatalCheck(err) if resp.Header.Get(acceptRangeHeader) == "" { @@ -145,6 +132,24 @@ func partCalculate(par int64, len int64, url string) []Part { return ret } +func ProxyAwareHttpClient(socks5_proxy string) *http.Client { + // setup a http client + httpTransport := &http.Transport{} + httpClient := &http.Client{Transport: httpTransport} + + // set our socks5 as the dialer + if len(socks5_proxy) > 0 { + // create a socks5 dialer + dialer, err := proxy.SOCKS5("tcp", d.proxy, nil, proxy.Direct) + if err != nil { + fmt.Fprintln(os.Stderr, "can't connect to the proxy:", err) + return httpClient + } + httpTransport.Dial = dialer.Dial + } + return httpClient +} + func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan chan error, interruptChan chan bool, stateSaveChan chan Part) { var ws sync.WaitGroup var bars []*pb.ProgressBar @@ -164,20 +169,7 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan for i, p := range d.parts { ws.Add(1) go func(d *HttpDownloader, loop int64, part Part) { - // setup a http client - httpTransport := &http.Transport{} - httpClient := &http.Client{Transport: httpTransport} - - // set our socks5 as the dialer - if len(d.proxy) > 0 { - // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", d.proxy, nil, proxy.Direct) - if err != nil { - fmt.Fprintln(os.Stderr, "can't connect to the proxy:", err) - os.Exit(1) - } - httpTransport.Dial = dialer.Dial - } + client := ProxyAwareHttpClient(d.proxy) defer ws.Done() var bar *pb.ProgressBar @@ -208,7 +200,7 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan } //write to file - resp, err := httpClient.Do(req) + resp, err := client.Do(req) if err != nil { errorChan <- err return From 64314c835ba12ee4f0fbf3c8dd6dc719c8839f08 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:03:21 +0430 Subject: [PATCH 04/37] Update http.go --- http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http.go b/http.go index a18ba74..3dd66e1 100644 --- a/http.go +++ b/http.go @@ -140,9 +140,9 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { // set our socks5 as the dialer if len(socks5_proxy) > 0 { // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", d.proxy, nil, proxy.Direct) + dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) if err != nil { - fmt.Fprintln(os.Stderr, "can't connect to the proxy:", err) + fmt.Fprintln(os.Stderr, "can't connect to the proxy: ", err) return httpClient } httpTransport.Dial = dialer.Dial From 4eea90b79399f31de7a76ef4ebebd88f4ec9092a Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:21:34 +0430 Subject: [PATCH 05/37] Update http.go --- http.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/http.go b/http.go index 3dd66e1..925c61c 100644 --- a/http.go +++ b/http.go @@ -136,14 +136,24 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { // setup a http client httpTransport := &http.Transport{} httpClient := &http.Client{Transport: httpTransport} - + dialer := nil // set our socks5 as the dialer if len(socks5_proxy) > 0 { - // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) - if err != nil { - fmt.Fprintln(os.Stderr, "can't connect to the proxy: ", err) - return httpClient + if strings.HasPrefix(socks5_proxy, "http"){ + proxyUrl, err := url.Parse(proxyServer) + if err != nil { + fmt.Fprintln(os.Stderr, "invalid proxy: ", err) + } + dialer, err = proxy.FromURL(proxyUrl, proxy.Direct) + if err != nil { + fmt.Fprintln(os.Stderr, "can't connect to the proxy: ", err) + } + }else{ + // create a socks5 dialer + dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) + if err != nil { + fmt.Fprintln(os.Stderr, "can't connect to the proxy: ", err) + } } httpTransport.Dial = dialer.Dial } From 9accef4be2d735cc768653c3ecbb3f46c2859b9c Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:23:22 +0430 Subject: [PATCH 06/37] Update main.go --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 129510d..bd33b72 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,7 @@ func main() { conn := flag.Int("n", runtime.NumCPU(), "connection") skiptls := flag.Bool("skip-tls", true, "skip verify certificate for https") - flag.StringVar(&proxy, "proxy", "", "socks5 proxy for downloading") + flag.StringVar(&proxy, "proxy", "", "proxy for downloading, ex \n\t-proxy '127.0.0.1:12345' for socks5 proxy\n\t-proxy 'http://proxy.com:8080' for http proxy") flag.Parse() args := flag.Args() @@ -134,7 +134,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string) { func usage() { Printf(`Usage: -hget [URL] [-n connection] [-skip-tls true] [-proxy socks5_proxy] +hget [URL] [-n connection] [-skip-tls true] [-proxy proxy_address] hget tasks hget resume [TaskName] `) From 37964d77f9d69b7f37fbfc33ca2386cfbbc7c638 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:26:02 +0430 Subject: [PATCH 07/37] Update http.go --- http.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/http.go b/http.go index 925c61c..04bbe45 100644 --- a/http.go +++ b/http.go @@ -136,26 +136,26 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { // setup a http client httpTransport := &http.Transport{} httpClient := &http.Client{Transport: httpTransport} - dialer := nil + // set our socks5 as the dialer if len(socks5_proxy) > 0 { if strings.HasPrefix(socks5_proxy, "http"){ - proxyUrl, err := url.Parse(proxyServer) + proxyUrl, err := stdurl.Parse(socks5_proxy) if err != nil { fmt.Fprintln(os.Stderr, "invalid proxy: ", err) } dialer, err = proxy.FromURL(proxyUrl, proxy.Direct) - if err != nil { - fmt.Fprintln(os.Stderr, "can't connect to the proxy: ", err) + if err == nil { + httpTransport.Dial = dialer.Dial } }else{ // create a socks5 dialer dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) - if err != nil { - fmt.Fprintln(os.Stderr, "can't connect to the proxy: ", err) + if err == nil { + httpTransport.Dial = dialer.Dial } } - httpTransport.Dial = dialer.Dial + } return httpClient } From 2a1c3d6829715ece562db6545d4ccd03d3b89c9f Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:27:05 +0430 Subject: [PATCH 08/37] Update http.go --- http.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/http.go b/http.go index 04bbe45..099c8a8 100644 --- a/http.go +++ b/http.go @@ -136,21 +136,23 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { // setup a http client httpTransport := &http.Transport{} httpClient := &http.Client{Transport: httpTransport} + var dialer proxy.Dialer + dialer = proxy.Direct - // set our socks5 as the dialer if len(socks5_proxy) > 0 { if strings.HasPrefix(socks5_proxy, "http"){ proxyUrl, err := stdurl.Parse(socks5_proxy) if err != nil { fmt.Fprintln(os.Stderr, "invalid proxy: ", err) } + // create a http dialer dialer, err = proxy.FromURL(proxyUrl, proxy.Direct) if err == nil { httpTransport.Dial = dialer.Dial } }else{ // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) + dialer, err = proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) if err == nil { httpTransport.Dial = dialer.Dial } From 1115cd83ed533e2608ca60ceea8e772c187b4ebb Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:28:49 +0430 Subject: [PATCH 09/37] Update http.go --- http.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http.go b/http.go index 099c8a8..7c0666d 100644 --- a/http.go +++ b/http.go @@ -140,7 +140,7 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { dialer = proxy.Direct if len(socks5_proxy) > 0 { - if strings.HasPrefix(socks5_proxy, "http"){ + if strings.HasPrefix(socks5_proxy, "http") { proxyUrl, err := stdurl.Parse(socks5_proxy) if err != nil { fmt.Fprintln(os.Stderr, "invalid proxy: ", err) @@ -150,9 +150,9 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { if err == nil { httpTransport.Dial = dialer.Dial } - }else{ + } else { // create a socks5 dialer - dialer, err = proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) + dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) if err == nil { httpTransport.Dial = dialer.Dial } From 7fb70cd15d272a52869bb83594e0a1aeb6f13ec5 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:30:59 +0430 Subject: [PATCH 10/37] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3bb3449..ca669e7 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,11 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/ ## Usage ``` -hget [Url] [-n parallel] [-skip-tls false] //to download url, with n connections, and not skip tls certificate +hget [Url] [-n parallel] [-skip-tls false] [-proxy proxy_server]://to download url, with n connections, and not skip tls certificate hget tasks //get interrupted tasks hget resume [TaskName | URL] //to resume task +hget -proxy "127.0.0.1:12345" https://releases.ubuntu.com/20.04.1/ubuntu-20.04.1-desktop-amd64.iso # to download using socks5 proxy +hget -proxy "http://sample-proxy.com:8080" https://releases.ubuntu.com/20.04.1/ubuntu-20.04.1-desktop-amd64.iso # to download using http proxy ``` To interrupt any on-downloading process, just ctrl-c or ctrl-d at the middle of the download, hget will safely save your data and you will be able to resume later From 5ed3d645d4d06593941fb5c29195c8b0bc33dc7d Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:31:41 +0430 Subject: [PATCH 11/37] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca669e7..701a269 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/ hget [Url] [-n parallel] [-skip-tls false] [-proxy proxy_server]://to download url, with n connections, and not skip tls certificate hget tasks //get interrupted tasks hget resume [TaskName | URL] //to resume task -hget -proxy "127.0.0.1:12345" https://releases.ubuntu.com/20.04.1/ubuntu-20.04.1-desktop-amd64.iso # to download using socks5 proxy -hget -proxy "http://sample-proxy.com:8080" https://releases.ubuntu.com/20.04.1/ubuntu-20.04.1-desktop-amd64.iso # to download using http proxy +hget -proxy "127.0.0.1:12345" link # to download using socks5 proxy +hget -proxy "http://sample-proxy.com:8080" link # to download using http proxy ``` To interrupt any on-downloading process, just ctrl-c or ctrl-d at the middle of the download, hget will safely save your data and you will be able to resume later From 81d344f10aa9a3799aad651edd05a3468211be7c Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:32:16 +0430 Subject: [PATCH 12/37] Update README.md --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 701a269..8da644c 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,11 @@ This project is my personal project to learn golang to build something useful. ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) - -**Build Status**: [![Build Status](https://travis-ci.org/huydx/hget.svg?branch=master)](https://travis-ci.org/huydx/hget) - ## Install ``` -$ go get -d github.com/huydx/hget -$ cd $GOPATH/src/github.com/huydx/hget +$ go get -d github.com/abzcoding/hget +$ cd $GOPATH/src/github.com/abzcoding/hget $ make clean install ``` From bcc3af9accd2e2837be747208451edb8e5e6ce22 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:37:34 +0430 Subject: [PATCH 13/37] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8da644c..8199c06 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This project is my personal project to learn golang to build something useful. ## Install -``` +```bash $ go get -d github.com/abzcoding/hget $ cd $GOPATH/src/github.com/abzcoding/hget $ make clean install @@ -16,10 +16,10 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/ ## Usage -``` -hget [Url] [-n parallel] [-skip-tls false] [-proxy proxy_server]://to download url, with n connections, and not skip tls certificate -hget tasks //get interrupted tasks -hget resume [TaskName | URL] //to resume task +```bash +hget [Url] [-n parallel] [-skip-tls false] [-proxy proxy_server] # to download url, with n connections, and not skip tls certificate +hget tasks # get interrupted tasks +hget resume [TaskName | URL] # to resume task hget -proxy "127.0.0.1:12345" link # to download using socks5 proxy hget -proxy "http://sample-proxy.com:8080" link # to download using http proxy ``` From 24607c6513169631785e5b2b1be01809f5308cff Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:50:57 +0430 Subject: [PATCH 14/37] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8199c06..6acc5df 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,11 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/ ## Usage ```bash -hget [Url] [-n parallel] [-skip-tls false] [-proxy proxy_server] # to download url, with n connections, and not skip tls certificate +hget [-n parallel] [-skip-tls false] [-proxy proxy_server] URL # to download url, with n connections, and not skip tls certificate hget tasks # get interrupted tasks hget resume [TaskName | URL] # to resume task -hget -proxy "127.0.0.1:12345" link # to download using socks5 proxy -hget -proxy "http://sample-proxy.com:8080" link # to download using http proxy +hget -proxy "127.0.0.1:12345" URL # to download using socks5 proxy +hget -proxy "http://sample-proxy.com:8080" URL # to download using http proxy ``` To interrupt any on-downloading process, just ctrl-c or ctrl-d at the middle of the download, hget will safely save your data and you will be able to resume later From f0a12d8d050083b96fa46f81ab928c762ca42347 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:58:54 +0430 Subject: [PATCH 15/37] Update http.go --- http.go | 100 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/http.go b/http.go index 7c0666d..26924ad 100644 --- a/http.go +++ b/http.go @@ -44,9 +44,9 @@ type HttpDownloader struct { func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) *HttpDownloader { var resumable = true - + client := ProxyAwareHttpClient(socks5_proxy) - + parsed, err := stdurl.Parse(url) FatalCheck(err) @@ -108,7 +108,8 @@ func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) * } func partCalculate(par int64, len int64, url string) []Part { - ret := make([]Part, 0) + // Pre-allocate, perf tunning + ret := make([]Part, par) for j := int64(0); j < par; j++ { from := (len / par) * j var to int64 @@ -125,10 +126,12 @@ func partCalculate(par int64, len int64, url string) []Part { os.Exit(1) } - fname := fmt.Sprintf("%s.part%d", file, j) + // Padding 0 before path name as filename will be sorted as string + fname := fmt.Sprintf("%s.part%06d", file, j) path := filepath.Join(folder, fname) // ~/.hget/download-file-name/part-name - ret = append(ret, Part{Url: url, Path: path, RangeFrom: from, RangeTo: to}) + ret[j] = Part{Index: j, Url: url, Path: path, RangeFrom: from, RangeTo: to} } + return ret } @@ -168,26 +171,32 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan var barpool *pb.Pool var err error - if DisplayProgressBar() { - bars = make([]*pb.ProgressBar, 0) - for i, part := range d.parts { - newbar := pb.New64(part.RangeTo - part.RangeFrom).SetUnits(pb.U_BYTES).Prefix(color.YellowString(fmt.Sprintf("%s-%d", d.file, i))) - bars = append(bars, newbar) + for _, p := range d.parts { + + if p.RangeTo <= p.RangeFrom { + fileChan <- p.Path + stateSaveChan <- Part{ + Index: p.Index, + Url: d.url, + Path: p.Path, + RangeFrom: p.RangeFrom, + RangeTo: p.RangeTo, + } + + continue + } + + var bar *pb.ProgressBar + + if DisplayProgressBar() { + bar = pb.New64(p.RangeTo - p.RangeFrom).SetUnits(pb.U_BYTES).Prefix(color.YellowString(fmt.Sprintf("%s-%d", d.file, p.Index))) + bars = append(bars, bar) } - barpool, err = pb.StartPool(bars...) - FatalCheck(err) - } - for i, p := range d.parts { ws.Add(1) - go func(d *HttpDownloader, loop int64, part Part) { + go func(d *HttpDownloader, bar *pb.ProgressBar, part Part) { client := ProxyAwareHttpClient(d.proxy) defer ws.Done() - var bar *pb.ProgressBar - - if DisplayProgressBar() { - bar = bars[loop] - } var ranges string if part.RangeTo != d.len { @@ -234,29 +243,42 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan writer = io.MultiWriter(f) } - //make copy interruptable by copy 100 bytes each loop current := int64(0) - for { - select { - case <-interruptChan: - stateSaveChan <- Part{Url: d.url, Path: part.Path, RangeFrom: current + part.RangeFrom, RangeTo: part.RangeTo} - return - default: - written, err := io.CopyN(writer, resp.Body, 100) - current += written - if err != nil { - if err != io.EOF { - errorChan <- err - } - bar.Finish() - fileChan <- part.Path - return - } - } + finishDownloadChan := make(chan bool) + + go func() { + written, _ := io.Copy(writer, resp.Body) + current += written + fileChan <- part.Path + finishDownloadChan <- true + }() + + select { + case <-interruptChan: + // interrupt download by forcefully close the input stream + resp.Body.Close() + <-finishDownloadChan + case <-finishDownloadChan: + } + + stateSaveChan <- Part{ + Index: part.Index, + Url: d.url, + Path: part.Path, + RangeFrom: current + part.RangeFrom, + RangeTo: part.RangeTo, + } + + if DisplayProgressBar() { + bar.Update() + bar.Finish() } - }(d, int64(i), p) + }(d, bar, p) } + barpool, err = pb.StartPool(bars...) + FatalCheck(err) + ws.Wait() doneChan <- true barpool.Stop() From 756bfcbed404b8909f67c0a238121c91c160885b Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 15:59:33 +0430 Subject: [PATCH 16/37] Update http_test.go --- http_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/http_test.go b/http_test.go index 44181eb..dac53e6 100644 --- a/http_test.go +++ b/http_test.go @@ -1,9 +1,9 @@ package main import ( - "testing" "os/user" "path/filepath" + "testing" ) func TestPartCalculate(t *testing.T) { @@ -13,15 +13,22 @@ func TestPartCalculate(t *testing.T) { if len(parts) != 10 { t.Fatalf("parts length should be 10") } + if parts[0].Url != "http://foo.bar/file" { t.Fatalf("part url was wrong") } + usr, _ := user.Current() - dir := filepath.Join(usr.HomeDir, dataFolder, "file/file.part0") - if parts[0].Path != dir { + dir := filepath.Join(usr.HomeDir, dataFolder, "file/file.part000001") + if parts[1].Path != dir { t.Fatalf("part path was wrong") } + if parts[0].RangeFrom != 0 && parts[0].RangeTo != 10 { t.Fatalf("part range was wrong") } + + if parts[1].Index != 1 { + t.Fatal("part index was wrong") + } } From b42e4fa5fb099e7ee2f5ec28b07747a6a87b0441 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 16:00:03 +0430 Subject: [PATCH 17/37] Update joiner.go --- joiner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joiner.go b/joiner.go index dee8793..bb86427 100644 --- a/joiner.go +++ b/joiner.go @@ -1,8 +1,8 @@ package main import ( - "gopkg.in/cheggaaa/pb.v1" "github.com/fatih/color" + "gopkg.in/cheggaaa/pb.v1" "io" "os" "sort" From 089aeb735a893938135f676436303844acade7a1 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 16:01:29 +0430 Subject: [PATCH 18/37] Update main.go --- main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index bd33b72..9a49d7e 100644 --- a/main.go +++ b/main.go @@ -63,7 +63,6 @@ func main() { func Execute(url string, state *State, conn int, skiptls bool, proxy string) { //otherwise is hget command - var err error signal_chan := make(chan os.Signal, 1) signal.Notify(signal_chan, @@ -122,7 +121,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string) { return } } else { - err = JoinFile(files, filepath.Base(url)) + err := JoinFile(files, filepath.Base(url)) FatalCheck(err) err = os.RemoveAll(FolderOf(url)) FatalCheck(err) @@ -134,7 +133,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string) { func usage() { Printf(`Usage: -hget [URL] [-n connection] [-skip-tls true] [-proxy proxy_address] +hget [-n connection] [-skip-tls true] [-proxy proxy_address] URL hget tasks hget resume [TaskName] `) From fe6883d013aa1fe14f150576a37cb337ef358100 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Wed, 19 Aug 2020 16:01:58 +0430 Subject: [PATCH 19/37] Update state.go --- state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/state.go b/state.go index 529b4d8..d3fd0d7 100644 --- a/state.go +++ b/state.go @@ -16,6 +16,7 @@ type State struct { } type Part struct { + Index int64 Url string Path string RangeFrom int64 From 327fc8bf7cddd2df48b462f4c78631523d5ab475 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Sun, 30 Aug 2020 19:53:07 +0430 Subject: [PATCH 20/37] Update Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 9f7afc9..532a5ee 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ deps: go get -d github.com/fatih/color go get -d gopkg.in/cheggaaa/pb.v1 go get -d github.com/mattn/go-isatty + go get -d github.com/imkira/go-task clean: @echo "====> Remove installed binary" From b5c791ba4e51f3934ed560970a0e759466088357 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Sun, 30 Aug 2020 19:54:25 +0430 Subject: [PATCH 21/37] Update main.go --- main.go | 47 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 9a49d7e..3a26a37 100644 --- a/main.go +++ b/main.go @@ -1,30 +1,60 @@ package main import ( + "bufio" "flag" + "io" "os" "os/signal" "path/filepath" "runtime" "syscall" + + "github.com/imkira/go-task" ) var displayProgress = true func main() { var err error - var proxy string + var proxy, filepath string conn := flag.Int("n", runtime.NumCPU(), "connection") skiptls := flag.Bool("skip-tls", true, "skip verify certificate for https") flag.StringVar(&proxy, "proxy", "", "proxy for downloading, ex \n\t-proxy '127.0.0.1:12345' for socks5 proxy\n\t-proxy 'http://proxy.com:8080' for http proxy") + flag.StringVar(&filepath, "file", "", "filepath that contains links in each line") flag.Parse() args := flag.Args() if len(args) < 1 { - Errorln("url is required") - usage() - os.Exit(1) + if len(filepath) > 1 { + // Creating a SerialGroup. + g1 := task.NewSerialGroup() + file, err := os.Open(filepath) + if err != nil { + FatalCheck(err) + } + + defer file.Close() + + reader := bufio.NewReader(file) + + for { + line, _, err := reader.ReadLine() + + if err == io.EOF { + break + } + + g1.AddChild(downloadTask(string(line), nil, *conn, *skiptls, proxy)) + } + g1.Run(nil) + return + } else { + Errorln("url is required") + usage() + os.Exit(1) + } } command := args[0] @@ -61,6 +91,13 @@ func main() { } } +func downloadTask(url string, state *State, conn int, skiptls bool, proxy string) task.Task { + run := func(t task.Task, ctx task.Context) { + Execute(url, state, conn, skiptls, proxy) + } + return task.NewTaskWithFunc(run) +} + func Execute(url string, state *State, conn int, skiptls bool, proxy string) { //otherwise is hget command @@ -133,7 +170,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string) { func usage() { Printf(`Usage: -hget [-n connection] [-skip-tls true] [-proxy proxy_address] URL +hget [-n connection] [-skip-tls true] [-proxy proxy_address] [-file filename] URL hget tasks hget resume [TaskName] `) From 69c59b2f75cd32dcc18f39ec038f8ba0d68d31df Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Sun, 30 Aug 2020 19:55:35 +0430 Subject: [PATCH 22/37] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6acc5df..d24f25f 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,12 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/ ## Usage ```bash -hget [-n parallel] [-skip-tls false] [-proxy proxy_server] URL # to download url, with n connections, and not skip tls certificate +hget [-n parallel] [-skip-tls false] [-proxy proxy_server] [-file filename] [URL] # to download url, with n connections, and not skip tls certificate hget tasks # get interrupted tasks hget resume [TaskName | URL] # to resume task hget -proxy "127.0.0.1:12345" URL # to download using socks5 proxy hget -proxy "http://sample-proxy.com:8080" URL # to download using http proxy +hget -file sample.txt # to download a list of files ``` To interrupt any on-downloading process, just ctrl-c or ctrl-d at the middle of the download, hget will safely save your data and you will be able to resume later From 44e2abcb6fc107d41f97a0cc4a9c1bcb4d4085e2 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Sun, 30 Aug 2020 19:58:06 +0430 Subject: [PATCH 23/37] Update README.md --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index d24f25f..f5b0f29 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,22 @@ hget -proxy "http://sample-proxy.com:8080" URL # to download using http proxy hget -file sample.txt # to download a list of files ``` +### Help +``` +[I] ➜ hget -h +Usage of hget: + -file string + filepath that contains links in each line + -n int + connection (default 16) + -proxy string + proxy for downloading, ex + -proxy '127.0.0.1:12345' for socks5 proxy + -proxy 'http://proxy.com:8080' for http proxy + -skip-tls + skip verify certificate for https (default true) +``` + To interrupt any on-downloading process, just ctrl-c or ctrl-d at the middle of the download, hget will safely save your data and you will be able to resume later ### Download From 9aade6b679a3fc079d961294f0a9041e97d48622 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Thu, 10 Sep 2020 19:55:32 +0430 Subject: [PATCH 24/37] added bandwidth limiting to hget --- Makefile | 2 ++ README.md | 7 ++++++- http.go | 43 ++++++++++++++++++++++++++++++------------- main.go | 17 +++++++++-------- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 532a5ee..cd43b05 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ deps: go get -d gopkg.in/cheggaaa/pb.v1 go get -d github.com/mattn/go-isatty go get -d github.com/imkira/go-task + go get -d github.com/fujiwara/shapeio + go get -d github.com/alecthomas/units clean: @echo "====> Remove installed binary" diff --git a/README.md b/README.md index f5b0f29..51edab2 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,13 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/ ## Usage ```bash -hget [-n parallel] [-skip-tls false] [-proxy proxy_server] [-file filename] [URL] # to download url, with n connections, and not skip tls certificate +hget [-n parallel] [-skip-tls false] [-rate bwRate] [-proxy proxy_server] [-file filename] [URL] # to download url, with n connections, and not skip tls certificate hget tasks # get interrupted tasks hget resume [TaskName | URL] # to resume task hget -proxy "127.0.0.1:12345" URL # to download using socks5 proxy hget -proxy "http://sample-proxy.com:8080" URL # to download using http proxy hget -file sample.txt # to download a list of files +hget -n 4 -rate 100KB URL # to download using 4 threads & limited to 100Kb per second ``` ### Help @@ -37,6 +38,10 @@ Usage of hget: proxy for downloading, ex -proxy '127.0.0.1:12345' for socks5 proxy -proxy 'http://proxy.com:8080' for http proxy + -rate string + bandwidth limit to use while downloading, ex + -rate 10kB + -rate 10MiB -skip-tls skip verify certificate for https (default true) ``` diff --git a/http.go b/http.go index 26924ad..83e7219 100644 --- a/http.go +++ b/http.go @@ -13,7 +13,9 @@ import ( "strings" "sync" + "github.com/alecthomas/units" "github.com/fatih/color" + "github.com/fujiwara/shapeio" "golang.org/x/net/proxy" pb "gopkg.in/cheggaaa/pb.v1" ) @@ -30,8 +32,10 @@ var ( contentLengthHeader = "Content-Length" ) +// HttpDownloader holds the required configurations type HttpDownloader struct { proxy string + rate int64 url string file string par int64 @@ -42,11 +46,11 @@ type HttpDownloader struct { resumable bool } -func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) *HttpDownloader { +func NewHttpDownloader(url string, par int, skipTls bool, proxy_server string, bwLimit string) *HttpDownloader { var resumable = true - - client := ProxyAwareHttpClient(socks5_proxy) - + + client := ProxyAwareHttpClient(proxy_server) + parsed, err := stdurl.Parse(url) FatalCheck(err) @@ -94,6 +98,12 @@ func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) * file := filepath.Base(url) ret := new(HttpDownloader) + ret.rate = 0 + bandwidthLimit, err := units.ParseStrictBytes(bwLimit) + if err == nil { + ret.rate = bandwidthLimit + Printf("Download with bandwidth limit set to %s[%d]\n", bwLimit, ret.rate) + } ret.url = url ret.file = file ret.par = int64(par) @@ -102,7 +112,7 @@ func NewHttpDownloader(url string, par int, skipTls bool, socks5_proxy string) * ret.skipTls = skipTls ret.parts = partCalculate(int64(par), len, url) ret.resumable = resumable - ret.proxy = socks5_proxy + ret.proxy = proxy_server return ret } @@ -135,16 +145,16 @@ func partCalculate(par int64, len int64, url string) []Part { return ret } -func ProxyAwareHttpClient(socks5_proxy string) *http.Client { +func ProxyAwareHttpClient(proxy_server string) *http.Client { // setup a http client httpTransport := &http.Transport{} httpClient := &http.Client{Transport: httpTransport} var dialer proxy.Dialer dialer = proxy.Direct - - if len(socks5_proxy) > 0 { - if strings.HasPrefix(socks5_proxy, "http") { - proxyUrl, err := stdurl.Parse(socks5_proxy) + + if len(proxy_server) > 0 { + if strings.HasPrefix(proxy_server, "http") { + proxyUrl, err := stdurl.Parse(proxy_server) if err != nil { fmt.Fprintln(os.Stderr, "invalid proxy: ", err) } @@ -155,12 +165,12 @@ func ProxyAwareHttpClient(socks5_proxy string) *http.Client { } } else { // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", socks5_proxy, nil, proxy.Direct) + dialer, err := proxy.SOCKS5("tcp", proxy_server, nil, proxy.Direct) if err == nil { httpTransport.Dial = dialer.Dial } } - + } return httpClient } @@ -247,7 +257,14 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan finishDownloadChan := make(chan bool) go func() { - written, _ := io.Copy(writer, resp.Body) + var written int64 + if d.rate != 0 { + reader := shapeio.NewReader(resp.Body) + reader.SetRateLimit(float64(d.rate)) + written, _ = io.Copy(writer, reader) + } else { + written, _ = io.Copy(writer, resp.Body) + } current += written fileChan <- part.Path finishDownloadChan <- true diff --git a/main.go b/main.go index 3a26a37..6b5b301 100644 --- a/main.go +++ b/main.go @@ -17,12 +17,13 @@ var displayProgress = true func main() { var err error - var proxy, filepath string + var proxy, filepath, bwLimit string conn := flag.Int("n", runtime.NumCPU(), "connection") skiptls := flag.Bool("skip-tls", true, "skip verify certificate for https") flag.StringVar(&proxy, "proxy", "", "proxy for downloading, ex \n\t-proxy '127.0.0.1:12345' for socks5 proxy\n\t-proxy 'http://proxy.com:8080' for http proxy") flag.StringVar(&filepath, "file", "", "filepath that contains links in each line") + flag.StringVar(&bwLimit, "rate", "", "bandwidth limit to use while downloading, ex\n\t -rate 10kB\n\t-rate 10MiB") flag.Parse() args := flag.Args() @@ -46,7 +47,7 @@ func main() { break } - g1.AddChild(downloadTask(string(line), nil, *conn, *skiptls, proxy)) + g1.AddChild(downloadTask(string(line), nil, *conn, *skiptls, proxy, bwLimit)) } g1.Run(nil) return @@ -79,7 +80,7 @@ func main() { state, err := Resume(task) FatalCheck(err) - Execute(state.Url, state, *conn, *skiptls, proxy) + Execute(state.Url, state, *conn, *skiptls, proxy, bwLimit) return } else { if ExistDir(FolderOf(command)) { @@ -87,18 +88,18 @@ func main() { err := os.RemoveAll(FolderOf(command)) FatalCheck(err) } - Execute(command, nil, *conn, *skiptls, proxy) + Execute(command, nil, *conn, *skiptls, proxy, bwLimit) } } -func downloadTask(url string, state *State, conn int, skiptls bool, proxy string) task.Task { +func downloadTask(url string, state *State, conn int, skiptls bool, proxy string, bwLimit string) task.Task { run := func(t task.Task, ctx task.Context) { - Execute(url, state, conn, skiptls, proxy) + Execute(url, state, conn, skiptls, proxy, bwLimit) } return task.NewTaskWithFunc(run) } -func Execute(url string, state *State, conn int, skiptls bool, proxy string) { +func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwLimit string) { //otherwise is hget command signal_chan := make(chan os.Signal, 1) @@ -122,7 +123,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string) { var downloader *HttpDownloader if state == nil { - downloader = NewHttpDownloader(url, conn, skiptls, proxy) + downloader = NewHttpDownloader(url, conn, skiptls, proxy, bwLimit) } else { downloader = &HttpDownloader{url: state.Url, file: filepath.Base(state.Url), par: int64(len(state.Parts)), parts: state.Parts, resumable: true} } From 326db66af95c03aecd4dd63bcc857564aecced92 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Thu, 10 Sep 2020 20:08:26 +0430 Subject: [PATCH 25/37] Update README.md --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 51edab2..5e31c1f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ # hget -This project is my personal project to learn golang to build something useful. - ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) +### Features +- Multithreading +- Resume (interrupt & task mangement) +- Proxy ( socks5 or http) +- Download from a file list +- Bandwidth limiting -## Install +### Install ```bash $ go get -d github.com/abzcoding/hget @@ -14,7 +18,7 @@ $ make clean install Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/bin and even `alias wget hget` to replace wget totally :P -## Usage +### Usage ```bash hget [-n parallel] [-skip-tls false] [-rate bwRate] [-proxy proxy_server] [-file filename] [URL] # to download url, with n connections, and not skip tls certificate From 0208775cbee786395500098936cf796888e1bec7 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Thu, 10 Sep 2020 20:09:32 +0430 Subject: [PATCH 26/37] Update .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9502fea..12defce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ sudo: false language: go go: - - 1.6.2 + - 1.15.1 - tip script: - make clean install - - go test \ No newline at end of file + - go test From dd774a77eff5e16e5d6fedbcc030616225721abd Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Thu, 10 Sep 2020 20:11:25 +0430 Subject: [PATCH 27/37] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5e31c1f..1ecc388 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) ### Features -- Multithreading -- Resume (interrupt & task mangement) -- Proxy ( socks5 or http) -- Download from a file list +- Fast (multithreading & stuff) +- Ability to interrupt/resume (task mangement) +- Support for proxies( socks5 or http) - Bandwidth limiting +- You can give it a file that contains list of urls to download ### Install From ceb836bb745707c78e9b08afd7900c589e5b3ab0 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Tue, 15 Jun 2021 19:15:53 +0430 Subject: [PATCH 28/37] fixed build issue --- Makefile | 7 ++++++- go.mod | 15 +++++++++++++++ go.sum | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/Makefile b/Makefile index cd43b05..44e4854 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,11 @@ clean: @echo "====> Remove installed binary" rm -f bin/hget -install: deps +build: deps @echo "====> Build hget in ./bin " go build -ldflags "-X main.GitCommit=\"$(COMMIT)\"" -o bin/hget + +install: build + @echo "====> Installing hget in /usr/local/bin/hget" + chmod +x ./bin/hget + sudo mv ./bin/hget /usr/local/bin/hget diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2bb2366 --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module github.com/abzcoding/hget + +go 1.16 + +require ( + github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 + github.com/fatih/color v1.12.0 + github.com/fujiwara/shapeio v1.0.0 + github.com/imkira/go-task v1.0.0 + github.com/mattn/go-colorable v0.1.8 + github.com/mattn/go-isatty v0.0.13 + github.com/mattn/go-runewidth v0.0.13 // indirect + golang.org/x/net v0.0.0-20210614182718-04defd469f4e + gopkg.in/cheggaaa/pb.v1 v1.0.28 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7fa56e5 --- /dev/null +++ b/go.sum @@ -0,0 +1,43 @@ +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fujiwara/shapeio v1.0.0 h1:xG5D9oNqCSUUbryZ/jQV3cqe1v2suEjwPIcEg1gKM8M= +github.com/fujiwara/shapeio v1.0.0/go.mod h1:LmEmu6L/8jetyj1oewewFb7bZCNRwE7wLCUNzDLaLVA= +github.com/imkira/go-task v1.0.0 h1:r8RN5nLcmVpYf/UB28d1w4XApVxDntWLAsiExNIptsY= +github.com/imkira/go-task v1.0.0/go.mod h1:xU9xcPxKeBOQTwx8ILmT8xLxrm/SFmyBhPO8SlCRyRI= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From fae557f3ab334adb92db619060ef74a1c6b22585 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Mon, 21 Jun 2021 22:01:14 +0000 Subject: [PATCH 29/37] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1ecc388..a1ec2a4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.com/abzcoding/hget.svg?branch=master)](https://travis-ci.com/abzcoding/hget) +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/abzcoding/hget/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/abzcoding/hget/?branch=master) # hget ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) From 7d0c6fb591b342d505889a7037033b9eec7057a9 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Mon, 21 Jun 2021 22:06:50 +0000 Subject: [PATCH 30/37] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a1ec2a4..8956e67 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ [![Build Status](https://travis-ci.com/abzcoding/hget.svg?branch=master)](https://travis-ci.com/abzcoding/hget) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/abzcoding/hget/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/abzcoding/hget/?branch=master) +[![Maintainability](https://api.codeclimate.com/v1/badges/936e2aacab5946478295/maintainability)](https://codeclimate.com/github/abzcoding/hget/maintainability) +[![Test Coverage](https://api.codeclimate.com/v1/badges/936e2aacab5946478295/test_coverage)](https://codeclimate.com/github/abzcoding/hget/test_coverage) + # hget ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) From c22b586a7a939627c63020d49b6fc64be0755745 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Mon, 21 Jun 2021 22:10:07 +0000 Subject: [PATCH 31/37] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8956e67..2bbb099 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/abzcoding/hget/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/abzcoding/hget/?branch=master) [![Maintainability](https://api.codeclimate.com/v1/badges/936e2aacab5946478295/maintainability)](https://codeclimate.com/github/abzcoding/hget/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/936e2aacab5946478295/test_coverage)](https://codeclimate.com/github/abzcoding/hget/test_coverage) +[![codebeat +badge](https://codebeat.co/badges/ea357ae8-4d84-4599-bff7-cffc4f28fd67)](https://codebeat.co/projects/github-com-abzcoding-hget-master) # hget ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) From 264c222da4e2dcce3fa5d53752d483cb7cf65322 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Tue, 22 Jun 2021 02:45:38 +0430 Subject: [PATCH 32/37] removed unused returns --- main.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 6b5b301..2c6b1a0 100644 --- a/main.go +++ b/main.go @@ -149,22 +149,19 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwL if downloader.resumable { Printf("Interrupted, saving state ... \n") s := &State{Url: url, Parts: parts} - err := s.Save() - if err != nil { + if err := s.Save(); err != nil { Errorf("%v\n", err) } - return } else { Warnf("Interrupted, but downloading url is not resumable, silently die") - return } } else { err := JoinFile(files, filepath.Base(url)) FatalCheck(err) err = os.RemoveAll(FolderOf(url)) FatalCheck(err) - return } + return } } } From c2f968836d0488acbac85640d52578f591cdcc70 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Tue, 22 Jun 2021 02:56:24 +0430 Subject: [PATCH 33/37] cleaned up the code a bit --- http.go | 37 ++++++++++++++++++++----------------- main.go | 4 ++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/http.go b/http.go index 83e7219..cccedfc 100644 --- a/http.go +++ b/http.go @@ -32,8 +32,8 @@ var ( contentLengthHeader = "Content-Length" ) -// HttpDownloader holds the required configurations -type HttpDownloader struct { +// HTTPDownloader holds the required configurations +type HTTPDownloader struct { proxy string rate int64 url string @@ -41,15 +41,16 @@ type HttpDownloader struct { par int64 len int64 ips []string - skipTls bool + skipTLS bool parts []Part resumable bool } -func NewHttpDownloader(url string, par int, skipTls bool, proxy_server string, bwLimit string) *HttpDownloader { +// NewHttpDownloader returns a ProxyAwareHttpClient with given configurations. +func NewHttpDownloader(url string, par int, skipTLS bool, proxyServer string, bwLimit string) *HTTPDownloader { var resumable = true - client := ProxyAwareHttpClient(proxy_server) + client := ProxyAwareHTTPClient(proxyServer) parsed, err := stdurl.Parse(url) FatalCheck(err) @@ -97,7 +98,7 @@ func NewHttpDownloader(url string, par int, skipTls bool, proxy_server string, b } file := filepath.Base(url) - ret := new(HttpDownloader) + ret := new(HTTPDownloader) ret.rate = 0 bandwidthLimit, err := units.ParseStrictBytes(bwLimit) if err == nil { @@ -109,10 +110,10 @@ func NewHttpDownloader(url string, par int, skipTls bool, proxy_server string, b ret.par = int64(par) ret.len = len ret.ips = ipstr - ret.skipTls = skipTls + ret.skipTLS = skipTLS ret.parts = partCalculate(int64(par), len, url) ret.resumable = resumable - ret.proxy = proxy_server + ret.proxy = proxyServer return ret } @@ -145,27 +146,28 @@ func partCalculate(par int64, len int64, url string) []Part { return ret } -func ProxyAwareHttpClient(proxy_server string) *http.Client { +// ProxyAwareHTTPClient will use http or socks5 proxy if given one. +func ProxyAwareHTTPClient(proxyServer string) *http.Client { // setup a http client httpTransport := &http.Transport{} httpClient := &http.Client{Transport: httpTransport} var dialer proxy.Dialer dialer = proxy.Direct - if len(proxy_server) > 0 { - if strings.HasPrefix(proxy_server, "http") { - proxyUrl, err := stdurl.Parse(proxy_server) + if len(proxyServer) > 0 { + if strings.HasPrefix(proxyServer, "http") { + proxyURL, err := stdurl.Parse(proxyServer) if err != nil { fmt.Fprintln(os.Stderr, "invalid proxy: ", err) } // create a http dialer - dialer, err = proxy.FromURL(proxyUrl, proxy.Direct) + dialer, err = proxy.FromURL(proxyURL, proxy.Direct) if err == nil { httpTransport.Dial = dialer.Dial } } else { // create a socks5 dialer - dialer, err := proxy.SOCKS5("tcp", proxy_server, nil, proxy.Direct) + dialer, err := proxy.SOCKS5("tcp", proxyServer, nil, proxy.Direct) if err == nil { httpTransport.Dial = dialer.Dial } @@ -175,7 +177,8 @@ func ProxyAwareHttpClient(proxy_server string) *http.Client { return httpClient } -func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan chan error, interruptChan chan bool, stateSaveChan chan Part) { +// Do is where the magic happens. +func (d *HTTPDownloader) Do(doneChan chan bool, fileChan chan string, errorChan chan error, interruptChan chan bool, stateSaveChan chan Part) { var ws sync.WaitGroup var bars []*pb.ProgressBar var barpool *pb.Pool @@ -204,8 +207,8 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan } ws.Add(1) - go func(d *HttpDownloader, bar *pb.ProgressBar, part Part) { - client := ProxyAwareHttpClient(d.proxy) + go func(d *HTTPDownloader, bar *pb.ProgressBar, part Part) { + client := ProxyAwareHTTPClient(d.proxy) defer ws.Done() var ranges string diff --git a/main.go b/main.go index 2c6b1a0..268620a 100644 --- a/main.go +++ b/main.go @@ -121,11 +121,11 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwL stateChan := make(chan Part, 1) interruptChan := make(chan bool, conn) - var downloader *HttpDownloader + var downloader *HTTPDownloader if state == nil { downloader = NewHttpDownloader(url, conn, skiptls, proxy, bwLimit) } else { - downloader = &HttpDownloader{url: state.Url, file: filepath.Base(state.Url), par: int64(len(state.Parts)), parts: state.Parts, resumable: true} + downloader = &HTTPDownloader{url: state.Url, file: filepath.Base(state.Url), par: int64(len(state.Parts)), parts: state.Parts, resumable: true} } go downloader.Do(doneChan, fileChan, errorChan, interruptChan, stateChan) From f94c9e392573f9cd05d070fc368c04b73320463f Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Tue, 22 Jun 2021 03:22:43 +0430 Subject: [PATCH 34/37] cleaned up the code a bit --- http.go | 12 +++++------- http_test.go | 2 +- main.go | 8 ++++---- state.go | 6 +++--- ui.go | 11 +++++++++++ util.go | 16 ++++++++++++---- 6 files changed, 36 insertions(+), 19 deletions(-) diff --git a/http.go b/http.go index cccedfc..b8c6517 100644 --- a/http.go +++ b/http.go @@ -46,10 +46,9 @@ type HTTPDownloader struct { resumable bool } -// NewHttpDownloader returns a ProxyAwareHttpClient with given configurations. -func NewHttpDownloader(url string, par int, skipTLS bool, proxyServer string, bwLimit string) *HTTPDownloader { +// NewHTTPDownloader returns a ProxyAwareHttpClient with given configurations. +func NewHTTPDownloader(url string, par int, skipTLS bool, proxyServer string, bwLimit string) *HTTPDownloader { var resumable = true - client := ProxyAwareHTTPClient(proxyServer) parsed, err := stdurl.Parse(url) @@ -69,7 +68,6 @@ func NewHttpDownloader(url string, par int, skipTLS bool, proxyServer string, bw if resp.Header.Get(acceptRangeHeader) == "" { Printf("Target url is not supported range download, fallback to parallel 1\n") - //fallback to par = 1 par = 1 } @@ -140,7 +138,7 @@ func partCalculate(par int64, len int64, url string) []Part { // Padding 0 before path name as filename will be sorted as string fname := fmt.Sprintf("%s.part%06d", file, j) path := filepath.Join(folder, fname) // ~/.hget/download-file-name/part-name - ret[j] = Part{Index: j, Url: url, Path: path, RangeFrom: from, RangeTo: to} + ret[j] = Part{Index: j, URL: url, Path: path, RangeFrom: from, RangeTo: to} } return ret @@ -190,7 +188,7 @@ func (d *HTTPDownloader) Do(doneChan chan bool, fileChan chan string, errorChan fileChan <- p.Path stateSaveChan <- Part{ Index: p.Index, - Url: d.url, + URL: d.url, Path: p.Path, RangeFrom: p.RangeFrom, RangeTo: p.RangeTo, @@ -283,7 +281,7 @@ func (d *HTTPDownloader) Do(doneChan chan bool, fileChan chan string, errorChan stateSaveChan <- Part{ Index: part.Index, - Url: d.url, + URL: d.url, Path: part.Path, RangeFrom: current + part.RangeFrom, RangeTo: part.RangeTo, diff --git a/http_test.go b/http_test.go index dac53e6..c7948ab 100644 --- a/http_test.go +++ b/http_test.go @@ -14,7 +14,7 @@ func TestPartCalculate(t *testing.T) { t.Fatalf("parts length should be 10") } - if parts[0].Url != "http://foo.bar/file" { + if parts[0].URL != "http://foo.bar/file" { t.Fatalf("part url was wrong") } diff --git a/main.go b/main.go index 268620a..9f91ef5 100644 --- a/main.go +++ b/main.go @@ -80,7 +80,7 @@ func main() { state, err := Resume(task) FatalCheck(err) - Execute(state.Url, state, *conn, *skiptls, proxy, bwLimit) + Execute(state.URL, state, *conn, *skiptls, proxy, bwLimit) return } else { if ExistDir(FolderOf(command)) { @@ -123,9 +123,9 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwL var downloader *HTTPDownloader if state == nil { - downloader = NewHttpDownloader(url, conn, skiptls, proxy, bwLimit) + downloader = NewHTTPDownloader(url, conn, skiptls, proxy, bwLimit) } else { - downloader = &HTTPDownloader{url: state.Url, file: filepath.Base(state.Url), par: int64(len(state.Parts)), parts: state.Parts, resumable: true} + downloader = &HTTPDownloader{url: state.URL, file: filepath.Base(state.URL), par: int64(len(state.Parts)), parts: state.Parts, resumable: true} } go downloader.Do(doneChan, fileChan, errorChan, interruptChan, stateChan) @@ -148,7 +148,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwL if isInterrupted { if downloader.resumable { Printf("Interrupted, saving state ... \n") - s := &State{Url: url, Parts: parts} + s := &State{URL: url, Parts: parts} if err := s.Save(); err != nil { Errorf("%v\n", err) } diff --git a/state.go b/state.go index d3fd0d7..38f15eb 100644 --- a/state.go +++ b/state.go @@ -11,13 +11,13 @@ var dataFolder = ".hget/" var stateFileName = "state.json" type State struct { - Url string + URL string Parts []Part } type Part struct { Index int64 - Url string + URL string Path string RangeFrom int64 RangeTo int64 @@ -26,7 +26,7 @@ type Part struct { func (s *State) Save() error { //make temp folder //only working in unix with env HOME - folder := FolderOf(s.Url) + folder := FolderOf(s.URL) Printf("Saving current download data in %s\n", folder) if err := MkdirIfNotExist(folder); err != nil { return err diff --git a/ui.go b/ui.go index d3d00a0..1cd066c 100644 --- a/ui.go +++ b/ui.go @@ -16,6 +16,7 @@ var ( Default UI = Console{Stdout: Stdout, Stderr: Stderr} ) +// UI represents a simple IO output. type UI interface { Printf(format string, a ...interface{}) (n int, err error) Println(a ...interface{}) (n int, err error) @@ -23,43 +24,53 @@ type UI interface { Errorln(a ...interface{}) (n int, err error) } +// Printf outputs information level logs func Printf(format string, a ...interface{}) (n int, err error) { return Default.Printf(color.CyanString("INFO: ")+format, a...) } +// Errorf outputs error level logs func Errorf(format string, a ...interface{}) (n int, err error) { return Default.Errorf(color.RedString("ERROR: ")+format, a...) } +// Warnf outputs warning level logs func Warnf(format string, a ...interface{}) (n int, err error) { return Default.Errorf(color.YellowString("WARN: ")+format, a...) } +// Errorln is non formatted error printer. func Errorln(a ...interface{}) (n int, err error) { return Default.Errorln(a...) } +// IsTerminal checks if we have tty func IsTerminal(f *os.File) bool { return isatty.IsTerminal(f.Fd()) } +// Console is an implementation of UI interface type Console struct { Stdout io.Writer Stderr io.Writer } +// Printf prints formatted information logs to Stdout func (c Console) Printf(format string, a ...interface{}) (n int, err error) { return fmt.Fprintf(c.Stdout, format, a...) } +// Println prints information logs to Stdout func (c Console) Println(a ...interface{}) (n int, err error) { return fmt.Fprintln(c.Stdout, a...) } +// Errorf prints formatted error logs to Stderr func (c Console) Errorf(format string, a ...interface{}) (n int, err error) { return fmt.Fprintf(c.Stderr, format, a...) } +// Errorln prints error logs to Stderr func (c Console) Errorln(a ...interface{}) (n int, err error) { return fmt.Fprintln(c.Stderr, a...) } diff --git a/util.go b/util.go index 72f9b1e..1a2743b 100644 --- a/util.go +++ b/util.go @@ -1,15 +1,16 @@ package main import ( + "errors" + "github.com/mattn/go-isatty" "net" + "net/url" "os" - "github.com/mattn/go-isatty" "path/filepath" - "errors" "strings" - "net/url" ) +// FatalCheck panics if err is not nil. func FatalCheck(err error) { if err != nil { Errorf("%v", err) @@ -17,6 +18,7 @@ func FatalCheck(err error) { } } +// FilterIPV4 returns parsed ipv4 string. func FilterIPV4(ips []net.IP) []string { var ret = make([]string, 0) for _, ip := range ips { @@ -27,6 +29,7 @@ func FilterIPV4(ips []net.IP) []string { return ret } +// MkdirIfNotExist creates `folder` directory if not available func MkdirIfNotExist(folder string) error { if _, err := os.Stat(folder); err != nil { if err = os.MkdirAll(folder, 0700); err != nil { @@ -36,15 +39,18 @@ func MkdirIfNotExist(folder string) error { return nil } +// ExistDir checks if `folder` is available func ExistDir(folder string) bool { _, err := os.Stat(folder) return err == nil } +// DisplayProgressBar shows a fancy progress bar func DisplayProgressBar() bool { return isatty.IsTerminal(os.Stdout.Fd()) && displayProgress } +// FolderOf makes sure you won't get LFI func FolderOf(url string) string { safePath := filepath.Join(os.Getenv("HOME"), dataFolder) fullQualifyPath, err := filepath.Abs(filepath.Join(os.Getenv("HOME"), dataFolder, filepath.Base(url))) @@ -58,13 +64,14 @@ func FolderOf(url string) string { FatalCheck(err) if strings.Contains(relative, "..") { - FatalCheck(errors.New("you may be a victim of directory traversal path attack\n")) + FatalCheck(errors.New("you may be a victim of directory traversal path attack")) return "" //return is redundant be cause in fatal check we have panic, but compiler does not able to check } else { return fullQualifyPath } } +// TaskFromUrl runs when you want to download a single url func TaskFromUrl(url string) string { //task is just download file name //so we get download file name on url @@ -72,6 +79,7 @@ func TaskFromUrl(url string) string { return filename } +// IsUrl checks if `s` is actually a parsable URL. func IsUrl(s string) bool { _, err := url.Parse(s) return err == nil From f6e4d04e966f5000126d15e4fb9914424e0aa39e Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Tue, 22 Jun 2021 03:37:07 +0430 Subject: [PATCH 35/37] cleaned up the code a bit --- joiner.go | 1 + main.go | 54 +++++++++++++++++++++++++++--------------------------- resume.go | 2 ++ state.go | 4 ++++ util.go | 12 ++++++------ 5 files changed, 40 insertions(+), 33 deletions(-) diff --git a/joiner.go b/joiner.go index bb86427..58ed84a 100644 --- a/joiner.go +++ b/joiner.go @@ -8,6 +8,7 @@ import ( "sort" ) +// JoinFile joins seperate chunks of file and forms the final downloaded artifact func JoinFile(files []string, out string) error { //sort with file name or we will join files with wrong order sort.Strings(files) diff --git a/main.go b/main.go index 9f91ef5..9881d83 100644 --- a/main.go +++ b/main.go @@ -28,34 +28,33 @@ func main() { flag.Parse() args := flag.Args() if len(args) < 1 { - if len(filepath) > 1 { - // Creating a SerialGroup. - g1 := task.NewSerialGroup() - file, err := os.Open(filepath) - if err != nil { - FatalCheck(err) - } + if len(filepath) < 2 { + Errorln("url is required") + usage() + os.Exit(1) + } + // Creating a SerialGroup. + g1 := task.NewSerialGroup() + file, err := os.Open(filepath) + if err != nil { + FatalCheck(err) + } - defer file.Close() + defer file.Close() - reader := bufio.NewReader(file) + reader := bufio.NewReader(file) - for { - line, _, err := reader.ReadLine() + for { + line, _, err := reader.ReadLine() - if err == io.EOF { - break - } - - g1.AddChild(downloadTask(string(line), nil, *conn, *skiptls, proxy, bwLimit)) + if err == io.EOF { + break } - g1.Run(nil) - return - } else { - Errorln("url is required") - usage() - os.Exit(1) + + g1.AddChild(downloadTask(string(line), nil, *conn, *skiptls, proxy, bwLimit)) } + g1.Run(nil) + return } command := args[0] @@ -72,8 +71,8 @@ func main() { } var task string - if IsUrl(args[1]) { - task = TaskFromUrl(args[1]) + if IsURL(args[1]) { + task = TaskFromURL(args[1]) } else { task = args[1] } @@ -99,11 +98,12 @@ func downloadTask(url string, state *State, conn int, skiptls bool, proxy string return task.NewTaskWithFunc(run) } +// Execute configures the HTTPDownloader and uses it to download stuff. func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwLimit string) { //otherwise is hget command - signal_chan := make(chan os.Signal, 1) - signal.Notify(signal_chan, + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, @@ -131,7 +131,7 @@ func Execute(url string, state *State, conn int, skiptls bool, proxy string, bwL for { select { - case <-signal_chan: + case <-signalChan: //send par number of interrupt for each routine isInterrupted = true for i := 0; i < conn; i++ { diff --git a/resume.go b/resume.go index ba0f51d..a5bbd82 100644 --- a/resume.go +++ b/resume.go @@ -8,6 +8,7 @@ import ( "strings" ) +// TaskPrint read and prints data about current download jobs func TaskPrint() error { downloading, err := ioutil.ReadDir(filepath.Join(os.Getenv("HOME"), dataFolder)) if err != nil { @@ -28,6 +29,7 @@ func TaskPrint() error { return nil } +// Resume gets back to a previously stopped task func Resume(task string) (*State, error) { return Read(task) } diff --git a/state.go b/state.go index 38f15eb..e11a60d 100644 --- a/state.go +++ b/state.go @@ -10,11 +10,13 @@ import ( var dataFolder = ".hget/" var stateFileName = "state.json" +// State holds information about url Parts type State struct { URL string Parts []Part } +// Part represents a chunk of downloaded file type Part struct { Index int64 URL string @@ -23,6 +25,7 @@ type Part struct { RangeTo int64 } +// Save stores downloaded file into disk func (s *State) Save() error { //make temp folder //only working in unix with env HOME @@ -45,6 +48,7 @@ func (s *State) Save() error { return ioutil.WriteFile(filepath.Join(folder, stateFileName), j, 0644) } +// Read loads data about the state of downloaded files func Read(task string) (*State, error) { file := filepath.Join(os.Getenv("HOME"), dataFolder, task, stateFileName) Printf("Getting data from %s\n", file) diff --git a/util.go b/util.go index 1a2743b..ac9e5de 100644 --- a/util.go +++ b/util.go @@ -66,21 +66,21 @@ func FolderOf(url string) string { if strings.Contains(relative, "..") { FatalCheck(errors.New("you may be a victim of directory traversal path attack")) return "" //return is redundant be cause in fatal check we have panic, but compiler does not able to check - } else { - return fullQualifyPath } + return fullQualifyPath + } -// TaskFromUrl runs when you want to download a single url -func TaskFromUrl(url string) string { +// TaskFromURL runs when you want to download a single url +func TaskFromURL(url string) string { //task is just download file name //so we get download file name on url filename := filepath.Base(url) return filename } -// IsUrl checks if `s` is actually a parsable URL. -func IsUrl(s string) bool { +// IsURL checks if `s` is actually a parsable URL. +func IsURL(s string) bool { _, err := url.Parse(s) return err == nil } From 7953a7a0ef191bdb06d01bd001e1580b09b5efc8 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Mon, 21 Jun 2021 23:16:35 +0000 Subject: [PATCH 36/37] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 2bbb099..9d8b43d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ [![Build Status](https://travis-ci.com/abzcoding/hget.svg?branch=master)](https://travis-ci.com/abzcoding/hget) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/abzcoding/hget/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/abzcoding/hget/?branch=master) [![Maintainability](https://api.codeclimate.com/v1/badges/936e2aacab5946478295/maintainability)](https://codeclimate.com/github/abzcoding/hget/maintainability) -[![Test Coverage](https://api.codeclimate.com/v1/badges/936e2aacab5946478295/test_coverage)](https://codeclimate.com/github/abzcoding/hget/test_coverage) -[![codebeat -badge](https://codebeat.co/badges/ea357ae8-4d84-4599-bff7-cffc4f28fd67)](https://codebeat.co/projects/github-com-abzcoding-hget-master) +[![Codebeat](https://codebeat.co/badges/ea357ae8-4d84-4599-bff7-cffc4f28fd67)](https://codebeat.co/projects/github-com-abzcoding-hget-master) # hget ![](https://i.gyazo.com/641166ab79e196e35d1a0ef3f9befd80.png) From ff8e0886516fae090b704ff2b65454fd5f24bcc6 Mon Sep 17 00:00:00 2001 From: Abouzar Parvan Date: Mon, 27 Dec 2021 11:19:27 +0330 Subject: [PATCH 37/37] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d8b43d..eb87982 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ hget tasks # get interrupted tasks hget resume [TaskName | URL] # to resume task hget -proxy "127.0.0.1:12345" URL # to download using socks5 proxy hget -proxy "http://sample-proxy.com:8080" URL # to download using http proxy -hget -file sample.txt # to download a list of files -hget -n 4 -rate 100KB URL # to download using 4 threads & limited to 100Kb per second +hget -file sample.txt # to download a list of urls +hget -n 4 -rate 100KB URL # to download using 4 threads & limited to 100KB per second ``` ### Help