diff --git a/client/client.go b/client/client.go index 9263276..6bd0e14 100644 --- a/client/client.go +++ b/client/client.go @@ -3,8 +3,8 @@ package main import ( "EkSukkel/moeggesukkel" "archive/tar" + "archive/zip" "bufio" - "bytes" "compress/gzip" "context" "fmt" @@ -21,9 +21,6 @@ import ( ) var log = flogging.MustGetLogger("MAIN") - -// var conn *grpc.ClientConn -// var mgc moeggesukkel.MoegGeSukkelClient var defaultBufSize int = 4096 func upload(address, path string, ttl int64) string { @@ -48,31 +45,15 @@ func upload(address, path string, ttl int64) string { // File // tar + gzip - var buf bytes.Buffer + // var buf bytes.Buffer + compressFileName := compress(path) + // err = compress(path, &buf) - err = compress(path, &buf) if err != nil { log.Panic("Error: ", err) } - // write the .tar.gzip - compressFileName := path + ".tar.gzip" - fileToWrite, err := os.Create(compressFileName) - - if err != nil { - log.Panic("Error: ", err) - } - - if _, err := io.Copy(fileToWrite, &buf); err != nil { - log.Panic("Error: ", err) - } - - err = fileToWrite.Close() - if err != nil { - log.Panic("Error: ", err) - } - - fileToWrite, err = os.Open(compressFileName) + fileToWrite, err := os.Open(compressFileName) if err != nil { log.Panic("Error: ", err) @@ -90,12 +71,7 @@ func upload(address, path string, ttl int64) string { fileName := fileInfo.Name() for { - _, err := buffReader.Read(dataArr) - if err == io.EOF { - // there is no more data to read - break - // return err - } + _, readFinsihed := buffReader.Read(dataArr) request := moeggesukkel.UploadRequest{ Filename: fileName, @@ -104,9 +80,16 @@ func upload(address, path string, ttl int64) string { } err = stream.Send(&request) + if err != nil { log.Panic("Error: ", err) } + + if readFinsihed == io.EOF { + // there is no more data to read + break + // return err + } } // close file @@ -121,35 +104,15 @@ func upload(address, path string, ttl int64) string { log.Panic("Error: ", err) } - return response.GetToken() -} - -// func upload2(address, path string) error { - -// // tar + gzip -// var buf bytes.Buffer - -// err := compress(path, &buf) -// if err != nil { -// return err -// } -// // write the .tar.gzip -// fileToWrite, err := os.OpenFile(path+".tar.gzip", os.O_CREATE|os.O_RDWR, os.FileMode(600)) - -// if err != nil { -// return err -// } + // log.Info("Deleting zip file : ", compressFileName) + err = os.Remove(compressFileName) -// if _, err := io.Copy(fileToWrite, &buf); err != nil { -// return err -// } - -// // untar write -// if err := untar(&buf, "./uncompressHere/"); err != nil { -// // probably delete uncompressHere? -// } + if err != nil { + log.Panic("Error: ", err) + } -// } + return response.GetToken() +} func download(address, token string) { // Create client and @@ -169,170 +132,462 @@ func download(address, token string) { Token: token, } - response, err := mgc.Download(ctx, &request) + stream, err := mgc.Download(ctx, &request) if err != nil { log.Panic("Error: ", err) } - log.Info(response) -} + firstResponse, err := stream.Recv() -// func client(address string) { //moeggesukkel.MoegGeSukkelClient { -// var err error -// conn, err = grpc.Dial(address) -// if err != nil { -// log.Panic("failed to create gRPC connection") -// } + if err != nil { + log.Panic("Error: ", err) + } -// mgc = moeggesukkel.NewMoegGeSukkelClient(conn) + homeDir, _ := os.UserHomeDir() + downloadFileName := filepath.Join(homeDir, "Downloads", firstResponse.GetFilename()) + fileToDownload, err := os.Create(downloadFileName) -// // return mgc -// } + if err != nil { + log.Panic("Error: ", err) + } -func compress(src string, buf io.Writer) error { - // tar > gzip > buf - zr := gzip.NewWriter(buf) - tw := tar.NewWriter(zr) + buffWriter := bufio.NewWriter(fileToDownload) + _, err = buffWriter.Write(firstResponse.GetData()) - // is file a folder? - fi, err := os.Stat(src) if err != nil { - return err + log.Panic("Error: ", err) } - mode := fi.Mode() - if mode.IsRegular() { - // get header - header, err := tar.FileInfoHeader(fi, src) + + for { + + res, err := stream.Recv() + if err != nil { - return err - } - // write header - if err := tw.WriteHeader(header); err != nil { - return err + break } - // get content - data, err := os.Open(src) + + _, err = buffWriter.Write(res.GetData()) + if err != nil { - return err - } - if _, err := io.Copy(tw, data); err != nil { - return err + log.Panic("Error: ", err) } - } else if mode.IsDir() { // folder + } + + err = buffWriter.Flush() + if err != nil { + log.Panic("Error: ", err) + } + + err = fileToDownload.Close() + if err != nil { + log.Panic("Error: ", err) + } +} + +func compress(path string) string { + + fileBase := filepath.Base(path) + ".zip" + + // log.Info("Creating zip file : ", fileBase) + + err := ZipSource(path, fileBase) + if err != nil { + log.Panic("Error: ", err) + } + // else { + // log.Info("Zip file created at : ", fileBase) + // } + + return fileBase - // walk through every file in the folder - filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { - // generate tar header - header, err := tar.FileInfoHeader(fi, file) +} + +// write the .tar.gzip +// compressFileName := path + ".tar.gzip" +// fileToWrite, err := os.Create(compressFileName) + +// if err != nil { +// log.Panic("Error: ", err) +// } + +// if _, err := io.Copy(fileToWrite, &buf); err != nil { +// log.Panic("Error: ", err) +// } + +// err = fileToWrite.Close() +// if err != nil { +// log.Panic("Error: ", err) +// } + +// targetTar, err := Tar(path, "./") + +// if err != nil { +// log.Panic("Error: ", err) +// } else { +// log.Info("Tar filed created at : ", targetTar) +// } + +// compressTar, err := Gzip(targetTar, "./") + +// if err != nil { +// log.Panic("Error: ", err) +// } else { +// log.Info("Gzip filed created at : ", compressTar) +// } + +// return compressTar +// } + +func Tar(source, target string) (string, error) { + filename := filepath.Base(source) + target = filepath.Join(target, fmt.Sprintf("%s.tar", filename)) + tarfile, err := os.Create(target) + if err != nil { + return "", err + } + defer tarfile.Close() + + tarball := tar.NewWriter(tarfile) + defer tarball.Close() + + info, err := os.Stat(source) + if err != nil { + return "", nil + } + + var baseDir string + if info.IsDir() { + baseDir = filepath.Base(source) + } + + err = filepath.Walk(source, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { return err } - // must provide real name - // (see https://golang.org/src/archive/tar/common.go?#L626) - header.Name = filepath.ToSlash(file) + if baseDir != "" { + header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) + } - // write header - if err := tw.WriteHeader(header); err != nil { + if err := tarball.WriteHeader(header); err != nil { return err } - // if not a dir, write file content - if !fi.IsDir() { - data, err := os.Open(file) - if err != nil { - return err - } - if _, err := io.Copy(tw, data); err != nil { - return err - } + + if info.IsDir() { + return nil } - return nil + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(tarball, file) + return err }) + + if err != nil { + return "", nil } else { - return fmt.Errorf("error: file type not supported") + return target, nil } +} - // produce tar - if err := tw.Close(); err != nil { +func Untar(tarball, target string) error { + reader, err := os.Open(tarball) + if err != nil { return err } - // produce gzip - if err := zr.Close(); err != nil { - return err + defer reader.Close() + tarReader := tar.NewReader(reader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + path := filepath.Join(target, header.Name) + info := header.FileInfo() + if info.IsDir() { + if err = os.MkdirAll(path, info.Mode()); err != nil { + return err + } + continue + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, tarReader) + if err != nil { + return err + } } - // + return nil } -// check for path traversal and correct forward slashes -func validRelPath(p string) bool { - if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { - return false +func Gzip(source, target string) (string, error) { + reader, err := os.Open(source) + if err != nil { + return "", err + } + + filename := filepath.Base(source) + target = filepath.Join(target, fmt.Sprintf("%s.gz", filename)) + writer, err := os.Create(target) + if err != nil { + return "", err + } + defer writer.Close() + + archiver := gzip.NewWriter(writer) + archiver.Name = filename + defer archiver.Close() + + _, err = io.Copy(archiver, reader) + + if err != nil { + return "", err + } else { + return target, nil + } +} + +func UnGzip(source, target string) error { + reader, err := os.Open(source) + if err != nil { + return err } - return true + defer reader.Close() + + archive, err := gzip.NewReader(reader) + if err != nil { + return err + } + defer archive.Close() + + target = filepath.Join(target, archive.Name) + writer, err := os.Create(target) + if err != nil { + return err + } + defer writer.Close() + + _, err = io.Copy(writer, archive) + return err } -func decompress(src io.Reader, dst string) error { - // ungzip - zr, err := gzip.NewReader(src) +func ZipSource(source, target string) error { + // 1. Create a ZIP file and zip.Writer + f, err := os.Create(target) if err != nil { return err } - // untar - tr := tar.NewReader(zr) + defer f.Close() - // uncompress each element - for { - header, err := tr.Next() - if err == io.EOF { - break // End of archive + writer := zip.NewWriter(f) + defer writer.Close() + + // 2. Go through all the files of the source + return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err } + + // 3. Create a local file header + header, err := zip.FileInfoHeader(info) if err != nil { return err } - target := header.Name - // validate name against path traversal - if !validRelPath(header.Name) { - return fmt.Errorf("tar contained invalid name error %q", target) + // set compression + header.Method = zip.Deflate + + // 4. Set relative path of a file as the header name + header.Name, err = filepath.Rel(filepath.Dir(source), path) + if err != nil { + return err + } + if info.IsDir() { + header.Name += "/" } - // add dst + re-format slashes according to system - target = filepath.Join(dst, header.Name) - // if no join is needed, replace with ToSlash: - // target = filepath.ToSlash(header.Name) + // 5. Create writer for the file header and save content of the file + headerWriter, err := writer.CreateHeader(header) + if err != nil { + return err + } - // check the type - switch header.Typeflag { + if info.IsDir() { + return nil + } - // if its a dir and it doesn't exist create it (with 0755 permission) - case tar.TypeDir: - if _, err := os.Stat(target); err != nil { - if err := os.MkdirAll(target, 0755); err != nil { - return err - } - } - // if it's a file create it (with same permission) - case tar.TypeReg: - fileToWrite, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) - if err != nil { - return err - } - // copy over contents - if _, err := io.Copy(fileToWrite, tr); err != nil { - return err - } - // manually close here after each file operation; defering would cause each file close - // to wait until all operations have completed. - fileToWrite.Close() + f, err := os.Open(path) + if err != nil { + return err } - } + defer f.Close() - // - return nil + _, err = io.Copy(headerWriter, f) + return err + }) } +// func compress(src string, buf io.Writer) error { +// // tar > gzip > buf +// zr := gzip.NewWriter(buf) +// tw := tar.NewWriter(zr) + +// // is file a folder? +// fi, err := os.Stat(src) +// if err != nil { +// return err +// } +// mode := fi.Mode() +// if mode.IsRegular() { +// // get header +// header, err := tar.FileInfoHeader(fi, src) +// if err != nil { +// return err +// } +// // write header +// if err := tw.WriteHeader(header); err != nil { +// return err +// } +// // get content +// data, err := os.Open(src) +// if err != nil { +// return err +// } +// if _, err := io.Copy(tw, data); err != nil { +// return err +// } +// } else if mode.IsDir() { // folder + +// // walk through every file in the folder +// filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { +// // generate tar header +// header, err := tar.FileInfoHeader(fi, file) +// if err != nil { +// return err +// } + +// // must provide real name +// // (see https://golang.org/src/archive/tar/common.go?#L626) +// header.Name = filepath.ToSlash(file) + +// // write header +// if err := tw.WriteHeader(header); err != nil { +// return err +// } +// // if not a dir, write file content +// if !fi.IsDir() { +// data, err := os.Open(file) +// if err != nil { +// return err +// } +// if _, err := io.Copy(tw, data); err != nil { +// return err +// } +// } +// return nil +// }) +// } else { +// return fmt.Errorf("error: file type not supported") +// } + +// // produce tar +// if err := tw.Close(); err != nil { +// return err +// } +// // produce gzip +// if err := zr.Close(); err != nil { +// return err +// } +// // +// return nil +// } + +// // check for path traversal and correct forward slashes +// func validRelPath(p string) bool { +// if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { +// return false +// } +// return true +// } + +// func decompress(src io.Reader, dst string) error { +// // ungzip +// zr, err := gzip.NewReader(src) +// if err != nil { +// return err +// } +// // untar +// tr := tar.NewReader(zr) + +// // uncompress each element +// for { +// header, err := tr.Next() +// if err == io.EOF { +// break // End of archive +// } +// if err != nil { +// return err +// } +// target := header.Name + +// // validate name against path traversal +// if !validRelPath(header.Name) { +// return fmt.Errorf("tar contained invalid name error %q", target) +// } + +// // add dst + re-format slashes according to system +// target = filepath.Join(dst, header.Name) +// // if no join is needed, replace with ToSlash: +// // target = filepath.ToSlash(header.Name) + +// // check the type +// switch header.Typeflag { + +// // if its a dir and it doesn't exist create it (with 0755 permission) +// case tar.TypeDir: +// if _, err := os.Stat(target); err != nil { +// if err := os.MkdirAll(target, 0755); err != nil { +// return err +// } +// } +// // if it's a file create it (with same permission) +// case tar.TypeReg: +// fileToWrite, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) +// if err != nil { +// return err +// } +// // copy over contents +// if _, err := io.Copy(fileToWrite, tr); err != nil { +// return err +// } +// // manually close here after each file operation; defering would cause each file close +// // to wait until all operations have completed. +// fileToWrite.Close() +// } +// } + +// // +// return nil +// } + func main() { // Silent var silent bool @@ -355,7 +610,7 @@ func main() { // Download downloadCmd := &cobra.Command{ - Use: "download [token]", + Use: "download [address] [token]", Short: "downloads the given token from the server", Long: "Passes the token to the Moeggesukkel server to download", Run: func(cmd *cobra.Command, args []string) { @@ -400,76 +655,104 @@ func main() { rootCmd.AddCommand(downloadCmd, uploadCmd) rootCmd.Execute() - // bar := progressbar.Default(100) - // for i := 0; i < 100; i++ { - // bar.Add(1) - // time.Sleep(40 * time.Millisecond) - // } - // writer := ansi.NewAnsiStdout() - - // log.Info("Wrting to the thigny, starting now") - - // bar1 := progressbar.NewOptions(1000, - // progressbar.OptionFullWidth(), - // progressbar.OptionSetWriter(writer), - // progressbar.OptionEnableColorCodes(true), - // progressbar.OptionShowBytes(true), - // progressbar.OptionSetWidth(15), - // progressbar.OptionSetDescription("[cyan][1/3][reset] Writing moshable file..."), - // progressbar.OptionSetTheme(progressbar.Theme{ - // Saucer: "[green]=[reset]", - // SaucerHead: "[green]>[reset]", - // SaucerPadding: " ", - // BarStart: "[", - // BarEnd: "]", - // })) - // for i := 0; i < 1000; i++ { - // bar1.Add(1) - // time.Sleep(5 * time.Millisecond) - // } +} - // fmt.Println() - // log.Info("\nWrting to the thigny, starting now") - - // bar2 := progressbar.NewOptions(1000, - // progressbar.OptionFullWidth(), - // progressbar.OptionSetWriter(writer), - // progressbar.OptionEnableColorCodes(true), - // progressbar.OptionShowBytes(true), - // progressbar.OptionSetWidth(15), - // progressbar.OptionSetDescription("[yellow][2/3] Second detached stage..."), - // progressbar.OptionSetTheme(progressbar.Theme{ - // Saucer: "[green]=[reset]", - // SaucerHead: "[green]>[reset]", - // SaucerPadding: " ", - // BarStart: "[", - // BarEnd: "]", - // })) - // for i := 0; i < 1000; i++ { - // bar2.Add(2) - // time.Sleep(5 * time.Millisecond) - // } - // fmt.Println() - // log.Info("\nWrting to the thigny, starting now") - - // bar := progressbar.NewOptions(1000, - // progressbar.OptionFullWidth(), - // progressbar.OptionSetWriter(writer), - // progressbar.OptionEnableColorCodes(true), - // progressbar.OptionShowBytes(true), - // progressbar.OptionSetWidth(15), - // progressbar.OptionSetDescription("[red][3/3] Deploying files..."), - // progressbar.OptionSetTheme(progressbar.Theme{ - // Saucer: "[green]=[reset]", - // SaucerHead: "[green]>[reset]", - // SaucerPadding: " ", - // BarStart: "[", - // BarEnd: "]", - // })) - // for i := 0; i < 1000; i++ { - // bar.Add(10) - // time.Sleep(5 * time.Millisecond) - // } +// func upload2(address, path string) error { - // io.MultiWriter() -} +// // tar + gzip +// var buf bytes.Buffer + +// err := compress(path, &buf) +// if err != nil { +// return err +// } +// // write the .tar.gzip +// fileToWrite, err := os.OpenFile(path+".tar.gzip", os.O_CREATE|os.O_RDWR, os.FileMode(600)) + +// if err != nil { +// return err +// } + +// if _, err := io.Copy(fileToWrite, &buf); err != nil { +// return err +// } + +// // untar write +// if err := untar(&buf, "./uncompressHere/"); err != nil { +// // probably delete uncompressHere? +// } + +// } + +// bar := progressbar.Default(100) +// for i := 0; i < 100; i++ { +// bar.Add(1) +// time.Sleep(40 * time.Millisecond) +// } +// writer := ansi.NewAnsiStdout() + +// log.Info("Wrting to the thigny, starting now") + +// bar1 := progressbar.NewOptions(1000, +// progressbar.OptionFullWidth(), +// progressbar.OptionSetWriter(writer), +// progressbar.OptionEnableColorCodes(true), +// progressbar.OptionShowBytes(true), +// progressbar.OptionSetWidth(15), +// progressbar.OptionSetDescription("[cyan][1/3][reset] Writing moshable file..."), +// progressbar.OptionSetTheme(progressbar.Theme{ +// Saucer: "[green]=[reset]", +// SaucerHead: "[green]>[reset]", +// SaucerPadding: " ", +// BarStart: "[", +// BarEnd: "]", +// })) +// for i := 0; i < 1000; i++ { +// bar1.Add(1) +// time.Sleep(5 * time.Millisecond) +// } + +// fmt.Println() +// log.Info("\nWrting to the thigny, starting now") + +// bar2 := progressbar.NewOptions(1000, +// progressbar.OptionFullWidth(), +// progressbar.OptionSetWriter(writer), +// progressbar.OptionEnableColorCodes(true), +// progressbar.OptionShowBytes(true), +// progressbar.OptionSetWidth(15), +// progressbar.OptionSetDescription("[yellow][2/3] Second detached stage..."), +// progressbar.OptionSetTheme(progressbar.Theme{ +// Saucer: "[green]=[reset]", +// SaucerHead: "[green]>[reset]", +// SaucerPadding: " ", +// BarStart: "[", +// BarEnd: "]", +// })) +// for i := 0; i < 1000; i++ { +// bar2.Add(2) +// time.Sleep(5 * time.Millisecond) +// } +// fmt.Println() +// log.Info("\nWrting to the thigny, starting now") + +// bar := progressbar.NewOptions(1000, +// progressbar.OptionFullWidth(), +// progressbar.OptionSetWriter(writer), +// progressbar.OptionEnableColorCodes(true), +// progressbar.OptionShowBytes(true), +// progressbar.OptionSetWidth(15), +// progressbar.OptionSetDescription("[red][3/3] Deploying files..."), +// progressbar.OptionSetTheme(progressbar.Theme{ +// Saucer: "[green]=[reset]", +// SaucerHead: "[green]>[reset]", +// SaucerPadding: " ", +// BarStart: "[", +// BarEnd: "]", +// })) +// for i := 0; i < 1000; i++ { +// bar.Add(10) +// time.Sleep(5 * time.Millisecond) +// } + +// io.MultiWriter()