add upload flag to channel download

Signed-off-by: benjamin.747 <[email protected]>

src/commands/channel.rs

@@ -6,10 +6,10 @@
 //!
 //!   Arguments:
 //!   - __domain__: you can choose your own upstream by adding this arugment in command
-//!   - __download-threads__: specify the download threads to parallel download, 
+//!   - __download-threads__: specify the download threads to parallel download,
 //!        this param can be changed in the configuration file or pass it here
 //!   - __no-progressbar__: not implemented
-//! 
+//!
 //! # download subcommand
 //!   - before each download, freighter will try to fetch the sha256 of the file and compare with local file if it exists
 //!         and will skip downloading if they are matching.
@@ -23,9 +23,9 @@
 //!
 //!   Arguments:
 //!   - __clean__: clean history files read by config file after download successfully.
-//!   - __version__: only download the version you specified, 
+//!   - __version__: only download the version you specified,
 //!         you can provide any version format supported by rust-org, such as stable, beta or nightly-2022-07-31.
-//! 
+//!
 //! # upload subcommand
 //!   upload file to Object Storage Service compatible with [AWS S3](https://aws.amazon.com/s3/)
 //!     - Digitalocean Spaces
@@ -35,15 +35,15 @@
 //!     - AWS S3
 //!     - minio
 //!     - Ceph
-//! 
+//!
 //!   Arguments:
 //!   - __bucket__: set the s3 bucket you want to upload files to, you must provide this param befor uplaod.
 //!   
 
 use clap::{arg, ArgMatches};
 use log::info;
 
-use crate::cloud::s3::{S3cmd, CloudStorage};
+use crate::cloud::s3::{CloudStorage, S3cmd};
 use crate::commands::command_prelude::*;
 use crate::config::Config;
 use crate::crates::channel::{sync_rust_toolchain, ChannelOptions};
@@ -54,6 +54,9 @@ pub fn cli() -> clap::Command {
         .subcommand(subcommand("download")
             .arg(flag("clean", "clean up historical versions"))
             .arg(arg!(-v --"version" <VALUE> "only download the version you specified"))
+            .arg(flag("upload", "upload every crate file after download"))
+            .arg(arg!(-b --"bucket" <VALUE> "set the s3 bucket name you want to upload files"))
+            .arg(flag("delete-after-upload", "this will delete file after upload"))
         )
         .subcommand(subcommand("upload")
         .arg(
@@ -122,15 +125,28 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
     info!("ChannelOptions info : {:#?}", opts);
 
     match args.subcommand() {
-        Some(("download", args)) => sync_rust_toolchain(&ChannelOptions {
-            clean: args.get_flag("clean"),
-            version: args.get_one::<String>("version").cloned(),
-            ..opts
-        })?,
+        Some(("download", args)) => {
+            let bucket_name = args.get_one::<String>("bucket").cloned();
+            let upload = args.get_flag("upload");
+            if upload && bucket_name.is_none() {
+                unreachable!("can not upload with empty bucket name")
+            }
+
+            sync_rust_toolchain(&ChannelOptions {
+                clean: args.get_flag("clean"),
+                version: args.get_one::<String>("version").cloned(),
+                upload: upload,
+                delete_after_upload: args.get_flag("delete-after-upload"),
+                bucket_name: bucket_name.unwrap(),
+                ..opts
+            })?
+        },
         Some(("upload", args)) => {
             let bucket_name = args.get_one::<String>("bucket").cloned().unwrap();
             let s3cmd = S3cmd::default();
-            s3cmd.upload_folder(opts.dist_path.to_str().unwrap(), &bucket_name).unwrap();
+            s3cmd
+                .upload_folder(opts.dist_path.to_str().unwrap(), &bucket_name)
+                .unwrap();
         }
         Some((cmd, _)) => {
             unreachable!("unexpected command {}", cmd)

src/commands/crates.rs

@@ -11,9 +11,9 @@
 //!   - __no-progressbar__: Whether to hide progress bar when start downloading
 //!
 //! # pull subcommand
-//! 
+//!
 //!   sync crates index from upstream to local:
-//! 
+//!
 //!   - The crates index is a git repository, and **cargo** clone and update from [GitHub](https://github.com/rust-lang/crates.io-index).
 //!     - The clone use `bare` mode, more details in the [cargo guide](https://github.com/rust-lang/cargo/blob/6b6b0b486d73c03ed952591d880debec1d47c534/src/doc/src/guide/cargo-home.md#directories)
 //!   
@@ -28,10 +28,12 @@
 //!         URL_s3_primary: "https://crates-io.s3-us-west-1.amazonaws.com/crates/{crate}/{crate}-{version}.crate"
 //!         URL_s3_fallback: "https://crates-io-fallback.s3-eu-west-1.amazonaws.com/crates/{crate}/{crate}-{version}.crate"
 //!      ```
-//! 
+//!
 //!   Arguments:
 //!   - __init__: Whether to download all the crates files for initialization.
 //!   - __upload__: Whether to upload single file to s3 after download success.
+//!   - __bucket__: set the s3 bucket you want to upload files to, you must provide this param befor uplaod.
+//!   - __delete-after-upload__: This optional parameter will be used to delete files after upload.
 //!
 //! # upload subcommand
 //!
@@ -48,7 +50,7 @@
 //!  
 
 use clap::{arg, ArgMatches};
-use log::{info, debug};
+use log::{debug, info};
 
 use crate::commands::command_prelude::*;
 use crate::config::Config;
@@ -76,7 +78,7 @@ pub fn cli() -> clap::Command {
             .arg(flag("init", "Start init download of crates file, this will traverse all index for full download"))
             .arg(flag("upload", "upload every crate file after download"))
             .arg(arg!(-b --"bucket" <VALUE> "set the s3 bucket name you want to upload files"))
-            .arg(flag("delete-after-download", "this will delete file after download"))
+            .arg(flag("delete-after-upload", "this will delete file after upload"))
         )
         .subcommand_required(true)
         .arg_required_else_help(true)
@@ -144,8 +146,12 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
         Some(("download", args)) => {
             opts.upload = args.get_flag("upload");
             opts.init_download = args.get_flag("init");
-            opts.delete_after_download = args.get_flag("delete-after-download");
-            opts.bucket_name = args.get_one::<String>("bucket").cloned().unwrap();
+            opts.delete_after_upload = args.get_flag("delete-after-upload");
+            let bucket_name = args.get_one::<String>("bucket").cloned();
+            if opts.upload && bucket_name.is_none() {
+                unreachable!("can not upload with empty bucket name")
+            }
+            opts.bucket_name = bucket_name.unwrap();
             debug!("download command opts: {:?}", opts);
             if let Some(source) = domain {
                 config.crates.domain = source;

src/config.default.toml

@@ -22,11 +22,8 @@ domain = "https://static.crates.io/crates"
 # Number of download threads
 download_threads = 2
 
-# when local file does not exist, redirect client to this domain
-redirect_domain = "https://rsproxy.cn"
-
 # when local file does not exist, will try to download from these doamins
-backup_domain = ["https://rsproxy.cn", "https://static.crates.io"]
+backup_domain = ["localhost", "https://rsproxy.cn", "https://static.crates.io"]
 
 [rustup]
 # download rustup from domain
@@ -134,11 +131,7 @@ sync_nightly_days = 30
 # days you want to keep for historical beta version
 sync_beta_days = 30
 
-
-# when local file does not exist, redirect client to this domain
-redirect_domain = "https://static.rust-lang.org"
-
 # when local file does not exist, will try to download from these doamins
-backup_domain = ["https://rsproxy.cn", "https://static.rust-lang.org"]
+backup_domain = ["localhost", "https://rsproxy.cn", "https://static.rust-lang.org"]
 
 

src/crates/channel.rs

@@ -5,18 +5,18 @@ use std::{
 };
 
 use chrono::{Duration, NaiveDate, Utc};
-use log::{info, error};
+use log::{error, info};
 use serde::Deserialize;
 use threadpool::ThreadPool;
 use walkdir::WalkDir;
 
 use crate::{
+    cloud::s3::{CloudStorage, S3cmd},
     config::RustUpConfig,
     download::{download_file, download_file_with_sha},
     errors::{FreightResult, FreighterError},
 };
 
-
 #[derive(Debug, Deserialize)]
 pub struct Channel {
     #[serde(alias = "manifest-version")]
@@ -56,6 +56,10 @@ pub struct ChannelOptions {
     pub dist_path: PathBuf,
 
     pub bucket_name: String,
+
+    pub upload: bool,
+
+    pub delete_after_upload: bool,
 }
 
 /// entrance function
@@ -87,8 +91,6 @@ pub fn sync_rust_toolchain(opts: &ChannelOptions) -> FreightResult {
     Ok(())
 }
 
-
-
 // sync the latest toolchain by given a channel name(stable, beta, nightly) or history verison by version number
 pub fn sync_channel(opts: &ChannelOptions, channel: &str) -> FreightResult {
     let channel_name;
@@ -116,8 +118,22 @@ pub fn sync_channel(opts: &ChannelOptions, channel: &str) -> FreightResult {
                         url.split('/').map(PathBuf::from).collect::<Vec<PathBuf>>()[4..].to_owned(),
                     )
                     .collect();
+                let (upload, dist_path, bucket_name, delete_after_upload) = (
+                    opts.upload,
+                    opts.dist_path.to_owned(),
+                    opts.bucket_name.to_owned(),
+                    opts.delete_after_upload,
+                );
+                let s3cmd = S3cmd::default();
                 pool.execute(move || {
-                    download_file(&url, &path, Some(&hash), false).unwrap();
+                    let downloaded = download_file(&url, &path, Some(&hash), false);
+                    if downloaded.is_ok() && upload {
+                        let uploaded =
+                            s3cmd.upload_folder(dist_path.to_str().unwrap(), &bucket_name);
+                        if uploaded.is_ok() && delete_after_upload {
+                            fs::remove_file(&path).unwrap();
+                        }
+                    }
                 });
             });
             pool.join();
@@ -204,4 +220,3 @@ pub fn compare_date(entry: &DirEntry, sync_days: i64) -> bool {
         false
     }
 }
-

src/crates/crates_file.rs

@@ -41,7 +41,7 @@ pub struct CratesOptions {
 
     pub bucket_name: String,
 
-    pub delete_after_download: bool,
+    pub delete_after_upload: bool,
 }
 
 /// Crate preserve the crates info parse from registry json file
@@ -251,9 +251,9 @@ pub fn download_crates_with_log(
             if download_succ && opts.upload {
                 let s3 = S3cmd::default();
                 let s3_path = file.to_str().unwrap().replace(opts.crates_path.to_str().unwrap(), "");
-                info!("s3_path: {}, {}", s3_path, opts.delete_after_download);
+                info!("s3_path: {}, {}", s3_path, opts.delete_after_upload);
                 let uploded = s3.upload_file(&file, &s3_path, &opts.bucket_name);
-                if uploded.is_ok() && opts.delete_after_download {
+                if uploded.is_ok() && opts.delete_after_upload {
                     fs::remove_file(file).unwrap();
                 }
             }