diff --git a/Cargo.toml b/Cargo.toml index a982613dc..3af4db115 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "packsquash" -version = "0.1.0" +version = "0.1.1" authors = ["AlexTMjugador"] edition = "2018" @@ -13,7 +13,6 @@ codegen-units = 1 lazy_static = "1.4.0" clap = "2.33.1" simple-error = "0.2.1" -pbr = "1.0.2" num_cpus = "1.13.0" threadpool = "1.8.1" tempfile = "3.1.0" diff --git a/README.md b/README.md index 11e889925..287256dcd 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ You can get a pre-built release for 64-bit Windows, Linux and macOS systems from ## Usage ``` -PackSquash 0.1.0 +PackSquash 0.1.1 AlexTMjugador Lossily compresses and prepares Minecraft resource packs for distribution @@ -25,21 +25,25 @@ USAGE: packsquash [FLAGS] [OPTIONS] [result ZIP file] FLAGS: - -c, --compress-compressed If specified, resource files that are already compressed by design (like PNG and OGG - files) will be losslessly compressed again in the result ZIP file, unless that yields - no significant space savings. This is disabled by default, as it is expected that - compressing already compressed file formats yields marginal space savings that do not - outweigh the time cost - -o, --zip-obfuscation If provided, the generated ZIP file will not be able to be read by some common - programs, like WinRAR, 7-Zip or unzip, but will function normally within Minecraft. It - will also not duplicate some metadata in the ZIP file, reducing its size a bit more. - You shouldn't rely on this feature to keep the resources safe from unauthorized - ripping, and there is a low possibility that a future version of Minecraft rejects - these ZIP files. Therefore, it is disabled by default - -h, --help Prints help information - -n, --skip-pack-icon If specified, the pack icon in pack.png will be skipped and not included in the - resulting file - -V, --version Prints version information + -c, --compress-compressed + If specified, resource files that are already compressed by design (like PNG and OGG files) will be + losslessly compressed again in the result ZIP file, unless that yields no significant space savings. This is + disabled by default, as it is expected that compressing already compressed file formats yields marginal + space savings that do not outweigh the time cost + -a, --do-not-ignore-system-and-hiden-files + If specified, filenames which start with a dot or are likely generated by a software system (Windows, + VCS...) won't be skipped without a message + -o, --zip-obfuscation + If provided, the generated ZIP file will not be able to be read by some common programs, like WinRAR, 7-Zip + or unzip, but will function normally within Minecraft. It will also not duplicate some metadata in the ZIP + file, reducing its size a bit more. You shouldn't rely on this feature to keep the resources safe from + unauthorized ripping, and there is a low possibility that a future version of Minecraft rejects these ZIP + files. Therefore, it is disabled by default + -h, --help Prints help information + -n, --skip-pack-icon + If specified, the pack icon in pack.png will be skipped and not included in the resulting file + + -V, --version Prints version information OPTIONS: -t, --threads The number of resource pack files to process in parallel, in different threads. By @@ -54,6 +58,7 @@ ARGS: ## Potentials for improvement These are some tweaks to the application which could further improve the compression it achieves, but are not scheduled for a release. Feel free to submit a PR with some (or all) of these changes! -* Parse JSON files to determine unused assets (models, textures, sounds...), and skip them from the result ZIP file -* Understand the TTF and OGG formats more, to remove metadata and further compress them -* Implement an "append mode", which only adds to a result ZIP file resource pack files which are newer than it (so the entire ZIP file isn't generated again if a single file changes) +* Determine unused assets (models, textures, sounds...) by analyzing JSON files, and skip them from the result ZIP file. +* Understand the OGG format more, to remove metadata from it. +* Implement TTF minification. +* Implement an "append mode", which only adds to a result ZIP file resource pack files which are newer than it (so the entire ZIP file isn't generated again if a single file changes). diff --git a/src/main.rs b/src/main.rs index d92ab8ed9..53a5b4cd4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,10 +2,9 @@ mod micro_zip; mod resource_pack_file; use std::error::Error; -use std::io::Write; +use std::ffi::OsStr; use std::iter::FromIterator; use std::path::PathBuf; -use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::{cmp, env, fs, process}; @@ -13,12 +12,17 @@ use std::{cmp, env, fs, process}; use threadpool::ThreadPool; use clap::{App, ArgMatches}; -use pbr::MultiBar; use simple_error::SimpleError; use micro_zip::MicroZip; use micro_zip::ZipFileType; +use lazy_static::lazy_static; + +lazy_static! { + static ref EMPTY_OS_STR: &'static OsStr = OsStr::new(""); +} + fn main() { let parameters = App::new("PackSquash") .version(env!("CARGO_PKG_VERSION")) @@ -29,6 +33,7 @@ fn main() { [number of threads] -t --threads=[THREADS] 'The number of resource pack files to process in parallel, in different threads. By default, a value appropriate to the CPU the program is running on is used, but you might want to decrease it to reduce memory usage' [ZIP obfuscation] -o --zip-obfuscation 'If provided, the generated ZIP file will not be able to be read by some common programs, like WinRAR, 7-Zip or unzip, but will function normally within Minecraft. It will also not duplicate some metadata in the ZIP file, reducing its size a bit more. You shouldn't rely on this feature to keep the resources safe from unauthorized ripping, and there is a low possibility that a future version of Minecraft rejects these ZIP files. Therefore, it is disabled by default' [Compress already compressed files] -c --compress-compressed 'If specified, resource files that are already compressed by design (like PNG and OGG files) will be losslessly compressed again in the result ZIP file, unless that yields no significant space savings. This is disabled by default, as it is expected that compressing already compressed file formats yields marginal space savings that do not outweigh the time cost' + [Do not ignore system and hidden files] -a --do-not-ignore-system-and-hiden-files 'If specified, filenames which start with a dot or are likely generated by a software system (Windows, VCS...) won't be skipped without a message' 'The directory where the resource pack to process is' [result ZIP file] 'The path to the resulting ZIP file, ready to be distributed' ").get_matches_from(env::args()); @@ -43,14 +48,14 @@ fn main() { } fn run(parameters: ArgMatches) -> Result<(), Box> { - let mut progress = MultiBar::new(); - let skip_pack_icon = parameters.is_present("skip pack icon"); let use_zip_obfuscation = parameters.is_present("ZIP obfuscation"); let compress_already_compressed = parameters.is_present("Compress already compressed files"); + let ignore_system_hidden_files = + !parameters.is_present("Do not ignore system and hidden files"); let threads = cmp::max( match parameters.value_of("number of threads") { - Some(threads_string) => FromStr::from_str(threads_string), + Some(threads_string) => threads_string.parse(), None => Ok(num_cpus::get() * 2) }?, 1 @@ -71,14 +76,14 @@ fn run(parameters: ArgMatches) -> Result<(), Box> { canonical_root_path, &file_count, &file_process_thread_pool, - &mut progress, skip_pack_icon, compress_already_compressed, - µ_zip + µ_zip, + ignore_system_hidden_files )?; - // Listen for progress bar changes and wait until all is done - progress.listen(); + // Wait until all the work is done + file_process_thread_pool.join(); // Append the central directory println!("> Finishing up resource pack ZIP file..."); @@ -96,15 +101,15 @@ fn run(parameters: ArgMatches) -> Result<(), Box> { /// Recursively processes all the resource pack files in the given path, /// storing the resulting processed resource pack file data in a vector. #[allow(clippy::too_many_arguments)] // Not really much point in grouping the arguments -fn process_directory( +fn process_directory( canonical_root_path: &PathBuf, current_path: &PathBuf, file_count: &Arc, thread_pool: &ThreadPool, - progress: &mut MultiBar, skip_pack_icon: bool, compress_already_compressed: bool, - micro_zip: &Arc + micro_zip: &Arc, + ignore_system_hidden_files: bool ) -> Result<(), Box> { for entry in fs::read_dir(current_path)? { let entry = entry?; @@ -114,6 +119,24 @@ fn process_directory( let is_directory = file_type.is_dir(); let is_file = file_type.is_file(); + // Check whether this is a system or hidden file, and if so skip it without printing anything + if ignore_system_hidden_files { + let file_name = match path.file_name().unwrap_or(&EMPTY_OS_STR).to_str() { + Some(file_name) => file_name, + None => { + return Err(Box::new(SimpleError::new( + "A file name contains invalid UTF-8 codepoints" + ))) + } + }; + + let dot_file = file_name.chars().next().unwrap_or('x') == '.'; + + if dot_file || is_system_file(file_name, is_directory, is_file) { + continue; + } + } + if is_directory || is_file { if is_directory { process_directory( @@ -121,10 +144,10 @@ fn process_directory( &path, file_count, &thread_pool, - progress, skip_pack_icon, compress_already_compressed, - micro_zip + micro_zip, + ignore_system_hidden_files )?; } else { let mut relative_path = relativize_path_for_zip_file(&canonical_root_path, &path)?; @@ -132,21 +155,13 @@ fn process_directory( Some(path) => String::from(path), None => { return Err(Box::new(SimpleError::new( - "The path contains invalid UTF-8 codepoints" + "A path contains invalid UTF-8 codepoints" ))) } }; let path_in_root = path.parent().unwrap() == canonical_root_path; - let mut file_progress = progress.create_bar(0); - file_progress.message(format!("> {}... ", relative_path_str).as_str()); - file_progress.show_tick = true; - file_progress.show_speed = false; - file_progress.show_percent = false; - file_progress.show_counter = false; - file_progress.tick(); - let file_count = file_count.clone(); let micro_zip = micro_zip.clone(); @@ -156,7 +171,7 @@ fn process_directory( skip_pack_icon, path_in_root ) { - let result = resource_pack_file.process(&mut file_progress); + let result = resource_pack_file.process(); if result.is_ok() { let (processed_bytes, message) = result.ok().unwrap(); @@ -169,33 +184,29 @@ fn process_directory( &relative_path, ZipFileType::RegularFile, &processed_bytes, - resource_pack_file.is_compressed() && !compress_already_compressed, - &mut file_progress + resource_pack_file.is_compressed() && !compress_already_compressed ); if add_result.is_ok() { - file_progress.finish_println(&format!( - "> {}: {}", - relative_path_str, message - )); + println!("> {}: {}", relative_path_str, message); file_count.fetch_add(1, Ordering::Relaxed); } else { - file_progress.finish_println(&format!( - "> {}: Error occurred while adding to the ZIP file: {}", + println!( + "> {}: An error occurred while adding to the ZIP file: {}", relative_path_str, add_result.err().unwrap() - )); + ); } } else { - file_progress.finish_println(&format!( - "> {}: Error occurred while processing: {}", + println!( + "> {}: An error occurred while processing: {}", relative_path_str, result.err().unwrap() - )); + ); } } else { - file_progress.finish_println(&format!("> {}: Skipped", relative_path_str)); + println!("> {}: Skipped", relative_path_str); } }); } @@ -227,3 +238,13 @@ fn relativize_path_for_zip_file( Ok(relativized_path) } + +/// Checks whether the specified path likely represents a file generated for use +/// with some specific software that is not Minecraft, like VCS or operating system components. +fn is_system_file(file_name: &str, _is_directory: bool, is_file: bool) -> bool { + ( + file_name == "desktop.ini" || + file_name == "thumbs.db" || + file_name == "README.md" + ) && is_file +} diff --git a/src/micro_zip.rs b/src/micro_zip.rs index a9203d9f0..e302f8ec9 100644 --- a/src/micro_zip.rs +++ b/src/micro_zip.rs @@ -15,8 +15,6 @@ use simple_error::SimpleError; use crc32fast::Hasher; -use pbr::ProgressBar; - /// Compression methods supported by Minecraft in a ZIP file. enum ZipCompressionMethod { DEFLATE, @@ -106,7 +104,7 @@ impl ZipFileType { fn to_ms_dos_attributes(&self) -> u8 { match self { ZipFileType::RegularFile => 0x01 // FILE_ATTRIBUTE_READONLY - //ZipFileType::Folder => 0x10 // FILE_ATTRIBUTE_DIRECTORY (16) + //ZipFileType::Folder => 0x10 // FILE_ATTRIBUTE_DIRECTORY (16) } } } @@ -153,7 +151,7 @@ struct CentralDirectoryFileHeader { impl Default for CentralDirectoryFileHeader { fn default() -> CentralDirectoryFileHeader { CentralDirectoryFileHeader { - signature: 0x02014b50, // Magic number from the spec + signature: 0x02014B50, // Magic number from the spec version_made_by: 20, // MS-DOS OS, 2.0 ZIP spec compliance version_needed_to_extract: 20, // MS-DOS OS. Zip spec version may be higher than actually needed general_purpose_bit_flag: 0, // Irrelevant metadata, always 0 @@ -225,7 +223,7 @@ struct LocalFileHeader<'a> { impl<'a> Default for LocalFileHeader<'a> { fn default() -> LocalFileHeader<'a> { LocalFileHeader { - signature: 0x04034b50, // Magic number from the spec + signature: 0x04034B50, // Magic number from the spec version_needed_to_extract: 20, general_purpose_bit_flag: 0, last_mod_time: 0, @@ -277,7 +275,7 @@ struct CentralDirectoryEndRecord { impl Default for CentralDirectoryEndRecord { fn default() -> CentralDirectoryEndRecord { CentralDirectoryEndRecord { - signature: 0x06054b50, // Magic number from the spec + signature: 0x06054B50, // Magic number from the spec disk_number: 0, // No multi-disk support central_directory_start_disk: 0, // No multi-disk support comment_length: 0, // No comments @@ -328,13 +326,12 @@ impl MicroZip { /// Adds a file to the to-be-finished ZIP file represented by this struct, /// returning an error if something went wrong during the operation. - pub fn add_file( + pub fn add_file( &self, relativized_path: &PathBuf, file_type: ZipFileType, data: &[u8], - skip_compression: bool, - progress: &mut ProgressBar + skip_compression: bool ) -> Result<(), Box> { if self.writing_central_directory.load(Ordering::SeqCst) { return Err(Box::new(SimpleError::new( @@ -383,9 +380,11 @@ impl MicroZip { (compressed_data, Some(crc)) => { (ZipCompressionMethod::STORE, compressed_data, crc) } - _ => return Err(Box::new( - SimpleError::new("The contract of compress was violated")) - ) + _ => { + return Err(Box::new(SimpleError::new( + "The contract of compress was violated" + ))) + } } } else { ZipCompressionMethod::best_compress(data) @@ -400,8 +399,6 @@ impl MicroZip { ))); } - progress.tick(); - // Generate the local file header let actual_path_str = path_str_arc.as_ref().as_bytes(); let local_file_header = LocalFileHeader { @@ -454,8 +451,6 @@ impl MicroZip { } }; - progress.tick(); - partial_central_directory_entries.push(PartialCentralDirectoryFileHeader { compression_method: zip_compression_method_field, crc_32: crc, diff --git a/src/resource_pack_file.rs b/src/resource_pack_file.rs index fdac42920..b2bd53e54 100644 --- a/src/resource_pack_file.rs +++ b/src/resource_pack_file.rs @@ -1,14 +1,12 @@ use std::error::Error; use std::ffi::OsStr; use std::fs::{self, File}; -use std::io::{BufReader, Write}; +use std::io::BufReader; use std::path::PathBuf; use std::sync::{Arc, Once, RwLock}; use simple_error::SimpleError; -use pbr::ProgressBar; - use imagequant::Attributes; use imagequant::Image; @@ -33,11 +31,11 @@ lazy_static! { static GSTREAMER_INIT: Once = Once::new(); -pub trait ResourcePackFile { - /// Processes this resource pack file, returning its processed byte contents - /// and updating the progress status. A descriptive string containing the performed - /// action with the file is also returned in the tuple. - fn process(&self, progress: &mut ProgressBar) -> Result<(Vec, String), Box>; +pub trait ResourcePackFile { + /// Processes this resource pack file, returning its processed byte contents. + /// A descriptive string containing the performed action with the file is also + /// returned in the tuple. + fn process(&self) -> Result<(Vec, String), Box>; /// Returns the canonical extension for this resource pack file, to use for /// the resulting ZIP file. @@ -49,11 +47,11 @@ pub trait ResourcePackFile { } /// Converts a path to a resource pack file object. -pub fn path_to_resource_pack_file( +pub fn path_to_resource_pack_file( path: &PathBuf, skip_pack_icon: bool, path_in_root: bool -) -> Option>> { +) -> Option> { let empty_os_str = OsStr::new(""); let extension = path.extension().unwrap_or(empty_os_str); @@ -62,8 +60,7 @@ pub fn path_to_resource_pack_file( path: path.to_path_buf() })) } else if extension == "png" { - if path_in_root && skip_pack_icon && path.file_name().unwrap_or(empty_os_str) == "pack.png" - { + if path_in_root && skip_pack_icon && path.file_name().unwrap_or(empty_os_str) == "pack.png" { // Ignore pack.png if desired, as it is not visible for server resource packs None } else { @@ -71,8 +68,7 @@ pub fn path_to_resource_pack_file( path: path.to_path_buf() })) } - } else if extension == "ogg" || extension == "oga" || extension == "flac" || extension == "wav" - { + } else if extension == "ogg" || extension == "oga" || extension == "flac" || extension == "wav" { Some(Box::new(OggFile { path: path.to_path_buf() })) @@ -98,12 +94,11 @@ struct JsonFile { path: PathBuf } -impl ResourcePackFile for JsonFile { - fn process(&self, progress: &mut ProgressBar) -> Result<(Vec, String), Box> { +impl ResourcePackFile for JsonFile { + fn process(&self) -> Result<(Vec, String), Box> { // Parse the JSON so we know how to serialize it again in a compact manner let json_value: Value = serde_json::from_reader(BufReader::new(File::open(&self.path)?))?; - progress.tick(); Ok(( json_value.to_string().into_bytes(), JSON_COMPACTED.to_string() @@ -124,12 +119,11 @@ struct PngFile { path: PathBuf } -impl ResourcePackFile for PngFile { - fn process(&self, progress: &mut ProgressBar) -> Result<(Vec, String), Box> { +impl ResourcePackFile for PngFile { + fn process(&self) -> Result<(Vec, String), Box> { // Read the image to a memory struct let image = lodepng::decode32_file(&self.path)?; let image_bytes = image.buffer.as_ref(); - progress.tick(); let mut compression_attributes = Attributes::new(); compression_attributes.set_max_colors(256); @@ -143,7 +137,6 @@ impl ResourcePackFile for PngFile { image.height, 0.0 )?; - progress.tick(); // Quantize the image and remap it, so it uses the computed palette let mut quantization_result = compression_attributes.quantize(&image)?; @@ -151,7 +144,6 @@ impl ResourcePackFile for PngFile { let (palette, image_bytes) = quantization_result.remapped(&mut image)?; let mut encoder = Encoder::new(); let color_mode = encoder.info_raw_mut(); - progress.tick(); // Store used palette information color_mode.colortype = ColorType::PALETTE; @@ -161,9 +153,8 @@ impl ResourcePackFile for PngFile { } let encoded_png = encoder.encode(&image_bytes, image.width(), image.height())?; - progress.tick(); - // Init OxiPNG optimization settings settings + // Init OxiPNG optimization settings let mut alpha_optimizations = IndexSet::with_capacity(6); alpha_optimizations.insert(AlphaOptim::Black); alpha_optimizations.insert(AlphaOptim::Down); @@ -180,8 +171,6 @@ impl ResourcePackFile for PngFile { optimization_filters.insert(4); optimization_filters.insert(5); - progress.tick(); - // Optimize the palette reduced PNG with Zopfli // compression and more things that LodePNG and imagequant // don't do @@ -207,7 +196,6 @@ impl ResourcePackFile for PngFile { } )?; - progress.tick(); Ok(( optimized_png, format!( @@ -231,8 +219,8 @@ struct OggFile { path: PathBuf } -impl ResourcePackFile for OggFile { - fn process(&self, progress: &mut ProgressBar) -> Result<(Vec, String), Box> { +impl ResourcePackFile for OggFile { + fn process(&self) -> Result<(Vec, String), Box> { GSTREAMER_INIT.call_once(|| { gstreamer::init().unwrap(); }); @@ -263,8 +251,6 @@ impl ResourcePackFile for OggFile { let sink_element = gstreamer::ElementFactory::make("appsink", None)?; sink_element.set_property("sync", &false)?; // Output at max speed, not realtime - progress.tick(); - let gstreamer_pipeline = gstreamer::Pipeline::new(None); gstreamer_pipeline.add_many(&[ @@ -289,8 +275,6 @@ impl ResourcePackFile for OggFile { enc_element.link(&mux_element)?; mux_element.link(&sink_element)?; - progress.tick(); - // Handle the demuxer receiving a source pad let dec_element_weak = dec_element.downgrade(); dec_element.connect_pad_added(move |_, src_pad| { @@ -370,26 +354,22 @@ impl ResourcePackFile for OggFile { gstreamer_pipeline.set_state(gstreamer::State::Playing)?; - progress.tick(); - // Handle errors and end of stream let bus = gstreamer_pipeline.get_bus().unwrap(); for msg in bus.iter_timed(gstreamer::CLOCK_TIME_NONE) { - match msg.view() { - MessageView::Eos(..) => break, - MessageView::Error(err) => { - gstreamer_pipeline.set_state(gstreamer::State::Null)?; - - return Err(Box::new(SimpleError::new( - err.get_debug().unwrap_or_else(|| String::from("unknown")) - ))); - } - _ => progress.tick() + let message_view = msg.view(); + + if let MessageView::Eos(..) = message_view { + break; + } else if let MessageView::Error(err) = message_view { + gstreamer_pipeline.set_state(gstreamer::State::Null)?; + + return Err(Box::new(SimpleError::new( + err.get_debug().unwrap_or_else(|| String::from("unknown")) + ))); } } - progress.tick(); - // Clean up state before returning gstreamer_pipeline.set_state(gstreamer::State::Null)?; @@ -420,8 +400,8 @@ struct PassthroughFile<'a> { is_compressed: bool } -impl<'a, T: Write> ResourcePackFile for PassthroughFile<'a> { - fn process(&self, _progress: &mut ProgressBar) -> Result<(Vec, String), Box> { +impl<'a> ResourcePackFile for PassthroughFile<'a> { + fn process(&self) -> Result<(Vec, String), Box> { // Just copy file contents to memory Ok((fs::read(&self.path)?, String::from(self.message))) }