Skip to content

Commit

Permalink
Added BOM sniffing check to file load
Browse files Browse the repository at this point in the history
  • Loading branch information
AngelOnFira committed May 25, 2024
1 parent e3c1d0b commit 04e6135
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 3 deletions.
10 changes: 10 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ async = ["async-trait"]
lazy_static = "1.4"
serde = "1.0"
nom = "7"
encoding_rs_io = "0.1"

async-trait = { version = "0.1", optional = true }
toml = { version = "0.8", optional = true }
Expand Down
15 changes: 12 additions & 3 deletions src/file/source/file.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
use std::env;
use std::error::Error;
use std::fs;
use std::io;
use std::io::Read;
use std::path::PathBuf;

use encoding_rs_io::DecodeReaderBytesBuilder;

use crate::file::{
format::ALL_EXTENSIONS, source::FileSourceResult, FileSource, FileStoredFormat, Format,
};
Expand Down Expand Up @@ -114,8 +116,15 @@ where
.and_then(|base| pathdiff::diff_paths(&filename, base))
.unwrap_or_else(|| filename.clone());

// Read contents from file
let text = fs::read_to_string(filename)?;
let mut text = String::new();

DecodeReaderBytesBuilder::new()
// On Windows, we need to check for added Byte Order Mark (BOM). To
// do this, we don't specify an encoding, and enable BOM sniffing.
.encoding(None)
.bom_sniffing(true)
.build(std::fs::File::open(filename)?)
.read_to_string(&mut text)?;

Ok(FileSourceResult {
uri: Some(uri.to_string_lossy().into_owned()),
Expand Down

0 comments on commit 04e6135

Please sign in to comment.