Skip to content

Commit

Permalink
chore(website): fix disk compile feature
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mendez committed Feb 7, 2025
1 parent 428957c commit cae5dd5
Show file tree
Hide file tree
Showing 11 changed files with 156 additions and 152 deletions.
280 changes: 144 additions & 136 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion spider/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider"
version = "2.27.44"
version = "2.27.47"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
1 change: 0 additions & 1 deletion spider/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ pub extern crate url;
pub extern crate async_job;
#[cfg(feature = "openai")]
pub extern crate async_openai;
#[cfg(feature = "encoding")]
pub extern crate auto_encoder;
#[cfg(feature = "flexbuffers")]
pub extern crate flexbuffers;
Expand Down
2 changes: 1 addition & 1 deletion spider/src/page.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2772,7 +2772,7 @@ pub fn get_html_encoded(html: &Option<Box<Bytes>>, label: &str) -> String {

#[cfg(not(feature = "encoding"))]
/// Get the content with proper encoding. Pass in a proper encoding label like SHIFT_JIS.
pub fn get_html_encoded(html: &Option<Bytes>, _label: &str) -> String {
pub fn get_html_encoded(html: &Option<Box<Bytes>>, _label: &str) -> String {
match html {
Some(b) => String::from_utf8_lossy(b).to_string(),
_ => Default::default(),
Expand Down
3 changes: 1 addition & 2 deletions spider/src/website.rs
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ impl Website {

/// Check if signature exists (ignore case). This does nothing with `disk` flag enabled.
#[cfg(not(feature = "disk"))]
async fn is_allowed_signature_disk(&self, _signature_to_check: &str) -> bool {
async fn is_allowed_signature_disk(&self, _signature_to_check: u64) -> bool {
true
}

Expand Down Expand Up @@ -2651,7 +2651,6 @@ impl Website {

self.configuration.configure_allowlist();


let semaphore = self.setup_semaphore();

let shared = Arc::new((
Expand Down
2 changes: 1 addition & 1 deletion spider_chrome/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_chrome"
version = "2.27.44"
version = "2.27.47"
rust-version = "1.70"
authors = [
"j-mendez <[email protected]>"
Expand Down
2 changes: 1 addition & 1 deletion spider_cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_cli"
version = "2.27.44"
version = "2.27.47"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
2 changes: 1 addition & 1 deletion spider_transformations/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_transformations"
version = "2.27.44"
version = "2.27.47"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
2 changes: 1 addition & 1 deletion spider_utils/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_utils"
version = "2.27.44"
version = "2.27.47"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
10 changes: 4 additions & 6 deletions spider_utils/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
use scraper::{ElementRef, Html, Selector};
use hashbrown::{hash_map::Entry, HashMap};
use lazy_static::lazy_static;
use tokio_stream::StreamExt;
use log::{self, warn};
use hashbrown::{hash_map::Entry, HashMap};
use scraper::{ElementRef, Html, Selector};
use std::{fmt::Debug, hash::Hash};
use sxd_document::parser;
use sxd_xpath::evaluate_xpath;
use tokio_stream::StreamExt;

/// The type of selectors that can be used to query.
#[derive(Default, Debug, Clone)]
Expand Down Expand Up @@ -255,9 +255,7 @@ where

/// Build valid css selectors for extracting. The hashmap takes items with the key for the object key and the value is the css selector.
#[cfg(not(feature = "indexset"))]
pub fn build_selectors<K, V>(
selectors: HashMap<K, hashbrown::HashSet<V>>,
) -> DocumentSelectors<K>
pub fn build_selectors<K, V>(selectors: HashMap<K, hashbrown::HashSet<V>>) -> DocumentSelectors<K>
where
K: AsRef<str> + Eq + Hash + Clone + Debug,
V: AsRef<str> + Debug + AsRef<str>,
Expand Down
2 changes: 1 addition & 1 deletion spider_worker/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_worker"
version = "2.27.44"
version = "2.27.47"
authors = [
"j-mendez <[email protected]>"
]
Expand Down

0 comments on commit cae5dd5

Please sign in to comment.