Skip to content

Commit

Permalink
chore(transform): fix default features [#264]
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mendez committed Feb 7, 2025
1 parent 0f3afb3 commit 428957c
Show file tree
Hide file tree
Showing 9 changed files with 41 additions and 33 deletions.
20 changes: 12 additions & 8 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion examples/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_examples"
version = "2.8.6"
version = "2.27.43"
authors = [
"j-mendez <[email protected]>",
]
Expand Down
2 changes: 1 addition & 1 deletion spider/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider"
version = "2.27.43"
version = "2.27.44"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
2 changes: 1 addition & 1 deletion spider_chrome/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_chrome"
version = "2.27.43"
version = "2.27.44"
rust-version = "1.70"
authors = [
"j-mendez <[email protected]>"
Expand Down
2 changes: 1 addition & 1 deletion spider_cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_cli"
version = "2.27.43"
version = "2.27.44"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
3 changes: 2 additions & 1 deletion spider_transformations/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_transformations"
version = "2.27.43"
version = "2.27.44"
authors = [
"j-mendez <[email protected]>"
]
Expand Down Expand Up @@ -33,6 +33,7 @@ spider_scraper = "0.1"
[dependencies.spider]
version = "2"
path = "../spider"
default-features = false

[dev-dependencies]
maud = "0.26"
Expand Down
15 changes: 9 additions & 6 deletions spider_utils/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_utils"
version = "2.27.43"
version = "2.27.44"
authors = [
"j-mendez <[email protected]>"
]
Expand All @@ -19,18 +19,21 @@ sxd-document = "0.3"
sxd-xpath = "0.4"
serde = { version = "1", optional = true, features = ["derive"] }
spider_scraper = "0.1"

[dependencies.spider]
version = "2"
path = "../spider"
lazy_static = "1"
tokio-stream = "0.1"
hashbrown = { version = "0.15", default-features = true }
log = "0.4"

[dependencies.spider_transformations]
version = "2"
path = "../spider_transformations"
optional = true

[dev-dependencies]
tokio = { version = "1", features = ["full"]}

[features]
default = []
indexset = ["indexmap"]
serde = ["dep:serde", "indexmap/serde", "spider/serde"]
serde = ["dep:serde", "indexmap/serde"]
transformations = ["dep:spider_transformations"]
26 changes: 13 additions & 13 deletions spider_utils/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
use scraper::{ElementRef, Html, Selector};
use spider::lazy_static::lazy_static;
use spider::tokio_stream::StreamExt;
use spider::utils::log;
use spider::{
hashbrown::{hash_map::Entry, HashMap},
tokio,
};
use lazy_static::lazy_static;
use tokio_stream::StreamExt;
use log::{self, warn};
use hashbrown::{hash_map::Entry, HashMap};
use std::{fmt::Debug, hash::Hash};
use sxd_document::parser;
use sxd_xpath::evaluate_xpath;
Expand Down Expand Up @@ -49,7 +46,7 @@ where
let mut map: CSSQueryMap = HashMap::new();

if !selectors.css.is_empty() {
let mut stream = spider::tokio_stream::iter(&selectors.css);
let mut stream = tokio_stream::iter(&selectors.css);
let fragment = Box::new(Html::parse_document(html));

while let Some(selector) = stream.next().await {
Expand Down Expand Up @@ -220,8 +217,8 @@ where
if is_valid_xpath(selector_str.as_ref()) {
selectors_vec_xpath.push(selector_str.as_ref().to_string())
} else {
log(
"",
warn!(
"{}",
format!(
"Failed to parse selector '{}': {:?}",
selector_str.as_ref(),
Expand Down Expand Up @@ -259,13 +256,13 @@ where
/// Build valid css selectors for extracting. The hashmap takes items with the key for the object key and the value is the css selector.
#[cfg(not(feature = "indexset"))]
pub fn build_selectors<K, V>(
selectors: HashMap<K, spider::hashbrown::HashSet<V>>,
selectors: HashMap<K, hashbrown::HashSet<V>>,
) -> DocumentSelectors<K>
where
K: AsRef<str> + Eq + Hash + Clone + Debug,
V: AsRef<str> + Debug + AsRef<str>,
{
build_selectors_base::<K, V, spider::hashbrown::HashSet<V>>(selectors)
build_selectors_base::<K, V, hashbrown::HashSet<V>>(selectors)
}

/// Build valid css selectors for extracting. The hashmap takes items with the key for the object key and the value is the css selector.
Expand All @@ -279,14 +276,15 @@ where
}

#[cfg(not(feature = "indexset"))]
pub type QueryCSSSelectSet<'a> = spider::hashbrown::HashSet<&'a str>;
pub type QueryCSSSelectSet<'a> = hashbrown::HashSet<&'a str>;
#[cfg(feature = "indexset")]
pub type QueryCSSSelectSet<'a> = indexmap::IndexSet<&'a str>;
#[cfg(not(feature = "indexset"))]
pub type QueryCSSMap<'a> = HashMap<&'a str, QueryCSSSelectSet<'a>>;
#[cfg(feature = "indexset")]
pub type QueryCSSMap<'a> = HashMap<&'a str, QueryCSSSelectSet<'a>>;

#[cfg(test)]
#[tokio::test]
async fn test_css_query_select_map_streamed() {
let map = QueryCSSMap::from([("list", QueryCSSSelectSet::from([".list", ".sub-list"]))]);
Expand All @@ -311,6 +309,7 @@ fn test_css_query_select_map() {
assert!(!data.is_empty(), "CSS extraction failed",);
}

#[cfg(test)]
#[tokio::test]
async fn test_css_query_select_map_streamed_multi_join() {
let map = QueryCSSMap::from([("list", QueryCSSSelectSet::from([".list", ".sub-list"]))]);
Expand All @@ -328,6 +327,7 @@ async fn test_css_query_select_map_streamed_multi_join() {
assert!(!data.is_empty(), "CSS extraction failed");
}

#[cfg(test)]
#[tokio::test]
async fn test_xpath_query_select_map_streamed() {
let map = QueryCSSMap::from([(
Expand Down
2 changes: 1 addition & 1 deletion spider_worker/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_worker"
version = "2.27.43"
version = "2.27.44"
authors = [
"j-mendez <[email protected]>"
]
Expand Down

0 comments on commit 428957c

Please sign in to comment.