Skip to content

Commit

Permalink
chore(website): fix with_headers building
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mendez committed Dec 9, 2023
1 parent 1a2d98f commit 3edd41b
Show file tree
Hide file tree
Showing 5 changed files with 17 additions and 25 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
edition = "2021"
name = "spider_rs"
version = "0.0.5"
version = "0.0.6"
description = "The fastest web crawler written in Rust ported to nodejs."
repository = "https://github.com/spider-rs/spider-nodejs"

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import asyncio
from spider_rs import Website

async def main():
website = Website("https://choosealicense.com", False)
website = Website("https://choosealicense.com", False).with_headers({ "authorization": "myjwttoken" })
website.crawl()
print(website.get_links())

Expand Down
1 change: 0 additions & 1 deletion bench/scrappy.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ class MySpider(CrawlSpider):
allowed_domains = ['rsseau.fr']
start_urls = ['https://rsseau.fr']
links = []

rules = (
Rule(LinkExtractor(), callback='parse_item', follow=True),
)
Expand Down
2 changes: 1 addition & 1 deletion examples/website.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from spider_rs import Website

async def main():
website = Website("https://choosealicense.com", False)
website = Website("https://choosealicense.com", False).with_headers({ "authorization": "myjwttoken"})
website.crawl()
print(website.get_links())

Expand Down
35 changes: 14 additions & 21 deletions src/website.rs
Original file line number Diff line number Diff line change
Expand Up @@ -527,34 +527,27 @@ impl Website {
mut slf: PyRefMut<'_, Self>,
headers: Option<PyObject>,
) -> PyRefMut<'_, Self> {
use pyo3::types::PyDict;
use std::str::FromStr;

match headers {
Some(obj) => {
let mut h = spider::reqwest::header::HeaderMap::new();
let py = slf.py();
let dict = obj.downcast::<pyo3::types::PyDict>(py);

match obj.as_ref(slf.py()).iter() {
match dict {
Ok(keys) => {
for key in keys.into_iter() {
match key {
Ok(k) => {
let key_name = k.to_string();
let header_key = spider::reqwest::header::HeaderName::from_str(&key_name);

match header_key {
Ok(hn) => match k.get_item(key_name) {
Ok(he) => {
let header_value = he.to_string();

match spider::reqwest::header::HeaderValue::from_str(&header_value) {
Ok(hk) => {
h.append(hn, hk);
}
_ => (),
}
}
_ => (),
},
let header_key = spider::reqwest::header::HeaderName::from_str(&key.0.to_string());

match header_key {
Ok(hn) => {
let header_value = key.1.to_string();

match spider::reqwest::header::HeaderValue::from_str(&header_value) {
Ok(hk) => {
h.append(hn, hk);
}
_ => (),
}
}
Expand Down

0 comments on commit 3edd41b

Please sign in to comment.