chore(params): add sitemap_only

spider-rs · Jan 21, 2025 · 728a7ee · 728a7ee
1 parent 296f640
commit 728a7ee
Show file tree

Hide file tree

Showing 14 changed files with 33 additions and 35 deletions.
diff --git a/.gitignore b/.gitignore
@@ -41,4 +41,5 @@ docs/api/
 # Miscellaneous
 *.log
 .DS_Store
-Thumbs.db
+Thumbs.db
+deploy.sh
diff --git a/book/book.toml b/book/book.toml
@@ -1,5 +1,5 @@
 [book]
-authors = ["Jeff Mendez <jeff@a11ywatch.com>"]
+authors = ["Jeff Mendez <jeff@spider.cloud>"]
 language = "en"
 multilingual = false
 src = "src"

diff --git a/cli/Cargo.lock b/cli/Cargo.lock
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -1,8 +1,8 @@
 [package]
 name = "spider-cloud-cli"
-version = "0.1.25"
+version = "0.1.26"
 edition = "2021"
-authors = [ "j-mendez <jeff@a11ywatch.com>"]
+authors = [ "j-mendez <jeff@spider.cloud>"]
 description = "The Spider Cloud CLI for web crawling and scraping"
 license = "MIT"
 readme = "README.md"

diff --git a/javascript/package-lock.json b/javascript/package-lock.json
diff --git a/javascript/package.json b/javascript/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.1.25",
+  "version": "0.1.26",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",
@@ -20,13 +20,13 @@
     "api",
     "llm scraping"
   ],
-  "author": "Jeff Mendez<jeff@a11ywatch.com>",
+  "author": "Jeff Mendez<jeff@spider.cloud>",
   "license": "MIT",
   "devDependencies": {
-    "@types/node": "22.10.1",
+    "@types/node": "22.10.7",
     "dotenv": "^16.4.7",
     "tsx": "^4.19.2",
-    "typescript": "5.7.2"
+    "typescript": "5.7.3"
   },
   "dependencies": {
     "exponential-backoff": "^3.1.1"

diff --git a/javascript/src/config.ts b/javascript/src/config.ts
@@ -354,9 +354,9 @@ export interface SpiderParams {
   sitemap?: boolean;
 
   /**
-   * Get page insights to determine information like request duration, accessibility, and other web vitals. Requires the `metadata` parameter to be set to `true`.
+   * Specifies whether to only use the sitemap links.
    */
-  page_insights?: boolean;
+  sitemap_only?: boolean;
 
   /**
    * External domains to include the crawl.

diff --git a/python/setup.py b/python/setup.py
@@ -8,7 +8,7 @@ def read_file(fname):
 
 setup(
     name="spider-client",
-    version="0.1.25",
+    version="0.1.26",
     url="https://github.com/spider-rs/spider-clients/tree/main/python",
     author="Spider",
     author_email="[email protected]",

diff --git a/python/spider/async_spider.py b/python/spider/async_spider.py
@@ -431,7 +431,7 @@ def _prepare_headers(
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": "AsyncSpider-Client/0.1.25",
+            "User-Agent": "AsyncSpider-Client/0.1.26",
         }
 
     async def _handle_error(self, response: ClientResponse, action: str) -> None:

diff --git a/python/spider/spider.py b/python/spider/spider.py
@@ -427,7 +427,7 @@ def _prepare_headers(self, content_type: str = "application/json"):
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": f"Spider-Client/0.1.25",
+            "User-Agent": f"Spider-Client/0.1.26",
         }
 
     def _post_request(self, url: str, data, headers, stream=False):

diff --git a/python/spider/spider_types.py b/python/spider/spider_types.py
@@ -238,8 +238,8 @@ class RequestParamsDict(TypedDict, total=False):
     # Specifies whether to use the sitemap links.
     sitemap: Optional[bool]
 
-    # Get page insights to determine information like request duration, accessibility, and other web vitals. Requires the `metadata` parameter to be set to `true`.
-    page_insights: Optional[bool]
+    # Specifies whether to only use the sitemap links.
+    sitemap_only: Optional[bool]
 
     # External domains to include in the crawl.
     external_domains: Optional[List[str]]

diff --git a/rust/Cargo.lock b/rust/Cargo.lock
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -1,8 +1,8 @@
 [package]
 name = "spider-client"
-version = "0.1.25"
+version = "0.1.26"
 edition = "2021"
-authors = [ "j-mendez <jeff@a11ywatch.com>"]
+authors = [ "j-mendez <jeff@spider.cloud>"]
 description = "Spider Cloud client"
 license = "MIT"
 readme = "README.md"

diff --git a/rust/src/lib.rs b/rust/src/lib.rs
@@ -339,9 +339,6 @@ pub struct RequestParams {
     /// Specifies whether to use the sitemap links.
     pub sitemap: Option<bool>,
     #[serde(default)]
-    /// Get page insights to determine information like request duration, accessibility, and other web vitals. Requires the `metadata` parameter to be set to `true`.
-    pub page_insights: Option<bool>,
-    #[serde(default)]
     /// External domains to include the crawl.
     pub external_domains: Option<Vec<String>>,
     #[serde(default)]