From 87def195bd77136108c596315b1f05388b010a02 Mon Sep 17 00:00:00 2001 From: Dan Claudiu Pop Date: Thu, 3 Mar 2022 19:00:30 +0200 Subject: [PATCH] ability to save cookies to disk (#8) * ability to save cookies to disk * bump version to v0.2.3 --- CHANGELOG.md | 3 +++ README.md | 17 +++++++++++++++++ pyproject.toml | 2 +- src/robox/_client.py | 16 ++++++++++++++++ tests/test_client.py | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 526241f..9a29ed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 0.2.3 (3th March, 2022) +* save and load cookies from disk + ## 0.2.2 (22th February, 2022) * ability to parse tables diff --git a/README.md b/README.md index 21a33b8..874ff99 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,23 @@ with Robox() as robox: ... ``` +An example on how to reuse authentication state with cookies: +```python +with Robox() as robox: + page = robox.open("https://news.ycombinator.com/login") + form = page.get_forms()[0] + form.fill_in("acct", value=os.getenv("PASSWORD")) + form.fill_in("pw", value=os.getenv("USERNAME")) + page.submit_form(form) + robox.save_cookies("cookies.json") + + +with Robox() as robox: + robox.load_cookies("cookies.json") + page = robox.open("https://news.ycombinator.com/") + assert page.parsed.find("a", attrs={"id": "logout"}) +``` + See [examples](https://github.com/danclaudiupop/robox/tree/main/examples) folder for more detailed examples. ## Installation diff --git a/pyproject.toml b/pyproject.toml index 8710c65..47c58d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "robox" -version = "0.2.2" +version = "0.2.3" description = "Robox is a simple library for exploring/scraping the web or testing a website you’re developing." authors = ["Dan Claudiu Pop "] license = "BSD 3" diff --git a/src/robox/_client.py b/src/robox/_client.py index ff23a0b..b81df39 100644 --- a/src/robox/_client.py +++ b/src/robox/_client.py @@ -1,8 +1,10 @@ import asyncio import itertools +import json import random import time import typing as tp +from pathlib import Path import httpx from httpx._client import USE_CLIENT_DEFAULT, UseClientDefault @@ -59,6 +61,20 @@ def current_url(self) -> URL: else: raise RoboxError("Not tracking history") + def save_cookies(self, filename: str) -> None: + cookies = {} + for cookie in self.cookies.jar: + cookies[cookie.name] = cookie.value + with open(filename, "w") as f: + json.dump(cookies, f) + + def load_cookies(self, filename: str) -> None: + if not Path(filename).is_file(): + return None + with open(filename, "r") as f: + cookies = httpx.Cookies(json.load(f)) + self.cookies = cookies + def _increment_request_counter(self) -> None: self.total_requests = next(self._request_counter) diff --git a/tests/test_client.py b/tests/test_client.py index 197e6b1..88c8e29 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,3 +1,5 @@ +import json +from http.cookiejar import Cookie, CookieJar from unittest.mock import MagicMock, patch import httpx @@ -150,3 +152,41 @@ def test_retry_recovarable(respx_mock): with Robox(options=Options(retry=True, retry_max_attempts=2)) as robox: page = robox.open(TEST_URL) assert page.status_code == 200 + + +def test_save_and_load_cookies(respx_mock, tmp_path): + cookies = CookieJar() + cookie = Cookie( + version=0, + name="example-name", + value="example-value", + port=None, + port_specified=False, + domain="", + domain_specified=False, + domain_initial_dot=False, + path="/", + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={"HttpOnly": ""}, + rfc2109=False, + ) + cookies.set_cookie(cookie) + + respx_mock.get(TEST_URL).respond(200) + with Robox(cookies=cookies) as robox: + robox.open(TEST_URL) + robox.save_cookies(tmp_path / "cookies.json") + with open(tmp_path / "cookies.json") as f: + loaded_cookies = json.load(f) + assert loaded_cookies == {"example-name": "example-value"} + assert len(robox.cookies) == 1 + + with Robox() as robox: + robox.load_cookies(tmp_path / "cookies.json") + robox.open(TEST_URL) + assert len(robox.cookies) == 1