refactor(src): change repo name

ma-pony · Nov 8, 2023 · c8ca21b · c8ca21b
1 parent a4e9ff4
commit c8ca21b
Show file tree

Hide file tree

Showing 20 changed files with 36 additions and 8 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,15 +1,15 @@
 [tool.poetry]
-name = "scrapy-kit"
-version = "0.1.12"
+name = "spider-tool-kit"
+version = "0.1.0"
 description = ""
 authors = ["Pony.Ma <[email protected]>"]
 readme = "README.md"
-packages = [{include = "scrapy_kit"}]
-homepage = "https://scrapy-kit.readthedocs.io/en/latest/"
-documentation = "https://scrapy-kit.readthedocs.io/en/latest/"
-repository = "https://github.com/ma-pony/scrapy-kit"
+packages = [{include = "spider_tool_kit"}]
+homepage = "https://spider-tool-kit.readthedocs.io/en/latest/"
+documentation = "https://spider-tool-kit.readthedocs.io/en/latest/"
+repository = "https://github.com/ma-pony/spider-tool-kit"
 license = "Apache-2.0"
-keywords = ["scrapy", "scrapy-kit", "scrapy-kit"]
+keywords = ["spider", "scrapy", "scrapy-kit", "spider-kit", "spider-tools", "spider-tool-kit"]
 
 
 [tool.poetry.dependencies]

diff --git a/scrapy_kit/__init__.py → spider_tool_kit/__init__.py b/scrapy_kit/__init__.py → spider_tool_kit/__init__.py
diff --git a/spider_tool_kit/contants.py b/spider_tool_kit/contants.py
@@ -0,0 +1 @@
+ENCODINGS = ['latin1', 'cp1252', 'iso-8859-1', 'gbk', 'ascii']
diff --git a/scrapy_kit/playwright/__init__.py → spider_tool_kit/scrapy/__init__.py b/scrapy_kit/playwright/__init__.py → spider_tool_kit/scrapy/__init__.py
diff --git a/scrapy_kit/middlewares/__init__.py → ...r_tool_kit/scrapy/middlewares/__init__.py b/scrapy_kit/middlewares/__init__.py → ...r_tool_kit/scrapy/middlewares/__init__.py
diff --git a/...kit/middlewares/proxy_connection_close.py → ...apy/middlewares/proxy_connection_close.py b/...kit/middlewares/proxy_connection_close.py → ...apy/middlewares/proxy_connection_close.py
diff --git a/scrapy_kit/pipelines/__init__.py → spider_tool_kit/scrapy/pipelines/__init__.py b/scrapy_kit/pipelines/__init__.py → spider_tool_kit/scrapy/pipelines/__init__.py
diff --git a/scrapy_kit/pipelines/crawl_one_pipeline.py → ...it/scrapy/pipelines/crawl_one_pipeline.py b/scrapy_kit/pipelines/crawl_one_pipeline.py → ...it/scrapy/pipelines/crawl_one_pipeline.py
diff --git a/scrapy_kit/pipelines/mongo_pipeline.py → ...ol_kit/scrapy/pipelines/mongo_pipeline.py b/scrapy_kit/pipelines/mongo_pipeline.py → ...ol_kit/scrapy/pipelines/mongo_pipeline.py
diff --git a/test/unit/pipelines/__init__.py → ...er_tool_kit/scrapy/playwright/__init__.py b/test/unit/pipelines/__init__.py → ...er_tool_kit/scrapy/playwright/__init__.py
diff --git a/scrapy_kit/playwright/request.py → spider_tool_kit/scrapy/playwright/request.py b/scrapy_kit/playwright/request.py → spider_tool_kit/scrapy/playwright/request.py
diff --git a/spider_tool_kit/tools/__init__.py b/spider_tool_kit/tools/__init__.py
@@ -0,0 +1 @@
+from .text_encode import *
diff --git a/spider_tool_kit/tools/text_encode.py b/spider_tool_kit/tools/text_encode.py
@@ -0,0 +1,16 @@
+from spider_tool_kit.contants import ENCODINGS
+
+
+def fix_encode(garbled_text: str, decoding='utf-8') -> (str, str):
+    """
+    修复乱码
+    :param garbled_text: 乱码文本
+    :param decoding: 解码方式
+    :return: 正常文本, 编码
+    """
+    for encoding in ENCODINGS:
+        try:
+            res = garbled_text.encode(encoding).decode(decoding)
+            return res, encoding
+        except (UnicodeEncodeError, UnicodeDecodeError):
+            continue
diff --git a/test/unit/playwright/__init__.py → test/unit/scrapy/__init__.py b/test/unit/playwright/__init__.py → test/unit/scrapy/__init__.py
diff --git a/test/unit/playwright/request.py → test/unit/scrapy/pipelines/__init__.py b/test/unit/playwright/request.py → test/unit/scrapy/pipelines/__init__.py
diff --git a/test/unit/pipelines/mongo.py → test/unit/scrapy/pipelines/mongo.py b/test/unit/pipelines/mongo.py → test/unit/scrapy/pipelines/mongo.py
@@ -4,7 +4,7 @@
 import pytest
 from pytest_twisted import ensureDeferred
 
-from scrapy_kit.pipelines.mongo_pipeline import MongoPipeline
+from spider_tool_kit.scrapy.pipelines.mongo_pipeline import MongoPipeline
 
 
 class TestMongoPipeline:

diff --git a/test/unit/scrapy/playwright/__init__.py b/test/unit/scrapy/playwright/__init__.py
diff --git a/test/unit/scrapy/playwright/request.py b/test/unit/scrapy/playwright/request.py
diff --git a/test/unit/tools/__init__.py b/test/unit/tools/__init__.py
diff --git a/test/unit/tools/text_encode.py b/test/unit/tools/text_encode.py
@@ -0,0 +1,10 @@
+from spider_tool_kit.tools import fix_encode
+
+
+class TestTextEncode:
+    def test_fix_encode(self):
+        garbled_text = "äº¿å…ƒ"
+        expected_text = "亿元"
+        expected_encoding = "cp1252"
+        result = fix_encode(garbled_text)
+        assert result == (expected_text, expected_encoding)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ENCODINGS = ['latin1', 'cp1252', 'iso-8859-1', 'gbk', 'ascii']