使用多管道的方法自己定义类同时在setting.py中设置

JLUVicent · Sep 16, 2021 · 12a44a0 · 12a44a0
1 parent c4374cc
commit 12a44a0
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 6 deletions.
diff --git a/scrapy_dangdang_040/scrapy_dangdang_040/pipelines.py b/scrapy_dangdang_040/scrapy_dangdang_040/pipelines.py
@@ -5,6 +5,7 @@
 
 
 # useful for handling different item types with a single interface
+import urllib.request
 from itemadapter import ItemAdapter
 
 # 如果想要使用管道必须在settings中开启管道
@@ -30,3 +31,22 @@ def process_item(self, item, spider):
 
     def close_spider(self, spider):
         self.fp.close()
+
+
+# 多条管道同时开启
+
+# （1）定义管道类
+# （2）在setting中开启管道
+# 'scrapy_dangdang_040.pipelines.DangDangDownloadPipeline':301
+
+
+class DangDangDownloadPipeline:
+    def process_item(self, item, spider):
+
+        url = 'https:'+item.get('src')
+        print('******************9999999999999999999999999999')
+        print(url)
+        filename = './books/'+item.get('name')+'.jpg'
+        print(filename)
+        urllib.request.urlretrieve(url=url, filename=filename)
+        return item
diff --git a/scrapy_dangdang_040/scrapy_dangdang_040/settings.py b/scrapy_dangdang_040/scrapy_dangdang_040/settings.py
@@ -62,9 +62,14 @@
 
 # Configure item pipelines
 # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-# ITEM_PIPELINES = {
-#    'scrapy_dangdang_040.pipelines.ScrapyDangdang040Pipeline': 300,
-# }
+ITEM_PIPELINES = {
+    # 管道可以有很多个，管道是有优先级的，优先级的范围是1-1000，值越小，优先级越高
+    #
+    'scrapy_dangdang_040.pipelines.ScrapyDangdang040Pipeline': 300,
+    # DangDangDownloadPipeline
+    'scrapy_dangdang_040.pipelines.DangDangDownloadPipeline': 301
+}
+
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/autothrottle.html

diff --git a/scrapy_dangdang_040/scrapy_dangdang_040/spiders/book.json b/scrapy_dangdang_040/scrapy_dangdang_040/spiders/book.json
@@ -1,8 +1,6 @@
 {'name': ' 顾漫：你是我的荣耀【赠海报+明信片2张】 杨洋、迪丽热巴主演原著，签名本随机掉落',
  'price': '¥18.00',
- 'src': '//img3m4.ddimg.cn/94/0/27869584-1_b_14.jpg'
-}
- {'name': ' 那个不为人知的故事',
+ 'src': '//img3m4.ddimg.cn/94/0/27869584-1_b_14.jpg'}{'name': ' 那个不为人知的故事',
  'price': '¥21.00',
  'src': '//img3m2.ddimg.cn/46/8/29124262-1_b_3.jpg'}{'name': ' 偷偷藏不住（共2册）',
  'price': '¥38.87',