diff --git a/scrapy_dangdang_040/scrapy_dangdang_040/pipelines.py b/scrapy_dangdang_040/scrapy_dangdang_040/pipelines.py index 05233a7..4958929 100644 --- a/scrapy_dangdang_040/scrapy_dangdang_040/pipelines.py +++ b/scrapy_dangdang_040/scrapy_dangdang_040/pipelines.py @@ -5,6 +5,7 @@ # useful for handling different item types with a single interface +import urllib.request from itemadapter import ItemAdapter # 如果想要使用管道必须在settings中开启管道 @@ -30,3 +31,22 @@ def process_item(self, item, spider): def close_spider(self, spider): self.fp.close() + + +# 多条管道同时开启 + +# (1)定义管道类 +# (2)在setting中开启管道 +# 'scrapy_dangdang_040.pipelines.DangDangDownloadPipeline':301 + + +class DangDangDownloadPipeline: + def process_item(self, item, spider): + + url = 'https:'+item.get('src') + print('******************9999999999999999999999999999') + print(url) + filename = './books/'+item.get('name')+'.jpg' + print(filename) + urllib.request.urlretrieve(url=url, filename=filename) + return item diff --git a/scrapy_dangdang_040/scrapy_dangdang_040/settings.py b/scrapy_dangdang_040/scrapy_dangdang_040/settings.py index 34dff05..1c3f91e 100644 --- a/scrapy_dangdang_040/scrapy_dangdang_040/settings.py +++ b/scrapy_dangdang_040/scrapy_dangdang_040/settings.py @@ -62,9 +62,14 @@ # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html -# ITEM_PIPELINES = { -# 'scrapy_dangdang_040.pipelines.ScrapyDangdang040Pipeline': 300, -# } +ITEM_PIPELINES = { + # 管道可以有很多个,管道是有优先级的,优先级的范围是1-1000,值越小,优先级越高 + # + 'scrapy_dangdang_040.pipelines.ScrapyDangdang040Pipeline': 300, + # DangDangDownloadPipeline + 'scrapy_dangdang_040.pipelines.DangDangDownloadPipeline': 301 +} + # Enable and configure the AutoThrottle extension (disabled by default) # See https://docs.scrapy.org/en/latest/topics/autothrottle.html diff --git a/scrapy_dangdang_040/scrapy_dangdang_040/spiders/book.json b/scrapy_dangdang_040/scrapy_dangdang_040/spiders/book.json index acbaeb4..dc8ae8d 100644 --- a/scrapy_dangdang_040/scrapy_dangdang_040/spiders/book.json +++ b/scrapy_dangdang_040/scrapy_dangdang_040/spiders/book.json @@ -1,8 +1,6 @@ {'name': ' 顾漫:你是我的荣耀【赠海报+明信片2张】 杨洋、迪丽热巴主演原著,签名本随机掉落', 'price': '¥18.00', - 'src': '//img3m4.ddimg.cn/94/0/27869584-1_b_14.jpg' -} - {'name': ' 那个不为人知的故事', + 'src': '//img3m4.ddimg.cn/94/0/27869584-1_b_14.jpg'}{'name': ' 那个不为人知的故事', 'price': '¥21.00', 'src': '//img3m2.ddimg.cn/46/8/29124262-1_b_3.jpg'}{'name': ' 偷偷藏不住(共2册)', 'price': '¥38.87',