Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Fixed Duplicate Pipeline + rename

Browse Source
This commit is contained in:
Jip J. Dekker 2014-05-08 15:45:42 +02:00
parent 2e654255c5
commit f193aac24a
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
FourmiCrawler/pipelines.py
View File

@ -6,7 +6,7 @@ import re
from scrapy.exceptions import DropItem
class FourmiPipeline(object):
class DuplicatePipeline(object):
def __init__(self):
self.known_values = set()
@ -18,7 +18,7 @@ class FourmiPipeline(object):
:param spider: The spider which scraped the spider
:return: :raise DropItem: Returns the item if unique or drops them if it's already known
"""
value = item['attribute'], item['value']
value = (item['attribute'], item['value'], item['conditions'])
if value in self.known_values:
raise DropItem("Duplicate item found: %s" % item) # #[todo] append sources of first item.
else:

2
FourmiCrawler/settings.py
View File

@ -12,7 +12,7 @@ SPIDER_MODULES = ['FourmiCrawler']
NEWSPIDER_MODULE = 'FourmiCrawler'
ITEM_PIPELINES = {
'FourmiCrawler.pipelines.AttributeSelectionPipeline': 100,
'FourmiCrawler.pipelines.FourmiPipeline': 200,
'FourmiCrawler.pipelines.DuplicatePipeline': 200,
}
FEED_URI = 'results.json'
FEED_FORMAT = 'jsonlines'