Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Merge branch 'hotfix/No_TABs'

Browse Source
This commit is contained in:
Jip J. Dekker 2014-04-02 14:21:34 +02:00
commit 9805bb5adb
4 changed files with 66 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
FourmiCrawler/pipelines.py
View File

@ -19,7 +19,7 @@ class FourmiPipeline(object):
""" """
value = item['attribute'], item['value'] value = item['attribute'], item['value']
if value in self.known_values: if value in self.known_values:
raise DropItem("Duplicate item found: %s" % item) raise DropItem("Duplicate item found: %s" % item) # #[todo] append sources of first item.
else: else:
self.known_values.add(value) self.known_values.add(value)
return item return item