Dekker1/Fourmi
Archived
1
0
Fork 0
Code

No two requests shall be the same!

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-04 19:12:08 +02:00
parent 0c9862d836
commit eb727bd6c4
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
FourmiCrawler/spider.py
View File

@ -10,7 +10,7 @@ class FourmiSpider(Spider):
""" """
name = "FourmiSpider" name = "FourmiSpider"
_sources = [] _sources = []
synonyms = [] synonyms = set()
def __init__(self, compound=None, selected_attributes=[".*"], *args, **kwargs): def __init__(self, compound=None, selected_attributes=[".*"], *args, **kwargs):
""" """
@ -19,7 +19,7 @@ class FourmiSpider(Spider):
:param selected_attributes: A list of regular expressions that the attributes should match. :param selected_attributes: A list of regular expressions that the attributes should match.
""" """
super(FourmiSpider, self).__init__(*args, **kwargs) super(FourmiSpider, self).__init__(*args, **kwargs)
self.synonyms.append(compound) self.synonyms.add(compound)
self.selected_attributes = selected_attributes self.selected_attributes = selected_attributes
def parse(self, response): def parse(self, response):
@ -42,6 +42,8 @@ class FourmiSpider(Spider):
:return: A list of Scrapy Request objects :return: A list of Scrapy Request objects
""" """
requests = [] requests = []
if compound not in self.synonyms:
self.synonyms.add(compound)
for parser in self._sources: for parser in self._sources:
parser_requests = parser.new_compound_request(compound) parser_requests = parser.new_compound_request(compound)
if parser_requests is not None: if parser_requests is not None: