Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Fixed the synonym requests function for the start requests

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-06 22:22:29 +02:00
parent b5ada123ec
commit 321295a395
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
FourmiCrawler/spider.py
View File

@ -35,14 +35,14 @@ class FourmiSpider(Spider):
return source.parse(response)
return None
def get_synonym_requests(self, compound):
def get_synonym_requests(self, compound, force=False):
"""
A function that generates new Scrapy Request for each source given a new synonym of a compound.
:param compound: A compound name
:return: A list of Scrapy Request objects
"""
requests = []
if compound not in self.synonyms:
if force or compound not in self.synonyms:
self.synonyms.add(compound)
for parser in self._sources:
parser_requests = parser.new_compound_request(compound)
@ -57,7 +57,7 @@ class FourmiSpider(Spider):
"""
requests = []
for synonym in self.synonyms:
requests.extend(self.get_synonym_requests(synonym))
requests.extend(self.get_synonym_requests(synonym, force=True))
return requests
def add_sources(self, sources):