Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Fixed the synonym requests function for the start requests

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-06 22:22:29 +02:00
parent b5ada123ec
commit 321295a395
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
FourmiCrawler/spider.py
View File

@ -35,14 +35,14 @@ class FourmiSpider(Spider):
return source.parse(response) return source.parse(response)
return None return None
def get_synonym_requests(self, compound): def get_synonym_requests(self, compound, force=False):
""" """
A function that generates new Scrapy Request for each source given a new synonym of a compound. A function that generates new Scrapy Request for each source given a new synonym of a compound.
:param compound: A compound name :param compound: A compound name
:return: A list of Scrapy Request objects :return: A list of Scrapy Request objects
""" """
requests = [] requests = []
if compound not in self.synonyms: if force or compound not in self.synonyms:
self.synonyms.add(compound) self.synonyms.add(compound)
for parser in self._sources: for parser in self._sources:
parser_requests = parser.new_compound_request(compound) parser_requests = parser.new_compound_request(compound)
@ -57,7 +57,7 @@ class FourmiSpider(Spider):
""" """
requests = [] requests = []
for synonym in self.synonyms: for synonym in self.synonyms:
requests.extend(self.get_synonym_requests(synonym)) requests.extend(self.get_synonym_requests(synonym, force=True))
return requests return requests
def add_sources(self, sources): def add_sources(self, sources):