Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Merge branch 'hotfix/no-requests' into develop

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-06 22:28:20 +02:00
commit 96e41868bd
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
FourmiCrawler/spider.py
View File

@ -35,14 +35,14 @@ class FourmiSpider(Spider):
return source.parse(response)
return None
def get_synonym_requests(self, compound):
def get_synonym_requests(self, compound, force=False):
"""
A function that generates new Scrapy Request for each source given a new synonym of a compound.
:param compound: A compound name
:return: A list of Scrapy Request objects
"""
requests = []
if compound not in self.synonyms:
if force or compound not in self.synonyms:
self.synonyms.add(compound)
for parser in self._sources:
parser_requests = parser.new_compound_request(compound)
@ -57,7 +57,7 @@ class FourmiSpider(Spider):
"""
requests = []
for synonym in self.synonyms:
requests.extend(self.get_synonym_requests(synonym))
requests.extend(self.get_synonym_requests(synonym, force=True))
return requests
def add_sources(self, sources):

5
tests/test_spider.py
View File

@ -43,7 +43,10 @@ class TestFoumiSpider(unittest.TestCase):
src2 = ChemSpider()
self.spi.add_source(src2)
self.assertIsNotNone(self.spi.start_requests())
requests = self.spi.start_requests()
self.assertGreater(len(requests), 0)
self.assertIsInstance(requests[0], Request)
def test_synonym_requests(self):
# A test for the synonym request function