Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Merge branch 'develop' into feature/sources-configuration

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-06 22:31:49 +02:00
commit c9f902993c
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
FourmiCrawler/spider.py
View File

@ -35,14 +35,14 @@ class FourmiSpider(Spider):
return source.parse(response) return source.parse(response)
return None return None
def get_synonym_requests(self, compound): def get_synonym_requests(self, compound, force=False):
""" """
A function that generates new Scrapy Request for each source given a new synonym of a compound. A function that generates new Scrapy Request for each source given a new synonym of a compound.
:param compound: A compound name :param compound: A compound name
:return: A list of Scrapy Request objects :return: A list of Scrapy Request objects
""" """
requests = [] requests = []
if compound not in self.synonyms: if force or compound not in self.synonyms:
self.synonyms.add(compound) self.synonyms.add(compound)
for parser in self._sources: for parser in self._sources:
parser_requests = parser.new_compound_request(compound) parser_requests = parser.new_compound_request(compound)
@ -57,7 +57,7 @@ class FourmiSpider(Spider):
""" """
requests = [] requests = []
for synonym in self.synonyms: for synonym in self.synonyms:
requests.extend(self.get_synonym_requests(synonym)) requests.extend(self.get_synonym_requests(synonym, force=True))
return requests return requests
def add_sources(self, sources): def add_sources(self, sources):

5
tests/test_spider.py
View File

@ -43,7 +43,10 @@ class TestFoumiSpider(unittest.TestCase):
src2 = ChemSpider() src2 = ChemSpider()
self.spi.add_source(src2) self.spi.add_source(src2)
self.assertIsNotNone(self.spi.start_requests()) requests = self.spi.start_requests()
self.assertGreater(len(requests), 0)
self.assertIsInstance(requests[0], Request)
def test_synonym_requests(self): def test_synonym_requests(self):
# A test for the synonym request function # A test for the synonym request function