diff --git a/FourmiCrawler/spider.py b/FourmiCrawler/spider.py index d1b99a7..60f7363 100644 --- a/FourmiCrawler/spider.py +++ b/FourmiCrawler/spider.py @@ -35,14 +35,14 @@ class FourmiSpider(Spider): return source.parse(response) return None - def get_synonym_requests(self, compound): + def get_synonym_requests(self, compound, force=False): """ A function that generates new Scrapy Request for each source given a new synonym of a compound. :param compound: A compound name :return: A list of Scrapy Request objects """ requests = [] - if compound not in self.synonyms: + if force or compound not in self.synonyms: self.synonyms.add(compound) for parser in self._sources: parser_requests = parser.new_compound_request(compound) @@ -57,7 +57,7 @@ class FourmiSpider(Spider): """ requests = [] for synonym in self.synonyms: - requests.extend(self.get_synonym_requests(synonym)) + requests.extend(self.get_synonym_requests(synonym, force=True)) return requests def add_sources(self, sources): diff --git a/tests/test_spider.py b/tests/test_spider.py index 66878eb..89d6cfc 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -43,7 +43,10 @@ class TestFoumiSpider(unittest.TestCase): src2 = ChemSpider() self.spi.add_source(src2) - self.assertIsNotNone(self.spi.start_requests()) + requests = self.spi.start_requests() + self.assertGreater(len(requests), 0) + self.assertIsInstance(requests[0], Request) + def test_synonym_requests(self): # A test for the synonym request function