From 321295a395da502ff8887735dca153cdb901a0e8 Mon Sep 17 00:00:00 2001 From: "Jip J. Dekker" Date: Fri, 6 Jun 2014 22:22:29 +0200 Subject: [PATCH] Fixed the synonym requests function for the start requests --- FourmiCrawler/spider.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FourmiCrawler/spider.py b/FourmiCrawler/spider.py index d1b99a7..60f7363 100644 --- a/FourmiCrawler/spider.py +++ b/FourmiCrawler/spider.py @@ -35,14 +35,14 @@ class FourmiSpider(Spider): return source.parse(response) return None - def get_synonym_requests(self, compound): + def get_synonym_requests(self, compound, force=False): """ A function that generates new Scrapy Request for each source given a new synonym of a compound. :param compound: A compound name :return: A list of Scrapy Request objects """ requests = [] - if compound not in self.synonyms: + if force or compound not in self.synonyms: self.synonyms.add(compound) for parser in self._sources: parser_requests = parser.new_compound_request(compound) @@ -57,7 +57,7 @@ class FourmiSpider(Spider): """ requests = [] for synonym in self.synonyms: - requests.extend(self.get_synonym_requests(synonym)) + requests.extend(self.get_synonym_requests(synonym, force=True)) return requests def add_sources(self, sources):