From f128c5431215f5011c3ae1d26f65a8191e4a0c2d Mon Sep 17 00:00:00 2001 From: "Jip J. Dekker" Date: Wed, 4 Jun 2014 18:34:31 +0200 Subject: [PATCH] Sources don't need to be mangled --- FourmiCrawler/spider.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/FourmiCrawler/spider.py b/FourmiCrawler/spider.py index 08abb6b..6a9a12e 100644 --- a/FourmiCrawler/spider.py +++ b/FourmiCrawler/spider.py @@ -9,7 +9,7 @@ class FourmiSpider(Spider): A spider writen for the Fourmi Project which calls upon all available sources to request and scrape data. """ name = "FourmiSpider" - __sources = [] + _sources = [] synonyms = [] def __init__(self, compound=None, selected_attributes=[".*"], *args, **kwargs): @@ -29,7 +29,7 @@ class FourmiSpider(Spider): :param response: A Scrapy Response object that should be parsed :return: A list of Result items and new Request to be handled by the scrapy core. """ - for source in self.__sources: + for source in self._sources: if re.match(source.website, response.url): log.msg("Url: " + response.url + " -> Source: " + source.website, level=log.DEBUG) return source.parse(response) @@ -42,7 +42,7 @@ class FourmiSpider(Spider): :return: A list of Scrapy Request objects """ requests = [] - for parser in self.__sources: + for parser in self._sources: parser_requests = parser.new_compound_request(compound) if parser_requests is not None: requests.append(parser_requests) @@ -71,5 +71,5 @@ class FourmiSpider(Spider): A function add a new Parser object to the list of available parsers. :param source: A Source Object """ - self.__sources.append(source) + self._sources.append(source) source.set_spider(self) \ No newline at end of file