From 68139b483931cbddb44091c1c958003d6e9619a1 Mon Sep 17 00:00:00 2001 From: RTB Date: Fri, 6 Jun 2014 19:27:27 +0200 Subject: [PATCH] added config to wikipedia along with reliability check --- FourmiCrawler/sources/WikipediaParser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/FourmiCrawler/sources/WikipediaParser.py b/FourmiCrawler/sources/WikipediaParser.py index d0dfd03..dd27e34 100644 --- a/FourmiCrawler/sources/WikipediaParser.py +++ b/FourmiCrawler/sources/WikipediaParser.py @@ -19,8 +19,15 @@ class WikipediaParser(Source): __spider = None searched_compounds = [] + cfg = {} + def __init__(self, config={}): Source.__init__(self, config) + self.cfg = config + if 'reliability' not in self.cfg or self.cfg['reliability'] == '': + log.msg('Reliability not set for Wikipedia', level=log.WARNING) + self.cfg['reliability'] = '' + def parse(self, response): """ Distributes the above described behaviour """