From 325febe834feaff06f9ceab4462fab17720902ce Mon Sep 17 00:00:00 2001 From: "Jip J. Dekker" Date: Fri, 28 Mar 2014 14:43:22 +0100 Subject: [PATCH] Added an basic parser class to extend, next step implementing the global function --- Fourmi.py | 2 ++ FourmiCrawler/spider.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Fourmi.py b/Fourmi.py index 533240e..f1bf1ba 100755 --- a/Fourmi.py +++ b/Fourmi.py @@ -12,6 +12,8 @@ from scrapy.utils.project import get_project_settings def setup_crawler(searchable): + # [TODO] - Initiate all parsers for the different websites and get + # allowed URLs. spider = FourmiSpider(compound=searchable) settings = get_project_settings() crawler = Crawler(settings) diff --git a/FourmiCrawler/spider.py b/FourmiCrawler/spider.py index 2805c8e..4c25df9 100644 --- a/FourmiCrawler/spider.py +++ b/FourmiCrawler/spider.py @@ -6,10 +6,11 @@ class FourmiSpider(Spider): def __init__(self, compound=None, *args, **kwargs): super(FourmiSpider, self).__init__(*args, **kwargs) - # [TODO] - Initiate all parsers for the different websites and get - # allowed URLs. def parse(self, reponse): # [TODO] - This function should delegate it's functionality to other # parsers. pass + + def add_parser(self, parser): + self.parsers.add(parser)