Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Added an basic parser class to extend, next step implementing the global function

Browse Source
This commit is contained in:
Jip J. Dekker 2014-03-28 14:43:22 +01:00
parent d91706d6e5
commit 325febe834
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Fourmi.py
View File

@ -12,6 +12,8 @@ from scrapy.utils.project import get_project_settings
def setup_crawler(searchable): def setup_crawler(searchable):
# [TODO] - Initiate all parsers for the different websites and get
# allowed URLs.
spider = FourmiSpider(compound=searchable) spider = FourmiSpider(compound=searchable)
settings = get_project_settings() settings = get_project_settings()
crawler = Crawler(settings) crawler = Crawler(settings)

5
FourmiCrawler/spider.py
View File

@ -6,10 +6,11 @@ class FourmiSpider(Spider):
def __init__(self, compound=None, *args, **kwargs): def __init__(self, compound=None, *args, **kwargs):
super(FourmiSpider, self).__init__(*args, **kwargs) super(FourmiSpider, self).__init__(*args, **kwargs)
# [TODO] - Initiate all parsers for the different websites and get
# allowed URLs.
def parse(self, reponse): def parse(self, reponse):
# [TODO] - This function should delegate it's functionality to other # [TODO] - This function should delegate it's functionality to other
# parsers. # parsers.
pass pass
def add_parser(self, parser):
self.parsers.add(parser)