Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Sources don't need to be mangled

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-04 18:34:31 +02:00
parent 1557d17877
commit f128c54312
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
FourmiCrawler/spider.py
View File

@ -9,7 +9,7 @@ class FourmiSpider(Spider):
A spider writen for the Fourmi Project which calls upon all available sources to request and scrape data.
"""
name = "FourmiSpider"
__sources = []
_sources = []
synonyms = []
def __init__(self, compound=None, selected_attributes=[".*"], *args, **kwargs):
@ -29,7 +29,7 @@ class FourmiSpider(Spider):
:param response: A Scrapy Response object that should be parsed
:return: A list of Result items and new Request to be handled by the scrapy core.
"""
for source in self.__sources:
for source in self._sources:
if re.match(source.website, response.url):
log.msg("Url: " + response.url + " -> Source: " + source.website, level=log.DEBUG)
return source.parse(response)
@ -42,7 +42,7 @@ class FourmiSpider(Spider):
:return: A list of Scrapy Request objects
"""
requests = []
for parser in self.__sources:
for parser in self._sources:
parser_requests = parser.new_compound_request(compound)
if parser_requests is not None:
requests.append(parser_requests)
@ -71,5 +71,5 @@ class FourmiSpider(Spider):
A function add a new Parser object to the list of available parsers.
:param source: A Source Object
"""
self.__sources.append(source)
self._sources.append(source)
source.set_spider(self)