Dekker1/Fourmi
Archived
1
0
Fork 0
Code

updated parse to use newresult function

Browse Source
This commit is contained in:
RTB 2014-06-06 20:16:25 +02:00
parent 69664d3ac0
commit 30f00b676d
1 changed files with 8 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
FourmiCrawler/sources/WikipediaParser.py
View File

@ -28,7 +28,6 @@ class WikipediaParser(Source):
log.msg('Reliability not set for Wikipedia', level=log.WARNING) log.msg('Reliability not set for Wikipedia', level=log.WARNING)
self.cfg['reliability'] = '' self.cfg['reliability'] = ''
def parse(self, response): def parse(self, response):
""" Distributes the above described behaviour """ """ Distributes the above described behaviour """
log.msg('A response from %s just arrived!' % response.url, level=log.DEBUG) log.msg('A response from %s just arrived!' % response.url, level=log.DEBUG)
@ -51,13 +50,10 @@ class WikipediaParser(Source):
prop_names = tr_list[::2] prop_names = tr_list[::2]
prop_values = tr_list[1::2] prop_values = tr_list[1::2]
for i, prop_name in enumerate(prop_names): for i, prop_name in enumerate(prop_names):
item = Result({ item = self.newresult(
'attribute': prop_name.extract().encode('utf-8'), attribute=prop_name.extract().encode('utf-8'),
'value': prop_values[i].extract().encode('utf-8'), value=prop_values[i].extract().encode('utf-8')
'source': "Wikipedia", )
'reliability': "Unknown",
'conditions': ""
})
items.append(item) items.append(item)
log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG)
@ -68,13 +64,10 @@ class WikipediaParser(Source):
log.msg('item: %s' % tablerow.xpath('./th').xpath('normalize-space(string())'), level=log.DEBUG) log.msg('item: %s' % tablerow.xpath('./th').xpath('normalize-space(string())'), level=log.DEBUG)
if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath( if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath(
'normalize-space(string())'): 'normalize-space(string())'):
item = Result({ item = self.newresult(
'attribute': tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'), attribute=tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'),
'value': tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'), value=tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'),
'source': "Wikipedia", )
'reliability': "Unknown",
'conditions': ""
})
items.append(item) items.append(item)
log.msg( log.msg(
'Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']), 'Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']),