Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Wikipediaparser now works on chemboxes as well

Browse Source
This commit is contained in:
Bas Vb 2014-05-01 16:05:37 +02:00
parent f8d390d3e6
commit 03e652d454
1 changed files with 15 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
FourmiCrawler/sources/WikipediaParser.py
View File

@ -46,27 +46,26 @@ class WikipediaParser(Source):
'attribute': prop_name.extract().encode('utf-8'), 'attribute': prop_name.extract().encode('utf-8'),
'value': prop_values[i].extract().encode('utf-8'), 'value': prop_values[i].extract().encode('utf-8'),
'source': "Wikipedia", 'source': "Wikipedia",
'reliability': "", 'reliability': "Unknown",
'conditions': "" 'conditions': ""
}) })
items.append(item) items.append(item)
log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG)
tr_list2 = sel.xpath('.//table[@class="infobox"]//tr').\ tr_list2 = sel.xpath('.//table[@class="infobox"]//tr')#.xpath('normalize-space(string())')
xpath('normalize-space(string())') log.msg('dit: %s' %tr_list2,level=log.DEBUG)
log.msg('%s' %tr_list2,level=log.DEBUG) for tablerow in tr_list2:
#prop_names = tr_list2[::2] log.msg('item: %s' %tablerow.xpath('./th').xpath('normalize-space(string())'),level=log.DEBUG)
#prop_values = tr_list2[1::2] if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath('normalize-space(string())'):
#for i, prop_name in enumerate(prop_names): item = Result({
# item = Result({ 'attribute': tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'),
# 'attribute': prop_name.extract().encode('utf-8'), 'value': tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'),
# 'value': prop_values[i].extract().encode('utf-8'), 'source': "Wikipedia",
# 'source': "Wikipedia", 'reliability': "Unknown",
# 'reliability': "", 'conditions': ""
# 'conditions': "" })
# }) items.append(item)
# items.append(item) log.msg('Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG)
# log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG)
items = filter(lambda a: a['value'] != '', items) # remove items with an empty value items = filter(lambda a: a['value'] != '', items) # remove items with an empty value
item_list = self.clean_items(items) item_list = self.clean_items(items)