Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Small fixes

Browse Source
This commit is contained in:
Bas Vb 2014-05-13 16:18:32 +02:00
parent 03e652d454
commit b54568bab0
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
FourmiCrawler/sources/WikipediaParser.py
View File

@ -36,7 +36,7 @@ class WikipediaParser(Source):
""" scrape data from infobox on wikipedia. """ """ scrape data from infobox on wikipedia. """
items = [] items = []
#be sure to get both chembox (wikipedia template) and drugbox (wikipedia template) to scrape #be sure to get chembox (wikipedia template)
tr_list = sel.xpath('.//table[@class="infobox bordered"]//td[not(@colspan)]'). \ tr_list = sel.xpath('.//table[@class="infobox bordered"]//td[not(@colspan)]'). \
xpath('normalize-space(string())') xpath('normalize-space(string())')
prop_names = tr_list[::2] prop_names = tr_list[::2]
@ -52,11 +52,13 @@ class WikipediaParser(Source):
items.append(item) items.append(item)
log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG)
tr_list2 = sel.xpath('.//table[@class="infobox"]//tr')#.xpath('normalize-space(string())') #scrape the drugbox (wikipedia template)
tr_list2 = sel.xpath('.//table[@class="infobox"]//tr')
log.msg('dit: %s' % tr_list2, level=log.DEBUG) log.msg('dit: %s' % tr_list2, level=log.DEBUG)
for tablerow in tr_list2: for tablerow in tr_list2:
log.msg('item: %s' % tablerow.xpath('./th').xpath('normalize-space(string())'), level=log.DEBUG) log.msg('item: %s' % tablerow.xpath('./th').xpath('normalize-space(string())'), level=log.DEBUG)
if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath('normalize-space(string())'): if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath(
'normalize-space(string())'):
item = Result({ item = Result({
'attribute': tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'), 'attribute': tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'),
'value': tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'), 'value': tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'),
@ -65,7 +67,9 @@ class WikipediaParser(Source):
'conditions': "" 'conditions': ""
}) })
items.append(item) items.append(item)
log.msg('Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) log.msg(
'Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']),
level=log.DEBUG)
items = filter(lambda a: a['value'] != '', items) # remove items with an empty value items = filter(lambda a: a['value'] != '', items) # remove items with an empty value
item_list = self.clean_items(items) item_list = self.clean_items(items)