Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Added extra request for chemspider link retreived from Wikipedia

Browse Source
This commit is contained in:
Nout van Deijck 2014-04-23 12:27:53 +02:00
parent f926f86d7d
commit b5c83125f7
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
FourmiCrawler/parsers/WikipediaParser.py
View File

@ -46,7 +46,10 @@ class WikipediaParser(Parser):
items.append(item) items.append(item)
log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG)
items=filter(lambda a: a['value']!='', items) #remove items with an empty value items=filter(lambda a: a['value']!='', items) #remove items with an empty value
return self.cleanitems(items) itemlist=self.cleanitems(items)
request=Request(self.getchemspider(sel))
itemlist.append(request)
return itemlist
def new_compound_request(self, compound): def new_compound_request(self, compound):
return Request(url=self.website[:-1] + compound, callback=self.parse) return Request(url=self.website[:-1] + compound, callback=self.parse)