Dekker1/Fourmi
Archived
1
0
Fork 0
Code

new function to clean up the datapoints

Browse Source
This commit is contained in:
Bas Vb 2014-04-16 16:23:33 +02:00
parent 74aa446f40
commit 6f82b117c9
1 changed files with 8 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
FourmiCrawler/parsers/WikipediaParser.py
View File

@ -33,21 +33,20 @@ class WikipediaParser(Parser):
item['value'] = prop_values[i].extract().encode('utf-8')
item['source'] = "Wikipedia"
items.append(item)
print "new: " + item['attribute']
print item['value']
#print "new: " + item['attribute']
#print item['value']
items=filter(lambda a: a['value']!='', items) #remove items with an empty value
print items
#print items
self.cleanitems(items)
return items
def new_compound_request(self, compound):
return Request(url=self.website[:-1] + compound, callback=self.parse)
def getmeltingpoint(self, sel):
item=Result()
item['attribute']="Melting point"
item['value']= sel.xpath('//tr/td/a[@title="Melting point"]/../../td[2]/text()').extract() # ('//tr[contains(@href, "/wiki/Melting_point")]/text()').extract()
item['source']= "Wikipedia"
return item
def cleanitems(self, items):
for item in items:
print item['value']
return items
def getboilingpoint(self, sel):
item=Result()