Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Both Boiling point and melting point are now parsed from chemical Wikipedia pages, there's one error about different types of attributes in the Result-items, this needs to be fixed by cleaning up the retrieved data.

Browse Source
This commit is contained in:
Bas Vb 2014-04-16 00:50:50 +02:00
parent 1ca3593ae1
commit cd1637b0fe
1 changed files with 18 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
FourmiCrawler/parsers/WikipediaParser.py
View File

@ -5,6 +5,10 @@ from FourmiCrawler.items import Result
class WikipediaParser(Parser): class WikipediaParser(Parser):
# General notes:
# Redirects seem to not matter as Wikipedia returns the page the redirect forwards to
# although this might lead to scraping both the original and the redirect with the same data.
website = "http://en.wikipedia.org/wiki/*" website = "http://en.wikipedia.org/wiki/*"
__spider = None __spider = None
@ -16,16 +20,16 @@ class WikipediaParser(Parser):
#self.log('A response from %s just arrived!' % response.url) #self.log('A response from %s just arrived!' % response.url)
sel = Selector(response) sel = Selector(response)
items = [] items = []
item = Result() meltingpoint = self.getmeltingpoint(sel)
item['attribute']="Melting point" items.append(meltingpoint)
item['value']= sel.xpath('//tr/td/a[@title="Melting point"]/../../td[2]/text()').extract() # ('//tr[contains(@href, "/wiki/Melting_point")]/text()').extract() boilingpoint = self.getboilingpoint(sel)
item['source']= "Wikipedia" print boilingpoint
items.append(item) items.append(boilingpoint)
print item['attribute']
print item['value']
print item['source']
return items return items
def new_compound_request(self, compound):
return Request(url=self.website[:-1] + compound, callback=self.parse)
def getmeltingpoint(self, sel): def getmeltingpoint(self, sel):
item=Result() item=Result()
item['attribute']="Melting point" item['attribute']="Melting point"
@ -33,5 +37,9 @@ class WikipediaParser(Parser):
item['source']= "Wikipedia" item['source']= "Wikipedia"
return item return item
def new_compound_request(self, compound): def getboilingpoint(self, sel):
return Request(url=self.website[:-1] + compound, callback=self.parse) item=Result()
item['attribute']="Boiling point"
item['value']= sel.xpath('//tr/td/a[@title="Boiling point"]/../../td[2]/text()').extract() # ('//tr[contains(@href, "/wiki/Melting_point")]/text()').extract()
item['source']= "Wikipedia"
return item