Dekker1/Fourmi
Archived
1
0
Fork 0
Code

created function to start scraping the aggregate data table

Browse Source
This commit is contained in:
RTB 2014-05-07 18:12:08 +02:00
parent 95e24f9c44
commit 85595ecf35
1 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
FourmiCrawler/sources/NIST.py
View File

@ -16,6 +16,8 @@ class NIST(Source):
def parse(self, response): def parse(self, response):
sel = Selector(response) sel = Selector(response)
requests = []
symbol_table = {} symbol_table = {}
tds = sel.xpath('//table[@class="symbol_table"]/tr/td') tds = sel.xpath('//table[@class="symbol_table"]/tr/td')
for (symbol_td, name_td) in zip(tds[::2], tds[1::2]): for (symbol_td, name_td) in zip(tds[::2], tds[1::2]):
@ -28,6 +30,8 @@ class NIST(Source):
for tables in sel.xpath('//table[@class="data"]'): for tables in sel.xpath('//table[@class="data"]'):
if tables.xpath('@summary').extract()[0] == 'One dimensional data': if tables.xpath('@summary').extract()[0] == 'One dimensional data':
log.msg('NIST table: Aggregrate data', level=log.DEBUG) log.msg('NIST table: Aggregrate data', level=log.DEBUG)
requests.extend(
self.parse_aggregate_data(tables, symbol_table))
elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1': elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1':
log.msg('NIST table; Enthalpy/entropy of phase transition', log.msg('NIST table; Enthalpy/entropy of phase transition',
level=log.DEBUG) level=log.DEBUG)
@ -46,6 +50,25 @@ class NIST(Source):
else: else:
log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) log.msg('NIST table: NOT SUPPORTED', level=log.WARNING)
continue #Assume unsupported continue #Assume unsupported
return requests
@staticmethod
def parse_aggregate_data(table, symbol_table):
results = []
for tr in table.xpath('tr[td]'):
data = []
for td in tr.xpath('td'):
data.append(''.join(td.xpath('node()').extract()))
result = Result({
'attribute': symbol_table[data[0]],
'value': data[1] + ' ' + data[2],
'source': 'NIST',
'reliability': 'Unknown',
'conditions': ''
})
log.msg('NIST: |%s|' % data, level=log.DEBUG)
results.append(result)
return results
def new_compound_request(self, compound): def new_compound_request(self, compound):
return Request(url=self.website[:-1] + self.search % compound, return Request(url=self.website[:-1] + self.search % compound,