Dekker1/Fourmi
Archived
1
0
Fork 0
Code

altered scraping of aggregate data to test for and request url to individual data points

Browse Source
This commit is contained in:
RTB 2014-05-09 12:36:54 +02:00
parent 74dddace88
commit 5e067fd572
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
FourmiCrawler/sources/NIST.py
View File

@ -60,10 +60,16 @@ class NIST(Source):
continue #Assume unsupported continue #Assume unsupported
return requests return requests
@staticmethod def parse_aggregate_data(self, table, symbol_table):
def parse_aggregate_data(table, symbol_table):
results = [] results = []
for tr in table.xpath('tr[td]'): for tr in table.xpath('tr[td]'):
extra_data_url = tr.xpath('td[last()][a="Individual data points"]'
'/a/@href').extract()
if extra_data_url:
request = Request(url=self.website[:-1] + extra_data_url[0],
callback=self.parse_individual_datapoints)
results.append(request)
continue
data = [] data = []
for td in tr.xpath('td'): for td in tr.xpath('td'):
data.append(''.join(td.xpath('node()').extract())) data.append(''.join(td.xpath('node()').extract()))
@ -145,6 +151,9 @@ class NIST(Source):
return results return results
def parse_individual_datapoints(self, response):
pass
def new_compound_request(self, compound): def new_compound_request(self, compound):
return Request(url=self.website[:-1] + self.search % compound, return Request(url=self.website[:-1] + self.search % compound,
callback=self.parse) callback=self.parse)