From 95e9675605deed13f3b2f53ffb47525fadcdbf17 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Thu, 1 May 2014 14:57:09 +0200 Subject: [PATCH 01/27] created stub for NIST parser --- FourmiCrawler/sources/NIST.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 FourmiCrawler/sources/NIST.py diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py new file mode 100644 index 0000000..57befff --- /dev/null +++ b/FourmiCrawler/sources/NIST.py @@ -0,0 +1,16 @@ +from scrapy import log +# from scrapy.http import Request + + +class NIST(Source): + website = "http://webbook.nist.gov/*" + + def __init__(self): + Source.__init__(self) + + def parse(self, reponse): + pass + + def new_compound_request(self, compound): + # return Request(url=self.website[:-1] + compound, callback=self.parse) + pass From e1e507f745b1dbc16a15b460594f69a3842ff2fe Mon Sep 17 00:00:00 2001 From: Rob tB Date: Thu, 1 May 2014 15:30:28 +0200 Subject: [PATCH 02/27] added several required imports --- FourmiCrawler/sources/NIST.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 57befff..cbcbeb4 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -1,5 +1,8 @@ +from source import Source from scrapy import log -# from scrapy.http import Request +from scrapy.http import Request +from scrapy.selector import Selector +from FourmiCrawler.items import Result class NIST(Source): From 0cec4bd2d8960bb9ffa1e8b3f99f3230bb694580 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Sun, 4 May 2014 20:48:51 +0200 Subject: [PATCH 03/27] new_compound_request now returns a Request with a searh URL --- FourmiCrawler/sources/NIST.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index cbcbeb4..2c4337c 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -8,6 +8,8 @@ from FourmiCrawler.items import Result class NIST(Source): website = "http://webbook.nist.gov/*" + search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTG=on&cTC=on&cTP=on' + def __init__(self): Source.__init__(self) @@ -15,5 +17,5 @@ class NIST(Source): pass def new_compound_request(self, compound): - # return Request(url=self.website[:-1] + compound, callback=self.parse) - pass + return Request(url=self.website[:-1] + self.search % compound, + callback=self.parse) From 930eb6cad588d49a46b2dea51d0cbe72565c4763 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Sun, 4 May 2014 21:20:46 +0200 Subject: [PATCH 04/27] NIST now scrapes the symbol table for later use --- FourmiCrawler/sources/NIST.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 2c4337c..44a8037 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -13,8 +13,17 @@ class NIST(Source): def __init__(self): Source.__init__(self) - def parse(self, reponse): - pass + def parse(self, response): + sel = Selector(response) + + symbol_table = {} + tds = sel.xpath('//table[@class="symbol_table"]/tr/td') + for (symbol_td, name_td) in zip(tds[::2], tds[1::2]): + symbol = ''.join(symbol_td.xpath('node()').extract()) + name = name_td.xpath('text()').extract()[0] + symbol_table[symbol] = name + log.msg('NIST symbol: |%s|, name: |%s|' % (symbol, name), + level=log.DEBUG) def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, From 9c80f291b6230f2a0d3958773df33f9f6e1e6f82 Mon Sep 17 00:00:00 2001 From: RTB Date: Wed, 7 May 2014 13:27:22 +0200 Subject: [PATCH 05/27] search NIST exclusively for phase change data --- FourmiCrawler/sources/NIST.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 44a8037..cd049a0 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -8,7 +8,7 @@ from FourmiCrawler.items import Result class NIST(Source): website = "http://webbook.nist.gov/*" - search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTG=on&cTC=on&cTP=on' + search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTP=on' def __init__(self): Source.__init__(self) From 95e24f9c44778a18d764e6bfc09d99d16b2fdb87 Mon Sep 17 00:00:00 2001 From: RTB Date: Wed, 7 May 2014 17:09:42 +0200 Subject: [PATCH 06/27] added code to recognize various table formats --- FourmiCrawler/sources/NIST.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index cd049a0..37f8d04 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -25,6 +25,28 @@ class NIST(Source): log.msg('NIST symbol: |%s|, name: |%s|' % (symbol, name), level=log.DEBUG) + for tables in sel.xpath('//table[@class="data"]'): + if tables.xpath('@summary').extract()[0] == 'One dimensional data': + log.msg('NIST table: Aggregrate data', level=log.DEBUG) + elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1': + log.msg('NIST table; Enthalpy/entropy of phase transition', + level=log.DEBUG) + elif tables.xpath('tr[1]/td'): + log.msg('NIST table: Horizontal table', level=log.DEBUG) + elif (tables.xpath('@summary').extract()[0] == + 'Antoine Equation Parameters'): + log.msg('NIST table: Antoine Equation Parameters', + level=log.DEBUG) + elif len(tables.xpath('tr[1]/th')) == 5: + log.msg('NIST table: generic 5 columns', level=log.DEBUG) + # Symbol (unit) Temperature (K) Method Reference Comment + elif len(tables.xpath('tr[1]/th')) == 4: + log.msg('NIST table: generic 4 columns', level=log.DEBUG) + # Symbol (unit) Temperature (K) Reference Comment + else: + log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) + continue #Assume unsupported + def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, callback=self.parse) From 85595ecf350e173692420f91eb439571444fed86 Mon Sep 17 00:00:00 2001 From: RTB Date: Wed, 7 May 2014 18:12:08 +0200 Subject: [PATCH 07/27] created function to start scraping the aggregate data table --- FourmiCrawler/sources/NIST.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 37f8d04..6e884ef 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -16,6 +16,8 @@ class NIST(Source): def parse(self, response): sel = Selector(response) + requests = [] + symbol_table = {} tds = sel.xpath('//table[@class="symbol_table"]/tr/td') for (symbol_td, name_td) in zip(tds[::2], tds[1::2]): @@ -28,6 +30,8 @@ class NIST(Source): for tables in sel.xpath('//table[@class="data"]'): if tables.xpath('@summary').extract()[0] == 'One dimensional data': log.msg('NIST table: Aggregrate data', level=log.DEBUG) + requests.extend( + self.parse_aggregate_data(tables, symbol_table)) elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1': log.msg('NIST table; Enthalpy/entropy of phase transition', level=log.DEBUG) @@ -46,6 +50,25 @@ class NIST(Source): else: log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) continue #Assume unsupported + return requests + + @staticmethod + def parse_aggregate_data(table, symbol_table): + results = [] + for tr in table.xpath('tr[td]'): + data = [] + for td in tr.xpath('td'): + data.append(''.join(td.xpath('node()').extract())) + result = Result({ + 'attribute': symbol_table[data[0]], + 'value': data[1] + ' ' + data[2], + 'source': 'NIST', + 'reliability': 'Unknown', + 'conditions': '' + }) + log.msg('NIST: |%s|' % data, level=log.DEBUG) + results.append(result) + return results def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, From 10dd74e02617401f80e32c29dd95513fe93248a7 Mon Sep 17 00:00:00 2001 From: RTB Date: Wed, 7 May 2014 21:58:52 +0200 Subject: [PATCH 08/27] added function to scrape transition tables --- FourmiCrawler/sources/NIST.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 6e884ef..0191fee 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -3,7 +3,7 @@ from scrapy import log from scrapy.http import Request from scrapy.selector import Selector from FourmiCrawler.items import Result - +import re class NIST(Source): website = "http://webbook.nist.gov/*" @@ -35,6 +35,8 @@ class NIST(Source): elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1': log.msg('NIST table; Enthalpy/entropy of phase transition', level=log.DEBUG) + requests.extend( + self.parse_transition_data(tables, symbol_table)) elif tables.xpath('tr[1]/td'): log.msg('NIST table: Horizontal table', level=log.DEBUG) elif (tables.xpath('@summary').extract()[0] == @@ -70,6 +72,28 @@ class NIST(Source): results.append(result) return results + @staticmethod + def parse_transition_data(table, symbol_table): + results = [] + + name = table.xpath('@summary').extract()[0] + unit = table.xpath('tr[1]/th[1]/node()').extract()[-1][2:-1] + + for tr in table.xpath('tr[td]'): + tds = tr.xpath('td/text()').extract() + result = Result({ + 'attribute': name, + 'value': tds[0] + ' ' + unit, + 'source': 'NIST', + 'reliability': 'Unknown', + 'conditions': '%s K, (%s -> %s)' % (tds[1], tds[2], tds[3]) + }) + log.msg('NIST: |%s|' % result, level=log.DEBUG) + results.append(result) + + + return results + def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, callback=self.parse) From 7abb491d3fdcd92661b9c7dc8a7ebfbd686e4cbb Mon Sep 17 00:00:00 2001 From: RTB Date: Wed, 7 May 2014 22:08:43 +0200 Subject: [PATCH 09/27] added function for most generic tables --- FourmiCrawler/sources/NIST.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 0191fee..5757546 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -46,9 +46,13 @@ class NIST(Source): elif len(tables.xpath('tr[1]/th')) == 5: log.msg('NIST table: generic 5 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Method Reference Comment + requests.extend( + self.parse_generic_data(tables)) elif len(tables.xpath('tr[1]/th')) == 4: log.msg('NIST table: generic 4 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Reference Comment + requests.extend( + self.parse_generic_data(tables)) else: log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) continue #Assume unsupported @@ -94,6 +98,26 @@ class NIST(Source): return results + @staticmethod + def parse_generic_data(table): + results = [] + + name = table.xpath('@summary').extract()[0] + unit = table.xpath('tr[1]/th[1]/node()').extract()[-1][2:-1] + + for tr in table.xpath('tr[td]'): + tds = tr.xpath('td/text()').extract() + result = Result({ + 'attribute': name, + 'value': tds[0] + ' ' + unit, + 'source': 'NIST', + 'reliability': 'Unknown', + 'conditions': '%s K' % tds[1] + }) + log.msg('NIST: |%s|' % result, level=log.DEBUG) + results.append(result) + return results + def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, callback=self.parse) From 151f1988a16d4fb58134320106b7712105083681 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Thu, 8 May 2014 14:54:30 +0200 Subject: [PATCH 10/27] added function to scrape table for Antoine equation parameters --- FourmiCrawler/sources/NIST.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 5757546..aba0ec2 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -43,6 +43,8 @@ class NIST(Source): 'Antoine Equation Parameters'): log.msg('NIST table: Antoine Equation Parameters', level=log.DEBUG) + requests.extend( + self.parse_antoine_data(tables)) elif len(tables.xpath('tr[1]/th')) == 5: log.msg('NIST table: generic 5 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Method Reference Comment @@ -118,6 +120,25 @@ class NIST(Source): results.append(result) return results + @staticmethod + def parse_antoine_data(table): + results = [] + + name = table.xpath('@summary').extract()[0] + + for tr in table.xpath('tr[td]'): + tds = tr.xpath('td/text()').extract() + result = Result({ + 'attribute': name, + 'value': 'A=%s, B=%s, C=%s' % (tds[1], tds[2], tds[3]), + 'source': 'NIST', + 'reliability': 'Unknown', + 'conditions': '%s K' % tds[0] + }) + results.append(result) + + return results + def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, callback=self.parse) From f6fa5e8adf283810ccb12389642015a37df44431 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Thu, 8 May 2014 15:22:48 +0200 Subject: [PATCH 11/27] fixed scraping of unit for two kinds of tables --- FourmiCrawler/sources/NIST.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index aba0ec2..0ce3a28 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -83,7 +83,11 @@ class NIST(Source): results = [] name = table.xpath('@summary').extract()[0] - unit = table.xpath('tr[1]/th[1]/node()').extract()[-1][2:-1] + tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) + m = re.search(r'\((.*)\)', tr_unit) + unit = '!' + if m: + unit = m.group(1) for tr in table.xpath('tr[td]'): tds = tr.xpath('td/text()').extract() @@ -105,7 +109,11 @@ class NIST(Source): results = [] name = table.xpath('@summary').extract()[0] - unit = table.xpath('tr[1]/th[1]/node()').extract()[-1][2:-1] + tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) + m = re.search(r'\((.*)\)', tr_unit) + unit = '!' + if m: + unit = m.group(1) for tr in table.xpath('tr[td]'): tds = tr.xpath('td/text()').extract() From 74dddace883e84b8bad787751ccf961f7125eca4 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Thu, 8 May 2014 15:42:53 +0200 Subject: [PATCH 12/27] removed logging of Result objects in debug messages because pointless --- FourmiCrawler/sources/NIST.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 0ce3a28..635a61b 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -98,7 +98,6 @@ class NIST(Source): 'reliability': 'Unknown', 'conditions': '%s K, (%s -> %s)' % (tds[1], tds[2], tds[3]) }) - log.msg('NIST: |%s|' % result, level=log.DEBUG) results.append(result) @@ -124,7 +123,6 @@ class NIST(Source): 'reliability': 'Unknown', 'conditions': '%s K' % tds[1] }) - log.msg('NIST: |%s|' % result, level=log.DEBUG) results.append(result) return results From 5e067fd57297769b63dc102352f858024233fec3 Mon Sep 17 00:00:00 2001 From: RTB Date: Fri, 9 May 2014 12:36:54 +0200 Subject: [PATCH 13/27] altered scraping of aggregate data to test for and request url to individual data points --- FourmiCrawler/sources/NIST.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 635a61b..1222c63 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -60,10 +60,16 @@ class NIST(Source): continue #Assume unsupported return requests - @staticmethod - def parse_aggregate_data(table, symbol_table): + def parse_aggregate_data(self, table, symbol_table): results = [] for tr in table.xpath('tr[td]'): + extra_data_url = tr.xpath('td[last()][a="Individual data points"]' + '/a/@href').extract() + if extra_data_url: + request = Request(url=self.website[:-1] + extra_data_url[0], + callback=self.parse_individual_datapoints) + results.append(request) + continue data = [] for td in tr.xpath('td'): data.append(''.join(td.xpath('node()').extract())) @@ -145,6 +151,9 @@ class NIST(Source): return results + def parse_individual_datapoints(self, response): + pass + def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, callback=self.parse) From 775a920b9bc72f2d6f7e08624a0203af4f0b0a22 Mon Sep 17 00:00:00 2001 From: RTB Date: Fri, 9 May 2014 13:00:22 +0200 Subject: [PATCH 14/27] NIST scraper now handles urls with individual data points --- FourmiCrawler/sources/NIST.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 1222c63..6ae6862 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -152,7 +152,30 @@ class NIST(Source): return results def parse_individual_datapoints(self, response): - pass + sel = Selector(response) + table = sel.xpath('//table[@class="data"]')[0] + + results = [] + + name = table.xpath('@summary').extract()[0] + tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) + m = re.search(r'\((.*)\)', tr_unit) + unit = '!' + if m: + unit = m.group(1) + + for tr in table.xpath('tr[td]'): + tds = tr.xpath('td/text()').extract() + result = Result({ + 'attribute': name, + 'value': '%s %s' % (tds[0], unit), + 'source': 'NIST', + 'reliability': 'Unknown', + 'conditions': '' + }) + results.append(result) + + return results def new_compound_request(self, compound): return Request(url=self.website[:-1] + self.search % compound, From 7e984b60d8a8d42d832a160554e356ab833a5419 Mon Sep 17 00:00:00 2001 From: RTB Date: Fri, 9 May 2014 14:24:08 +0200 Subject: [PATCH 15/27] added uncertainty to results from scraping individual data points urls --- FourmiCrawler/sources/NIST.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 6ae6862..2d3c672 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -166,9 +166,14 @@ class NIST(Source): for tr in table.xpath('tr[td]'): tds = tr.xpath('td/text()').extract() + uncertainty = '' + m = re.search('Uncertainty assigned by TRC = (.*?) ', tds[-1]) + if m: + uncertainty = '+- %s ' % m.group(1) + # [TODO]: get the plusminus sign working in here result = Result({ 'attribute': name, - 'value': '%s %s' % (tds[0], unit), + 'value': '%s %s%s' % (tds[0], uncertainty, unit), 'source': 'NIST', 'reliability': 'Unknown', 'conditions': '' From 50c79e3b1f7357b9100b3aefa097a036696ace2f Mon Sep 17 00:00:00 2001 From: Rob tB Date: Wed, 14 May 2014 13:44:43 +0200 Subject: [PATCH 16/27] conditions in name (split by ' at ') are now moved to condition field for individual value page and aggregate data table --- FourmiCrawler/sources/NIST.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 2d3c672..5a9b544 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -73,12 +73,21 @@ class NIST(Source): data = [] for td in tr.xpath('td'): data.append(''.join(td.xpath('node()').extract())) + + name = symbol_table[data[0]] + condition = '' + + m = re.match(r'(.*) at (.*)', name) + if m: + name = m.group(1) + condition = m.group(2) + result = Result({ - 'attribute': symbol_table[data[0]], + 'attribute': name, 'value': data[1] + ' ' + data[2], 'source': 'NIST', 'reliability': 'Unknown', - 'conditions': '' + 'conditions': condition }) log.msg('NIST: |%s|' % data, level=log.DEBUG) results.append(result) @@ -158,6 +167,12 @@ class NIST(Source): results = [] name = table.xpath('@summary').extract()[0] + condition = '' + m = re.match(r'(.*) at (.*)', name) + if m: + name = m.group(1) + condition = m.group(2) + tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) m = re.search(r'\((.*)\)', tr_unit) unit = '!' @@ -176,7 +191,7 @@ class NIST(Source): 'value': '%s %s%s' % (tds[0], uncertainty, unit), 'source': 'NIST', 'reliability': 'Unknown', - 'conditions': '' + 'conditions': condition }) results.append(result) From 98f58ea4e26f6ef8dfbd44467b3d47ebee64d283 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Thu, 15 May 2014 14:29:28 +0200 Subject: [PATCH 17/27] added scraping for generic info except for synonyms --- FourmiCrawler/sources/NIST.py | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 5a9b544..ddb7a09 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -18,6 +18,8 @@ class NIST(Source): requests = [] + requests.extend(self.parse_generic_info(sel)) + symbol_table = {} tds = sel.xpath('//table[@class="symbol_table"]/tr/td') for (symbol_td, name_td) in zip(tds[::2], tds[1::2]): @@ -60,6 +62,41 @@ class NIST(Source): continue #Assume unsupported return requests + def parse_generic_info(self, sel): + ul = sel.xpath('body/ul[li/strong="IUPAC Standard InChI:"]') + li = ul.xpath('li') + + data = {} + + raw_formula = ul.xpath('li[strong/a="Formula"]//text()').extract() + data['Chemical formula'] = ''.join(raw_formula[2:]).strip() + + raw_mol_weight = ul.xpath('li[strong/a="Molecular weight"]/text()') + data['Molecular weight'] = raw_mol_weight.extract()[0].strip() + + raw_inchi = ul.xpath('li[strong="IUPAC Standard InChI:"]//tt/text()') + data['IUPAC Standard InChI'] = raw_inchi.extract()[0] + + raw_inchikey = ul.xpath('li[strong="IUPAC Standard InChIKey:"]' + '/tt/text()') + data['IUPAC Standard InChIKey'] = raw_inchikey.extract()[0] + + raw_cas_number = ul.xpath('li[strong="CAS Registry Number:"]/text()') + data['CAS Registry Number'] = raw_cas_number.extract()[0].strip() + + requests = [] + for key, value in data.iteritems(): + result = Result({ + 'attribute': key, + 'value': value, + 'source': 'NIST', + 'reliability': 'Unknown', + 'conditions': '' + }) + requests.append(result) + + return requests + def parse_aggregate_data(self, table, symbol_table): results = [] for tr in table.xpath('tr[td]'): From 56ee6b1ad347c475ef24f077377c35577b4fbfc5 Mon Sep 17 00:00:00 2001 From: RTB Date: Sat, 17 May 2014 14:09:10 +0200 Subject: [PATCH 18/27] added ignore list --- FourmiCrawler/sources/NIST.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index ddb7a09..17552e5 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -5,11 +5,15 @@ from scrapy.selector import Selector from FourmiCrawler.items import Result import re +# [TODO]: values can be '128.', perhaps remove the dot in that case? + class NIST(Source): website = "http://webbook.nist.gov/*" search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTP=on' + ignore_list = set() + def __init__(self): Source.__init__(self) @@ -235,5 +239,7 @@ class NIST(Source): return results def new_compound_request(self, compound): - return Request(url=self.website[:-1] + self.search % compound, - callback=self.parse) + if compound not in self.ignore_list: + self.ignore_list.update(compound) + return Request(url=self.website[:-1] + self.search % compound, + callback=self.parse) From afc1106838120503bd4cde2af80781e3e2738c9d Mon Sep 17 00:00:00 2001 From: RTB Date: Sat, 17 May 2014 14:11:03 +0200 Subject: [PATCH 19/27] NIST now logs an error if chemical name is not found --- FourmiCrawler/sources/NIST.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 17552e5..d5eaa76 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -20,6 +20,11 @@ class NIST(Source): def parse(self, response): sel = Selector(response) + title = sel.xpath('head/title/text()').extract()[0] + if title == 'Name Not Found': + log.msg('NIST: Chemical not found!', level=log.ERROR) + return + requests = [] requests.extend(self.parse_generic_info(sel)) From b46c7a309d8132efbed111ba13d4d52fa122d70f Mon Sep 17 00:00:00 2001 From: RTB Date: Sat, 17 May 2014 14:21:11 +0200 Subject: [PATCH 20/27] if synonym name matched in search instead of primary name, emit primary name as synonym --- FourmiCrawler/sources/NIST.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index d5eaa76..a969384 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -24,6 +24,10 @@ class NIST(Source): if title == 'Name Not Found': log.msg('NIST: Chemical not found!', level=log.ERROR) return + if title not in self.ignore_list: + self.ignore_list.update(title) + log.msg('NIST emit synonym: %s' % title, level=log.DEBUG) + self._spider.get_synonym_requests(title) requests = [] From 472aae86be443c8372c356b0509e13cabd3b1c9d Mon Sep 17 00:00:00 2001 From: RTB Date: Sat, 17 May 2014 19:32:20 +0200 Subject: [PATCH 21/27] synonyms are now scraped --- FourmiCrawler/sources/NIST.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index a969384..4bb8e30 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -79,6 +79,12 @@ class NIST(Source): ul = sel.xpath('body/ul[li/strong="IUPAC Standard InChI:"]') li = ul.xpath('li') + raw_synonyms = ul.xpath('li[strong="Other names:"]/text()').extract() + for synonym in raw_synonyms[0].strip().split(';\n'): + log.msg('NIST synonym: %s' % synonym, level=log.DEBUG) + self.ignore_list.update(synonym) + self._spider.get_synonym_requests(synonym) + data = {} raw_formula = ul.xpath('li[strong/a="Formula"]//text()').extract() From 81719a38fbb72a4344cd25c49c7361e8907779e6 Mon Sep 17 00:00:00 2001 From: RTB Date: Tue, 20 May 2014 19:32:06 +0200 Subject: [PATCH 22/27] Added comments for the class and functions --- FourmiCrawler/sources/NIST.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 4bb8e30..a2fb425 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -6,8 +6,15 @@ from FourmiCrawler.items import Result import re # [TODO]: values can be '128.', perhaps remove the dot in that case? +# [TODO]: properties have references and comments which do not exist in the +# Result item, but should be included eventually. class NIST(Source): + """NIST Scraper plugin + + This plugin manages searching for a chemical on the NIST website + and parsing the resulting page if the chemical exists on NIST. + """ website = "http://webbook.nist.gov/*" search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTP=on' @@ -76,6 +83,9 @@ class NIST(Source): return requests def parse_generic_info(self, sel): + """Parses: synonyms, chemical formula, molecular weight, InChI, + InChiKey, CAS number + """ ul = sel.xpath('body/ul[li/strong="IUPAC Standard InChI:"]') li = ul.xpath('li') @@ -117,6 +127,9 @@ class NIST(Source): return requests def parse_aggregate_data(self, table, symbol_table): + """Parses the table(s) which contain possible links to individual + data points + """ results = [] for tr in table.xpath('tr[td]'): extra_data_url = tr.xpath('td[last()][a="Individual data points"]' @@ -151,6 +164,7 @@ class NIST(Source): @staticmethod def parse_transition_data(table, symbol_table): + """Parses the table containing properties regarding phase changes""" results = [] name = table.xpath('@summary').extract()[0] @@ -176,6 +190,11 @@ class NIST(Source): @staticmethod def parse_generic_data(table): + """Parses the common tables of 4 and 5 rows. Assumes they are of the + form: + Symbol (unit)|Temperature (K)|Method|Reference|Comment + Symbol (unit)|Temperature (K)|Reference|Comment + """ results = [] name = table.xpath('@summary').extract()[0] @@ -199,6 +218,7 @@ class NIST(Source): @staticmethod def parse_antoine_data(table): + """Parse table containing parameters for the Antione equation""" results = [] name = table.xpath('@summary').extract()[0] @@ -217,6 +237,7 @@ class NIST(Source): return results def parse_individual_datapoints(self, response): + """Parses the page linked from aggregate data""" sel = Selector(response) table = sel.xpath('//table[@class="data"]')[0] From 95565042cae6ba0eec9a58ff4308fa6e18a28765 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Wed, 21 May 2014 10:22:03 +0200 Subject: [PATCH 23/27] removed unused variable symbol_table from parse_transition_table --- FourmiCrawler/sources/NIST.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index a2fb425..7770efc 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -57,8 +57,7 @@ class NIST(Source): elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1': log.msg('NIST table; Enthalpy/entropy of phase transition', level=log.DEBUG) - requests.extend( - self.parse_transition_data(tables, symbol_table)) + requests.extend(self.parse_transition_data(tables)) elif tables.xpath('tr[1]/td'): log.msg('NIST table: Horizontal table', level=log.DEBUG) elif (tables.xpath('@summary').extract()[0] == @@ -163,7 +162,7 @@ class NIST(Source): return results @staticmethod - def parse_transition_data(table, symbol_table): + def parse_transition_data(table): """Parses the table containing properties regarding phase changes""" results = [] From 429ffd74221ff2262dde1ab1113bbbfaffe23001 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Wed, 21 May 2014 10:28:54 +0200 Subject: [PATCH 24/27] renamed tables to table in parse() --- FourmiCrawler/sources/NIST.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 7770efc..d04b0d0 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -49,33 +49,30 @@ class NIST(Source): log.msg('NIST symbol: |%s|, name: |%s|' % (symbol, name), level=log.DEBUG) - for tables in sel.xpath('//table[@class="data"]'): - if tables.xpath('@summary').extract()[0] == 'One dimensional data': + for table in sel.xpath('//table[@class="data"]'): + if table.xpath('@summary').extract()[0] == 'One dimensional data': log.msg('NIST table: Aggregrate data', level=log.DEBUG) requests.extend( - self.parse_aggregate_data(tables, symbol_table)) - elif tables.xpath('tr/th="Initial Phase"').extract()[0] == '1': + self.parse_aggregate_data(table, symbol_table)) + elif table.xpath('tr/th="Initial Phase"').extract()[0] == '1': log.msg('NIST table; Enthalpy/entropy of phase transition', level=log.DEBUG) - requests.extend(self.parse_transition_data(tables)) - elif tables.xpath('tr[1]/td'): + requests.extend(self.parse_transition_data(table)) + elif table.xpath('tr[1]/td'): log.msg('NIST table: Horizontal table', level=log.DEBUG) - elif (tables.xpath('@summary').extract()[0] == + elif (table.xpath('@summary').extract()[0] == 'Antoine Equation Parameters'): log.msg('NIST table: Antoine Equation Parameters', level=log.DEBUG) - requests.extend( - self.parse_antoine_data(tables)) - elif len(tables.xpath('tr[1]/th')) == 5: + requests.extend(self.parse_antoine_data(table)) + elif len(table.xpath('tr[1]/th')) == 5: log.msg('NIST table: generic 5 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Method Reference Comment - requests.extend( - self.parse_generic_data(tables)) - elif len(tables.xpath('tr[1]/th')) == 4: + requests.extend(self.parse_generic_data(table)) + elif len(table.xpath('tr[1]/th')) == 4: log.msg('NIST table: generic 4 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Reference Comment - requests.extend( - self.parse_generic_data(tables)) + requests.extend(self.parse_generic_data(table)) else: log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) continue #Assume unsupported From c0af24644b922eaa3223b7889ae094cd83290e1c Mon Sep 17 00:00:00 2001 From: Rob tB Date: Wed, 21 May 2014 10:31:19 +0200 Subject: [PATCH 25/27] added summary variable in parse() --- FourmiCrawler/sources/NIST.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index d04b0d0..cadf6dc 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -50,7 +50,8 @@ class NIST(Source): level=log.DEBUG) for table in sel.xpath('//table[@class="data"]'): - if table.xpath('@summary').extract()[0] == 'One dimensional data': + summary = table.xpath('@summary').extract()[0] + if summary == 'One dimensional data': log.msg('NIST table: Aggregrate data', level=log.DEBUG) requests.extend( self.parse_aggregate_data(table, symbol_table)) @@ -60,8 +61,7 @@ class NIST(Source): requests.extend(self.parse_transition_data(table)) elif table.xpath('tr[1]/td'): log.msg('NIST table: Horizontal table', level=log.DEBUG) - elif (table.xpath('@summary').extract()[0] == - 'Antoine Equation Parameters'): + elif summary == 'Antoine Equation Parameters'): log.msg('NIST table: Antoine Equation Parameters', level=log.DEBUG) requests.extend(self.parse_antoine_data(table)) From 6cd8edaf222f2bdfdc97ab59de914cc65d05369e Mon Sep 17 00:00:00 2001 From: Rob tB Date: Wed, 21 May 2014 10:36:42 +0200 Subject: [PATCH 26/27] included summary variable in call to transition_table, antoine table and generic table --- FourmiCrawler/sources/NIST.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index cadf6dc..76caca3 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -58,21 +58,21 @@ class NIST(Source): elif table.xpath('tr/th="Initial Phase"').extract()[0] == '1': log.msg('NIST table; Enthalpy/entropy of phase transition', level=log.DEBUG) - requests.extend(self.parse_transition_data(table)) + requests.extend(self.parse_transition_data(table, summary)) elif table.xpath('tr[1]/td'): log.msg('NIST table: Horizontal table', level=log.DEBUG) - elif summary == 'Antoine Equation Parameters'): + elif summary == 'Antoine Equation Parameters': log.msg('NIST table: Antoine Equation Parameters', level=log.DEBUG) - requests.extend(self.parse_antoine_data(table)) + requests.extend(self.parse_antoine_data(table, summary)) elif len(table.xpath('tr[1]/th')) == 5: log.msg('NIST table: generic 5 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Method Reference Comment - requests.extend(self.parse_generic_data(table)) + requests.extend(self.parse_generic_data(table, summary)) elif len(table.xpath('tr[1]/th')) == 4: log.msg('NIST table: generic 4 columns', level=log.DEBUG) # Symbol (unit) Temperature (K) Reference Comment - requests.extend(self.parse_generic_data(table)) + requests.extend(self.parse_generic_data(table, summary)) else: log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) continue #Assume unsupported @@ -159,7 +159,7 @@ class NIST(Source): return results @staticmethod - def parse_transition_data(table): + def parse_transition_data(table, summary): """Parses the table containing properties regarding phase changes""" results = [] @@ -185,7 +185,7 @@ class NIST(Source): return results @staticmethod - def parse_generic_data(table): + def parse_generic_data(table, summary): """Parses the common tables of 4 and 5 rows. Assumes they are of the form: Symbol (unit)|Temperature (K)|Method|Reference|Comment @@ -213,7 +213,7 @@ class NIST(Source): return results @staticmethod - def parse_antoine_data(table): + def parse_antoine_data(table, summary): """Parse table containing parameters for the Antione equation""" results = [] From 6ce5ff23359786ae24274fcac3e11a7934a453e6 Mon Sep 17 00:00:00 2001 From: Rob tB Date: Wed, 21 May 2014 10:40:44 +0200 Subject: [PATCH 27/27] replaced name variable with summary variable --- FourmiCrawler/sources/NIST.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 76caca3..0b75b17 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -163,7 +163,6 @@ class NIST(Source): """Parses the table containing properties regarding phase changes""" results = [] - name = table.xpath('@summary').extract()[0] tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) m = re.search(r'\((.*)\)', tr_unit) unit = '!' @@ -173,7 +172,7 @@ class NIST(Source): for tr in table.xpath('tr[td]'): tds = tr.xpath('td/text()').extract() result = Result({ - 'attribute': name, + 'attribute': summary, 'value': tds[0] + ' ' + unit, 'source': 'NIST', 'reliability': 'Unknown', @@ -193,7 +192,6 @@ class NIST(Source): """ results = [] - name = table.xpath('@summary').extract()[0] tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) m = re.search(r'\((.*)\)', tr_unit) unit = '!' @@ -203,7 +201,7 @@ class NIST(Source): for tr in table.xpath('tr[td]'): tds = tr.xpath('td/text()').extract() result = Result({ - 'attribute': name, + 'attribute': summary, 'value': tds[0] + ' ' + unit, 'source': 'NIST', 'reliability': 'Unknown', @@ -217,12 +215,10 @@ class NIST(Source): """Parse table containing parameters for the Antione equation""" results = [] - name = table.xpath('@summary').extract()[0] - for tr in table.xpath('tr[td]'): tds = tr.xpath('td/text()').extract() result = Result({ - 'attribute': name, + 'attribute': summary, 'value': 'A=%s, B=%s, C=%s' % (tds[1], tds[2], tds[3]), 'source': 'NIST', 'reliability': 'Unknown',