Dekker1/Fourmi
Archived
1
0
Fork 0
Code

chemspider now scrapes for reference data on synonyms

Browse Source
This commit is contained in:
RTB 2014-04-18 13:16:22 +02:00
parent 119d48890d
commit ae21fa7c67
1 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
FourmiCrawler/parsers/ChemSpider.py
View File

@ -102,13 +102,26 @@ class ChemSpider(Parser):
language = language[0][1:-1] language = language[0][1:-1]
else: else:
language = 'English' language = 'English'
log.msg('CS synonym: %s (%s) (%s)' % (name, category, language),
level=log.DEBUG)
references = []
for ref in sel.xpath('span[@class="synonym_ref"]'):
refname = ref.xpath('normalize-space(string())')
references.append({'name': refname.extract()[0][1:-1], 'URI': ''})
for ref in sel.xpath('a[@class="synonym_ref"]'):
references.append({
'name': ref.xpath('@title').extract()[0],
'URI': ref.xpath('@href').extract()[0]
})
for ref in references:
log.msg('CS synonym ref: %s %s' % (ref['name'], ref['URI']),
level=log.DEBUG)
synonym = { synonym = {
'name': name, 'name': name,
'category': category, 'category': category,
'language': language 'language': language,
'references': references
} }
log.msg('CS synonym: %s (%s) (%s)' % (name, category, language),
level=log.DEBUG)
return synonym return synonym
def parse_extendedinfo(self, response): def parse_extendedinfo(self, response):