Dekker1/Fourmi
Archived
1
0
Fork 0
Code

added scraping of synonym language

Browse Source
This commit is contained in:
RTB 2014-04-17 22:37:37 +02:00
parent 4f2c046c9c
commit ce5eeb56a6
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
FourmiCrawler/parsers/ChemSpider.py
View File

@ -94,13 +94,20 @@ class ChemSpider(Parser):
return requests return requests
def new_synonym(self, sel, name, reliability): def new_synonym(self, sel, name, category):
log.msg('CS synonym: %s (%s)' % (name, reliability), level=log.DEBUG)
self.ignore_list.append(name) self.ignore_list.append(name)
language = sel.xpath('span[@class="synonym_language"]/text()').extract()
if language:
language = language[0][1:-1]
else:
language = 'English'
synonym = { synonym = {
'name': name, 'name': name,
'category': category, 'category': category,
'language': language
} }
log.msg('CS synonym: %s (%s) (%s)' % (name, category, language),
level=log.DEBUG)
return synonym return synonym
def parse_extendedinfo(self, response): def parse_extendedinfo(self, response):