Dekker1/Fourmi
Archived
1
0
Fork 0
Code

moved parsing of synonyms to 'parse_synonyms' function

Browse Source
This commit is contained in:
RTB 2014-04-13 23:14:23 +02:00
parent 859a18c61a
commit 5565c28a1e
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
FourmiCrawler/parsers/ChemSpider.py
View File

@ -19,6 +19,13 @@ class ChemSpider(Parser):
def parse(self, response): def parse(self, response):
sel = Selector(response) sel = Selector(response)
requests = []
requests_synonyms = self.parse_synonyms(sel)
requests.extend(requests_synonyms)
return requests
def parse_synonyms(self, sel):
requests = []
synonyms = [] synonyms = []
for syn in sel.xpath('//p[@class="syn"]/strong/text()').extract(): for syn in sel.xpath('//p[@class="syn"]/strong/text()').extract():
synonyms.append( self.new_synonym( syn, 'high' ) ) synonyms.append( self.new_synonym( syn, 'high' ) )
@ -27,7 +34,7 @@ class ChemSpider(Parser):
for syn in sel.xpath('//p[@class="syn"]/span[@class=""]/text()').extract(): for syn in sel.xpath('//p[@class="syn"]/span[@class=""]/text()').extract():
synonyms.append( self.new_synonym( syn, 'low' ) ) synonyms.append( self.new_synonym( syn, 'low' ) )
return synonyms return requests
def new_synonym(self, name, reliability): def new_synonym(self, name, reliability):
log.msg('CS synonym: %s (%s)' % (name, reliability), level=log.WARNING) log.msg('CS synonym: %s (%s)' % (name, reliability), level=log.WARNING)