added scraping for wikipedia links in synonym tab

2014-04-13 23:35:25 +02:00 · 2014-04-13 23:35:25 +02:00 · 0ad98905e3
commit 0ad98905e3
parent 5565c28a1e
1 changed files with 3 additions and 0 deletions
--- a/FourmiCrawler/parsers/ChemSpider.py
+++ b/FourmiCrawler/parsers/ChemSpider.py
@ -22,6 +22,9 @@ class ChemSpider(Parser):
        requests = []
        requests_synonyms = self.parse_synonyms(sel)
        requests.extend(requests_synonyms)
+        for wiki_url in sel.xpath('.//a[@title="Wiki"]/@href').extract():
+            requests.append( Request(url=wiki_url) )
+
        return requests

    def parse_synonyms(self, sel):