diff --git a/FourmiCrawler/sources/ChemSpider.py b/FourmiCrawler/sources/ChemSpider.py index 6ca5382..b4bf6f0 100644 --- a/FourmiCrawler/sources/ChemSpider.py +++ b/FourmiCrawler/sources/ChemSpider.py @@ -277,8 +277,8 @@ class ChemSpider(Source): log.msg('ChemSpider found multiple substances, taking first ' 'element', level=log.DEBUG) csid = csids[0] - structure_url = self.website[:-1] + self.structure % csid - extendedinfo_url = self.website[:-1] + self.extendedinfo % csid + structure_url = self.website[:-2].replace("\\", "") + self.structure % csid + extendedinfo_url = self.website[:-2].replace("\\", "") + self.extendedinfo % csid log.msg('chemspider URL: %s' % structure_url, level=log.DEBUG) return [Request(url=structure_url, callback=self.parse), diff --git a/FourmiCrawler/sources/NIST.py b/FourmiCrawler/sources/NIST.py index 4ad93f5..691b062 100644 --- a/FourmiCrawler/sources/NIST.py +++ b/FourmiCrawler/sources/NIST.py @@ -164,7 +164,7 @@ class NIST(Source): extra_data_url = tr.xpath('td[last()][a="Individual data points"]' '/a/@href').extract() if extra_data_url: - request = Request(url=self.website[:-1] + extra_data_url[0], + request = Request(url=self.website[:-2].replace("\\", "") + extra_data_url[0], callback=self.parse_individual_datapoints) results.append(request) continue diff --git a/FourmiCrawler/sources/PubChem.py b/FourmiCrawler/sources/PubChem.py index 5947e54..0768612 100644 --- a/FourmiCrawler/sources/PubChem.py +++ b/FourmiCrawler/sources/PubChem.py @@ -51,7 +51,7 @@ class PubChem(Source): self._spider.get_synonym_requests(synonym) log.msg('Raw synonyms found: %s' % raw_synonyms, level=log.DEBUG) - n = re.search(r'cid=(\d+)',response.url) + n = re.search(r'cid=(\d+)', response.url) if n: cid = n.group(1) log.msg('cid: %s' % cid, level=log.DEBUG) #getting the right id of the compound with which it can reach