Dekker1/Fourmi
Archived
1
0
Fork 0
Code

now returns good results, with property values and corresponding sources

Browse Source
This commit is contained in:
Nout van Deijck 2014-06-04 15:44:53 +02:00
parent ba8f845178
commit 291547a5ad
1 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
FourmiCrawler/sources/PubChem.py
View File

@ -61,14 +61,34 @@ class PubChem(Source):
requests = [] requests = []
sel = Selector(response) sel = Selector(response)
# props = sel.xpath('.//div') props = sel.xpath('//div')
prop_values = sel.xpath('//div//a/text()').extract()
prop_names = sel.xpath('//div//a/ancestor::div/b/text()').extract()
print prop_values for prop in props:
print prop_names prop_name = ''.join(prop.xpath('b/text()').extract())
if prop.xpath('a'):
# print props prop_source = ''.join(prop.xpath('a/@title').extract())
prop_value = ''.join(prop.xpath('a/text()').extract())
new_prop = Result({
'attribute': prop_name,
'value': prop_value,
'source': prop_source,
'reliability': 'Unknown',
'conditions': ''
})
requests.append(new_prop)
elif prop.xpath('ul'):
prop_values = prop.xpath('ul//li')
for prop_li in prop_values:
prop_value = ''.join(prop_li.xpath('a/text()').extract())
prop_source = ''.join(prop_li.xpath('a/@title').extract())
new_prop = Result({
'attribute': prop_name,
'value': prop_value,
'source': prop_source,
'reliability': 'Unknown',
'conditions': ''
})
requests.append(new_prop)
return requests return requests