Dekker1/Fourmi
Archived
1
0
Fork 0
Code

Code inspection

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-04 19:43:33 +02:00
parent 046fbed3cd
commit 242e0bf628
4 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
FourmiCrawler/pipelines.py
View File

@ -9,7 +9,8 @@ class RemoveNonePipeline(object):
def __init__(self): def __init__(self):
pass pass
def process_item(self, item, spider): @staticmethod
def process_item(item, spider):
""" """
Processing the items so None values are replaced by empty strings Processing the items so None values are replaced by empty strings
:param item: The incoming item :param item: The incoming item
@ -43,9 +44,10 @@ class DuplicatePipeline(object):
class AttributeSelectionPipeline(object): class AttributeSelectionPipeline(object):
def __init__(self): def __init__(self):
pass; pass
def process_item(self, item, spider): @staticmethod
def process_item(item, spider):
""" """
The items are processed using the selected attribute list available in the spider, The items are processed using the selected attribute list available in the spider,
items that don't match the selected items are dropped. items that don't match the selected items are dropped.

4
FourmiCrawler/sources/ChemSpider.py
View File

@ -61,9 +61,7 @@ class ChemSpider(Source):
prop_conditions = '' prop_conditions = ''
# Test for properties without values, with one hardcoded exception # Test for properties without values, with one hardcoded exception
if (not re.match(r'^\d', prop_value) or if not re.match(r'^\d', prop_value) or (prop_name == 'Polarizability' and prop_value == '10-24cm3'):
(prop_name == 'Polarizability' and
prop_value == '10-24cm3')):
continue continue
# Match for condition in parentheses # Match for condition in parentheses

5
FourmiCrawler/sources/NIST.py
View File

@ -78,7 +78,7 @@ class NIST(Source):
requests.extend(self.parse_generic_data(table, summary)) requests.extend(self.parse_generic_data(table, summary))
else: else:
log.msg('NIST table: NOT SUPPORTED', level=log.WARNING) log.msg('NIST table: NOT SUPPORTED', level=log.WARNING)
continue #Assume unsupported continue # Assume unsupported
return requests return requests
def parse_generic_info(self, sel): def parse_generic_info(self, sel):
@ -230,7 +230,8 @@ class NIST(Source):
return results return results
def parse_individual_datapoints(self, response): @staticmethod
def parse_individual_datapoints(response):
"""Parses the page linked from aggregate data""" """Parses the page linked from aggregate data"""
sel = Selector(response) sel = Selector(response)
table = sel.xpath('//table[@class="data"]')[0] table = sel.xpath('//table[@class="data"]')[0]

12
tests/test_pipeline.py
View File

@ -10,7 +10,7 @@ class TestPipelines(unittest.TestCase):
def setUp(self): def setUp(self):
self.testItem = items.Result() self.testItem = items.Result()
def test_NonePipeline(self): def test_none_pipeline(self):
# Testing the pipeline that replaces the None values in items. # Testing the pipeline that replaces the None values in items.
self.testItem["value"] = "abc" self.testItem["value"] = "abc"
pipe = pipelines.RemoveNonePipeline() pipe = pipelines.RemoveNonePipeline()
@ -23,7 +23,7 @@ class TestPipelines(unittest.TestCase):
if key is not "value": if key is not "value":
self.assertIs(processed[key], "") self.assertIs(processed[key], "")
def test_DuplicatePipeline(self): def test_duplicate_pipeline(self):
# Testing the pipeline that removes duplicates. # Testing the pipeline that removes duplicates.
self.testItem["attribute"] = "test" self.testItem["attribute"] = "test"
self.testItem["value"] = "test" self.testItem["value"] = "test"
@ -33,11 +33,11 @@ class TestPipelines(unittest.TestCase):
self.assertEqual(pipe.process_item(self.testItem, spider.FourmiSpider()), self.testItem) self.assertEqual(pipe.process_item(self.testItem, spider.FourmiSpider()), self.testItem)
self.assertRaises(DropItem, pipe.process_item, self.testItem, spider.FourmiSpider()) self.assertRaises(DropItem, pipe.process_item, self.testItem, spider.FourmiSpider())
otherItem = copy.deepcopy(self.testItem) other_item = copy.deepcopy(self.testItem)
otherItem["value"] = "test1" other_item["value"] = "test1"
self.assertEqual(pipe.process_item(otherItem, spider.FourmiSpider()), otherItem) self.assertEqual(pipe.process_item(other_item, spider.FourmiSpider()), other_item)
def test_AttributeSelection(self): def test_attribute_selection(self):
# Testing the pipeline that selects attributes. # Testing the pipeline that selects attributes.
item1 = copy.deepcopy(self.testItem) item1 = copy.deepcopy(self.testItem)
item2 = copy.deepcopy(self.testItem) item2 = copy.deepcopy(self.testItem)