Merge branch 'feature/parsers-are-sources' into develop

FourmiCrawler/sources/ChemSpider.py

@@ -1,4 +1,4 @@
-from parser import Parser
+from source import Source
 from scrapy import log
 from scrapy.http import Request
 from scrapy.selector import Selector
@@ -8,7 +8,7 @@ import re
 # [TODO] - Maybe clean up usage of '.extract()[0]', because of possible IndexError exception.
 
 
-class ChemSpider(Parser):
+class ChemSpider(Source):
     """ChemSpider scraper for synonyms and properties
 
     This parser will manage searching for chemicals through the
@@ -18,7 +18,7 @@ class ChemSpider(Parser):
     """
 
     def __init__(self):
-        pass
+        Source.__init__(self)
 
     website = 'http://www.chemspider.com/*'
 

FourmiCrawler/sources/source.py

@@ -2,7 +2,7 @@ from scrapy import log
 # from scrapy.http import Request
 
 
-class Parser:
+class Source:
     website = "http://something/*"  # Regex of URI's the source is able to parse
     _spider = None
 

FourmiCrawler/spider.py

@@ -15,7 +15,7 @@ class FourmiSpider(Spider):
     def parse(self, reponse):
         for parser in self.__parsers:
             if re.match(parser.website, reponse.url):
-                log.msg("Url: " + reponse.url + " -> Parser: " + parser.website, level=log.DEBUG)
+                log.msg("Url: " + reponse.url + " -> Source: " + parser.website, level=log.DEBUG)
                 return parser.parse(reponse)
         return None
 

sourceloader.py

@@ -1,13 +1,13 @@
 import inspect
 import os
 import re
-from FourmiCrawler.parsers.parser import Parser
+from FourmiCrawler.sources.source import Source
 
 
 class SourceLoader:
     sources = []
 
-    def __init__(self, rel_dir="FourmiCrawler/parsers"):
+    def __init__(self, rel_dir="FourmiCrawler/sources"):
         path = os.path.dirname(os.path.abspath(__file__))
         path += "/" + rel_dir
         known_parser = set()
@@ -16,7 +16,7 @@ class SourceLoader:
             mod = __import__('.'.join([rel_dir.replace("/", "."), py]), fromlist=[py])
             classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))]
             for cls in classes:
-                if issubclass(cls, Parser) and cls not in known_parser:
+                if issubclass(cls, Source) and cls not in known_parser:
                     self.sources.append(cls())  # [review] - Would we ever need arguments for the parsers?
                     known_parser.add(cls)