diff --git a/Fourmi.py b/Fourmi.py index c411b4a..a71400c 100755 --- a/Fourmi.py +++ b/Fourmi.py @@ -18,7 +18,7 @@ def load_parsers(rel_dir="FourmiCrawler/parsers"): for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']: mod = __import__('.'.join(["FourmiCrawler.parsers", py]), fromlist=[py]) # [todo] - This module name should be derived from the rel_dir variable - classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))] + classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))] # [fix] - This also finds classes that are imported. for cls in classes: parsers.append(cls()) # [review] - Would we ever need arguments for the parsers? return parsers diff --git a/FourmiCrawler/parsers/parser.py b/FourmiCrawler/parsers/parser.py index 68f73cf..a3710c5 100644 --- a/FourmiCrawler/parsers/parser.py +++ b/FourmiCrawler/parsers/parser.py @@ -1,5 +1,5 @@ from scrapy import log -from scrapy.http import Request +# from scrapy.http import Request class Parser: diff --git a/FourmiCrawler/spider.py b/FourmiCrawler/spider.py index edd74a9..3fc5ce0 100644 --- a/FourmiCrawler/spider.py +++ b/FourmiCrawler/spider.py @@ -30,5 +30,5 @@ class FourmiSpider(Spider): self.add_parser(parser) def add_parser(self, parser): - self.__parsers.add(parser) + self.__parsers.append(parser) parser.set_spider(self) \ No newline at end of file