Dekker1/Fourmi
Archived
1
0
Fork 0
Code

The logging now using the scrapy setting overrides

Browse Source
This commit is contained in:
Jip J. Dekker 2014-06-15 19:50:31 +02:00
parent 2eb8f3e0af
commit 4672903c9b
2 changed files with 28 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fourmi.py
View File

@ -58,7 +58,7 @@ def search(docopt_arguments, source_loader):
:param source_loader: An initiated SourceLoader object pointed at the directory with the sources. :param source_loader: An initiated SourceLoader object pointed at the directory with the sources.
""" """
conf = Configurator() conf = Configurator()
conf.start_log(docopt_arguments["--log"], docopt_arguments["--verbose"]) conf.start_log(docopt_arguments["--log"], docopt_arguments["-v"])
conf.set_output(docopt_arguments["--output"], docopt_arguments["--format"]) conf.set_output(docopt_arguments["--output"], docopt_arguments["--format"])
setup_crawler(docopt_arguments["<compound>"], conf.scrapy_settings, source_loader, docopt_arguments["--attributes"].split(',')) setup_crawler(docopt_arguments["<compound>"], conf.scrapy_settings, source_loader, docopt_arguments["--attributes"].split(','))
reactor.run() reactor.run()
@ -69,8 +69,6 @@ if __name__ == '__main__':
arguments = docopt.docopt(__doc__, version='Fourmi - V0.5.0') arguments = docopt.docopt(__doc__, version='Fourmi - V0.5.0')
loader = SourceLoader() loader = SourceLoader()
print arguments["-v"]
if arguments["--include"]: if arguments["--include"]:
loader.include(arguments["--include"].split(',')) loader.include(arguments["--include"].split(','))
elif arguments["--exclude"]: elif arguments["--exclude"]:

39
utils/configurator.py
View File

@ -1,6 +1,8 @@
import ConfigParser
from scrapy import log from scrapy import log
from scrapy.utils.project import get_project_settings from scrapy.utils.project import get_project_settings
import ConfigParser
class Configurator: class Configurator:
""" """
@ -33,20 +35,33 @@ class Configurator:
def start_log(self, logfile, verbose): def start_log(self, logfile, verbose):
""" """
This function starts the logging functionality of Scrapy using the settings given by the CLI. This function changes the default settings of Scapy's logging functionality
using the settings given by the CLI.
:param logfile: The location where the logfile will be saved. :param logfile: The location where the logfile will be saved.
:param verbose: A boolean value to switch between loglevels. :param verbose: A integer value to switch between loglevels.
""" """
if logfile is not None: if verbose != 0:
if verbose: self.scrapy_settings.overrides["LOG_ENABLED"] = True
log.start(logfile=logfile, logstdout=False, loglevel=log.DEBUG)
else:
log.start(logfile=logfile, logstdout=True, loglevel=log.WARNING)
else: else:
if verbose: self.scrapy_settings.overrides["LOG_ENABLED"] = False
log.start(logstdout=False, loglevel=log.DEBUG)
else: if verbose == 1:
log.start(logstdout=True, loglevel=log.WARNING) self.scrapy_settings.overrides["LOG_LEVEL"] = "WARNING"
elif verbose == 2:
self.scrapy_settings.overrides["LOG_LEVEL"] = "INFO"
else:
self.scrapy_settings.overrides["LOG_LEVEL"] = "DEBUG"
if verbose > 1:
self.scrapy_settings.overrides["LOG_STDOUT"] = False
else:
self.scrapy_settings.overrides["LOG_STDOUT"] = True
if logfile is not None:
self.scrapy_settings.overrides["LOG_FILE"] = logfile
else:
self.scrapy_settings.overrides["LOG_FILE"] = None
@staticmethod @staticmethod
def read_sourceconfiguration(): def read_sourceconfiguration():