44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
from scrapy import log
|
|
from scrapy.utils.project import get_project_settings
|
|
|
|
|
|
class Configurator:
|
|
|
|
def __init__(self):
|
|
self.scrapy_settings = get_project_settings()
|
|
|
|
|
|
def set_output(self, filename, format):
|
|
"""
|
|
This function manipulates the Scrapy settings that normally would be set in the settings file. In the Fourmi
|
|
project these are command line arguments.
|
|
:param docopt_arguments: A dictionary generated by docopt containing all CLI arguments.
|
|
"""
|
|
|
|
if filename != 'result.*format*':
|
|
self.scrapy_settings.overrides["FEED_URI"] = format
|
|
elif format == "jsonlines":
|
|
self.scrapy_settings.overrides["FEED_URI"] = "results.json"
|
|
elif format is not None:
|
|
self.scrapy_settings.overrides["FEED_URI"] = "results." + format
|
|
|
|
if format is not None:
|
|
self.scrapy_settings.overrides["FEED_FORMAT"] = format
|
|
|
|
|
|
def start_log(self, logfile, verbose):
|
|
"""
|
|
This function starts the logging functionality of Scrapy using the settings given by the CLI.
|
|
:param docopt_arguments: A dictionary generated by docopt containing all CLI arguments.
|
|
"""
|
|
if logfile is not None:
|
|
if verbose:
|
|
log.start(logfile=logfile, logstdout=False, loglevel=log.DEBUG)
|
|
else:
|
|
log.start(logfile=logfile, logstdout=True, loglevel=log.WARNING)
|
|
else:
|
|
if verbose:
|
|
log.start(logstdout=False, loglevel=log.DEBUG)
|
|
else:
|
|
log.start(logstdout=True, loglevel=log.WARNING)
|