Feature - Stop long running scrapers.

__init__.py

@@ -0,0 +1,343 @@
+# coding: utf-8
+import logging
+from logging.config import dictConfig
+import platform
+import re
+import time
+import traceback
+
+from flask import Flask, current_app, render_template, url_for
+from flask_compress import Compress
+from logparser import __version__ as LOGPARSER_VERSION
+
+from .__version__ import __url__, __version__
+from .common import handle_metadata
+from .models import Metadata, db
+from .vars import PYTHON_VERSION, SQLALCHEMY_BINDS, SQLALCHEMY_DATABASE_URI
+# from .utils.scheduler import scheduler
+
+
+# https://stackoverflow.com/questions/18820274/how-to-suppress-sqlalchemy-engine-base-engine-logging-to-stdout
+# logging.getLogger('sqlalchemy.engine.base.Engine').propagate = False
+logging.getLogger('sqlalchemy.engine.base.Engine').setLevel(logging.WARNING)
+# http://flask.pocoo.org/docs/1.0/logging/#basic-configuration
+dictConfig({
+    'version': 1,
+    'formatters': {'default': {
+        'format': '[%(asctime)s] %(levelname)-8s in %(name)s: %(message)s',
+    }},
+    'handlers': {'wsgi': {
+        'class': 'logging.StreamHandler',
+        'stream': 'ext://flask.logging.wsgi_errors_stream',
+        'formatter': 'default'
+    }},
+    'root': {
+        'level': 'DEBUG',
+        'handlers': ['wsgi']
+    }
+})
+
+# Comment out the dictConfig above first
+# https://docs.sqlalchemy.org/en/latest/core/engines.html#configuring-logging
+# https://apscheduler.readthedocs.io/en/latest/userguide.html#troubleshooting
+# logging.basicConfig()
+# logging.getLogger('apscheduler').setLevel(logging.DEBUG)
+# logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
+
+
+def internal_server_error(error):
+    kwargs = dict(
+        error=error,
+        traceback=traceback.format_exc(),
+        url_issues=__url__ + '/issues',
+        os=platform.platform(),
+        python_version=PYTHON_VERSION,
+        scrapydweb_version=__version__,
+        logparser_version=LOGPARSER_VERSION,
+        scrapyd_servers_amount=len(current_app.config.get('SCRAPYD_SERVERS', []))
+    )
+    return render_template('500.html', **kwargs), 500
+
+
+def create_app(test_config=None):
+    app = Flask(__name__, instance_relative_config=True)
+    app.config.from_mapping(
+        SECRET_KEY='dev',
+    )
+    # http://flask.pocoo.org/docs/1.0/config/#configuring-from-files
+    app.config.from_object('scrapydweb.default_settings')
+
+    if test_config is None:
+        # load the instance config, if it exists, when not testing
+        app.config.from_pyfile('config.py', silent=True)
+    else:
+        # load the test config if passed in
+        app.config.from_mapping(test_config)
+
+    @app.route('/hello')
+    def hello():
+        return 'Hello, World!'
+
+    handle_db(app)
+    handle_route(app)
+    handle_template_context(app)
+
+    # @app.errorhandler(404)
+    # def handle_error(error):
+        # return ('Nothing Found', 404)
+    # http://flask.pocoo.org/docs/1.0/patterns/errorpages/
+    app.register_error_handler(500, internal_server_error)
+
+    # https://ansible-docs.readthedocs.io/zh/stable-2.0/rst/playbooks_filters.html#other-useful-filters
+    # https://stackoverflow.com/questions/12791216/how-do-i-use-regular-expressions-in-jinja2
+    # https://www.michaelcho.me/article/custom-jinja-template-filters-in-flask
+    # http://flask.pocoo.org/docs/1.0/api/#flask.Flask.template_filter
+    @app.template_filter()
+    def regex_replace(s, find, replace):
+        return re.sub(find, replace, s)
+    app.jinja_env.variable_start_string = '{{ '
+    app.jinja_env.variable_end_string = ' }}'
+
+    compress = Compress()
+    compress.init_app(app)
+
+    app.logger.setLevel(logging.DEBUG)
+    return app
+
+
+def handle_db(app):
+    # https://flask-sqlalchemy.palletsprojects.com/en/master/config/
+    app.config['SQLALCHEMY_DATABASE_URI'] = SQLALCHEMY_DATABASE_URI
+    app.config['SQLALCHEMY_BINDS'] = SQLALCHEMY_BINDS
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False  # https://stackoverflow.com/a/33790196/10517783
+    app.config['SQLALCHEMY_ECHO'] = True  # http://flask-sqlalchemy.pocoo.org/2.3/config/
+
+    # flask_sqlalchemy/__init__.py
+    # class SQLAlchemy(object):
+    #     def __init__(self, app=None
+    #         self.app = app
+    #         if app is not None:
+    #             self.init_app(app)
+    db.app = app  # https://github.com/viniciuschiele/flask-apscheduler/blob/master/examples/flask_context.py
+    db.init_app(app)  # http://flask-sqlalchemy.pocoo.org/2.3/contexts/
+    db.create_all()
+
+    # https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-vii-error-handling
+    @app.teardown_request
+    def handle_db_session(exception):
+        if exception:
+            db.session.rollback()
+        db.session.remove()
+
+    with db.app.app_context():
+        if not Metadata.query.filter_by(version=__version__).first():
+            metadata = Metadata(version=__version__)
+            db.session.add(metadata)
+            db.session.commit()
+    if time.time() - handle_metadata().get('last_check_update_timestamp', time.time()) > 3600 * 24 * 30:
+        handle_metadata('last_check_update_timestamp', time.time())
+        handle_metadata('pageview', 0)
+    else:
+        handle_metadata('pageview', 1)
+    # print(Metadata.query.filter_by(version=__version__).first())
+
+
+def handle_route(app):
+    def register_view(view, endpoint, url_defaults_list, with_node=True, trailing_slash=True):
+        view_func = view.as_view(endpoint)
+        for url, defaults in url_defaults_list:
+            rule = '/<int:node>/%s' % url if with_node else '/%s' % url
+            if trailing_slash:
+                rule += '/'
+            if not with_node:
+                if defaults:
+                    defaults['node'] = 1
+                else:
+                    defaults = dict(node=1)
+            app.add_url_rule(rule, defaults=defaults, view_func=view_func)
+
+    from .views.index import IndexView
+    index_view = IndexView.as_view('index')
+    app.add_url_rule('/<int:node>/', view_func=index_view)
+    app.add_url_rule('/', defaults=dict(node=1), view_func=index_view)
+
+    from .views.api import ApiView
+    register_view(ApiView, 'api', [
+        ('api/<opt>/<project>/<version_spider_job>', None),
+        ('api/<opt>/<project>', dict(version_spider_job=None)),
+        ('api/<opt>', dict(project=None, version_spider_job=None))
+    ])
+
+    from .views.baseview import MetadataView
+    register_view(MetadataView, 'metadata', [('metadata', None)])
+
+    # Overview
+    from .views.overview.servers import ServersView
+    register_view(ServersView, 'servers', [
+        ('servers/getreports/<project>/<spider>/<version_job>', dict(opt='getreports')),
+        ('servers/<opt>/<project>/<version_job>/<spider>', None),
+        ('servers/<opt>/<project>/<version_job>', dict(spider=None)),
+        ('servers/<opt>/<project>', dict(version_job=None, spider=None)),
+        ('servers/<opt>', dict(project=None, version_job=None, spider=None)),
+        ('servers', dict(opt=None, project=None, version_job=None, spider=None))
+    ])
+
+    from .views.overview.multinode import MultinodeView
+    register_view(MultinodeView, 'multinode', [
+        ('multinode/<opt>/<project>/<version_job>', None),
+        ('multinode/<opt>/<project>', dict(version_job=None))
+    ])
+
+    from .views.overview.tasks import TasksView, TasksXhrView
+    register_view(TasksView, 'tasks', [
+        ('tasks/<int:task_id>/<int:task_result_id>', None),
+        ('tasks/<int:task_id>', dict(task_result_id=None)),
+        ('tasks', dict(task_id=None, task_result_id=None))
+    ])
+    register_view(TasksXhrView, 'tasks.xhr', [
+        ('tasks/xhr/<action>/<int:task_id>/<int:task_result_id>', None),
+        ('tasks/xhr/<action>/<int:task_id>', dict(task_result_id=None)),
+        ('tasks/xhr/<action>', dict(task_id=None, task_result_id=None))
+    ])
+
+    from .views.overview.tasks import bp as bp_tasks_history
+    app.register_blueprint(bp_tasks_history)
+
+    # Dashboard
+    from .views.dashboard.jobs import JobsView, JobsXhrView
+    register_view(JobsView, 'jobs', [('jobs', None)])
+    register_view(JobsXhrView, 'jobs.xhr', [('jobs/xhr/<action>/<int:id>', None)])
+
+    from .views.dashboard.node_reports import NodeReportsView
+    register_view(NodeReportsView, 'nodereports', [('nodereports', None)])
+
+    from .views.dashboard.cluster_reports import ClusterReportsView
+    register_view(ClusterReportsView, 'clusterreports', [
+        ('clusterreports/<project>/<spider>/<job>', None),
+        ('clusterreports', dict(project=None, spider=None, job=None))
+    ])
+
+    # Operations
+    from .views.operations.deploy import DeployView, DeployUploadView, DeployXhrView
+    register_view(DeployView, 'deploy', [('deploy', None)])
+    register_view(DeployUploadView, 'deploy.upload', [('deploy/upload', None)])
+    register_view(DeployXhrView, 'deploy.xhr', [('deploy/xhr/<eggname>/<project>/<version>', None)])
+
+    from .views.operations.schedule import (ScheduleView, ScheduleCheckView, ScheduleRunView,
+                                            ScheduleXhrView, ScheduleTaskView)
+    register_view(ScheduleView, 'schedule', [
+        ('schedule/<project>/<version>/<spider>', None),
+        ('schedule/<project>/<version>', dict(spider=None)),
+        ('schedule/<project>', dict(version=None, spider=None)),
+        ('schedule', dict(project=None, version=None, spider=None))
+    ])
+    register_view(ScheduleCheckView, 'schedule.check', [('schedule/check', None)])
+    register_view(ScheduleRunView, 'schedule.run', [('schedule/run', None)])
+    register_view(ScheduleXhrView, 'schedule.xhr', [('schedule/xhr/<filename>', None)])
+    register_view(ScheduleTaskView, 'schedule.task', [('schedule/task', None)])
+
+    from .views.operations.schedule import bp as bp_schedule_history
+    app.register_blueprint(bp_schedule_history)
+
+    # Files
+    from .views.files.log import LogView
+    register_view(LogView, 'log', [('log/<opt>/<project>/<spider>/<job>', None)])
+
+    from .views.files.logs import LogsView
+    register_view(LogsView, 'logs', [
+        ('logs/<project>/<spider>', None),
+        ('logs/<project>', dict(spider=None)),
+        ('logs', dict(project=None, spider=None))
+    ])
+
+    from .views.files.items import ItemsView
+    register_view(ItemsView, 'items', [
+        ('items/<project>/<spider>', None),
+        ('items/<project>', dict(spider=None)),
+        ('items', dict(project=None, spider=None))
+    ])
+
+    from .views.files.projects import ProjectsView
+    register_view(ProjectsView, 'projects', [
+        ('projects/<opt>/<project>/<version_spider_job>', None),
+        ('projects/<opt>/<project>', dict(version_spider_job=None)),
+        ('projects', dict(opt='listprojects', project=None, version_spider_job=None))
+    ])
+
+    # Parse Log
+    from .views.utilities.parse import UploadLogView, UploadedLogView
+    register_view(UploadLogView, 'parse.upload', [('parse/upload', None)])
+    register_view(UploadedLogView, 'parse.uploaded', [('parse/uploaded/<filename>', None)])
+
+    from .views.utilities.parse import bp as bp_parse_source
+    app.register_blueprint(bp_parse_source)
+
+    # Send text
+    from .views.utilities.send_text import SendTextView, SendTextApiView
+    register_view(SendTextView, 'sendtext', [('sendtext', None)])
+    register_view(SendTextApiView, 'sendtextapi', [
+        ('slack/<channel_chatid_subject>/<text>', dict(opt='slack')),
+        ('slack/<text>', dict(opt='slack', channel_chatid_subject=None)),
+        ('slack', dict(opt='slack', channel_chatid_subject=None, text=None)),
+        ('telegram/<channel_chatid_subject>/<text>', dict(opt='telegram')),
+        ('telegram/<text>', dict(opt='telegram', channel_chatid_subject=None)),
+        ('telegram', dict(opt='telegram', channel_chatid_subject=None, text=None)),
+        ('tg/<channel_chatid_subject>/<text>', dict(opt='tg')),
+        ('tg/<text>', dict(opt='tg', channel_chatid_subject=None)),
+        ('tg', dict(opt='tg', channel_chatid_subject=None, text=None)),
+        ('email/<channel_chatid_subject>/<text>', dict(opt='email')),
+        ('email/<text>', dict(opt='email', channel_chatid_subject=None)),
+        ('email', dict(opt='email', channel_chatid_subject=None, text=None)),
+    ], with_node=False, trailing_slash=False)
+
+    # System
+    from .views.system.settings import SettingsView
+    register_view(SettingsView, 'settings', [('settings', None)])
+
+
+def handle_template_context(app):
+    STATIC = 'static'
+    VERSION = 'v' + __version__.replace('.', '')
+    # MUST be commented out for released version
+    # VERSION = 'v131dev'
+
+    @app.context_processor
+    def inject_variable():
+        return dict(
+            CHECK_LATEST_VERSION_FREQ=100,
+            GITHUB_URL=__url__,
+            PYTHON_VERSION=PYTHON_VERSION,
+            SCRAPYDWEB_VERSION=__version__,
+
+            # static_css_common=url_for(STATIC, filename='%s/css/common.css' % VERSION),
+            static_css_dropdown=url_for(STATIC, filename='%s/css/dropdown.css' % VERSION),
+            static_css_dropdown_mobileui=url_for(STATIC, filename='%s/css/dropdown_mobileui.css' % VERSION),
+            static_css_icon_upload_icon_right=url_for(STATIC,
+                                                      filename='%s/css/icon_upload_icon_right.css' % VERSION),
+            static_css_multinode=url_for(STATIC, filename='%s/css/multinode.css' % VERSION),
+            static_css_stacktable=url_for(STATIC, filename='%s/css/stacktable.css' % VERSION),
+            static_css_stats=url_for(STATIC, filename='%s/css/stats.css' % VERSION),
+            static_css_style=url_for(STATIC, filename='%s/css/style.css' % VERSION),
+            static_css_style_mobileui=url_for(STATIC, filename='%s/css/style_mobileui.css' % VERSION),
+            static_css_utf8=url_for(STATIC, filename='%s/css/utf8.css' % VERSION),
+            static_css_utf8_mobileui=url_for(STATIC, filename='%s/css/utf8_mobileui.css' % VERSION),
+
+            static_css_element_ui_index=url_for(STATIC,
+                                                filename='%s/[email protected]/lib/theme-chalk/index.css' % VERSION),
+            static_js_element_ui_index=url_for(STATIC, filename='%s/[email protected]/lib/index.js' % VERSION),
+
+            static_js_common=url_for(STATIC, filename='%s/js/common.js' % VERSION),
+            static_js_echarts_min=url_for(STATIC, filename='%s/js/echarts.min.js' % VERSION),
+            static_js_icons_menu=url_for(STATIC, filename='%s/js/icons_menu.js' % VERSION),
+            # static_js_github_buttons_html=url_for(STATIC, filename='%s/js/github_buttons.html' % VERSION),
+            static_js_github_buttons=url_for(STATIC, filename='%s/js/github_buttons.js' % VERSION),
+            static_js_jquery_min=url_for(STATIC, filename='%s/js/jquery.min.js' % VERSION),
+            static_js_multinode=url_for(STATIC, filename='%s/js/multinode.js' % VERSION),
+            static_js_stacktable=url_for(STATIC, filename='%s/js/stacktable.js' % VERSION),
+            static_js_stats=url_for(STATIC, filename='%s/js/stats.js' % VERSION),
+            static_js_vue_min=url_for(STATIC, filename='%s/js/vue.min.js' % VERSION),
+
+            static_icon=url_for(STATIC, filename='%s/icon/fav.ico' % VERSION),
+            static_icon_shortcut=url_for(STATIC, filename='%s/icon/fav.ico' % VERSION),
+            static_icon_apple_touch=url_for(STATIC, filename='%s/icon/spiderman.png' % VERSION),
+        )

__version__.py

@@ -0,0 +1,10 @@
+# coding: utf-8
+
+__title__ = 'scrapydweb'
+__version__ = '1.4.0'
+__author__ = 'my8100'
+__author_email__ = '[email protected]'
+__url__ = 'https://github.com/my8100/scrapydweb'
+__license__ = 'GNU General Public License v3.0'
+__description__ = ("Web app for Scrapyd cluster management, "
+                   "with support for Scrapy log analysis & visualization.")