added remove all jobs button

.bumpversion.cfg

@@ -0,0 +1,5 @@
+[bumpversion]
+current_version = 0.3.1
+files = SpiderKeeper/__init__.py
+commit = True
+

CHANGELOG.md

@@ -1,34 +1,16 @@
-# SpiderKeeper Changelog
-## 1.2.0 (2017-07-24)
-- support chose server manually
-- support set cron exp manually
-- fix log Chinese decode problem
-- fix scheduler trigger not fire problem
-- fix not delete project on scrapyd problem
+# SpiderKeeper-2 Changelog
 
-## 1.1.0 (2017-04-25)
-- support basic auth
-- show spider crawl time info (last_runtime,avg_runtime)
-- optimized for mobile
+## 0.3.0 (2018-09-14)
+- spiderkeepr was integrated into scrapyd service
 
-## 1.0.3 (2017-04-17)
-- support view log
+## 0.2.5 (2018-09-14)
+- refactoring flask application
 
-## 1.0.0 (2017-03-30)
-- refactor
-- support py3
-- optimized api
-- optimized scheduler
-- more scalable (can support access multiply spider service)
-- show running stats
-
-## 0.2.0 (2016-04-13)
-- support view job of multi daemons.
-- support run on multi daemons.
-- support choice running daemon automaticaly.
-
-## 0.1.1 (2016-02-16)
-- add status monitor(https://github.com/afaqurk/linux-dash)
-
-## 0.1.0 (2016-01-18)
-- initial.
+## 0.2.0 (2017-10-26)
+- SpiderKeeper was forked to Spiderkeeper-2
+- Add button for removing all periodic jobs
+- All tasks show stats now.
+- When you run spiderkeeper under wsgi you should not use background scheduler, [see issue](https://github.com/agronholm/apscheduler/issues/160), you should run scheduler in separated process. So, scheduler was separated to own module
+- Add foreign constraints to models.
+- No need to create project now, all projects will be synchronized automatically with scrapyd.
+- Fix bugs.

README.md

@@ -1,8 +1,9 @@
-# SpiderKeeper
+# SpiderKeeper-2 
+#### This is a fork of [SpiderKeeper](https://github.com/DormyMo/SpiderKeeper). See [changelog](https://github.com/kalombos/SpiderKeeper/blob/master/CHANGELOG.md) for new features
 
-[![Latest Version](http://img.shields.io/pypi/v/SpiderKeeper.svg)](https://pypi.python.org/pypi/SpiderKeeper)
-[![Python Versions](http://img.shields.io/pypi/pyversions/SpiderKeeper.svg)](https://pypi.python.org/pypi/SpiderKeeper)
-[![The MIT License](http://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/DormyMo/SpiderKeeper/blob/master/LICENSE)
+[![Latest Version](http://img.shields.io/pypi/v/SpiderKeeper-2.svg)](https://pypi.python.org/pypi/SpiderKeeper-2)
+[![Python Versions](https://img.shields.io/pypi/pyversions/SpiderKeeper-2.svg)](https://pypi.python.org/pypi/SpiderKeeper-2)
+![The MIT License](http://img.shields.io/badge/license-MIT-blue.svg)
 
 A scalable admin ui for spider service 
 
@@ -12,10 +13,10 @@ A scalable admin ui for spider service
 - With a single click deploy the scrapy project
 - Show spider running stats
 - Provide api
+- Integrated in scrapyd
+
 
 
-Current Support spider service
-- [Scrapy](https://github.com/scrapy/scrapy) ( with [scrapyd](https://github.com/scrapy/scrapyd))
 
 ## Screenshot
 ![job dashboard](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/screenshot_1.png)
@@ -29,83 +30,38 @@ Current Support spider service
 
 
 ```
-pip install spiderkeeper
+pip install spiderkeeper-2
 ```
 
-### Deployment
-
-``` 
-
-spiderkeeper [options]
-
-Options:
-
-  -h, --help            show this help message and exit
-  --host=HOST           host, default:0.0.0.0
-  --port=PORT           port, default:5000
-  --username=USERNAME   basic auth username ,default: admin
-  --password=PASSWORD   basic auth password ,default: admin
-  --type=SERVER_TYPE    access spider server type, default: scrapyd
-  --server=SERVERS      servers, default: ['http://localhost:6800']
-  --database-url=DATABASE_URL
-                        SpiderKeeper metadata database default: sqlite:////home/souche/SpiderKeeper.db
-  --no-auth             disable basic auth
-  -v, --verbose         log level
-
-
-example:
-
-spiderkeeper --server=http://localhost:6800
-
-```
 
 ## Usage
 
-```
-Visit: 
-
-- web ui : http://localhost:5000
 
-1. Create Project
 
-2. Use [scrapyd-client](https://github.com/scrapy/scrapyd-client) to generate egg file 
+1. Run ```spiderkeeper```
 
-   scrapyd-deploy --build-egg output.egg
+2. Visit http://localhost:5000/
 
-2. upload egg file (make sure you started scrapyd server)
+3. upload egg file (make sure you started scrapyd server)
 
-3. Done & Enjoy it
+4. Done & Enjoy it
 
 - api swagger: http://localhost:5000/api.html
 
-```
-
-## TODO
-- [ ] Job dashboard support filter
-- [x] User Authentication
-- [ ] Collect & Show scrapy crawl stats
-- [ ] Optimize load balancing
-
-## Versioning
-
-We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/DormyMo/SpiderKeeper/tags). 
+``` 
 
 ## Authors
 
 - *Initial work* - [DormyMo](https://github.com/DormyMo)
+- *Fork author* - [kalombo](https://github.com/kalombos/)
 
-See also the list of [contributors](https://github.com/DormyMo/SpiderKeeper/contributors) who participated in this project.
 
 ## License
 
-This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details
+This project is licensed under the MIT License.
 
 ## Contributing
 
 Contributions are welcomed!
 
-## 交流反馈
-![Contact](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/qqgroup_qrcode.png)
 
-## 捐赠
-![Contact](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/donate_wechat.png)

SpiderKeeper/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '1.2.0'
-__author__ = 'Dormy Mo'
+__version__ = '0.3.1'
+__author__ = 'kalombo'

SpiderKeeper/app/__init__.py

@@ -1,129 +1,96 @@
-# Import flask and template operators
-import logging
+import datetime
 import traceback
-
-import apscheduler
-from apscheduler.schedulers.background import BackgroundScheduler
-from flask import Flask
-from flask import jsonify
+from flask import Flask, session, jsonify
 from flask_basicauth import BasicAuth
-from flask_restful import Api
-from flask_restful_swagger import swagger
-from flask_sqlalchemy import SQLAlchemy
 from werkzeug.exceptions import HTTPException
 
 import SpiderKeeper
-from SpiderKeeper import config
-
-# Define the WSGI application object
-app = Flask(__name__)
-# Configurations
-app.config.from_object(config)
-
-# Logging
-log = logging.getLogger('werkzeug')
-log.setLevel(logging.ERROR)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-handler = logging.StreamHandler()
-handler.setFormatter(formatter)
-app.logger.setLevel(app.config.get('LOG_LEVEL', "INFO"))
-app.logger.addHandler(handler)
-
-# swagger
-api = swagger.docs(Api(app), apiVersion=SpiderKeeper.__version__, api_spec_url="/api",
-                   description='SpiderKeeper')
-# Define the database object which is imported
-# by modules and controllers
-db = SQLAlchemy(app, session_options=dict(autocommit=False, autoflush=True))
-
-
-@app.teardown_request
-def teardown_request(exception):
-    if exception:
-        db.session.rollback()
-        db.session.remove()
-    db.session.remove()
-
-# Define apscheduler
-scheduler = BackgroundScheduler()
-
-
-class Base(db.Model):
-    __abstract__ = True
-
-    id = db.Column(db.Integer, primary_key=True)
-    date_created = db.Column(db.DateTime, default=db.func.current_timestamp())
-    date_modified = db.Column(db.DateTime, default=db.func.current_timestamp(),
-                              onupdate=db.func.current_timestamp())
-
-
-# Sample HTTP error handling
-# @app.errorhandler(404)
-# def not_found(error):
-#     abort(404)
-
-
-@app.errorhandler(Exception)
-def handle_error(e):
-    code = 500
-    if isinstance(e, HTTPException):
-        code = e.code
-    app.logger.error(traceback.print_exc())
-    return jsonify({
-        'code': code,
-        'success': False,
-        'msg': str(e),
-        'data': None
-    })
-
-
-# Build the database:
-from SpiderKeeper.app.spider.model import *
-
-
-def init_database():
-    db.init_app(app)
-    db.create_all()
-
-
-# regist spider service proxy
-from SpiderKeeper.app.proxy.spiderctrl import SpiderAgent
+from SpiderKeeper.app.blueprints.dashboard.views import dashboard_bp
+from SpiderKeeper.app.blueprints.dashboard.api import api
+from SpiderKeeper.app.blueprints.dashboard.model import Project, SpiderInstance
+from SpiderKeeper.app.proxy import agent
 from SpiderKeeper.app.proxy.contrib.scrapy import ScrapydProxy
+from SpiderKeeper.app.extensions.sqlalchemy import db
 
-agent = SpiderAgent()
 
-
-def regist_server():
+def register_server(app):
     if app.config.get('SERVER_TYPE') == 'scrapyd':
         for server in app.config.get("SERVERS"):
             agent.regist(ScrapydProxy(server))
 
 
-from SpiderKeeper.app.spider.controller import api_spider_bp
-
-# Register blueprint(s)
-app.register_blueprint(api_spider_bp)
-
-# start sync job status scheduler
-from SpiderKeeper.app.schedulers.common import sync_job_execution_status_job, sync_spiders, \
-    reload_runnable_spider_job_execution
-
-scheduler.add_job(sync_job_execution_status_job, 'interval', seconds=5, id='sys_sync_status')
-scheduler.add_job(sync_spiders, 'interval', seconds=10, id='sys_sync_spiders')
-scheduler.add_job(reload_runnable_spider_job_execution, 'interval', seconds=30, id='sys_reload_job')
-
-
-def start_scheduler():
-    scheduler.start()
-
-
-def init_basic_auth():
+def init_basic_auth(app):
     if not app.config.get('NO_AUTH'):
-        basic_auth = BasicAuth(app)
+        BasicAuth(app)
 
 
-def initialize():
-    init_database()
-    regist_server()
-    start_scheduler()
-    init_basic_auth()
+def init_database(app):
+    db.init_app(app)
+    with app.app_context():
+        # Extensions like Flask-SQLAlchemy now know what the "current" app
+        # is while within this block. Therefore, you can now run........
+        db.create_all()
+
+
+def register_extensions(app):
+    init_database(app)
+    init_basic_auth(app)
+    api.init_app(app)
+
+
+def register_blueprints(app):
+    # Register blueprint(s)
+    app.register_blueprint(dashboard_bp)
+
+
+def create_flask_application(config):
+    # Define the WSGI application object
+    app = Flask(__name__)
+    # Configurations
+    app.config.from_object(config)
+    app.jinja_env.globals['sk_version'] = SpiderKeeper.__version__
+    register_extensions(app)
+    register_blueprints(app)
+    register_server(app)
+
+    @app.context_processor
+    def inject_common():
+        return dict(now=datetime.datetime.now(),
+                    servers=agent.servers)
+
+    @app.context_processor
+    def inject_project():
+        project_context = {}
+        project = None
+        projects = Project.query.all()
+        project_context['project_list'] = projects
+        if projects:
+            project = projects[0]
+
+        project_id = session.get('project_id')
+        if isinstance(project_id, int):
+            project = Project.query.get(project_id) or project
+
+        if project:
+            session['project_id'] = project.id
+            project_context['project'] = project
+            project_context['spider_list'] = [
+                spider_instance.to_dict() for spider_instance in
+                SpiderInstance.query.filter_by(project_id=project.id).all()]
+        else:
+            project_context['project'] = {}
+        return project_context
+
+    @app.errorhandler(Exception)
+    def handle_error(e):
+        code = 500
+        if isinstance(e, HTTPException):
+            code = e.code
+        app.logger.error(traceback.print_exc())
+        return jsonify({
+            'code': code,
+            'success': False,
+            'msg': str(e),
+            'data': None
+        })
+    return app