From dff243497e48430afcdd47e81d46696999fdc357 Mon Sep 17 00:00:00 2001 From: Jack McCracken Date: Thu, 10 Sep 2015 12:17:24 -0400 Subject: [PATCH] Change around to read only from fs --- ckan_multisite/admin.py | 10 +++--- ckan_multisite/app.py | 1 + ckan_multisite/config.py.template | 2 +- ckan_multisite/router.py | 31 ++--------------- ckan_multisite/site.py | 57 ++++++++++++++----------------- uwsgi.ini | 11 ++++++ 6 files changed, 46 insertions(+), 66 deletions(-) diff --git a/ckan_multisite/admin.py b/ckan_multisite/admin.py index e607aa6..eb15e28 100644 --- a/ckan_multisite/admin.py +++ b/ckan_multisite/admin.py @@ -11,7 +11,7 @@ from datacats.environment import Environment from datacats.validate import DATACATS_NAME_RE -from ckan_multisite.site import Site, sites +from ckan_multisite.site import Site, get_sites from ckan_multisite.router import nginx from ckan_multisite.config import MAIN_ENV_NAME from ckan_multisite.task import create_site_task, remove_site_task @@ -57,7 +57,7 @@ def _handle_view(self, name, **kwargs): def delete_model(self, site): remove_site_task.apply_async(args=(site,)) - return sites.remove(site) + return True def create_model(self, form): # Sites start not having their data finished. @@ -73,13 +73,14 @@ def update_model(self, form, site): nginx.update_site(site) def get_list(self, page, sort_field, sort_desc, search, filters): + print 'HELLO {}'.format(get_sites()) # `page` is zero-based if not sort_field: sort_field = 'name' page_start = page*SitesView.page_size page_end = page_start + SitesView.page_size - slice_unsorted = sites[page_start:page_end] + slice_unsorted = get_sites()[page_start:page_end] slice_sorted = sorted( slice_unsorted, # Magic to get a specific sort field out of a site @@ -90,7 +91,7 @@ def get_list(self, page, sort_field, sort_desc, search, filters): def get_one(self, id): # ids come in as strs (unicode) - return sites[int(id)] if int(id) < len(sites) else None + return get_sites()[int(id)] if int(id) < len(get_sites()) else None def scaffold_form(self): return SiteAddForm @@ -110,6 +111,7 @@ def scaffold_sortable_columns(self): return dict(zip(SitesView.column_sortable_list, SitesView.column_sortable_list)) def get_pk_value(self, model): + sites = get_sites() if model in sites: return sites.index(model) else: diff --git a/ckan_multisite/app.py b/ckan_multisite/app.py index 5bb9ae7..c3b90f2 100644 --- a/ckan_multisite/app.py +++ b/ckan_multisite/app.py @@ -14,6 +14,7 @@ from ckan_multisite.login import bp as login_bp app = Flask(__name__) +app.config['PROPAGATE_EXCEPTIONS'] = True app.secret_key = SECRET_KEY admin.init_app(app) app.register_blueprint(api_bp) diff --git a/ckan_multisite/config.py.template b/ckan_multisite/config.py.template index 699696b..f2ca66d 100644 --- a/ckan_multisite/config.py.template +++ b/ckan_multisite/config.py.template @@ -38,4 +38,4 @@ PORT = 5000 # THIS MUST BE FALSE ON A PRODUCTION SERVER DEBUG = True # This says that we should generate the default nginx configuration. -GENERATE_NGINX_DEFAULT = True +GENERATE_NGINX_DEFAULT = False diff --git a/ckan_multisite/router.py b/ckan_multisite/router.py index 60e2fa9..0503bb9 100644 --- a/ckan_multisite/router.py +++ b/ckan_multisite/router.py @@ -158,41 +158,14 @@ def remove_site(self, site): self.sites.remove(name) self.reload_nginx() - def sync_sites(self, authoritative_sites): - """ - Ensures that our list of sites (and the ones in nginx) is correct - - :param authoritative_sites: The source (probably the datadir) which we - should consider the source for correct information. - """ - # In this case we actually need syncing - if len(authoritative_sites) != len(self.sites): - print 'Unbalanced datacats sites and nginx sites... Attempting to sync them.' - authoritative_set = set(authoritative_sites) - # This is the set that's in the nginx configuration - our_set = set(self.sites) - - # These are the sites which should no longer have a config - # file because they were purged outside the app - outdated_sites = our_set - authoritative_set - for site in outdated_sites: - self.remove_site(site) - - # These are sites which have been created outside of the app - new_sites = authoritative_set - our_set - for site in new_sites: - self.add_site(site) - - print 'Sync successful!' - def regenerate_config(self): """ Regenerates all configuration files with new settings. """ # Avoid a recursive import - from ckan_multisite.site import sites + from ckan_multisite.site import get_sites self.update_default() - for site in sites: + for site in get_sites(): self.update_site(site) diff --git a/ckan_multisite/site.py b/ckan_multisite/site.py index cfc34ec..e04dbe6 100644 --- a/ckan_multisite/site.py +++ b/ckan_multisite/site.py @@ -11,10 +11,33 @@ MULTISITE_CONFIG_NAME = '.multisite-config' -sites = [] +SITES_PATH = expanduser(path_join(DATACATS_DIRECTORY, MAIN_ENV_NAME, 'sites')) + +def get_sites(): + if not exists(path_join(DATACATS_DIRECTORY)): + mkdir(path_join(DATACATS_DIRECTORY)) + dcats_listing = listdir(SITES_PATH) + sites = [] + # Primary child isn't mean for them + if 'primary' in dcats_listing: + dcats_listing.remove('primary') + for s in dcats_listing: + # Since Flask-admin does things in unicode convert to unicode strings for + config_path = path_join(SITES_PATH, s, MULTISITE_CONFIG_NAME) + if not exists(config_path): + print 'Making up a name for site {}: {}'.format(s, s.capitalize()) + with open(config_path, 'w') as wf: + wf.write(s.capitalize()) + with open(config_path) as f: + sites.append(Site(unicode(s), f.read(), sort=False)) + + # Sort the list initially + sites.sort() + + return sites def site_by_name(name): - return next(site for site in sites if site.name == name) + return next(site for site in get_sites() if site.name == name) class Site(object): def __init__(self, name, display_name, finished_create=True, sort=True): @@ -34,13 +57,6 @@ def __init__(self, name, display_name, finished_create=True, sort=True): self.finished_create = finished_create self.celery_task = None - if not sort: - sites.append(self) - else: - # This assumes the `sites` list is already sorted and inserts using an O(log n) - # search. - insort_left(sites, self) - def __repr__(self): return self.name.__repr__() @@ -59,26 +75,3 @@ def serialize_display_name(self): def __gt__(self, site): return self.name > site.name -SITES_PATH = expanduser(path_join(DATACATS_DIRECTORY, MAIN_ENV_NAME, 'sites')) -if not exists(path_join(DATACATS_DIRECTORY)): - mkdir(path_join(DATACATS_DIRECTORY)) -# This is just here to initially populate the list -__dcats_listing = listdir(SITES_PATH) -# Primary child isn't mean for them -if 'primary' in __dcats_listing: - __dcats_listing.remove('primary') -for s in __dcats_listing: - # Since Flask-admin does things in unicode convert to unicode strings for - __config_path = path_join(SITES_PATH, s, MULTISITE_CONFIG_NAME) - if not exists(__config_path): - print 'Making up a name for site {}: {}'.format(s, s.capitalize()) - with open(__config_path, 'w') as wf: - wf.write(s.capitalize()) - with open(__config_path) as f: - Site(unicode(s), f.read(), sort=False) - -# Sort the list initially -sites.sort() - -# the router will attempt to sync the configuration files with us -nginx.sync_sites(sites) diff --git a/uwsgi.ini b/uwsgi.ini index 86d71b9..eebf6ff 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -4,3 +4,14 @@ processes = 4 master = 1 module = ckan_multisite.app:app chmod-socket = 666 +logto=uwsgi.log +harakiri-verbose = False +log-maxsize = 10485760 +master = True + +max-requests = 5000 +buffer-size = 32768 +post-buffering = 4096 +processes = 4 +stats = :1717 +enable-threads = True