Merge pull request #128 from scality/bugfix/PTFE-600-crash-on-boot

[PTFE-600] Crash on boot

Author: tcarmet

MANIFEST.in

@@ -1,3 +1,5 @@
 recursive-include bert_e/templates *
+recursive-include bert_e/server/templates *
+recursive-include bert_e/server/static *
 recursive-include bert_e/docs *
 include requirements.txt

README.md

@@ -22,7 +22,7 @@ All above instructions will assume you are inside the codespace environment
 ```shell
 $ cp settings.sample.yml settings.yml
 # Configure settings.yml to your liking
-$ tox -e run
+$ tox run -e run
 ```
 
 ### Run local tests

bert_e/git_host/github/schema.py

@@ -18,10 +18,15 @@
 used by Bert-E) are declared.
 
 """
-from marshmallow import Schema, fields
+from marshmallow import Schema, fields, EXCLUDE
 
 
-class User(Schema):
+class GitHubSchema(Schema):
+    class Meta:
+        unknown = EXCLUDE
+
+
+class User(GitHubSchema):
     id = fields.Int(required=True)
     login = fields.Str(required=True)
     # Note: the "printable" name can be absent in most API call results.
@@ -30,7 +35,7 @@ class User(Schema):
     type = fields.Str()
 
 
-class Repo(Schema):
+class Repo(GitHubSchema):
     name = fields.Str(required=True)
     owner = fields.Nested(User, required=True)
     full_name = fields.Str(required=True)
@@ -41,7 +46,7 @@ class Repo(Schema):
     default_branch = fields.Str()
 
 
-class CreateRepo(Schema):
+class CreateRepo(GitHubSchema):
     name = fields.Str(required=True)
     description = fields.Str()
     homepage = fields.Url()
@@ -54,14 +59,14 @@ class CreateRepo(Schema):
     licence_template = fields.Str()
 
 
-class Status(Schema):
+class Status(GitHubSchema):
     state = fields.Str(required=True)
     target_url = fields.Str(required=True, allow_none=True)
     description = fields.Str(required=True, allow_none=True)
     context = fields.Str(required=True)
 
 
-class AggregatedStatus(Schema):
+class AggregatedStatus(GitHubSchema):
     # The most convenient way to get a pull request's build status is to
     # query github's API for an aggregated status.
     state = fields.Str()
@@ -70,23 +75,23 @@ class AggregatedStatus(Schema):
     statuses = fields.Nested(Status, many=True, required=True)
 
 
-class Branch(Schema):
+class Branch(GitHubSchema):
     label = fields.Str()  # user:ref or org:ref
     ref = fields.Str()
     sha = fields.Str()
     user = fields.Nested(User)
     repo = fields.Nested(Repo)
 
 
-class App(Schema):
+class App(GitHubSchema):
     id = fields.Int()
     slug = fields.Str()
     owner = fields.Nested(User)
     name = fields.Str()
     description = fields.Str()
 
 
-class CheckSuite(Schema):
+class CheckSuite(GitHubSchema):
     id = fields.Integer()
     head_sha = fields.Str()
     head_branch = fields.Str()
@@ -98,20 +103,20 @@ class CheckSuite(Schema):
     app = fields.Nested(App)
 
 
-class AggregateCheckSuites(Schema):
+class AggregateCheckSuites(GitHubSchema):
     total_count = fields.Integer()
     check_suites = fields.Nested(CheckSuite, many=True)
 
 
-class CheckRun(Schema):
+class CheckRun(GitHubSchema):
     id = fields.Integer()
     head_sha = fields.Str()
     status = fields.Str()
     conclusion = fields.Str(allow_none=True)
     html_url = fields.Url()
 
 
-class WorkflowRun(Schema):
+class WorkflowRun(GitHubSchema):
     id = fields.Integer()
     head_sha = fields.Str()
     head_branch = fields.Str()
@@ -121,17 +126,17 @@ class WorkflowRun(Schema):
     event = fields.Str()
 
 
-class AggregateWorkflowRuns(Schema):
+class AggregateWorkflowRuns(GitHubSchema):
     total_count = fields.Integer()
     workflow_runs = fields.Nested(WorkflowRun, many=True)
 
 
-class AggregateCheckRuns(Schema):
+class AggregateCheckRuns(GitHubSchema):
     total_count = fields.Integer()
     check_runs = fields.Nested(CheckRun, many=True)
 
 
-class PullRequest(Schema):
+class PullRequest(GitHubSchema):
     number = fields.Int(required=True)
     url = fields.Url()
     html_url = fields.Url()
@@ -149,23 +154,23 @@ class PullRequest(Schema):
     merged_at = fields.DateTime(allow_none=True)
 
 
-class CreatePullRequest(Schema):
+class CreatePullRequest(GitHubSchema):
     title = fields.Str(required=True)
     head = fields.Str(required=True)
     base = fields.Str(required=True)
     body = fields.Str()
     maintainer_can_modify = fields.Bool()
 
 
-class UpdatePullRequest(Schema):
+class UpdatePullRequest(GitHubSchema):
     title = fields.Str()
     body = fields.Str()
     state = fields.Str()
     base = fields.Str()
     maintainer_can_modify = fields.Bool()
 
 
-class Comment(Schema):
+class Comment(GitHubSchema):
     id = fields.Int(required=True)
     body = fields.Str()
     created_at = fields.DateTime()
@@ -174,61 +179,61 @@ class Comment(Schema):
     url = fields.Url()
 
 
-class CreateComment(Schema):
+class CreateComment(GitHubSchema):
     body = fields.Str(required=True)
 
 
-class Review(Schema):
+class Review(GitHubSchema):
     id = fields.Int(allow_none=True)
     body = fields.Str(allow_none=True)
     commit_id = fields.Str()
     state = fields.Str()
     user = fields.Nested(User)
 
 
-class DraftReview(Schema):
+class DraftReview(GitHubSchema):
     path = fields.Str()
     position = fields.Int()
     body = fields.Str()
 
 
-class CreateReview(Schema):
+class CreateReview(GitHubSchema):
     body = fields.Str(allow_none=True)
     event = fields.Str()
 
 
-class PullRequestEvent(Schema):
+class PullRequestEvent(GitHubSchema):
     action = fields.Str(required=True)
     number = fields.Int()
     pull_request = fields.Nested(PullRequest)
 
 
-class Issue(Schema):
+class Issue(GitHubSchema):
     number = fields.Int()
     title = fields.Str()
     # If this dict is present and non-empty, then the issue is a pull request.
     pull_request = fields.Dict(optional=True, default={})
 
 
-class IssueCommentEvent(Schema):
+class IssueCommentEvent(GitHubSchema):
     action = fields.Str()
     issue = fields.Nested(Issue)
 
 
-class PullRequestReviewEvent(Schema):
+class PullRequestReviewEvent(GitHubSchema):
     action = fields.Str()
     pull_request = fields.Nested(PullRequest)
 
 
-class StatusEvent(Schema):
+class StatusEvent(GitHubSchema):
     sha = fields.Str()
     state = fields.Str()
     context = fields.Str()
     description = fields.Str(allow_none=True)
     target_url = fields.Str(allow_none=True)
 
 
-class CheckSuiteEvent(Schema):
+class CheckSuiteEvent(GitHubSchema):
     action = fields.Str()
     check_suite = fields.Nested(CheckSuite)
     repository = fields.Nested(Repo)

bert_e/lib/schema.py

@@ -37,8 +37,9 @@ def load(cls: Schema, data, **kwargs):
         the result of any @post_load processing.
 
     """
-    res, errors = cls(**kwargs).load(data)
-    if errors:
+    try:
+        res = cls(**kwargs).load(data)
+    except Exception as errors:
         raise SchemaError(errors)
     return res
 

bert_e/server/__init__.py

@@ -40,8 +40,6 @@
 def setup_bert_e(settings_file, debug):
     """Create and configure Bert-E instance."""
     settings = setup_settings(settings_file)
-    settings['robot_password'] = os.environ['BERT_E_GITHOST_PWD']
-    settings['jira_token'] = os.environ['BERT_E_JIRA_TOKEN']
     settings['backtrace'] = True
 
     bert_e = BertE(settings)

bert_e/settings.py

@@ -4,7 +4,8 @@
 import logging
 import os
 from marshmallow import (
-    Schema, fields, post_load, pre_load, validates_schema, ValidationError)
+    Schema, fields, post_load, pre_load, validates_schema, ValidationError,
+    EXCLUDE)
 
 from bert_e.exceptions import (IncorrectSettingsFile,
                                SettingsFileNotFound,
@@ -122,6 +123,9 @@ def deserialize(self, value, attr=None, data=None, **kwargs):
 
 
 class SettingsSchema(Schema):
+    class Meta:
+        unknown = EXCLUDE
+
     # Settings defined in config files
     always_create_integration_pull_requests = fields.Bool(
         required=False, load_default=True)

charts/bert-e/templates/secrets.yaml

@@ -10,7 +10,7 @@ metadata:
     heritage: "{{ .Release.Service }}"
 type: Opaque
 data:
-  BERT_E_GITHOST_PWD: {{ .Values.bertE.robot.password | b64enc | quote }}
+  BERT_E_ROBOT_PASSWORD: {{ .Values.bertE.robot.password | b64enc | quote }}
   WEBHOOK_LOGIN: {{ .Values.bertE.webhook.username | b64enc | quote }}
   WEBHOOK_PWD: {{ .Values.bertE.webhook.password | b64enc | quote }}
   BERT_E_JIRA_TOKEN: {{ .Values.bertE.jira.token | b64enc | quote }}

manifests/base/secrets.env

@@ -1,6 +1,6 @@
 BERT_E_CLIENT_SECRET=vault:secret/data/bert-e#BERT_E_GITHUB_OAUTH_CLIENT_SECRET
 BERT_E_CLIENT_ID=vault:secret/data/bert-e#BERT_E_GITHUB_OAUTH_CLIENT_ID
-BERT_E_GITHOST_PWD=vault:secret/data/bert-e#BERT_E_GITHUB_TOKEN
+BERT_E_ROBOT_PASSWORD=vault:secret/data/bert-e#BERT_E_GITHUB_TOKEN
 BERT_E_GITHUB_TOKEN=vault:secret/data/bert-e#BERT_E_GITHUB_TOKEN
 BERT_E_JIRA_TOKEN=vault:secret/data/bert-e#BERT_E_JIRA_TOKEN
 BERT_E_JIRA_USER=vault:secret/data/bert-e#BERT_E_JIRA_USERNAME

settings.sample.yml

@@ -5,20 +5,20 @@ frontend_url: https://bert-e.mydomain.com/bitbucket/my_company/my_repository/ber
 
 # repository_host [MANDATORY]:
 #   The git hosting provider for the repository. Either bitbucket or github.
-repository_host: bitbucket
+repository_host: github
 
 # repository_owner [MANDATORY]:
 #   The name of the owner of the Bitbucket/GitHub repository to work on
 #
 #   (a.k.a. Bitbucket team or GitHub organization)
 #
-repository_owner: my_company
+repository_owner: scality
 
 
 # repository_slug [MANDATORY]:
 #   The slug of the Bitbucket/GitHub repository to work on
 #
-repository_slug: my_repository
+repository_slug: bert-e
 
 
 # robot [MANDATORY]:

setup.py

@@ -4,7 +4,7 @@
 import os
 import pip
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 
 # Besides not advised,
@@ -44,16 +44,7 @@ def requires():
     url='https://github.com/scality/bert-e',
     license='Apache',
     include_package_data=True,
-    packages=[
-        'bert_e',
-        'bert_e.jobs',
-        'bert_e.lib',
-        'bert_e.bin',
-        'bert_e.git_host',
-        'bert_e.git_host.github',
-        'bert_e.workflow',
-        'bert_e.workflow.gitwaterflow',
-    ],
+    packages=find_packages(),
     install_requires=requires(),
     entry_points={
         'console_scripts': [