Continue @tierra's issue-sync branch

README.md

@@ -4,13 +4,14 @@ Trac - GitHub integration
 Features
 --------
 
-This Trac plugin performs four functions:
+This Trac plugin performs five functions:
 
 1. update the local git mirror used by Trac after each push to GitHub, and
    notify the new changesets to Trac;
 2. authenticate users with their GitHub account;
-3. direct changeset TracLinks to GitHub's repository browser.
-4. sync GitHub teams to Trac permission groups
+3. direct changeset TracLinks to GitHub's repository browser;
+4. sync GitHub teams to Trac permission groups;
+5. sync any new GitHub issues and pull requests into Trac tickets.
 
 The notification of new changesets is strictly equivalent to the command
 described in Trac's setup guide:
@@ -301,6 +302,74 @@ If you do not want to store the API secrets for `access_token` and
 `webhook_secret` in trac.ini, you can use the same alternatives as for
 `client_id` and `client_secret` documented [above](#authentication).
 
+### Syncing Issues and Pull Requests
+
+**`tracext.github.GitHubIssueHook`** implements a a few GitHub hooks called
+when a new issue or pull request is opened, commented on, or changed.
+
+It will open a new Trac ticket with the corresponding issue or pull request
+title and description, setting the reporter to "username (GitHub)". It will use
+all default ticket fields for everything else. It will also automatically
+attach a patch file for any pull requests, along with any new patches when
+someone pushes new commits to any existing pull request.
+
+Additionally, if any comments are left on the issue or pull request (including
+inline patch comments on pulls) on GitHub, they will be posted to the Trac
+ticket as well.
+
+Unlike the post-commit hook used for syncing any GitHub repos, these hooks are
+required to configure a hook "secret" used to verify that the hooks were sent
+from GitHub. This is to prevent spam tickets and comments. So first, you should
+generate a random secret to be used with this hook. Ideally, it should be a
+random string about 40 characters long containing only `[0-9a-f]` characters.
+You can generate this on the command line by running this if you prefer:
+
+    $ ruby -rsecurerandom -e 'puts SecureRandom.hex(20)'
+
+It should look like this (don't just use this string though!):
+`cc6f7dddec47e4e10a423dcfbab5c102f506f72d`
+
+Save that somewhere safe, you will use this as your `hook_secret`.
+
+Edit your `trac.ini` as follows to configure syncing:
+
+    [components]
+    tracext.github.GitHubIssueHook = enabled
+
+    [github]
+    hook_secret = <your_hook_secret_here>
+
+Reload the web server, browse to the home page of your project in Trac and
+append `/github-issues` to the URL. You should see the following message:
+
+    Endpoint is ready to accept GitHub notifications.
+
+This is the URL of the issues endpoint we'll use for the hook.
+
+If you get a Trac error page saying "No handler matched request to
+/github-issues" instead, the plugin isn't installed properly. Make sure you've
+followed the installation instructions correctly and search Trac's logs for
+errors.
+
+This hook supports creating tickets for multiple GitHub repos if you want it to
+watch issues and pull requests from several of them at the same time. So it's
+possible to create an "organization hook", which will fire for all organization
+repositories, or just create any number of "repository hooks" for the repos you
+want it to watch. The configuration for either is exactly the same.
+
+Go to your organization's or repository's settings page on GitHub. In the
+"Webhooks & Services" tab, click "Add webhook". Put the URL of the endpoint in
+the "Payload URL" field, leave the "Content type" as "application/json", and set
+the "Secret" to the 40 character `hook_secret` you generated earlier. Now select
+the "Let me select individual events" radio option, and check the following
+hooks: "Issue Comments", "Issues", "Pull Request", and "Pull Request review
+comment". Then click "Add webhook", and you're done.
+
+If you click on the webhook you just created, at the bottom of the page, you
+should see that a "ping" payload was successufully delivered to Trac.
+
+If you already have existing issues or pull requests, they will not be synced
+to Trac. Only new issues and pull requests will be synced.
 
 Advanced setup
 --------------
@@ -518,6 +587,7 @@ Changelog
 * Add configuration option for path prefix of login and logout. (#127)
 * Add `GitHubPolicy` permission policy to make `[timeline]`
   `changeset_show_file` option work correctly. (#126)
+* Add support for syncing issues and pull requests.
 
 ### 2.3
 

tracext/github.py

@@ -0,0 +1,488 @@
+import fnmatch
+from hashlib import sha1
+import hmac
+import json
+import os
+import re
+import urllib2
+
+from genshi.builder import tag
+
+from trac.attachment import Attachment
+from trac.db import with_transaction
+from trac.config import ListOption, Option
+from trac.core import Component, implements
+from trac.ticket.model import Ticket
+from trac.util.translation import _
+from trac.versioncontrol.api import is_default, NoSuchChangeset, RepositoryManager
+from trac.versioncontrol.web_ui.changeset import ChangesetModule
+from trac.web.api import IRequestHandler
+from trac.web.auth import LoginModule
+
+
+class GitHubMixin(object):
+
+    def get_gh_repo(self, reponame):
+        key = 'repository' if is_default(reponame) else '%s.repository' % reponame
+        return self.config.get('github', key)
+
+    def get_branches(self, reponame):
+        key = 'branches' if is_default(reponame) else '%s.branches' % reponame
+        return self.config.getlist('github', key, sep=' ')
+
+    def _config(self, key):
+        assert key in ('client_id', 'client_secret', 'hook_secret')
+        value = self.config.get('github', key)
+        if re.match('[0-9a-f]+', value):
+            return value
+        elif value.isupper():
+            return os.environ.get(value, '')
+        else:
+            with open(value) as f:
+                return f.read.strip()
+
+    def verify_signature(self, req, body):
+        full_signature = req.get_header('X-Hub-Signature')
+        if not full_signature or not full_signature.find('='):
+            return False
+        sha_name, signature = full_signature.split('=')
+        if sha_name != 'sha1':
+            return False
+        hook_secret = str(self._config('hook_secret'))
+        mac = hmac.new(hook_secret, msg = str(body), digestmod = sha1)
+        return hmac.compare_digest(mac.hexdigest(), signature)
+
+
+class GitHubLoginModule(GitHubMixin, LoginModule):
+
+    # INavigationContributor methods
+
+    def get_active_navigation_item(self, req):
+        return 'github_login'
+
+    def get_navigation_items(self, req):
+        if req.authname and req.authname != 'anonymous':
+            # Use the same names as LoginModule to avoid duplicates.
+            yield ('metanav', 'login', _('logged in as %(user)s',
+                                         user=req.authname))
+            yield ('metanav', 'logout',
+                   tag.a(_('Logout'), href=req.href.github('logout')))
+        else:
+            # Use a different name from LoginModule to allow both in parallel.
+            yield ('metanav', 'github_login',
+                   tag.a(_('GitHub Login'), href=req.href.github('login')))
+
+    # IRequestHandler methods
+
+    def match_request(self, req):
+        return re.match('/github/(login|oauth|logout)/?$', req.path_info)
+
+    def process_request(self, req):
+        if req.path_info.startswith('/github/login'):
+            self._do_login(req)
+        elif req.path_info.startswith('/github/oauth'):
+            self._do_oauth(req)
+        elif req.path_info.startswith('/github/logout'):
+            self._do_logout(req)
+        self._redirect_back(req)
+
+    # Internal methods
+
+    def _do_login(self, req):
+        oauth = self._oauth_session(req)
+        authorization_url, state = oauth.authorization_url(
+            'https://github.com/login/oauth/authorize')
+        req.session['oauth_state'] = state
+        req.redirect(authorization_url)
+
+    def _do_oauth(self, req):
+        oauth = self._oauth_session(req)
+        authorization_response = req.abs_href(req.path_info) + '?' + req.query_string
+        client_secret = self._config('client_secret')
+        oauth.fetch_token(
+            'https://github.com/login/oauth/access_token',
+            authorization_response=authorization_response,
+            client_secret=client_secret)
+
+        user = oauth.get('https://api.github.com/user').json()
+        # Small hack to pass the username to _do_login.
+        req.environ['REMOTE_USER'] = user['login']
+        # Save other available values in the session.
+        req.session.setdefault('name', user.get('name') or '')
+        req.session.setdefault('email', user.get('email') or '')
+
+        return super(GitHubLoginModule, self)._do_login(req)
+
+    def _oauth_session(self, req):
+        client_id = self._config('client_id')
+        redirect_uri = req.abs_href.github('oauth')
+        # Inner import to avoid a hard dependency on requests-oauthlib.
+        from requests_oauthlib import OAuth2Session
+        return OAuth2Session(client_id, redirect_uri=redirect_uri, scope=[])
+
+
+class GitHubBrowser(GitHubMixin, ChangesetModule):
+
+    repository = Option('github', 'repository', '',
+            doc="Repository name on GitHub (<user>/<project>)")
+
+    # IRequestHandler methods
+
+    def match_request(self, req):
+        match = self._request_re.match(req.path_info)
+        if match:
+            rev, path = match.groups()
+            req.args['rev'] = rev
+            req.args['path'] = path or '/'
+            return True
+
+    def process_request(self, req):
+        rev = req.args.get('rev')
+        path = req.args.get('path')
+
+        rm = RepositoryManager(self.env)
+        reponame, repos, path = rm.get_repository_by_path(path)
+        gh_repo = self.get_gh_repo(reponame)
+
+        rev = repos.normalize_rev(rev)
+
+        if path and path != '/':
+            path = path.lstrip('/')
+            # GitHub will s/blob/tree/ if the path is a directory
+            url = 'https://github.com/%s/blob/%s/%s' % (gh_repo, rev, path)
+        else:
+            url = 'https://github.com/%s/commit/%s' % (gh_repo, rev)
+        req.redirect(url)
+
+    # ITimelineEventProvider methods
+
+    def get_timeline_events(self, req, start, stop, filters):
+        for event in super(GitHubBrowser, self).get_timeline_events(req, start, stop, filters):
+            assert event[0] == 'changeset'
+            viewable_changesets, show_location, show_files = event[3]
+            filtered_changesets = []
+            for cset, cset_resource, (reponame,) in viewable_changesets:
+                branches = self.get_branches(reponame)
+                if rev_in_branches(cset, branches):
+                    filtered_changesets.append((cset, cset_resource, [reponame]))
+            if filtered_changesets:
+                cset = filtered_changesets[-1][0]
+                yield ('changeset', cset.date, cset.author,
+                        (filtered_changesets, show_location, show_files))
+
+
+class GitHubPostCommitHook(GitHubMixin, Component):
+    implements(IRequestHandler)
+
+    branches = ListOption('github', 'branches', sep=' ',
+            doc="Notify only commits on these branches to Trac")
+
+    # IRequestHandler methods
+
+    _request_re = re.compile(r"/github(/.*)?$")
+
+    def match_request(self, req):
+        match = self._request_re.match(req.path_info)
+        if match:
+            req.args['path'] = match.group(1) or '/'
+            return True
+
+    def process_request(self, req):
+        path = req.args['path']
+
+        rm = RepositoryManager(self.env)
+        reponame, repos, path = rm.get_repository_by_path(path)
+
+        if repos is None or path != '/':
+            msg = u'No such repository (%s)\n' % path
+            self.log.warning(msg.rstrip('\n'))
+            req.send(msg.encode('utf-8'), 'text/plain', 400)
+
+        if req.method != 'POST':
+            msg = u'Endpoint is ready to accept GitHub notifications.\n'
+            self.log.warning(u'Method not allowed (%s)' % req.method)
+            req.send(msg.encode('utf-8'), 'text/plain', 405)
+
+        event = req.get_header('X-GitHub-Event')
+        if event == 'ping':
+            payload = json.loads(req.read())
+            req.send(payload['zen'].encode('utf-8'), 'text/plain', 200)
+        elif event != 'push':
+            msg = u'Only ping and push are supported\n'
+            self.log.warning(msg.rstrip('\n'))
+            req.send(msg.encode('utf-8'), 'text/plain', 400)
+
+        output = u'Running hook on %s\n' % (reponame or '(default)')
+
+        output += u'* Updating clone\n'
+        try:
+            git = repos.git.repo             # GitRepository
+        except AttributeError:
+            git = repos.repos.git.repo       # GitCachedRepository
+        git.remote('update', '--prune')
+
+        # Ensure that repos.get_changeset can find the new changesets.
+        output += u'* Synchronizing with clone\n'
+        repos.sync()
+
+        try:
+            payload = json.loads(req.read())
+            revs = [commit['id']
+                    for commit in payload['commits'] if commit['distinct']]
+        except (ValueError, KeyError):
+            msg = u'Invalid payload\n'
+            self.log.warning(msg.rstrip('\n'))
+            req.send(msg.encode('utf-8'), 'text/plain', 400)
+
+        branches = self.get_branches(reponame)
+        added, skipped, unknown = classify_commits(revs, repos, branches)
+
+        if added:
+            output += u'* Adding %s\n' % describe_commits(added)
+            # This is where Trac gets notified of the commits in the changeset
+            rm.notify('changeset_added', reponame, added)
+
+        if skipped:
+            output += u'* Skipping %s\n' % describe_commits(skipped)
+
+        if unknown:
+            output += u'* Unknown %s\n' % describe_commits(unknown)
+            self.log.error(u'Payload contains unknown %s',
+                    describe_commits(unknown))
+
+        for line in output.splitlines():
+            self.log.debug(line)
+
+        req.send(output.encode('utf-8'), 'text/plain', 200 if output else 204)
+
+
+def classify_commits(revs, repos, branches):
+    added, skipped, unknown = [], [], []
+    for rev in revs:
+        try:
+            cset = repos.get_changeset(rev)
+        except NoSuchChangeset:
+            unknown.append(rev)
+        else:
+            if rev_in_branches(cset, branches):
+                added.append(rev)
+            else:
+                skipped.append(rev)
+    return added, skipped, unknown
+
+
+def rev_in_branches(changeset, branches):
+    if not branches:            # no branches filter configured
+        return True
+    return any(fnmatch.fnmatchcase(cset_branch, branch)
+        for cset_branch, _ in changeset.get_branches() for branch in branches)
+
+
+def describe_commits(revs):
+    if len(revs) == 1:
+        return u'commit %s' % revs[0]
+    else:
+        return u'commits %s' % u', '.join(revs)
+
+
+class GitHubIssueHook(GitHubMixin, Component):
+    implements(IRequestHandler)
+
+    _request_re = re.compile(r"/github-issues/?$")
+
+    # IRequestHandler method
+    def match_request(self, req):
+        match = self._request_re.match(req.path_info)
+        if match:
+            return True
+
+    # IRequestHandler method
+    def process_request(self, req):
+        if req.method == 'GET':
+            req.send('Endpoint is ready to accept GitHub notifications.', 'text/plain', 200)
+            return
+        if req.method != 'POST':
+            msg = u'Method not allowed (%s)\n' % req.method
+            self.log.warning(msg.rstrip('\n'))
+            req.send(msg.encode('utf-8'), 'text/plain', 405)
+            return
+
+        body = req.read()
+
+        if not self.verify_signature(req, body):
+            msg = u'Invalid hook signature from %s, ignoring request.\n' % req.remote_addr
+            self.log.warning(msg.rstrip('\n'))
+            req.send(msg.encode('utf-8'), 'text/plain', 400)
+            return
+
+        event = req.get_header('X-GitHub-Event')
+        if event == 'ping':
+            payload = json.loads(body)
+            req.send(payload['zen'].encode('utf-8'), 'text/plain', 200)
+            return
+        if event not in ['issue_comment', 'issues', 'pull_request', 'pull_request_review_comment']:
+            msg = u'Unsupported event recieved (%s), ignoring request.\n' % event
+            self.log.warning(msg.rstrip('\n'))
+            req.send(msg.encode('utf-8'), 'text/plain', 400)
+            return
+
+        event_method = getattr(self, '_event_' + event)
+        event_method(req, json.loads(body))
+
+    def _event_issue_comment(self, req, data):
+        comment = data['comment']
+        author = comment['user']['login'] + ' (GitHub)'
+        issue = data['repository']['full_name'] + '#' + str(data['issue']['number'])
+        issue = issue.encode('utf-8')
+
+        ticket_id = self.find_ticket(issue)
+
+        if data['action'] == 'created':
+            ticket = Ticket(self.env, ticket_id)
+            ticket.save_changes(author, comment['body'])
+            req.send('Comment added to ticket #%d.' % ticket_id, 'text/plain', 200)
+
+        else:
+            req.send('No action taken for this hook.', 'text/plain', 200)
+
+    def _event_issues(self, req, data):
+        author = data['sender']['login'] + ' (GitHub)'
+        issue = data['repository']['full_name'] + '#' + str(data['issue']['number'])
+        issue = issue.encode('utf-8')
+
+        if data['action'] == 'opened':
+            ticket = Ticket(self.env)
+            ticket['reporter'] = author
+            ticket['summary'] = data['issue']['title']
+            ticket['description'] = data['issue']['body']
+            ticket['description'] += "\n\n!GitHub Issue: %s" % data['issue']['html_url']
+            ticket['status'] = 'new'
+            ticket_id = ticket.insert()
+
+            self.mark_github_issue(ticket_id, issue)
+
+            req.send('Synced to new ticket #%d' % ticket_id, 'text/plain', 200)
+
+        elif data['action'] == 'closed':
+            pass
+
+        elif data['action'] == 'reopened':
+            pass
+
+        else:
+            req.send('No action taken for this hook.', 'text/plain', 200)
+
+    def _event_pull_request(self, req, data):
+        pull = data['pull_request']
+        author = data['sender']['login'] + ' (GitHub)'
+        issue = data['repository']['full_name'] + '#' + str(pull['number'])
+        issue = issue.encode('utf-8')
+
+        if data['action'] == 'opened':
+            ticket = Ticket(self.env)
+            ticket['reporter'] = author
+            ticket['summary'] = pull['title']
+            ticket['description'] = pull['body']
+            ticket['description'] += "\n\nPull Request: %s" % pull['html_url']
+            ticket['status'] = 'new'
+            ticket_id = ticket.insert()
+
+            self.mark_github_issue(ticket_id, issue)
+
+            response = urllib2.urlopen(pull['patch_url'])
+            self.create_attachment(ticket_id, pull['number'], response.read(), author)
+
+            req.send('Synced to new ticket #%d' % ticket_id, 'text/plain', 200)
+
+        elif data['action'] == 'synchronize':
+            ticket_id = self.find_ticket(issue)
+
+            if ticket_id:
+                response = urllib2.urlopen(pull['patch_url'])
+                self.create_attachment(ticket_id, pull['number'], response.read(), author)
+                req.send('Attached new patch to ticket #%d.' % ticket_id, 'text/plain', 200)
+
+            # This happens if sync is turned on after existing PRs were opened.
+            req.send('No ticket to sync patch for issue %s.' % issue, 'text/plain', 200)
+
+        elif data['action'] == 'closed':
+            pass
+
+        elif data['action'] == 'reopened':
+            pass
+
+        else:
+            req.send('No action taken for this hook.', 'text/plain', 200)
+
+    def _event_pull_request_review_comment(self, req, data):
+        comment = data['comment']
+        pull = data['pull_request']
+        author = comment['user']['login'] + ' (GitHub)'
+        issue = data['repository']['full_name'] + '#' + str(pull['number'])
+        issue = issue.encode('utf-8')
+
+        ticket_id = self.find_ticket(issue)
+
+        if data['action'] == 'created':
+            ticket = Ticket(self.env, ticket_id)
+            ticket.save_changes(author,
+                "%s\n\n[%s See Inline Patch Context]" %
+                (comment['body'], comment['html_url'])
+            )
+            req.send('Comment added to ticket #%d.' % ticket_id, 'text/plain', 200)
+
+        else:
+            req.send('No action taken for this hook.', 'text/plain', 200)
+
+    def mark_github_issue(self, ticket_id, github_issue):
+        """
+        Manually save a custom field on the ticket (github_issue) with the
+        canonical GitHub address for the issue.
+
+        We manually do this instead of instructing users to manually setup a
+        custom field since it's only necessary internally. End users can still
+        configure it if they want though. Hopefully, this also prevents ticket
+        split/copy plugins from duplicating the github_issue field since this
+        only supports syncing with one ticket.
+
+        Several GitHub repos could be saving here, so don't only use number.
+        """
+
+        @with_transaction(self.env)
+        def sql_transaction(db):
+            cursor = db.cursor()
+            cursor.execute(
+                "DELETE FROM ticket_custom WHERE ticket = %d AND name = %%s" %
+                ticket_id, ['github_issue']
+            )
+            cursor.execute(
+                "INSERT INTO ticket_custom VALUES (%d, %%s, %%s)" %
+                ticket_id, ['github_issue', github_issue]
+            )
+
+    def find_ticket(self, github_issue):
+        """
+        Return the ticket_id for the given GitHub issue if it exists.
+        """
+
+        rows = self.env.db_query(
+            "SELECT ticket FROM ticket_custom WHERE name = %s AND value = %s",
+            ['github_issue', github_issue]
+        )
+        return int(rows[0][0]) if rows else None
+
+    def create_attachment(self, ticket_id, pull_id, patch, author = None):
+        if len(patch) > self.env.config.get('attachment', 'max_size'):
+            self.log.warning('GitHub patch (#%d) too big to attach to ticket #%d' % (pull_id, ticket_id))
+            return
+
+        # Create a temp file object for Trac attachments.
+        temp_fd = os.tmpfile()
+        temp_fd.write(patch)
+        temp_fd.seek(0)
+
+        attachment = Attachment(self.env, 'ticket', ticket_id)
+        if author:
+            attachment.author = author
+        filename = 'github-pull-%d.patch' % pull_id
+        attachment.insert(filename, temp_fd, len(patch))