diff --git a/.github/workflows/pyinstaller.yml b/.github/workflows/pyinstaller.yml index c867ab3..6b6dc5e 100644 --- a/.github/workflows/pyinstaller.yml +++ b/.github/workflows/pyinstaller.yml @@ -15,7 +15,7 @@ jobs: output: ${{ steps.latest_tag.outputs.tag }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: latest_tag run: | git fetch -a @@ -31,23 +31,23 @@ jobs: os: [macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: '3.11.2' + python-version: '3.x' # install requirements and build # - note: `--hidden-import` is required for GUI mode until https://github.com/hustcc/timeago/issues/40 is fixed - run: | python -m pip install --upgrade pip python -m pip install wheel - python -m pip install -r requirements.txt pyinstaller + python -m pip install -r requirements-core.txt -r requirements-gui.txt pyinstaller python -m PyInstaller --clean --noconsole --onefile --hidden-import timeago.locales.en_short --icon icon.png emailproxy.py - + # add license, documentation, configuration file sample and disclaimer # - note: `move [...] alias move=mv [...]` is to support using the same commands on all platforms - run: | - move LICENSE . 2> nul || { shopt -s expand_aliases; alias move=mv; } + move LICENSE . 2> nul || { shopt -s expand_aliases; alias move=mv; } move LICENSE dist/ move README.md dist/ move emailproxy.config dist/ @@ -55,20 +55,20 @@ jobs: echo 'but is not tested, and is not officially supported. Using the main emailproxy.py script ' >> _.txt echo 'directly is recommended for best results: https://github.com/simonrob/email-oauth2-proxy/' >> _.txt move _.txt dist/GettingStarted.txt - + # on macOS `--onefile` creates bundle *and* binary; we don't need both (and the .app also contains the binary) - if: runner.os == 'macOS' run: rm dist/emailproxy # zip the built output, naming according to tag and OS - - uses: thedoctor0/zip-release@0.7.1 + - uses: thedoctor0/zip-release@0.7.6 with: type: zip directory: dist/ filename: emailproxy-${{ needs.get_tag.outputs.output }}_pyinstaller-${{ runner.os }}.zip # append the zip to the latest release - - uses: xresloader/upload-to-github-release@v1 + - uses: xresloader/upload-to-github-release@v1.3.12 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..89ec8dc --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,65 @@ +name: Deploy to PyPI and add to latest release + +on: + push: + +permissions: + contents: write + id-token: write + +# see: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + + # set up build environment and package the proxy + - run: | + python -m pip install --upgrade pip + python -m pip install build + python -m build + + - uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + needs: build + + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/emailproxy + + steps: + - uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + # upload the built packages to PyPI (we use a token rather than trusted publishing due to our unconventional build branch method) + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI }} + skip-existing: true # avoid failing when repeating this action + + # sign the built packages + - uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + + # append the built packages to the latest release + - uses: xresloader/upload-to-github-release@v1.3.12 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + file: dist/*.whl;dist/*.tar.gz + update_latest_release: true diff --git a/README.md b/README.md index 9e39fd3..16ebd00 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The proxy works in the background with a menu bar/taskbar helper or as a system ### Example use-cases - You need to use an Office 365 email account, but don't get on with Outlook. -The email client you like doesn't support OAuth 2.0, which is mandatory [from January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437). +The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437). - You used to use Gmail via IMAP/POP/SMTP with your raw account credentials (i.e., your real password), but cannot do this now that Google has disabled this method, and don't want to use an [App Password](https://support.google.com/accounts/answer/185833) (or cannot enable this option). - You have an account already set up in an email client, and you need to switch it to OAuth 2.0 authentication. You can edit the server details, but the client forces you to delete and re-add the account to enable OAuth 2.0, and you don't want to do this. @@ -23,27 +23,37 @@ For commercial support or feature requests, please also consider [sponsoring thi ## Getting started -After cloning or [downloading](https://github.com/simonrob/email-oauth2-proxy/releases/latest) (and starring :-) this repository, start by editing the file `emailproxy.config` to add configuration details for each email server and account that you want to use with the proxy. -[Guidance and example account configurations](emailproxy.config) are provided for Office 365, Gmail and several other providers, though you will need to insert your own client credentials for each one (see the [documentation below](#oauth-20-client-credentials)). +Begin by downloading the proxy via one of the following methods: + +
    +
  1. Pick a pre-built release for your platform (no installation needed); or,
    2. +
  2. Install from PyPI: python -m pip install emailproxy[gui] to set up, then python -m emailproxy to run; or,
    3. +
  3. Clone or download (and star :-) the GitHub repository, then: python -m pip install -r requirements-core.txt -r requirements-gui.txt to install requirements, and python emailproxy.py to run
    4. +
+ +If you choose download option (A) or (B), you should also [download the sample `emailproxy.config` file](https://github.com/simonrob/email-oauth2-proxy/raw/main/emailproxy.config) and place this into the directory you will run the proxy from. +Next, edit the `emailproxy.config` file to add configuration details for each email server and account that you want to use with the proxy. +[Guidance and example account configurations](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) are provided for Office 365, Gmail and several other providers, though you will need to insert your own client credentials for each one (see the [documentation below](https://github.com/simonrob/email-oauth2-proxy/#oauth-20-client-credentials)). You can remove details from the sample configuration file for services you don't use, or add additional ones for any other OAuth 2.0-authenticated IMAP/POP/SMTP servers you would like to use with the proxy. -Next, from a terminal, install the script's requirements: `python -m pip install -r requirements.txt`, and start the proxy: `python emailproxy.py` – a menu bar/taskbar icon should appear. -If instead of the icon you see an error in the terminal, it is likely that your system is missing dependencies for the `pywebview` or `pystray` packages. -See the [dependencies and setup](#dependencies-and-setup) section below for help resolving this, and also the [advanced configuration](#advanced-configuration) section for additional options (including fully headless deployments and integration with a secrets manager). +You can now start the proxy: depending on which installation option you chose, either open the application or use the appropriate run command listed above. +A menu bar/taskbar icon should appear. +If instead of the icon you see an error notification, it is likely that your system is missing dependencies for the `pywebview` or `pystray` packages. +See the [dependencies and setup](https://github.com/simonrob/email-oauth2-proxy/#dependencies-and-setup) section below for help resolving this, and also the [advanced configuration](https://github.com/simonrob/email-oauth2-proxy/#advanced-configuration) section for additional options (including fully headless deployments and integration with a secrets manager). Finally, open your email client and configure its server details to match the ones you set in the proxy's configuration file. The correct server to use with an account is identified using the port number you select in your client – for example, to use the sample Office 365 details, this would be `localhost` on port `1993` for IMAP, port `1995` for POP and port `1587` for SMTP. The proxy supports multiple accounts simultaneously, and all accounts associated with the same provider can share the same proxy server. The local connection in your email client should be configured as unencrypted to allow the proxy to operate, but the connection between the proxy and your email server is always secure (implicit SSL/TLS for IMAP and POP; implicit or explicit (STARTTLS) SSL/TLS for SMTP). -See the [sample configuration file](emailproxy.config) for additional documentation about advanced features, including local encryption, account configuration inheritance and support for running in a container. +See the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for additional documentation about advanced features, including local encryption, account configuration inheritance and support for running in a container. The first time your email client makes a request you should see a notification from the proxy about authorising your account. Click the proxy's menu bar icon, select your account name in the `Authorise account` submenu, and then log in via the popup browser window that appears. The window will close itself once the process is complete. -See the various [optional arguments](#optional-arguments-and-configuration) below for completing authentication if running without a GUI. +See the various [optional arguments](https://github.com/simonrob/email-oauth2-proxy/#optional-arguments-and-configuration) below for completing authentication if running without a GUI. After successful authentication and authorisation you should have IMAP/POP/SMTP access to your account as normal. -Make sure you keep the proxy running at all times to allow it to authorise your email client's background activity – enable `Start at login` from the proxy's menu, or see the [instructions below](#starting-the-proxy-automatically) about how to configure this in various different setups. +Make sure you keep the proxy running at all times to allow it to authorise your email client's background activity – enable `Start at login` from the proxy's menu, or see the [instructions below](https://github.com/simonrob/email-oauth2-proxy/#starting-the-proxy-automatically) about how to configure this in various different setups. After your accounts are fully set-up and authorised, no further proxy interaction should be required unless your account needs authorising again. It will notify you if this is the case. @@ -58,7 +68,7 @@ If you do not have access to credentials for an existing client you will need to The process to do this is different for each provider, but the registration guides for several common ones are linked below. In all cases, when registering, make sure your client is set up to use an OAuth scope that will give it permission to access IMAP/POP/SMTP as desired. It is also highly recommended to use a scope that will grant "offline" access (i.e., a way to [refresh the OAuth 2.0 authentication token](https://oauth.net/2/refresh-tokens/) without user intervention). -The [sample configuration file](emailproxy.config) provides example scope values for several common providers. +The [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) provides example scope values for several common providers. - Office 365: register a new [Microsoft identity application](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) - Gmail / Google Workspace: register a [Google API desktop app client](https://developers.google.com/identity/protocols/oauth2/native-app) @@ -67,7 +77,7 @@ The [sample configuration file](emailproxy.config) provides example scope values The proxy also supports the [client credentials grant](https://learn.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow) and [resource owner password credentials grant](https://learn.microsoft.com/en-us/azure/active-directory/develop/v2-oauth-ropc) OAuth 2.0 flows if needed. Please note that currently only Office 365 is known to support these methods. In addition, when using the client credentials grant flow, Office 365 only supports IMAP/POP, [_not_ SMTP](https://learn.microsoft.com/en-us/exchange/client-developer/legacy-protocols/how-to-authenticate-an-imap-pop-smtp-application-by-using-oauth#use-client-credentials-grant-flow-to-authenticate-imap-and-pop-connections) (use [smtp2graph](https://github.com/EvanTrow/smtp2graph) instead here). -See the [sample configuration file](emailproxy.config) for further details. +See the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for further details. ## Optional arguments and configuration @@ -89,38 +99,38 @@ This argument is identical to enabling external authorisation mode from the `Aut The `--external-auth` option is ignored in this mode. To authorise your account, visit the link that is provided, authenticate, and proceed until you are presented with a success webpage from the proxy. Please note that while authentication links can actually be visited from anywhere to log in and authorise access, by default the final redirection target (i.e., a link starting with your account's `redirect_uri` value) must be accessed from the machine hosting the proxy itself so that the local server can receive the authorisation result. -See the [sample configuration file](emailproxy.config) for advanced options to configure this (via `redirect_listen_address`). +See the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for advanced options to configure this (via `redirect_listen_address`). -- `--config-file` allows you to specify the location of a [configuration file](emailproxy.config) that the proxy should load. +- `--config-file` allows you to specify the location of a [configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) that the proxy should load. By default, the proxy also saves its cached OAuth 2.0 tokens back to this file, so it must be writable. See the `--cache-store` option, if you would rather store configuration and cached values separately. If this argument is not provided, the proxy will look for `emailproxy.config` in the same directory as the script itself. - `--cache-store` is used to specify a separate location in which to cache authorised OAuth 2.0 tokens and associated metadata. -The value of this argument can either be the full path to a local file (which must be writable), or an identifier for an external store such as a secrets manager (see the [documentation below](#advanced-configuration)). +The value of this argument can either be the full path to a local file (which must be writable), or an identifier for an external store such as a secrets manager (see the [documentation below](https://github.com/simonrob/email-oauth2-proxy/#advanced-configuration)). If this argument is not provided, credentials will be cached in the current configuration file. - `--log-file` allows you to specify the location of a file to send log output to (full path required). Log files are rotated at 32MB and 10 older log files are kept. -This option overrides the proxy's default behaviour, which varies by platform (see [below](#troubleshooting) for details). +This option overrides the proxy's default behaviour, which varies by platform (see [below](https://github.com/simonrob/email-oauth2-proxy/#troubleshooting) for details). -- `--debug` enables debug mode, printing more verbose output to the log as [discussed below](#troubleshooting). +- `--debug` enables debug mode, printing more verbose output to the log as [discussed below](https://github.com/simonrob/email-oauth2-proxy/#troubleshooting). This argument is identical to enabling debug mode from the proxy's menu bar icon. ### Advanced configuration -The [example configuration file](emailproxy.config) contains further documentation for various additional features of the proxy, including catch-all (wildcard) accounts, locally-encrypted connections and advanced Office 365 OAuth 2.0 flows. +The [example configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) contains further documentation for various additional features of the proxy, including catch-all (wildcard) accounts, locally-encrypted connections and advanced Office 365 OAuth 2.0 flows. The proxy caches authenticated OAuth 2.0 tokens and associated metadata back to its own configuration file by default, but can alternatively be configured to use either a separate local file or a secrets manager service for this purpose. Currently only [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) is supported for remote token storage. -To use this feature, set the [`--cache-store`](#optional-arguments-and-configuration) parameter to either a full ARN or a secret name, prefixing the value with `aws:` to identify its type to the proxy. +To use this feature, set the [`--cache-store`](https://github.com/simonrob/email-oauth2-proxy/#optional-arguments-and-configuration) parameter to either a full ARN or a secret name, prefixing the value with `aws:` to identify its type to the proxy. You must also install the AWS SDK for Python: `python -m pip install boto3` and [set up authentication credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html#configuration) (including a region). The minimum required permissions for the associated AWS IAM user are `secretsmanager:GetSecretValue` and `secretsmanager:PutSecretValue`. If the named AWS Secret does not yet exist, the proxy will attempt to create it; here, the `secretsmanager:CreateSecret` permission is also required. If you are using the proxy in a non-GUI environment it is possible to skip installation of dependencies that apply only to the interactive version. -To do this, install the script's requirements via `python -m pip install -r requirements-no-gui.txt`, and pass the [`--no-gui`](#optional-arguments-and-configuration) argument when starting the proxy. +To do this, install the script via `python -m pip install emailproxy` (i.e., without the `[gui]` variant option), and pass the [`--no-gui`](https://github.com/simonrob/email-oauth2-proxy/#optional-arguments-and-configuration) argument when starting the proxy. Please note that the proxy was designed as a GUI-based tool from the outset due to the inherently interactive nature of OAuth 2.0 authorisation, and there are limits to its ability to support fully no-GUI operation. -See the [optional arguments and configuration](#optional-arguments-and-configuration) section of this file for further details. +See the [optional arguments and configuration](https://github.com/simonrob/email-oauth2-proxy/#optional-arguments-and-configuration) section of this file for further details. If your network requires connections to use an existing proxy, you can instruct the script to use this by setting the [proxy handler](https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler) environment variable `https_proxy` (and/or `http_proxy`) – for example, `https_proxy=localhost python emailproxy.py`. @@ -142,7 +152,7 @@ On macOS, the file `~/Library/LaunchAgents/ac.robinson.email-oauth2-proxy.plist` If you stop the proxy's service (i.e., `Quit Email OAuth 2.0 Proxy` from the menu bar), you can restart it using `launchctl start ac.robinson.email-oauth2-proxy` from a terminal. You can stop, disable or remove the service from your startup items either via the menu bar icon option, or using `launchctl unload [plist path]`. If you edit the plist file manually, make sure you `unload` and then `load` it to update the system with your changes. -If the `Start at login` option appears not to be working for you on macOS, see the [known issues](#known-issues) section below for potential solutions. +If the `Start at login` option appears not to be working for you on macOS, see the [known issues](https://github.com/simonrob/email-oauth2-proxy/#known-issues) section below for potential solutions. On Windows the auto-start functionality is achieved via a shortcut in your user account's startup folder. Pressing `⊞ Win` + `r` and entering `shell:startup` (and then clicking OK) will open this folder – from here you can either double-click the `ac.robinson.email-oauth2-proxy.cmd` file to relaunch the proxy, edit it to configure, or delete this file (either manually or by deselecting the option in the proxy's menu) to remove the script from your startup items. @@ -168,16 +178,17 @@ Because of this, if you are concerned about debug mode and security you can use It is often helpful to be able to view the raw connection details when debugging (i.e., without using your email client). This can be achieved using `telnet`, [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/) or similar. -For example, to test the Office 365 IMAP server from the [example configuration](emailproxy.config), first open a connection using `telnet localhost 1993`, and then send a login command: `a1 login e@mail.com password`, replacing `e@mail.com` with your email address, and `password` with any value you like during testing (see above for why the password is irrelevant). +For example, to test the Office 365 IMAP server from the [example configuration](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config), first open a connection using `telnet localhost 1993`, and then send a login command: `a1 login e@mail.com password`, replacing `e@mail.com` with your email address, and `password` with any value you like during testing (see above for why the password is irrelevant). If you have already authorised your account with the proxy you should see a response starting with `a1 OK`; if not, this command should trigger a notification from the proxy about authorising your account. -If you are using a [secure local connection](emailproxy.config) the interaction with the remote email server is the same as above, but you will need to use a local debugging tool that supports encryption. +If you are using a [secure local connection](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) the interaction with the remote email server is the same as above, but you will need to use a local debugging tool that supports encryption. The easiest approach here is to use [OpenSSL](https://www.openssl.org/): `openssl s_client -crlf -connect localhost:1993`. -If you are having trouble actually connecting to the proxy, it is always worth double-checking the `local_address` that you are using. -The proxy defaults to `::` for this parameter, which in most cases resolves to `localhost` for both IPv4 and IPv6 configurations, but it is possible that this differs depending on your environment. -If you are unable to connect to the proxy from your client, it is worth setting this value explicitly – see the [sample configuration file](emailproxy.config) for further details about how to do this. -Please try to connect to both IPv4 (i.e., `127.0.0.1`) and IPv6 (i.e., `::1`) loopback addresses before reporting any connection issues with the proxy. +If you are having trouble actually connecting to the proxy, it is always worth double-checking the `local_address` values that you are using. +The [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) sets this parameter to `127.0.0.1` for all servers. +If you remove this value and do not provide your own, the proxy defaults to `::` – in most cases this resolves to `localhost` for both IPv4 and IPv6 configurations, but it is possible that this differs depending on your environment. +If you are unable to connect to the proxy from your client, it is always worth first specifying this value explicitly – see the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for further details about how to do this. +Please try setting and connecting to both IPv4 (i.e., `127.0.0.1`) and IPv6 (i.e., `::1`) loopback addresses before reporting any connection issues with the proxy. ### Dependencies and setup On macOS the setup and installation instructions above should automatically install all required dependencies. @@ -214,12 +225,12 @@ Please feel free to [open an issue](https://github.com/simonrob/email-oauth2-pro ## Advanced / experimental features -The [plugins branch](https://github.com/simonrob/email-oauth2-proxy/tree/plugins) has a semi-experimental new feature that enables the use of separate scripts to modify IMAP/POP/SMTP commands when they are received from the client or server before passing through to the other side of the connection. +The [plugins variant](https://github.com/simonrob/email-oauth2-proxy/tree/plugins) has a semi-experimental new feature that enables the use of separate scripts to modify IMAP/POP/SMTP commands when they are received from the client or server before passing through to the other side of the connection. This allows a wide range of additional capabilities or triggers to be added the proxy. For example, the [IMAPIgnoreSentMessageUpload plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/IMAPIgnoreSentMessageUpload.py) intercepts any client commands to add emails to the IMAP sent messages mailbox, which resolves message duplication issues for servers that automatically do this when emails are received via SMTP (e.g., Office 365, Gmail, etc). The [IMAPCleanO365ATPLinks plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/IMAPCleanO365ATPLinks.py) restores links modified by Office 365 Advanced Threat Protection to their original URLs. The [SMTPBlackHole plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/SMTPBlackHole.py) gives the impression emails are being sent but actually silently discards them, which is useful for testing email sending tools. -See the documentation and examples in this branch for further details, additional sample plugins and setup instructions. +See the [documentation and examples](https://github.com/simonrob/email-oauth2-proxy/tree/plugins/plugins) for further details, additional sample plugins and setup instructions. ## Potential improvements (pull requests welcome) @@ -227,7 +238,6 @@ See the documentation and examples in this branch for further details, additiona - Full feature parity on different platforms (e.g., live menu updating; monitoring network status; clickable notifications) - STARTTLS for IMAP/POP? - Python 2 support? (see [discussion](https://github.com/simonrob/email-oauth2-proxy/issues/38)) -- Releases packaged as .app/.exe etc? ## Related projects and alternatives @@ -245,4 +255,4 @@ This proxy was developed to work around these limitations for providers that do ## License -[Apache 2.0](LICENSE) +[Apache 2.0](https://github.com/simonrob/email-oauth2-proxy/blob/main/LICENSE) diff --git a/emailproxy.config b/emailproxy.config index deb8b87..9909f1a 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -26,7 +26,7 @@ documentation = Local servers are specified as demonstrated below where, for exa behalf (i.e., do not enable STARTTLS in your client). IMAP STARTTLS and POP STARTTLS are not currently supported. - The `local_address` property can be used to set an IP address or hostname for the proxy to listen on. Both IPv4 - and IPv6 are supported. If not specified, this value is set to `::` (i.e., dual-stack IPv4 and IPv6 `localhost`). + and IPv6 are supported. If not specified, this value is set to `::` (i.e., dual-stack IPv4/IPv6 on all interfaces). When a hostname is set the proxy will first resolve this to an IP address, preferring IPv6 over IPv4 if both are available. When running in an IPv6 environment with dual-stack support, the proxy will attempt to listen on both IPv4 and IPv6 hosts simultaneously. Note that tools such as `netstat` do not always accurately show dual-stack mode; @@ -42,27 +42,33 @@ documentation = Local servers are specified as demonstrated below where, for exa [IMAP-1993] server_address = outlook.office365.com server_port = 993 +local_address = 127.0.0.1 [POP-1995] server_address = outlook.office365.com server_port = 995 +local_address = 127.0.0.1 [SMTP-1587] server_address = smtp.office365.com server_port = 587 starttls = True +local_address = 127.0.0.1 [IMAP-2993] server_address = imap.gmail.com server_port = 993 +local_address = 127.0.0.1 [POP-2995] server_address = pop.gmail.com server_port = 995 +local_address = 127.0.0.1 [SMTP-2465] server_address = smtp.gmail.com server_port = 465 +local_address = 127.0.0.1 [Account setup] @@ -196,7 +202,9 @@ documentation = The parameters below control advanced options for the proxy. In password typo does not give the false impression that the proxy has somehow made the account inaccessible. However, if the proxy is used in a headless (often also public-facing) context, where authentication flows are more likely to be laborious or need administrator intervention, this can potentially result in a denial-of-service issue, whether - malicious or not. Set to False and the proxy will instead return an error when an incorrect password is provided. + malicious or not. It can also be the source of confusion if using a client (such as Firefox) that stores a separate + password per protocol for each account, but does not make this clear when changing account passwords. Set this + option to False and the proxy will instead return an error when an incorrect password is provided. - encrypt_client_secret_on_first_use (default = False): The proxy encrypts sensitive configuration values (e.g., cached access tokens) using the password that is given when accessing an account via IMAP/POP/SMTP. It does not do diff --git a/emailproxy.py b/emailproxy.py index 10391af..831d7d9 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,8 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2023 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2023-09-06' # ISO 8601 (YYYY-MM-DD) +__version__ = '2023-11-01' # ISO 8601 (YYYY-MM-DD) +__package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc import argparse @@ -50,65 +51,79 @@ import asyncore # for encrypting/decrypting the locally-stored credentials -from cryptography.fernet import Fernet, InvalidToken +from cryptography.fernet import Fernet, MultiFernet, InvalidToken from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC -# for macOS-specific unified logging -if sys.platform == 'darwin': - # pyoslog *is* present; see youtrack.jetbrains.com/issue/PY-11963 (same for others with this suppressed inspection) - # noinspection PyPackageRequirements - import pyoslog - # by default the proxy is a GUI application with a menu bar/taskbar icon, but it is also useful in 'headless' contexts -# where not having to install GUI-only requirements can be helpful - see the proxy's readme and requirements-no-gui.txt -no_gui_parser = argparse.ArgumentParser(add_help=False) -no_gui_parser.add_argument('--no-gui', action='store_true') -no_gui_parser.add_argument('--external-auth', action='store_true') -no_gui_args = no_gui_parser.parse_known_args()[0] -if not no_gui_args.no_gui: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - # noinspection PyDeprecation - import pkg_resources # from setuptools - to change to importlib.metadata and packaging.version once min. is 3.8 +# where not having to install GUI-only requirements can be helpful - see the proxy's readme (the `--no-gui` option) +MISSING_GUI_REQUIREMENTS = [] + +try: import pystray # the menu bar/taskbar GUI - import timeago # the last authenticated activity hint +except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) + + + class DummyPystray: # dummy implementation allows initialisation to complete + class Icon: + pass + + + pystray = DummyPystray # this is just to avoid unignorable IntelliJ warnings about naming and spacing + +try: + # noinspection PyUnresolvedReferences from PIL import Image, ImageDraw, ImageFont # draw the menu bar icon from the TTF font stored in APP_ICON +except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) + +try: + # noinspection PyUnresolvedReferences + import timeago # the last authenticated activity hint +except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) - # noinspection PyPackageRequirements +try: + # noinspection PyUnresolvedReferences import webview # the popup authentication window (in default and GUI `--external-auth` modes only) +except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) - # for macOS-specific functionality - if sys.platform == 'darwin': - # noinspection PyPackageRequirements - import AppKit # retina icon, menu update on click, native notifications and receiving system events - import PyObjCTools # SIGTERM handling (only needed when in GUI mode; `signal` is sufficient otherwise) - import SystemConfiguration # network availability monitoring +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + try: + # noinspection PyDeprecation,PyUnresolvedReferences + import pkg_resources # from setuptools - to change to importlib.metadata and packaging.version once min. is 3.8 + except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) -else: - # dummy implementations to allow use regardless of whether pystray or AppKit are available - # noinspection PyPep8Naming - class pystray: - class Icon: - pass +# for macOS-specific functionality +if sys.platform == 'darwin': + try: + # PyUnresolvedReferences; see: youtrack.jetbrains.com/issue/PY-11963 (same for others with this suppression) + # noinspection PyPackageRequirements,PyUnresolvedReferences + import PyObjCTools # SIGTERM handling (only needed when in GUI mode; `signal` is sufficient otherwise) + except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) + try: + # noinspection PyPackageRequirements,PyUnresolvedReferences + import SystemConfiguration # network availability monitoring + except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) - class AppKit: - class NSObject: - pass + try: + # noinspection PyPackageRequirements + import AppKit # retina icon, menu update on click, native notifications and receiving system events + except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) - if no_gui_args.external_auth: - try: - # prompt_toolkit is a recent dependency addition that is only required in no-GUI external authorisation - # mode, but may not be present if only the proxy script itself has been updated - import prompt_toolkit - except ModuleNotFoundError: - sys.exit('Unable to load prompt_toolkit, which is a requirement when using `--external-auth` in `--no-gui` ' - 'mode. Please run `python -m pip install -r requirements-no-gui.txt`') -del no_gui_parser -del no_gui_args + class AppKit: # dummy implementation allows initialisation to complete + class NSObject: + pass APP_NAME = 'Email OAuth 2.0 Proxy' APP_SHORT_NAME = 'emailproxy' @@ -123,13 +138,16 @@ class NSObject: J6Sp0urC5fCken5STr0KDoUlyhjVd4nxSUvq3tCftEn8r2ro+mxUDIaCMQmQrGZGHmi53tAT3rPGH1e3qF0p9w7LtcohwuyvnRxWZ8sZUej6WvlhXSk1 7k+POJ1iR73N/+w2xN0f4+GJcHtfqoWzgfi6cuZscC54lSq3SbN1tmzC4MXtcwN/zOC78r9BIfNc3M=''' # TTF ('e') -> zlib -> base64 +CENSOR_CREDENTIALS = True CENSOR_MESSAGE = b'[[ Credentials removed from proxy log ]]' # replaces actual credentials; must be a byte-type string script_path = sys.executable if getattr(sys, 'frozen', False) else os.path.realpath(__file__) # for pyinstaller etc if sys.platform == 'darwin' and '.app/Contents/MacOS/' in script_path: # pyinstaller .app binary is within the bundle script_path = '/'.join(script_path.split('Contents/MacOS/')[0].split('/')[:-1]) -CONFIG_FILE_PATH = CACHE_STORE = os.path.join(os.path.dirname(script_path), '%s.config' % APP_SHORT_NAME) +script_path = os.getcwd() if __package__ is not None else os.path.dirname(script_path) # for packaged version (PyPI) +CONFIG_FILE_PATH = CACHE_STORE = os.path.join(script_path, '%s.config' % APP_SHORT_NAME) CONFIG_SERVER_MATCHER = re.compile(r'^(?P(IMAP|POP|SMTP))-(?P\d+)$') +del script_path MAX_CONNECTIONS = 0 # maximum concurrent IMAP/POP/SMTP connections; 0 = no limit; limit is per server @@ -159,7 +177,6 @@ class NSObject: REQUEST_QUEUE = queue.Queue() # requests for authentication RESPONSE_QUEUE = queue.Queue() # responses from user -WEBVIEW_QUEUE = queue.Queue() # authentication window events (macOS only) QUEUE_SENTINEL = object() # object to send to signify queues should exit loops MENU_UPDATE = object() # object to send to trigger a force-refresh of the GUI menu (new catch-all account added) @@ -209,7 +226,7 @@ class Log: _HANDLER = None _DATE_FORMAT = '%Y-%m-%d %H:%M:%S:' _SYSLOG_MESSAGE_FORMAT = '%s: %%(message)s' % APP_NAME - _MACOS_USE_SYSLOG = not pyoslog.is_supported() if sys.platform == 'darwin' else False + _MACOS_USE_SYSLOG = False @staticmethod def initialise(log_file=None): @@ -220,19 +237,25 @@ def initialise(log_file=None): os.path.realpath(__file__)), APP_SHORT_NAME), maxBytes=LOG_FILE_MAX_SIZE, backupCount=LOG_FILE_MAX_BACKUPS) handler.setFormatter(logging.Formatter('%(asctime)s: %(message)s')) + elif sys.platform == 'darwin': + # noinspection PyPackageRequirements + import pyoslog # for macOS-specific unified logging + Log._MACOS_USE_SYSLOG = not pyoslog.is_supported() if Log._MACOS_USE_SYSLOG: # syslog prior to 10.12 handler = logging.handlers.SysLogHandler(address='/var/run/syslog') handler.setFormatter(logging.Formatter(Log._SYSLOG_MESSAGE_FORMAT)) else: # unified logging in 10.12+ handler = pyoslog.Handler() handler.setSubsystem(APP_PACKAGE) + else: if os.path.exists('/dev/log'): handler = logging.handlers.SysLogHandler(address='/dev/log') handler.setFormatter(logging.Formatter(Log._SYSLOG_MESSAGE_FORMAT)) else: handler = logging.StreamHandler() + Log._HANDLER = handler Log._LOGGER.addHandler(Log._HANDLER) Log.set_level(logging.INFO) @@ -460,8 +483,8 @@ class AppConfig: _PARSER_LOCK = threading.Lock() # note: removing the unencrypted version of `client_secret_encrypted` is not automatic with --cache-store (see docs) - _CACHED_OPTION_KEYS = ['token_salt', 'access_token', 'access_token_expiry', 'refresh_token', 'last_activity', - 'client_secret_encrypted'] + _CACHED_OPTION_KEYS = ['access_token', 'access_token_expiry', 'refresh_token', 'token_salt', 'token_iterations', + 'client_secret_encrypted', 'last_activity'] # additional cache stores may be implemented by extending CacheStore and adding a prefix entry in this dict _EXTERNAL_CACHE_STORES = {'aws:': AWSSecretsManagerCacheStore} @@ -569,6 +592,69 @@ def _save_cache(cache_store_identifier, output_config_parser): Log.error('Error saving state to cache store file at', cache_store_identifier, '- is the file writable?') +class Cryptographer: + ITERATIONS = 870_000 # taken from cryptography's suggestion of using Django's defaults + LEGACY_ITERATIONS = 100_000 # fallback when the iteration count is not in the config file (versions < 2023-10-17) + + def __init__(self, config, username, password): + """Creates a cryptographer which allows encrypting and decrypting sensitive information for this account, + (such as stored tokens), and also supports increasing the encryption/decryption iterations (i.e., strength)""" + self._salt = None + + token_salt = config.get(username, 'token_salt', fallback=None) + if token_salt: + try: + self._salt = base64.b64decode(token_salt.encode('utf-8')) # catch incorrect third-party proxy guide + except (binascii.Error, UnicodeError): + Log.info('%s: Invalid `token_salt` value found in config file entry for account %s - this value is not ' + 'intended to be manually created; generating new `token_salt`' % (APP_NAME, username)) + + if not self._salt: + self._salt = os.urandom(16) # either a failed decode or the initial run when no salt exists + + # the iteration count is stored with the credentials, so could if required be user-edited (see PR #198 comments) + iterations = config.getint(username, 'token_iterations', fallback=self.LEGACY_ITERATIONS) + + # with MultiFernet each fernet is tried in order to decrypt a value, but encryption always uses the first + # fernet, so sort unique iteration counts in descending order (i.e., use the best available encryption) + self._iterations_options = sorted({self.ITERATIONS, iterations, self.LEGACY_ITERATIONS}, reverse=True) + + # generate encrypter/decrypter based on the password and salt + self._fernets = [Fernet(base64.urlsafe_b64encode( + PBKDF2HMAC(algorithm=hashes.SHA256(), length=32, salt=self._salt, iterations=iterations, + backend=default_backend()).derive(password.encode('utf-8')))) for iterations in + self._iterations_options] + self.fernet = MultiFernet(self._fernets) + + @property + def salt(self): + return base64.b64encode(self._salt).decode('utf-8') + + @property + def iterations(self): + return self._iterations_options[0] + + def encrypt(self, value): + return self.fernet.encrypt(value.encode('utf-8')).decode('utf-8') + + def decrypt(self, value): + return self.fernet.decrypt(value.encode('utf-8')).decode('utf-8') + + def requires_rotation(self, value): + try: + self._fernets[0].decrypt(value.encode('utf-8')) # if the first fernet works, everything is up-to-date + return False + except InvalidToken: + try: # check to see if any fernet can decrypt the value - if so we can upgrade the encryption strength + self.decrypt(value) + return True + except InvalidToken: + return False + + def rotate(self, value): + return self.fernet.rotate(value.encode('utf-8')).decode('utf-8') + + class OAuth2Helper: class TokenRefreshError(Exception): pass @@ -631,7 +717,6 @@ def get_account_with_catch_all_fallback(option): 'otherwise, if authentication fails, please double-check this value is correct') current_time = int(time.time()) - token_salt = config.get(username, 'token_salt', fallback=None) access_token = config.get(username, 'access_token', fallback=None) access_token_expiry = config.getint(username, 'access_token_expiry', fallback=current_time) refresh_token = config.get(username, 'refresh_token', fallback=None) @@ -641,37 +726,37 @@ def get_account_with_catch_all_fallback(option): AppConfig.unload() return OAuth2Helper.get_oauth2_credentials(username, password, reload_remote_accounts=False) - # we hash locally-stored tokens with the given password - if not token_salt: - token_salt = base64.b64encode(os.urandom(16)).decode('utf-8') - - # generate encrypter/decrypter based on password and random salt - try: - decoded_salt = base64.b64decode(token_salt.encode('utf-8')) # catch incorrect third-party proxy guide - except binascii.Error: - return (False, '%s: Invalid `token_salt` value found in config file entry for account %s - this value is ' - 'not intended to be manually created; please remove and retry' % (APP_NAME, username)) - key_derivation_function = PBKDF2HMAC(algorithm=hashes.SHA256(), length=32, salt=decoded_salt, iterations=100000, - backend=default_backend()) - fernet = Fernet(base64.urlsafe_b64encode(key_derivation_function.derive(password.encode('utf-8')))) + cryptographer = Cryptographer(config, username, password) + rotatable_values = { + 'access_token': access_token, + 'refresh_token': refresh_token, + 'client_secret_encrypted': client_secret_encrypted + } + if any(value and cryptographer.requires_rotation(value) for value in rotatable_values.values()): + Log.info('Rotating stored secrets for account', username, 'to use new cryptographic parameters') + for key, value in rotatable_values.items(): + if value: + config.set(username, key, cryptographer.rotate(value)) + + config.set(username, 'token_iterations', str(cryptographer.iterations)) + AppConfig.save() try: # if both secret values are present we use the unencrypted version (as it may have been user-edited) if client_secret_encrypted and not client_secret: - client_secret = OAuth2Helper.decrypt(fernet, client_secret_encrypted) + client_secret = cryptographer.decrypt(client_secret_encrypted) if access_token or refresh_token: # if possible, refresh the existing token(s) if not access_token or access_token_expiry - current_time < TOKEN_EXPIRY_MARGIN: if refresh_token: response = OAuth2Helper.refresh_oauth2_access_token(token_url, client_id, client_secret, - OAuth2Helper.decrypt(fernet, refresh_token)) + cryptographer.decrypt(refresh_token)) access_token = response['access_token'] - config.set(username, 'access_token', OAuth2Helper.encrypt(fernet, access_token)) + config.set(username, 'access_token', cryptographer.encrypt(access_token)) config.set(username, 'access_token_expiry', str(current_time + response['expires_in'])) if 'refresh_token' in response: - config.set(username, 'refresh_token', - OAuth2Helper.encrypt(fernet, response['refresh_token'])) + config.set(username, 'refresh_token', cryptographer.encrypt(response['refresh_token'])) AppConfig.save() else: @@ -682,7 +767,7 @@ def get_account_with_catch_all_fallback(option): # very infrequently, we don't add the extra complexity for just 10 extra minutes of token life) access_token = None # avoid trying invalid (or soon to be) tokens else: - access_token = OAuth2Helper.decrypt(fernet, access_token) + access_token = cryptographer.decrypt(access_token) if not access_token: auth_result = None @@ -709,12 +794,13 @@ def get_account_with_catch_all_fallback(option): if username not in config.sections(): config.add_section(username) # in catch-all mode the section may not yet exist REQUEST_QUEUE.put(MENU_UPDATE) # make sure the menu shows the newly-added account - config.set(username, 'token_salt', token_salt) - config.set(username, 'access_token', OAuth2Helper.encrypt(fernet, access_token)) + config.set(username, 'token_salt', cryptographer.salt) + config.set(username, 'token_iterations', str(cryptographer.iterations)) + config.set(username, 'access_token', cryptographer.encrypt(access_token)) config.set(username, 'access_token_expiry', str(current_time + response['expires_in'])) if 'refresh_token' in response: - config.set(username, 'refresh_token', OAuth2Helper.encrypt(fernet, response['refresh_token'])) + config.set(username, 'refresh_token', cryptographer.encrypt(response['refresh_token'])) elif permission_url: # ignore this situation with client credentials flow - it is expected Log.info('Warning: no refresh token returned for', username, '- you will need to re-authenticate', 'each time the access token expires (does your `oauth2_scope` value allow `offline` use?)') @@ -723,7 +809,7 @@ def get_account_with_catch_all_fallback(option): if client_secret: # note: save to the `username` entry even if `user_domain` exists, avoiding conflicts when using # incompatible `encrypt_client_secret_on_first_use` and `allow_catch_all_accounts` options - config.set(username, 'client_secret_encrypted', OAuth2Helper.encrypt(fernet, client_secret)) + config.set(username, 'client_secret_encrypted', cryptographer.encrypt(client_secret)) config.remove_option(username, 'client_secret') AppConfig.save() @@ -742,6 +828,7 @@ def get_account_with_catch_all_fallback(option): if not has_access_token: # if this is already a second failure, remove the refresh token as well, and force re-authentication config.remove_option(username, 'token_salt') + config.remove_option(username, 'token_iterations') config.remove_option(username, 'refresh_token') AppConfig.save() @@ -755,6 +842,7 @@ def get_account_with_catch_all_fallback(option): config.remove_option(username, 'access_token') config.remove_option(username, 'access_token_expiry') config.remove_option(username, 'token_salt') + config.remove_option(username, 'token_iterations') config.remove_option(username, 'refresh_token') AppConfig.save() @@ -775,14 +863,6 @@ def get_account_with_catch_all_fallback(option): return False, '%s: Login failed for account %s - please check your internet connection and retry' % ( APP_NAME, username) - @staticmethod - def encrypt(cryptographer, byte_input): - return cryptographer.encrypt(byte_input.encode('utf-8')).decode('utf-8') - - @staticmethod - def decrypt(cryptographer, byte_input): - return cryptographer.decrypt(byte_input.encode('utf-8')).decode('utf-8') - @staticmethod def oauth2_url_escape(text): return urllib.parse.quote(text, safe='~-._') # see https://tools.ietf.org/html/rfc3986#section-2.3 @@ -1018,8 +1098,8 @@ def decode_credentials(str_data): class SSLAsyncoreDispatcher(asyncore.dispatcher_with_send): - def __init__(self, connection=None, socket_map=None): - asyncore.dispatcher_with_send.__init__(self, sock=connection, map=socket_map) + def __init__(self, connection_socket=None, socket_map=None): + asyncore.dispatcher_with_send.__init__(self, sock=connection_socket, map=socket_map) self.ssl_handshake_errors = (ssl.SSLWantReadError, ssl.SSLWantWriteError, ssl.SSLEOFError, ssl.SSLZeroReturnError) self.ssl_connection, self.ssl_handshake_attempts, self.ssl_handshake_completed = self._reset() @@ -1144,17 +1224,17 @@ class OAuth2ClientConnection(SSLAsyncoreDispatcher): """The base client-side connection that is subclassed to handle IMAP/POP/SMTP client interaction (note that there is some protocol-specific code in here, but it is not essential, and only used to avoid logging credentials)""" - def __init__(self, proxy_type, connection, socket_map, connection_info, server_connection, proxy_parent, - custom_configuration): - SSLAsyncoreDispatcher.__init__(self, connection, socket_map) + def __init__(self, proxy_type, connection_socket, socket_map, proxy_parent, custom_configuration): + SSLAsyncoreDispatcher.__init__(self, connection_socket=connection_socket, socket_map=socket_map) self.receive_buffer = b'' self.proxy_type = proxy_type - self.connection_info = connection_info - self.server_connection = server_connection - self.local_address = proxy_parent.local_address - self.server_address = server_connection.server_address + self.server_connection = None self.proxy_parent = proxy_parent + self.local_address = proxy_parent.local_address + self.server_address = proxy_parent.server_address self.custom_configuration = custom_configuration + self.debug_address_string = '%s-{%s}-%s' % tuple(map(Log.format_host_port, ( + connection_socket.getpeername(), connection_socket.getsockname(), self.server_address))) self.censor_next_log = False # try to avoid logging credentials self.authenticated = False @@ -1163,11 +1243,11 @@ def __init__(self, proxy_type, connection, socket_map, connection_info, server_c bool(custom_configuration['local_certificate_path'] and custom_configuration['local_key_path'])) def info_string(self): - debug_string = '; %s->%s' % (Log.format_host_port(self.connection_info), Log.format_host_port( - self.server_address)) if Log.get_level() == logging.DEBUG else '' + debug_string = self.debug_address_string if Log.get_level() == logging.DEBUG else \ + Log.format_host_port(self.local_address) account = '; %s' % self.server_connection.authenticated_username if \ self.server_connection and self.server_connection.authenticated_username else '' - return '%s (%s%s%s)' % (self.proxy_type, Log.format_host_port(self.local_address), debug_string, account) + return '%s (%s%s)' % (self.proxy_type, debug_string, account) def handle_read(self): byte_data = self.recv(RECEIVE_BUFFER_SIZE) @@ -1214,7 +1294,7 @@ def handle_read(self): log_data = re.sub(b'(%s)?( )?(AUTH)(ENTICATE)? (PLAIN|LOGIN) (.*)\r\n' % tag_pattern, br'\1\2\3\4 \5 ' + CENSOR_MESSAGE + b'\r\n', log_data, flags=re.IGNORECASE) - Log.debug(self.info_string(), '-->', log_data) + Log.debug(self.info_string(), '-->', log_data if CENSOR_CREDENTIALS else line) try: self.process_data(line) except AttributeError: # AttributeError("'NoneType' object has no attribute 'username'"), etc @@ -1263,9 +1343,8 @@ def close(self): class IMAPOAuth2ClientConnection(OAuth2ClientConnection): """The client side of the connection - intercept LOGIN/AUTHENTICATE commands and replace with OAuth 2.0 SASL""" - def __init__(self, connection, socket_map, connection_info, server_connection, proxy_parent, custom_configuration): - super().__init__('IMAP', connection, socket_map, connection_info, server_connection, proxy_parent, - custom_configuration) + def __init__(self, connection_socket, socket_map, proxy_parent, custom_configuration): + super().__init__('IMAP', connection_socket, socket_map, proxy_parent, custom_configuration) self.authentication_tag = None self.authentication_command = None self.awaiting_credentials = False @@ -1393,9 +1472,8 @@ class STATE(enum.Enum): XOAUTH2_AWAITING_CONFIRMATION = 5 XOAUTH2_CREDENTIALS_SENT = 6 - def __init__(self, connection, socket_map, connection_info, server_connection, proxy_parent, custom_configuration): - super().__init__('POP', connection, socket_map, connection_info, server_connection, proxy_parent, - custom_configuration) + def __init__(self, connection_socket, socket_map, proxy_parent, custom_configuration): + super().__init__('POP', connection_socket, socket_map, proxy_parent, custom_configuration) self.connection_state = self.STATE.PENDING def process_data(self, byte_data, censor_server_log=False): @@ -1472,9 +1550,8 @@ class STATE(enum.Enum): XOAUTH2_AWAITING_CONFIRMATION = 6 XOAUTH2_CREDENTIALS_SENT = 7 - def __init__(self, connection, socket_map, connection_info, server_connection, proxy_parent, custom_configuration): - super().__init__('SMTP', connection, socket_map, connection_info, server_connection, proxy_parent, - custom_configuration) + def __init__(self, connection_socket, socket_map, proxy_parent, custom_configuration): + super().__init__('SMTP', connection_socket, socket_map, proxy_parent, custom_configuration) self.connection_state = self.STATE.PENDING def process_data(self, byte_data, censor_server_log=False): @@ -1549,16 +1626,17 @@ def send_authentication_request(self): class OAuth2ServerConnection(SSLAsyncoreDispatcher): """The base server-side connection that is subclassed to handle IMAP/POP/SMTP server interaction""" - def __init__(self, proxy_type, socket_map, server_address, connection_info, proxy_parent, custom_configuration): + def __init__(self, proxy_type, connection_socket, socket_map, proxy_parent, custom_configuration): SSLAsyncoreDispatcher.__init__(self, socket_map=socket_map) # note: establish connection later due to STARTTLS self.receive_buffer = b'' self.proxy_type = proxy_type - self.connection_info = connection_info self.client_connection = None - self.local_address = proxy_parent.local_address - self.server_address = server_address self.proxy_parent = proxy_parent + self.local_address = proxy_parent.local_address + self.server_address = proxy_parent.server_address self.custom_configuration = custom_configuration + self.debug_address_string = '%s-{%s}-%s' % tuple(map(Log.format_host_port, ( + connection_socket.getpeername(), connection_socket.getsockname(), self.server_address))) self.authenticated_username = None # used only for showing last activity in the menu self.last_activity = 0 @@ -1573,10 +1651,10 @@ def create_socket(self, socket_family=socket.AF_UNSPEC, socket_type=socket.SOCK_ return def info_string(self): - debug_string = '; %s->%s' % (Log.format_host_port(self.connection_info), Log.format_host_port( - self.server_address)) if Log.get_level() == logging.DEBUG else '' + debug_string = self.debug_address_string if Log.get_level() == logging.DEBUG else \ + Log.format_host_port(self.local_address) account = '; %s' % self.authenticated_username if self.authenticated_username else '' - return '%s (%s%s%s)' % (self.proxy_type, Log.format_host_port(self.local_address), debug_string, account) + return '%s (%s%s)' % (self.proxy_type, debug_string, account) def handle_connect(self): Log.debug(self.info_string(), '--> [ Client connected ]') @@ -1643,7 +1721,8 @@ def process_data(self, byte_data): def send(self, byte_data, censor_log=False): if not self.client_connection.authenticated: # after authentication these are identical to server-side logs - Log.debug(self.info_string(), ' -->', b'%s\r\n' % CENSOR_MESSAGE if censor_log else byte_data) + Log.debug(self.info_string(), ' -->', + b'%s\r\n' % CENSOR_MESSAGE if CENSOR_CREDENTIALS and censor_log else byte_data) return super().send(byte_data) def handle_error(self): @@ -1691,8 +1770,8 @@ class IMAPOAuth2ServerConnection(OAuth2ServerConnection): # IMAP: https://tools.ietf.org/html/rfc3501 # IMAP SASL-IR: https://tools.ietf.org/html/rfc4959 - def __init__(self, socket_map, server_address, connection_info, proxy_parent, custom_configuration): - super().__init__('IMAP', socket_map, server_address, connection_info, proxy_parent, custom_configuration) + def __init__(self, connection_socket, socket_map, proxy_parent, custom_configuration): + super().__init__('IMAP', connection_socket, socket_map, proxy_parent, custom_configuration) def process_data(self, byte_data): # note: there is no reason why IMAP STARTTLS (https://tools.ietf.org/html/rfc2595) couldn't be supported here @@ -1733,8 +1812,8 @@ class POPOAuth2ServerConnection(OAuth2ServerConnection): # POP3 CAPA: https://tools.ietf.org/html/rfc2449 # POP3 AUTH: https://tools.ietf.org/html/rfc1734 # POP3 SASL: https://tools.ietf.org/html/rfc5034 - def __init__(self, socket_map, server_address, connection_info, proxy_parent, custom_configuration): - super().__init__('POP', socket_map, server_address, connection_info, proxy_parent, custom_configuration) + def __init__(self, connection_socket, socket_map, proxy_parent, custom_configuration): + super().__init__('POP', connection_socket, socket_map, proxy_parent, custom_configuration) self.capa = [] self.username = None self.password = None @@ -1818,8 +1897,8 @@ class STARTTLS(enum.Enum): NEGOTIATING = 2 COMPLETE = 3 - def __init__(self, socket_map, server_address, connection_info, proxy_parent, custom_configuration): - super().__init__('SMTP', socket_map, server_address, connection_info, proxy_parent, custom_configuration) + def __init__(self, connection_socket, socket_map, proxy_parent, custom_configuration): + super().__init__('SMTP', connection_socket, socket_map, proxy_parent, custom_configuration) self.ehlo = None if self.custom_configuration['starttls']: self.starttls_state = self.STARTTLS.PENDING @@ -1927,26 +2006,26 @@ def handle_accept(self): else: Log.debug('Ignoring incoming connection to', self.info_string(), '- no connection information') - def handle_accepted(self, connection, address): + def handle_accepted(self, connection_socket, address): if MAX_CONNECTIONS <= 0 or len(self.client_connections) < MAX_CONNECTIONS: new_server_connection = None try: - Log.debug('Accepting new connection to', self.info_string(), 'via', connection.getpeername()) + Log.info('Accepting new connection from', Log.format_host_port(connection_socket.getpeername()), + 'to', self.info_string()) socket_map = {} server_class = globals()['%sOAuth2ServerConnection' % self.proxy_type] - new_server_connection = server_class(socket_map, self.server_address, address, self, - self.custom_configuration) + new_server_connection = server_class(connection_socket, socket_map, self, self.custom_configuration) client_class = globals()['%sOAuth2ClientConnection' % self.proxy_type] - new_client_connection = client_class(connection, socket_map, address, new_server_connection, self, - self.custom_configuration) + new_client_connection = client_class(connection_socket, socket_map, self, self.custom_configuration) new_server_connection.client_connection = new_client_connection + new_client_connection.server_connection = new_server_connection self.client_connections.append(new_client_connection) threading.Thread(target=OAuth2Proxy.run_server, args=(new_client_connection, socket_map), name='EmailOAuth2Proxy-connection-%d' % address[1], daemon=True).start() except Exception: - connection.close() + connection_socket.close() if new_server_connection: new_server_connection.close() raise @@ -1954,8 +2033,8 @@ def handle_accepted(self, connection, address): error_text = '%s rejecting new connection above MAX_CONNECTIONS limit of %d' % ( self.info_string(), MAX_CONNECTIONS) Log.error(error_text) - connection.send(b'%s\r\n' % self.bye_message(error_text).encode('utf-8')) - connection.close() + connection_socket.send(b'%s\r\n' % self.bye_message(error_text).encode('utf-8')) + connection_socket.close() @staticmethod def run_server(client, socket_map): @@ -2182,14 +2261,14 @@ def _assert_image(self): class App: """Manage the menu bar icon, server loading, authorisation and notifications, and start the main proxy thread""" - def __init__(self): + def __init__(self, args=None): global CONFIG_FILE_PATH, CACHE_STORE parser = argparse.ArgumentParser(description='%s: transparently add OAuth 2.0 support to IMAP/POP/SMTP client ' 'applications, scripts or any other email use-cases that don\'t ' 'support this authentication method.' % APP_NAME, add_help=False, epilog='Full readme and guide: https://github.com/simonrob/email-oauth2-proxy') group_gui = parser.add_argument_group(title='appearance') - group_gui.add_argument('--no-gui', action='store_true', + group_gui.add_argument('--no-gui', action='store_false', dest='gui', help='start the proxy without a menu bar icon (note: account authorisation requests ' 'will fail unless a pre-authorised `--config-file` is used, or you use ' '`--external-auth` or `--local-server-auth` and monitor log/terminal output)') @@ -2220,11 +2299,10 @@ def __init__(self): help='show the proxy\'s version string and exit') group_debug.add_argument('-h', '--help', action='help', help='show this help message and exit') - self.args = parser.parse_args() + self.args = parser.parse_args(args) Log.initialise(self.args.log_file) - if self.args.debug: - Log.set_level(logging.DEBUG) + self.toggle_debug(self.args.debug, log_message=False) if self.args.config_file: CONFIG_FILE_PATH = CACHE_STORE = self.args.config_file @@ -2235,26 +2313,44 @@ def __init__(self): self.authorisation_requests = [] self.web_view_started = False + self.macos_web_view_queue = queue.Queue() # authentication window events (macOS only) self.init_platforms() - if self.args.no_gui: - self.icon = None - self.post_create(None) - else: + if not self.args.gui and self.args.external_auth: + try: + # prompt_toolkit is a relatively recent dependency addition that is only required in no-GUI external + # authorisation mode, but may not be present if only the proxy script itself has been updated + import prompt_toolkit + except ImportError: + Log.error('Unable to load prompt_toolkit, which is a requirement when using `--external-auth` in', + '`--no-gui` mode. Please run `python -m pip install -r requirements-core.txt`') + self.exit(None) + return + + if self.args.gui and len(MISSING_GUI_REQUIREMENTS) > 0: + Log.error('Unable to load all GUI requirements:', MISSING_GUI_REQUIREMENTS, '- did you mean to run in', + '`--no-gui` mode? If not, please run `python -m pip install -r requirements-gui.txt`') + self.exit(None) + return + + if self.args.gui: self.icon = self.create_icon() try: self.icon.run(self.post_create) except NotImplementedError: - Log.error('Unable to initialise icon - did you mean to run in --no-gui mode?') + Log.error('Unable to initialise icon - did you mean to run in `--no-gui` mode?') self.exit(None) # noinspection PyProtectedMember self.icon._Icon__queue.put(False) # pystray sets up the icon thread even in dummy mode; need to exit + else: + self.icon = None + self.post_create(None) # PyAttributeOutsideInit inspection suppressed because init_platforms() is itself called from __init__() # noinspection PyUnresolvedReferences,PyAttributeOutsideInit def init_platforms(self): - if sys.platform == 'darwin' and not self.args.no_gui: + if sys.platform == 'darwin' and self.args.gui: # hide dock icon (but not LSBackgroundOnly as we need input via webview) info = AppKit.NSBundle.mainBundle().infoDictionary() info['LSUIElement'] = '1' @@ -2298,6 +2394,7 @@ def init_platforms(self): PyObjCTools.MachSignals.signal(signal.SIGTERM, lambda _signum: self.exit(self.icon)) PyObjCTools.MachSignals.signal(signal.SIGQUIT, lambda _signum: self.exit(self.icon)) PyObjCTools.MachSignals.signal(signal.SIGHUP, lambda _signum: self.load_and_start_servers(self.icon)) + PyObjCTools.MachSignals.signal(signal.SIGUSR1, lambda _: self.toggle_debug(Log.get_level() == logging.INFO)) else: # for other platforms, or in no-GUI mode, just try to exit gracefully if SIGINT/SIGTERM/SIGQUIT is received @@ -2309,6 +2406,10 @@ def init_platforms(self): # allow config file reloading without having to stop/start - e.g.: pkill -SIGHUP -f emailproxy.py # (we don't use linux_restart() here as it exits then uses nohup to restart, which may not be desirable) signal.signal(signal.SIGHUP, lambda _signum, _frame: self.load_and_start_servers(self.icon)) + if hasattr(signal, 'SIGUSR1'): + # use SIGUSR1 as a toggle for debug mode (e.g.: pkill -USR1 -f emailproxy.py) - please note that the + # proxy's handling of this signal may change in future if other actions are seen as more suitable + signal.signal(signal.SIGUSR1, lambda _signum, _fr: self.toggle_debug(Log.get_level() == logging.INFO)) # noinspection PyUnresolvedReferences,PyAttributeOutsideInit def macos_nsworkspace_notification_listener_(self, notification): @@ -2327,7 +2428,7 @@ def macos_nsworkspace_notification_listener_(self, notification): # noinspection PyDeprecation def create_icon(self): - # temporary fix for pystray <= 0.19.4 incompatibility with PIL 10.0.0+; fixed once pystray PR #147 is released + # fix pystray <= 0.19.4 incompatibility with PIL 10.0.0+; resolved in 0.19.5 and later via pystray PR #147 with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) pystray_version = pkg_resources.get_distribution('pystray').version @@ -2341,7 +2442,8 @@ def create_icon(self): pystray.MenuItem('Authorise account', pystray.Menu(self.create_authorisation_menu)), pystray.Menu.SEPARATOR, pystray.MenuItem('Start at login', self.toggle_start_at_login, checked=self.started_at_login), - pystray.MenuItem('Debug mode', self.toggle_debug, checked=lambda _: Log.get_level() == logging.DEBUG), + pystray.MenuItem('Debug mode', lambda _, item: self.toggle_debug(not item.checked), + checked=lambda _: Log.get_level() == logging.DEBUG), pystray.Menu.SEPARATOR, pystray.MenuItem('Quit %s' % APP_NAME, self.exit))) @@ -2521,7 +2623,7 @@ def authorise_account(self, _, item): forced_gui = 'mshtml' if sys.platform == 'win32' and self.args.external_auth else None webview.start(gui=forced_gui, debug=Log.get_level() == logging.DEBUG) else: - WEBVIEW_QUEUE.put(request) # future requests need to use the same thread + self.macos_web_view_queue.put(request) # future requests need to use the same thread return self.notify(APP_NAME, 'There are no pending authorisation requests') @@ -2571,7 +2673,7 @@ def handle_authorisation_windows(self): dummy_window.hide() # hidden=True (above) doesn't seem to work in all cases while True: - data = WEBVIEW_QUEUE.get() # note: blocking call + data = self.macos_web_view_queue.get() # note: blocking call if data is QUEUE_SENTINEL: # app is closing break self.create_authorisation_window(data) @@ -2756,9 +2858,12 @@ def started_at_login(_): return False - @staticmethod - def toggle_debug(_, item): - Log.set_level(logging.INFO if item.checked else logging.DEBUG) + def toggle_debug(self, enable_debug_mode, log_message=True): + Log.set_level(logging.DEBUG if enable_debug_mode else logging.INFO) + if log_message: + Log.info('Setting debug mode:', Log.get_level() == logging.DEBUG) + if hasattr(self, 'icon') and self.icon: + self.icon.update_menu() # noinspection PyUnresolvedReferences def notify(self, title, text): @@ -2806,7 +2911,9 @@ def stop_servers(self): def load_and_start_servers(self, icon=None, reload=True): # we allow reloading, so must first stop any existing servers self.stop_servers() - Log.info('Initialising', APP_NAME, '(version %s)' % __version__, 'from config file', CONFIG_FILE_PATH) + Log.info('Initialising', APP_NAME, + '(version %s)%s' % (__version__, ' in debug mode' if Log.get_level() == logging.DEBUG else ''), + 'from config file', CONFIG_FILE_PATH) if reload: AppConfig.unload() config = AppConfig.get() @@ -2861,6 +2968,9 @@ def load_and_start_servers(self, icon=None, reload=True): else: error_text = 'Invalid' if len(AppConfig.servers()) > 0 else 'No' Log.error(error_text, 'server configuration(s) found in', CONFIG_FILE_PATH, '- exiting') + if not os.path.exists(CONFIG_FILE_PATH): + Log.error(APP_NAME, 'config file not found - see https://github.com/simonrob/email-oauth2-proxy', + 'for full documentation and example configurations to help get started') self.notify(APP_NAME, error_text + ' server configuration(s) found. ' + 'Please verify your account and server details in %s' % CONFIG_FILE_PATH) AppConfig.unload() # so we don't overwrite the invalid file with a blank configuration @@ -2877,6 +2987,7 @@ def load_and_start_servers(self, icon=None, reload=True): @staticmethod def terminal_external_auth_input(prompt_session, prompt_stop_event, data): with contextlib.suppress(Exception): # cancel any other prompts; thrown if there are none to cancel + # noinspection PyUnresolvedReferences prompt_toolkit.application.current.get_app().exit(exception=EOFError) time.sleep(1) # seems to be needed to allow prompt_toolkit to clean up between prompts @@ -2928,6 +3039,7 @@ def terminal_external_auth_timeout(prompt_session, prompt_stop_event): time.sleep(1) # seems to be needed to allow prompt_toolkit to clean up between prompts def terminal_external_auth_prompt(self, data): + # noinspection PyUnresolvedReferences prompt_session = prompt_toolkit.PromptSession() prompt_stop_event = threading.Event() threading.Thread(target=self.terminal_external_auth_input, args=(prompt_session, prompt_stop_event, data), @@ -2962,7 +3074,7 @@ def post_create(self, icon): data['username']) data['local_server_auth'] = True RESPONSE_QUEUE.put(data) # local server auth is handled by the client/server connections - elif self.args.external_auth and self.args.no_gui: + elif self.args.external_auth and not self.args.gui: if sys.stdin and sys.stdin.isatty(): self.notify(APP_NAME, 'No-GUI external auth mode: please authorise a request for account ' '%s' % data['username']) @@ -3001,7 +3113,7 @@ def exit(self, icon, restart_callback=None): AppConfig.save() - if sys.platform == 'darwin' and not self.args.no_gui: + if sys.platform == 'darwin' and self.args.gui: # noinspection PyUnresolvedReferences SystemConfiguration.SCNetworkReachabilityUnscheduleFromRunLoop(self.macos_reachability_target, SystemConfiguration.CFRunLoopGetCurrent(), @@ -3009,9 +3121,9 @@ def exit(self, icon, restart_callback=None): REQUEST_QUEUE.put(QUEUE_SENTINEL) RESPONSE_QUEUE.put(QUEUE_SENTINEL) - WEBVIEW_QUEUE.put(QUEUE_SENTINEL) if self.web_view_started: + self.macos_web_view_queue.put(QUEUE_SENTINEL) for window in webview.windows[:]: # iterate over a copy; remove (in destroy()) from original window.show() window.destroy() @@ -3021,6 +3133,10 @@ def exit(self, icon, restart_callback=None): proxy.stop() if icon: + # work around a pystray issue with removing the macOS status bar icon when started from a parent script + if sys.platform == 'darwin': + # noinspection PyProtectedMember + icon._status_item.button().setImage_(None) icon.stop() # for the 'Start at login' option we need a callback to restart the script the first time this preference is @@ -3031,9 +3147,11 @@ def exit(self, icon, restart_callback=None): restart_callback() # macOS Launch Agents need reloading when changed; unloading exits immediately so this must be our final action - if sys.platform == 'darwin' and not self.args.no_gui and self.macos_unload_plist_on_exit: + if sys.platform == 'darwin' and self.args.gui and self.macos_unload_plist_on_exit: self.macos_launchctl('unload') + EXITING = False # to allow restarting when imported from parent scripts (or an interpreter) + if __name__ == '__main__': App() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d0f1faf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=61.0", "pyasyncore; python_version >= '3.12'", "cryptography"] # core requirements are needed for version detection, which requires importing the script +build-backend = "setuptools.build_meta" + +[project] +name = "emailproxy" +authors = [ + { name = "Simon Robinson", email = "simon@robinson.ac" } +] +description = "Transparently add OAuth 2.0 support to IMAP/POP/SMTP clients that don't support this authentication method." +readme = { file = "README.md", content-type = "text/markdown" } +license = { text = "Apache License 2.0" } +requires-python = ">=3.6" +classifiers = [ + "Operating System :: OS Independent", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Topic :: Communications :: Email", + "Topic :: Communications :: Email :: Mail Transport Agents", + "Topic :: Communications :: Email :: Post-Office", + "Topic :: Communications :: Email :: Post-Office :: IMAP", + "Topic :: Communications :: Email :: Post-Office :: POP3", + "Development Status :: 6 - Mature", + "License :: OSI Approved :: Apache Software License" +] +dynamic = ["dependencies", "optional-dependencies", "version"] + +[project.scripts] +emailproxy = "emailproxy:App" + +[project.urls] +"Homepage" = "https://github.com/simonrob/email-oauth2-proxy" +"Changelog" = "https://github.com/simonrob/email-oauth2-proxy/releases" +"Documentation" = "https://github.com/simonrob/email-oauth2-proxy#email-oauth-20-proxy" +"Bug Tracker" = "https://github.com/simonrob/email-oauth2-proxy/issues" +"Source Code" = "https://github.com/simonrob/email-oauth2-proxy" + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements-core.txt"] } +optional-dependencies.gui = { file = ["requirements-gui.txt"] } +version = { attr = "emailproxy.__package_version__" } diff --git a/requirements-no-gui.txt b/requirements-core.txt similarity index 64% rename from requirements-no-gui.txt rename to requirements-core.txt index b591c41..a06ef5e 100644 --- a/requirements-no-gui.txt +++ b/requirements-core.txt @@ -1,7 +1,9 @@ # this file contains the proxy's core dependencies beyond inbuilt python packages -# note that to use this file instead of the default requirements.txt you *must* pass the `--no-gui` option when starting -# the proxy - see the script's readme for further details -cryptography +# note that to use the proxy with only these requirements you *must* pass the `--no-gui` option when starting - see the +# script's readme for further details + +# 2.2 or later required for MultiFernet support +cryptography>=2.2 # provide the previously standard library module `asyncore`, removed in Python 3.12 (https://peps.python.org/pep-0594/) pyasyncore; python_version >= '3.12' diff --git a/requirements.txt b/requirements-gui.txt similarity index 81% rename from requirements.txt rename to requirements-gui.txt index 8cee259..0726fce 100644 --- a/requirements.txt +++ b/requirements-gui.txt @@ -1,5 +1,5 @@ -# include core requirements (separated in order to support usage without GUI-only dependencies) --r requirements-no-gui.txt +# the standard way to install the proxy and dependencies is `python -m pip install emailproxy` (i.e., direct from PyPI) +# to install requirements directly, use: `python -m pip install -r requirements-core.txt -r requirements-gui.txt` pillow # to create the menu bar icon image from a TTF icon setuptools # for pkg_resources (checking dependency versions)