diff --git a/.gitignore b/.gitignore index 404c9b86..5cc86b3d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__/ .tox/ *.egg-info/ db.sqlite3 +site/ diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..09e12a7b --- /dev/null +++ b/docs/config.md @@ -0,0 +1,861 @@ +# Configuration {: #configuration } + +Freezeyt is primarily configured using a dictionary of options, +usually loaded from a YAML (or JSON) file given on the command line (CLI) +using the `-c/--config` argument, for example: + +```console +$ python -m freezeyt my_app _build -c freezeyt.yaml +``` + +See [below](#example) for an example of what goes in the file. + +Instead of a file, you can also put the configuration in a Python +dictionary and tell Freezeyt to import it using the `-C/--import-config` +argument. +Like the application to freeze, the argument takes the name of an importable +module and the name of a variable in that module, separated by a colon. +For example: + +```console +$ python -m freezeyt my_app _build -C my_app:freezeyt_config +``` + +## CLI + +Most command-line arguments correspond directly to an configuration option. +Unless documented otherwise, CLI arguments will override values +from a file or dictionary. + +Here is a full list of CLI arguments: + +| CLI argument | Option name | Meaning | +|----------|------------|----------| +| APP (positional) | `app` | [Application to freeze](#conf-app) | +| `-o`, `--output`, positional | `output` | [Output directory](#conf-output) | +| `-c`, `--config` | --- | [Configuration file](#configuration) | +| `-C`, `--import-config` | --- | [Configuration variable](#configuration) | +| `--prefix` | `prefix` | [URL prefix](#conf-prefix) | +| `--extra-page` | `extra_pages` | [Extra pages](#conf-extra_pages) | +| `--progress` | (plugins) | [Progress bar and logging](#conf-cli-progress) | +| `--gh-pages` | `gh_pages` | [Github Pages Plugin](#conf-gh_pages) | +| `--no-cleanup` | `cleanup` | Don't [clean up](#conf-cleanup) | +| `-x`, `--fail-fast` | `fail_fast` | [Fail fast](#conf-fail_fast) | +| `--help` | --- | Show help and exit | + +## Example {: #example } + +Here's an example YAML configuration file. +See below for descriptions of the individual options. + +```yaml +output: ./_build/ # The website will be saved to this directory +prefix: https://mysite.example.com/subpage/ +extra_pages: + # Let Freezeyt know about URLs that are not linked from elsewhere + /robots.txt + /easter-egg.html +extra_files: + # Include additional files in the output: + # Static files + static: + copy_from: static/ + # Web host configuration + CNAME: mysite.example.com + ".nojekyll": '' + googlecc704f0f191eda8f.html: + copy_from: google-verification.html +status_handlers: + # If a redirect page (HTTP status 3xx) is found, warn but don't fail + "3xx": warn +``` + +## Overview of the options + +The following options are configurable: + +| Option name | Meaning | Example | +|----------|---------|---------| +| `app` | [Application to freeze](#conf-app) | `'module:wsgi_app'` | +| `output` | [Output directory](#conf-output) | `'./_build/'` | +| `prefix` | [URL prefix](#conf-prefix) | `'https://mysite.example.com/subpage/'` | +| `extra_pages` | [Extra pages](#conf-extra_pages) | (list) | +| `extra_files` | [Extra files](#conf-extra_files) | (dict) | +| `cleanup` | [Clean up](#conf-cleanup) | `False` | +| `fail_fast` | [Fail fast](#conf-fail_fast) | `True` | +| `gh_pages` | [Github Pages Plugin](#conf-gh_pages) | `True` | +| `default_mimetype` | [Default MIME type](#conf-default_mimetype) | `text/plain` | +| `get_mimetype` | [MIME type getter](#conf-default_mimetype) | `module:your_function` | +| `mime_db_file` | [MIME type database](#conf-mime_db_file) | `path/to/mime-db.json` | +| `version` | [Configuration version](#conf-version) | `1` | +| `plugins` | [Plugins](#conf-plugins) | (dict) | +| `hooks` | [Hooks](#conf-hooks) | (dict) | +| `status_handlers` | [HTTP Status handling](#conf-status_handlers) | (dict) | +| `url_finders` | [URL finders](#conf-url_finders) | (dict) | +| `use_default_url_finders` | [Use default URL finders](#conf-use_default_url_finders) | `False` | +| `urls_from_link_headers` | [Find URLs in Link headers](#conf-urls_from_link_headers) | `False` | +| `url_to_path` | [Path generation](#conf-url_to_path) | `my_module:url_to_path` | +| `static_mode` | [Middleware static mode](#conf-static_mode) | `True` | + + +## Basic options + + +### Configuration version {: #conf-version } + +To ensure that your configuration will work unchanged in newer versions of Freezeyt, +you should add the current version number, `1`, to your configuration like this: + +```yaml +version: 1 +``` + +This is not mandatory. If the version is not given, the configuration may +not work in future versions of Freezeyt. + + +### Application to freeze {: #conf-app } + +The name of importable Python module that contains the application must be +given in the configuration, or on the command line as first argument. + +Inside the module, Freezeyt looks for the variable *app* by default. +A different variable can be specified after the module name, separated by +a colon (`:`). +When the module is specified both on the command line and in the config file, +an error is raised. + +When the configuration is a Python dict, `app` can be given directly as +the WSGI application object, rather than a string. + +#### Examples + +Freezeyt looks for the variable `app` inside the module by default. +In YAML, it looks like this: + +```yaml +app: app_module +``` + +If `app` is in a submodule, separate package names with a dot: +```yaml +app: app_package.wsgi +``` + +A different variable name can be specified by using `:`. +```yaml +app: app_module:wsgi_application +``` + +In Python, the app can be given directly: + +```python +my_app = Flask(__name__) +... + +freezeyt_config = {'app': my_app} +``` + +### Output {: #conf-output } + +The `output` option conifgures the output directory, where the +result is saved: + +```yaml +output: ./_build/ +``` + +Alternatively, the output directory can be specified on the command line, +either by the `--output` (`-o`) argument or as a second positional argument. + +The output must be specified in only one way; providing both the config +option and CLI argument is an error. + +If there is any existing content in the output directory, +Freezeyt will either remove it (if the content looks like a previously +frozen website) or raise an error. +Best practice is to remove the output directory before freezing. + +#### Output to dictionary + +*Freezeyt* can return the result in a dictionary, +rather than save it to disk. +Note that this stores the entire frozen website in memory, +so it is mostly useful for testing. +This can be configured by setting `output` to the dictionary +`{'type': 'dict'}`, rather than to a string: + +```yaml +output: + type: dict +``` + +In this case, the `freeze()` function returns a dictionary of filenames +and their contents. +For example, a site with `/`, `/second_page/` and `/images/smile.png` +will be represented as: + +```python +{ + 'index.html': b'...', + 'second_page': { + 'index.html': b'...', + }, + 'images': { + 'smile.png': b'\x89PNG\r\n\x1a\n\x00...', + }, +} +``` + +#### Explicitly output to disk + +It is possible to explicitly request Freezeyt to output to a directory +using a dict like this: + +```yaml +output: + type: dir + dir: ./_build/ +``` + +This is equivalent to the string `./_build/`, or passing `./_build/` as the +CLI argument. + +There are currently no output types other that `dir` and `dict`. + + +### URL prefix {: #conf-prefix } + +The URL where the application will be deployed can be +specified with `prefix`, for example: + +```yaml +prefix: http://localhost:8000/ +``` +or +```yaml +prefix: https://mysite.example.com/subpage/ +``` + +The *prefix* URL must end with a slash. + +The page at the *prefix* URL is considered the application's “home page”, +and will always be frozen. + +Freezeyt considers all pages under the prefix to be part of +the application. +For example, with the second prefix above: + +- `https://mysite.example.com/subpage/blog.html` would be followed, + and the page would be frozen at `/blog.html`; +- `https://mysite.example.com/about.html` would be considered an external link, + and ignored. + +The prefix is also passed to the application as the server and script name, +which should be used whenever the app generates absolute URLs. + +The prefix can also be specified on thecommand line with e.g.: +`--prefix=http://localhost:8000/`. +The CLI argument has priority over the config file. + + +## Extra content + +Usually, Freezeyt saves pages that are reachable by links from the app's +home page. +There are two cases when this is not enough, and you need to specify +extra content manually: + +- Extra *pages* are part of the application, but not reachable by following + links. For example, a an old URL that redirects to a new location should + be configured as an extra page. See [Known URLs][known-urls] for details. + +- Extra *files* are not part of the application. + Typically, these are used to configure the static page server, like + a `CNAME` file GitHub's or `.htaccess` for Apache. + + +### Extra pages {: #conf-extra_pages } + +URLs of pages that are not reachable by following links from the homepage +can specified as “extra” pages in the configuration: + +```yaml +extra_pages: + - extra/ + - extra2.html +``` + +The URLs should be relative (to the [prefix](#conf-prefix)). +Absolute URLs are allowed, but they must start with the prefix. + +Freezeyt will handle these pages as if it found them as links. +For example, by default it will follow links in extra pages. + +Extra pages may also be given with the `--extra-page` command line argument, +which can be repeated (for example, +`--extra-page extra/ --extra-page extra2.html`). +The lists from CLI and the config file are merged together. + +You can also specify extra pages using a Python function, +specified using a module name and function name as follows: + +```yaml +extra_pages: + - generator: my_app:generate_extra_pages +``` + +This function should take the application as argument and return an iterable +of URLs as strings. + +When using the Python API, this function can be specified +directly as a Python object, for example: + +```python +def generate_extra_pages(app): + yield 'extra/' + yield 'extra2.html' + +config = { + ... + 'extra_pages': [{'generator': generate_extra_pages}], +} +another_config = { + ... + 'extra_pages': [generate_extra_pages], +} +``` + +### Extra files {: #conf-extra_files } + +Extra files to be included in the output can be specified, +along with their content. + +For example, the following config will add 3 files to the output: + +```yaml +extra_files: + CNAME: mysite.example.com + ".nojekyll": '' + config/xyz: abc +``` + +The files will be: + +- `/CNAME`, with the content `mysite.example.com`; +- `/.nojekyll`, empty; +- `/config/xyz`, with the content `abc`. + +These files are not considered part of the application. +*Freezeyt* will not retrieve them from the app, and it will not try to find +links in them. + +Extra files are mainly useful for configuration of your static server. +For files that are part of the website, +such as a [favicon](https://en.wikipedia.org/wiki/Favicon), we recommend +adding them to your application, and either link to them or add them as +[extra *pages*](#conf-extra_pages). + +You can also specify extra file content using the Base64 encoding (`base64`) or +as a filesystem path to be copied (`copy_from`), like so: + +```yaml +extra_files: + config.dat: + base64: "YWJjZAASNA==" + config2.dat: + copy_from: included/config2.dat +``` + +If the `copy_from` path names a directory, it will be copied recursively. + +In the file name, Freezeyt treats both backslashes and forward slashes +as path separators. + +Extra files cannot be specified on the CLI. + + +## Debugging options + +The following options are useful when debugging your application, +or its integration with Freezeyt. + + +### Clean up {: #conf-cleanup } + +By default, if an error occurs during freezing, Freezeyt will delete +the incomplete output directory. +This is meant to prevent uploading incomplete results to web hosting by mistake. + +If you want to keep the incomplete directory (for example, +to help debugging), you can use the `--no-cleanup` command line switch +or the `cleanup` configuration option: + +```shell +$ freezeyt app -o ./build/ --no-cleanup +``` + +```yaml +cleanup: False +``` + +The command line switch has priority over the configuration. +Use `--cleanup` to override `cleanup: False` from the config. + + +### Fail fast {: #conf-fail_fast } + +By default, Freezeyt collects errors it finds on individual pages, +and presents them all when done. +To stop the process early when the first error occurs, use the +the `--fail-fast` (`-x`) command line switch or the `fail_fast` configuration option: + +```shell +$ freezeyt app -o ./build/ --fail-fast +``` + +```yaml +fail_fast: True +``` + +The command line switch has priority over the configuration. +Use `--no-fail-fast` to override `fail_fast: True` from the config. + + +## Customizing the process + +Here are ways to configure details of how Freezeyt saves pages. + + +### MIME type checking + +For static sites to be served correctly on most servers, the MIME +`Content-Type` of a page must match the saved file's extension. +See [MIME type mapping][mime-type-mapping] for details. + +To ensure that the application will work as intended when frozen and served +with such a server, Freezeyt verifies that the extensions of saved files +correspond to the MIME types served by the app. +This funtionality is provided by [`freezeyt.Middleware`][freezeyt.Middleware]. + +The exact mapping between extensions and `Content-Type` values varies +between servers. +*Freezeyt* uses Python's `mimetypes` module by default, but provides +several ways to customize it. + +#### Default MIME type {: #conf-default_mimetype } + +Files without an extension are, by default, served as `application/octet-stream` +(arbitrary binary data). +This can be configured using the `default_mimetype` option. +For example, if your static page server defaults to plain text files, +use: + +```yaml +default_mimetype=text/plain +``` + + +#### MIME type getter {: #conf-get_mimetype } + +The most flexible way to map file extensions to MIME types is with +a custom function, which you can specify using the `get_mimetype` option. +For example: + +```yaml +get_mimetype=module:your_function +``` + +`get_mimetype` can be defined as a string in the form `"module:function"`, +which names the function to call, or as a Python function +(if configuring Freezeyt using a Python dict). + +The function will be called with one argument, the file path as a string, and +it should returns a list of corresponding MIME types +(for example, `["text/html"]` or `["audio/wav", "audio/wave"]`). + +If `get_mimetype` instead returns `None`, Freezeyt will use the +[default MIME type](#conf-default_mimetype). + +By default, Freezeyt calls the Python function +[`mimetypes.guess_type`](https://docs.python.org/3/library/mimetypes.html#mimetypes.guess_type) +and uses the `type` (the first element) of the result: + +```python +def default_mimetype(url: str) -> Optional[List[str]]: + file_mimetype, encoding = guess_type(url) + if file_mimetype is None: + # Freezeyt should use the default + return None + else: + # A one-element list + return [file_mimetype] +``` + + +#### Using a mime-db database {: #conf-mime_db_file } + +There is an option to use [the MIME type database from the `jshttp` project](https://github.com/jshttp/mime-db/blob/master/db.json) +(the database used by GitHub Pages), +or a database with the same structure, for mapping file names to MIME types. + +To use it, add the path to the JSON file to Freezeyt configuration: +```yaml +mime_db_file=path/to/mime-db.json +``` +This is equivalent to setting `get_mimetype` to a function that maps +extensions to filetypes according to the database. + + +### URL finding {: #conf-url_finders } + +Freezeyt discovers new pages in the application by searching for URLs +in pages it processes. The search is done by *URL finders*, +functions that find URLs in a specific page type. + +You can configure which finder is used for which MIME type using +the `url_finders` configuration key. +In the default configuration, Freezeyt finds links in HTML and CSS files. +The default could be configured like this: + +```yaml +url_finders: + text/html: freezeyt.url_finders:get_html_links + text/css: freezeyt.url_finders:get_css_links +``` + +Keys in the `url_finders` dict are MIME types. +Values are functions, which can be defined as: + +* Strings in the form `"module:function"`, which name the finder + function to call. +* Python functions (if configuring Freezeyt from Python). +* Strings without a colon (`:`), which name a function from the + `freezeyt.url_finders` module. Using this shortcut, the default configuration + could also be written as: + + url_finders: + text/html: get_html_links + text/css: get_css_links + + + The `freezeyt.url_finders` module includes these finders: + + - `get_html_links`, the default finder for HTML + - `get_css_links`, the default finder for CSS + - `get_html_links_async` and `get_css_links_async`, asynchronous variants + of the above + - `none`, a finder that doesn't find any links. + +An URL finder function gets these arguments: + +* The page content, as a binary file open for reading (for example, + `io.BinaryIO`), +* the absolute URL of the page, as a `str`, and +* the HTTP headers, as a list of 2-tuples (as in WSGI). + +The function should return an iterator of all URLs (as strings) found +in the page's contents, as they would appear in `href` or `src` attributes. +Specifically: + +- The URLs can be relative. +- External URLs (i.e. those not beginning with the [`prefix`](#conf-prefix)) + should be included. + +Finder functions may be asynchronous. +If the function returns a coroutine (for example, if it's defined with +`async def`, Freezeyt will use `await` on the result. +If the function returns an asynchronous generator (for example, if it's +defined with `async def` and uses `yield`), Freezeyt will use async iteration +to handle it. + + +#### URL finder header {: #conf-header-Freezeyt-URL-Finder } + +You can specify a finder as a string in the `Freezeyt-URL-Finder` HTTP header. +If given, it overrides the `url_finders` configuration. + +#### Default `get_html_links` + +The default URL finder for HTML pages looks in `src` and `href` attributes +of all tags in the document. +It currently does not handle other links, such as embedded CSS, but it +may be improved in the future. + +#### Default `get_css_links` + +The default URL finder for CSS uses the [`cssutils`](https://pypi.org/project/cssutils/) +library to find all links in a stylesheet. +This may be changed in the future. + +#### Disabling default URL finders {: #conf-use_default_url_finders } + +If a finder is not explictly specified in the configuration file, +Freezeyt will use the default. For example, if you specify +`text/html: my_custom_finder` only, Freezeyt will use the default finder +for `text/css`. + +You can disable this behavior: + +```yaml +use_default_url_finders: false +``` + + +#### Finding URLs in Link headers {: #conf-urls_from_link_headers } + +By default, Freezeyt will follow URLs in `Link` HTTP headers. +To disable this, specify: + +```yaml +urls_from_link_headers: false +``` + + +### Freeze actions {: #freeze-actions } + +For each page it finds, Freezeyt will take an *action*: save the page, +ignore it, or treat it as an error. + +By default, Freezeyt will save the pages with a `200 OK` HTTP status code, +and raise an error for any other status code. + +You can configure the for an individual page from within the application +(or middleware), by setting the `Freezeyt-Action` HTTP header to one of +these strings: + +* `'save'`: Freezeyt will save the body of the page. +* `'ignore'`: Freezeyt will not save any content for the page +* `'warn'`: will save the content and send warn message to stdout +* `'follow'`: Freezeyt will save content from the redirected location. + This requires a `Location` header, which is usually added for redirects. + Redirects to external pages are not supported. +* `'error'`: fail; the page will not be saved and `freeze()` will raise + an exception. + + +#### HTTP Status handling {: #conf-status_handlers } + +If the `Freezeyt-Action` header is not set, Freezeyt will determine what to +do based on the HTTP status. The behavior can be customized using the +`status_handlers` setting. +For example, to ignore pages with the `404 NOT FOUND` status, rather than +treat them as errors, set the `404` handler to `'ignore'`: + +```yaml +status_handlers: + '404': ignore +``` + +More varied `status_handlers` could be specified as: + +```yaml +status_handlers: + '202': warn + '301': follow + '404': ignore + '418': my_module:custom_action # see below + '429': ignore + '5xx': error +``` + +Note that the status code must be a string. +In a YAML file, it needs to be quoted. + +A range of statuses can be specified as one number (`1` to `5`) followed by +lowercase `xx`. +(Other "wildcards" like `50x` are not supported.) + + +[](){#custom-actions} +#### Custom freeze actions + +You can also define a custom action in `status_handlers` as: + +* a string in the form `'my_module:custom_action'`, which names a handler + function to call, or +* a Python function (if configuring Freezeyt from Python). + +The action function takes one argument a [`TaskInfo`][freezeyt.TaskInfo] +with information about the page being frozen. +Freezeyt's predefined actions, like `follow`, can be imported from +`freezeyt.actions`. +A custom action should call one of these default actions and return the return value from it. + + +### Path generation {: #conf-url_to_path } + +It is possible to customize the filenames that pages are saved under +using the `url_to_path` configuration key, for example: + +```yaml +url_to_path: my_module:url_to_path +``` + +The value can be: + +* A string in the form `"module:function"`, which names the + function to call. The function can be omitted along with the colon, + and defaults to `url_to_path`. +* A Python function, if configuring Freezeyt from Python. + +The function receives one string: the *path* portion of the URL to save, +relative to the `prefix`. +It should return a path to the saved file, relative to the build directory, +as a string. + +The default function, available as [`freezeyt.url_to_path`][freezeyt.url_to_path], +adds `"index.html"` if the URL ends with `/`. + + +### Plugins {: #conf-plugins } + +It is possible to extend Freezeyt with *plugins*, +either [“built-in” ones](#built-in-plugins) that ship with Freezeyt +or [external ones](#custom-plugins). + +Plugins are added using configuration like: + +```yaml +plugins: + - freezeyt.plugins:ProgressBarPlugin + - mymodule:my_plugin +``` + + +## Built-in plugins {: #built-in-plugins } + + +### Github Pages Plugin {: #conf-gh_pages } + +To make it easier to upload frozen pages to the [Github Pages service](https://pages.github.com/), +you can activate the GitHub Pages plugin using the `--gh-pages` CLI argument +or the `gh_pages` key in the configuration. +This creates a `gh-pages` git branch in the output directory. + +By default, the Github Pages Plugin is not active. However, if you have +activated it in your configuration, you can override the choice in the CLI with +`--no-gh-pages`. + +Configuration example: +```yaml +gh_pages: True +``` + +This is a shortcut for adding `freezeyt.plugins:GHPagesPlugin` +to [`plugins`](#conf-plugins). + +To deploy a site to Github, you can then work with the git repository directly +in the output directory or pull the files into another repository/directory. +You can then pull/fetch files from the newly created `gh-pages` git branch in +many ways, for example: +```console +$ git fetch output_dir gh-pages +$ git branch --force gh-pages FETCH_HEAD +``` +Note that this will overwrite the current contents of the `gh-pages` branch, +because of the `--force` switch. + + +### Progress bar and logging {: #conf-cli-progress } + +The CLI argument `--progress` controls what Freezeyt outputs as it +handles pages: + +* `--progress=log`: Output a message about each frozen page to stdout. +* `--progress=bar`: Draw a status bar in the terminal. Messages about + each frozen page are *also* printed to stdout, as with `log`. +* `--progress=none`: Don't do any of this. + +The default is `bar` if stdout is a terminal, and `log` otherwise. + +Alternately, it is possible to configure logging by adding one of the following +[`plugins`](#conf-plugins): + +* `freezeyt.progressbar:ProgressBarPlugin` +* `freezeyt.progressbar:LogPlugin` + + +## Middleware static mode {: #conf-static_mode } + +When using the [Freezeyt middleware][middleware], you can enable *static mode*, +which simulates behaviour after the app is saved to static pages: + +```yaml +static_mode: true +``` + +Currently, in static mode: + +- HTTP methods other than GET and HEAD are disallowed. +- URL parameters are removed +- Request bodies are discarded +- Non-default WSGI environ keys are removed + +Other restrictions and features may be added in the future, without regard +to backwards compatibility. +The static mode is intended for interactive use -- testing your app without +having to freeze all of it after each change. + + + +## Extending Freezeyt + + +### Custom plugins {: #custom-plugins } + +A plugin is a function that Freezeyt will call before starting to +freeze pages. + +It is passed a [`FreezeInfo`][freezeyt.FreezeInfo] object as argument. +Usually, the plugin will its [`add_hook`][freezeyt.FreezeInfo.add_hook] method +to register additional functions. + + +### Hooks {: #conf-hooks } + +It is possible to register *hooks*, functions that are called when +specific events happen in the freezing process. + +For example, if `mymodule` defines functions `start` and `page_frozen`, +you can make Freezeyt call them using this configuration: + +```yaml +hooks: + start: + - mymodule:start + page_frozen: + - mymodule:page_frozen +``` + +When configuring Freezeyt from Python, a function can be used directly +instead of a string. + +The available hooks are: + +#### `start` + +Called when the freezing process starts, before any other hooks. + +Takes one argument: a [`FreezeInfo`][freezeyt.FreezeInfo] object. + + +#### `page_frozen` + +Called whenever a page is processed successfully. + +Takes one argument: a [`TaskInfo`][freezeyt.TaskInfo] object. + + +#### `page_failed` + +Called whenever a page is not saved due to an exception. + +Takes one argument: a [`TaskInfo`][freezeyt.TaskInfo] object. + + +#### `success` + +Called after the app is successfully frozen. + +Takes one argument: a [`FreezeInfo`][freezeyt.FreezeInfo] object. diff --git a/docs/contrib.md b/docs/contrib.md new file mode 100644 index 00000000..e0e15425 --- /dev/null +++ b/docs/contrib.md @@ -0,0 +1,179 @@ + +# Contributing + +Freezeyt is developed on [GitHub](https://github.com/encukou/freezeyt). + +Contributions, issues and feature requests are welcome. +Feel free to check out the [issues page] if you'd like to +contribute. + +[issues page]: https://github.com/encukou/freezeyt/issues + + +## Quick guide + +1. Clone this repository to your local + computer: + + $ git clone https://github.com/encukou/freezeyt + +2. Then fork this repo to your GitHub account +3. Add your forked repo as a new remote to your local computer: + + $ git remote add https://github.com//freezeyt + +4. Create a new branch at your local computer + + $ git branch + +5. Switch to your new branch + + $ git switch + +6. Update the code +7. Push the changes to your forked repo on GitHub + + $ git push + +8. Finally, make a pull request from your GitHub account to origin + + +## Installing for development + +Freezeyt can be installed from the current directory: + +```console +$ python -m pip install -e . +``` + +It also has several groups of extra dependecies: + +* `blog` for the project blog +* `dev` for development and running tests +* `typecheck` for [mypy] type checks + +Each group can be installed separately: + +```console +$ python -m pip install -e ."[typecheck]" +``` + +or you can install more groups at once: +```console +$ python -m pip install -e ."[blog, dev, typecheck]" +``` + +[mypy]: https://www.mypy-lang.org/ + + +## Using an in-development copy of Freezeyt + +* Set `PYTHONPATH` to the directory with Freezeyt, for example: + * Unix: `export PYTHONPATH="/home/name/freezeyt"` + * Windows: `set PYTHONPATH=C:\Users\Name\freezeyt` + +* Install the web application you want to freeze. Either: + * install the application using `pip`, if possible, or + * install the application's dependencies and `cd` to the app's directory. + +* Run Freezeyt, for example: + * `python -m freezeyt demo_app_url_for _build --prefix http://freezeyt.test/foo/` + + + +## Tests + +For testing the project it's necessary to install additional requirements: + +```console +$ python -m pip install .[dev] +``` + +To run tests in your current environment, use pytest: + +```console +$ python -m pytest +``` + +To run tests with multiple Python versions (if you have them installed), +install `tox` using `python -m pip install tox` and run it: + +```console +$ tox +``` + +### Environ variables for tests + +Some test scenarios compare Freezeyt's results with expected output. +When the files with expected output don't exist yet, +they can be created by setting the environment variable +`TEST_CREATE_EXPECTED_OUTPUT` to `1`: + +**Unix** + +```console +$ export TEST_CREATE_EXPECTED_OUTPUT=1 +``` + +**Windows** + +```doscon +> set TEST_CREATE_EXPECTED_OUTPUT=1 +``` + +If you set the variable to any different value or leave it unset +then the files will not be recreated +(tests will fail if the files are not up to date). + +When output changes, you need to first delete the expected output, +regenerate it by running tests with `TEST_CREATE_EXPECTED_OUTPUT=1`, +and check that the difference is correct. + + +## How to watch progress + +Unfortunately our progress of development can be watched only in Czech language. + +Watch the progress on our [Youtube playlist](https://www.youtube.com/playlist?list=PLFt-PM7J_H3EU5Oez3ZSVjY5pZJttP2lT). + +Other communication channels and info can be found +in a [Google doc](https://tinyurl.com/freezeyt) (in Czech). + + +## Freezeyt Blog + +We keep a blog about the development of Freezeyt. +It is available [here](https://encukou.github.io/freezeyt/). + +**Be warned:** some of it is in the Czech language. + +### Blog development + +The blog was tested on Python version 3.8. + +The blog is a Flask application. +To run it, install additional dependecies with +`python -m pip install .[blog]` and run the Flask server: + +```console +$ python -m pip install .[blog] +$ flask --app freezeyt_blog/app.py run --debug +``` + +The URL where your blog is running will be printed on the terminal. + +Once you're satisfied with how the blog looks, you can freeze it with: + +```console +$ python -m freezeyt freezeyt_blog.app freezeyt_blog/build +``` + +### Adding new articles to the blog + +Articles are writen in the `Markdown` language. + +**Article** - save to directory `../freezeyt/freezeyt_blog/articles` + +**Images to articles** - save to directory `../freezeyt/freezeyt_blog/static/images` + +If the files are saved elsewhere, the blog will not work correctly. diff --git a/docs/css/pygments-friendly.css b/docs/css/pygments-friendly.css new file mode 100644 index 00000000..3c8a979f --- /dev/null +++ b/docs/css/pygments-friendly.css @@ -0,0 +1,78 @@ +[data-bs-theme="light"] { + --string-color: #4070a0; + pre { line-height: 125%; } + td.linenos .normal { color: #666666; background-color: transparent; padding-left: 5px; padding-right: 5px; } + span.linenos { color: #666666; background-color: transparent; padding-left: 5px; padding-right: 5px; } + td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } + span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } + .codehilite .hll { background-color: #ffffcc } + .codehilite { background: #f0f0f0; } + .codehilite .c { color: #60a0b0; font-style: italic } /* Comment */ + .codehilite .err { border: 1px solid #FF0000 } /* Error */ + .codehilite .k { color: #007020; font-weight: bold } /* Keyword */ + .codehilite .o { color: #666666 } /* Operator */ + .codehilite .ch { color: #60a0b0; font-style: italic } /* Comment.Hashbang */ + .codehilite .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ + .codehilite .cp { color: #007020 } /* Comment.Preproc */ + .codehilite .cpf { color: #60a0b0; font-style: italic } /* Comment.PreprocFile */ + .codehilite .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ + .codehilite .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ + .codehilite .gd { color: #A00000 } /* Generic.Deleted */ + .codehilite .ge { font-style: italic } /* Generic.Emph */ + .codehilite .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */ + .codehilite .gr { color: #FF0000 } /* Generic.Error */ + .codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */ + .codehilite .gi { color: #00A000 } /* Generic.Inserted */ + .codehilite .go { color: #888888 } /* Generic.Output */ + .codehilite .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ + .codehilite .gs { font-weight: bold } /* Generic.Strong */ + .codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ + .codehilite .gt { color: #0044DD } /* Generic.Traceback */ + .codehilite .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ + .codehilite .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ + .codehilite .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ + .codehilite .kp { color: #007020 } /* Keyword.Pseudo */ + .codehilite .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ + .codehilite .kt { color: #902000 } /* Keyword.Type */ + .codehilite .m { color: #40a070 } /* Literal.Number */ + .codehilite .s { color: var(--string-color) } /* Literal.String */ + .codehilite .na { color: var(--string-color) } /* Name.Attribute */ + .codehilite .nb { color: #007020 } /* Name.Builtin */ + .codehilite .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ + .codehilite .no { color: #60add5 } /* Name.Constant */ + .codehilite .nd { color: #555555; font-weight: bold } /* Name.Decorator */ + .codehilite .ni { color: #d55537; font-weight: bold } /* Name.Entity */ + .codehilite .ne { color: #007020 } /* Name.Exception */ + .codehilite .nf { color: #06287e } /* Name.Function */ + .codehilite .nl { color: #002070; font-weight: bold } /* Name.Label */ + .codehilite .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ + .codehilite .nt { color: #062873; font-weight: bold } /* Name.Tag */ + .codehilite .nv { color: #bb60d5 } /* Name.Variable */ + .codehilite .ow { color: #007020; font-weight: bold } /* Operator.Word */ + .codehilite .w { color: #bbbbbb } /* Text.Whitespace */ + .codehilite .mb { color: #40a070 } /* Literal.Number.Bin */ + .codehilite .mf { color: #40a070 } /* Literal.Number.Float */ + .codehilite .mh { color: #40a070 } /* Literal.Number.Hex */ + .codehilite .mi { color: #40a070 } /* Literal.Number.Integer */ + .codehilite .mo { color: #40a070 } /* Literal.Number.Oct */ + .codehilite .sa { color: var(--string-color) } /* Literal.String.Affix */ + .codehilite .sb { color: var(--string-color) } /* Literal.String.Backtick */ + .codehilite .sc { color: var(--string-color) } /* Literal.String.Char */ + .codehilite .dl { color: var(--string-color) } /* Literal.String.Delimiter */ + .codehilite .sd { color: var(--string-color); font-style: italic } /* Literal.String.Doc */ + .codehilite .s2 { color: var(--string-color) } /* Literal.String.Double */ + .codehilite .se { color: var(--string-color); font-weight: bold } /* Literal.String.Escape */ + .codehilite .sh { color: var(--string-color) } /* Literal.String.Heredoc */ + .codehilite .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ + .codehilite .sx { color: #c65d09 } /* Literal.String.Other */ + .codehilite .sr { color: #235388 } /* Literal.String.Regex */ + .codehilite .s1 { color: var(--string-color) } /* Literal.String.Single */ + .codehilite .ss { color: #517918 } /* Literal.String.Symbol */ + .codehilite .bp { color: #007020 } /* Name.Builtin.Pseudo */ + .codehilite .fm { color: #06287e } /* Name.Function.Magic */ + .codehilite .vc { color: #bb60d5 } /* Name.Variable.Class */ + .codehilite .vg { color: #bb60d5 } /* Name.Variable.Global */ + .codehilite .vi { color: #bb60d5 } /* Name.Variable.Instance */ + .codehilite .vm { color: #bb60d5 } /* Name.Variable.Magic */ + .codehilite .il { color: #40a070 } /* Literal.Number.Integer.Long */ +} diff --git a/docs/css/pygments-nord.css b/docs/css/pygments-nord.css new file mode 100644 index 00000000..51e2ec86 --- /dev/null +++ b/docs/css/pygments-nord.css @@ -0,0 +1,87 @@ +[data-bs-theme="dark"] { + pre { line-height: 125%; } + td.linenos .normal { color: #D8DEE9; background-color: #242933; padding-left: 5px; padding-right: 5px; } + span.linenos { color: #D8DEE9; background-color: #242933; padding-left: 5px; padding-right: 5px; } + td.linenos .special { color: #242933; background-color: #D8DEE9; padding-left: 5px; padding-right: 5px; } + span.linenos.special { color: #242933; background-color: #D8DEE9; padding-left: 5px; padding-right: 5px; } + .codehilite .hll { background-color: #3B4252 } + .codehilite { background: #2E3440; color: #d8dee9 } + .codehilite .c { color: #616e87; font-style: italic } /* Comment */ + .codehilite .err { color: #bf616a } /* Error */ + .codehilite .esc { color: #d8dee9 } /* Escape */ + .codehilite .g { color: #d8dee9 } /* Generic */ + .codehilite .k { color: #81a1c1; font-weight: bold } /* Keyword */ + .codehilite .l { color: #d8dee9 } /* Literal */ + .codehilite .n { color: #d8dee9 } /* Name */ + .codehilite .o { color: #81a1c1; font-weight: bold } /* Operator */ + .codehilite .x { color: #d8dee9 } /* Other */ + .codehilite .p { color: #eceff4 } /* Punctuation */ + .codehilite .ch { color: #616e87; font-style: italic } /* Comment.Hashbang */ + .codehilite .cm { color: #616e87; font-style: italic } /* Comment.Multiline */ + .codehilite .cp { color: #5e81ac; font-style: italic } /* Comment.Preproc */ + .codehilite .cpf { color: #616e87; font-style: italic } /* Comment.PreprocFile */ + .codehilite .c1 { color: #616e87; font-style: italic } /* Comment.Single */ + .codehilite .cs { color: #616e87; font-style: italic } /* Comment.Special */ + .codehilite .gd { color: #bf616a } /* Generic.Deleted */ + .codehilite .ge { color: #d8dee9; font-style: italic } /* Generic.Emph */ + .codehilite .ges { color: #d8dee9; font-weight: bold; font-style: italic } /* Generic.EmphStrong */ + .codehilite .gr { color: #bf616a } /* Generic.Error */ + .codehilite .gh { color: #88c0d0; font-weight: bold } /* Generic.Heading */ + .codehilite .gi { color: #a3be8c } /* Generic.Inserted */ + .codehilite .go { color: #d8dee9 } /* Generic.Output */ + .codehilite .gp { color: #616e88; font-weight: bold } /* Generic.Prompt */ + .codehilite .gs { color: #d8dee9; font-weight: bold } /* Generic.Strong */ + .codehilite .gu { color: #88c0d0; font-weight: bold } /* Generic.Subheading */ + .codehilite .gt { color: #bf616a } /* Generic.Traceback */ + .codehilite .kc { color: #81a1c1; font-weight: bold } /* Keyword.Constant */ + .codehilite .kd { color: #81a1c1; font-weight: bold } /* Keyword.Declaration */ + .codehilite .kn { color: #81a1c1; font-weight: bold } /* Keyword.Namespace */ + .codehilite .kp { color: #81a1c1 } /* Keyword.Pseudo */ + .codehilite .kr { color: #81a1c1; font-weight: bold } /* Keyword.Reserved */ + .codehilite .kt { color: #81a1c1 } /* Keyword.Type */ + .codehilite .ld { color: #d8dee9 } /* Literal.Date */ + .codehilite .m { color: #b48ead } /* Literal.Number */ + .codehilite .s { color: #a3be8c } /* Literal.String */ + .codehilite .na { color: #8fbcbb } /* Name.Attribute */ + .codehilite .nb { color: #81a1c1 } /* Name.Builtin */ + .codehilite .nc { color: #8fbcbb } /* Name.Class */ + .codehilite .no { color: #8fbcbb } /* Name.Constant */ + .codehilite .nd { color: #d08770 } /* Name.Decorator */ + .codehilite .ni { color: #d08770 } /* Name.Entity */ + .codehilite .ne { color: #bf616a } /* Name.Exception */ + .codehilite .nf { color: #88c0d0 } /* Name.Function */ + .codehilite .nl { color: #d8dee9 } /* Name.Label */ + .codehilite .nn { color: #8fbcbb } /* Name.Namespace */ + .codehilite .nx { color: #d8dee9 } /* Name.Other */ + .codehilite .py { color: #d8dee9 } /* Name.Property */ + .codehilite .nt { color: #81a1c1 } /* Name.Tag */ + .codehilite .nv { color: #d8dee9 } /* Name.Variable */ + .codehilite .ow { color: #81a1c1; font-weight: bold } /* Operator.Word */ + .codehilite .pm { color: #eceff4 } /* Punctuation.Marker */ + .codehilite .w { color: #d8dee9 } /* Text.Whitespace */ + .codehilite .mb { color: #b48ead } /* Literal.Number.Bin */ + .codehilite .mf { color: #b48ead } /* Literal.Number.Float */ + .codehilite .mh { color: #b48ead } /* Literal.Number.Hex */ + .codehilite .mi { color: #b48ead } /* Literal.Number.Integer */ + .codehilite .mo { color: #b48ead } /* Literal.Number.Oct */ + .codehilite .sa { color: #a3be8c } /* Literal.String.Affix */ + .codehilite .sb { color: #a3be8c } /* Literal.String.Backtick */ + .codehilite .sc { color: #a3be8c } /* Literal.String.Char */ + .codehilite .dl { color: #a3be8c } /* Literal.String.Delimiter */ + .codehilite .sd { color: #616e87 } /* Literal.String.Doc */ + .codehilite .s2 { color: #a3be8c } /* Literal.String.Double */ + .codehilite .se { color: #ebcb8b } /* Literal.String.Escape */ + .codehilite .sh { color: #a3be8c } /* Literal.String.Heredoc */ + .codehilite .si { color: #a3be8c } /* Literal.String.Interpol */ + .codehilite .sx { color: #a3be8c } /* Literal.String.Other */ + .codehilite .sr { color: #ebcb8b } /* Literal.String.Regex */ + .codehilite .s1 { color: #a3be8c } /* Literal.String.Single */ + .codehilite .ss { color: #a3be8c } /* Literal.String.Symbol */ + .codehilite .bp { color: #81a1c1 } /* Name.Builtin.Pseudo */ + .codehilite .fm { color: #88c0d0 } /* Name.Function.Magic */ + .codehilite .vc { color: #d8dee9 } /* Name.Variable.Class */ + .codehilite .vg { color: #d8dee9 } /* Name.Variable.Global */ + .codehilite .vi { color: #d8dee9 } /* Name.Variable.Instance */ + .codehilite .vm { color: #d8dee9 } /* Name.Variable.Magic */ + .codehilite .il { color: #b48ead } /* Literal.Number.Integer.Long */ +} diff --git a/docs/css/pygments.css b/docs/css/pygments.css new file mode 100644 index 00000000..eb4c32ab --- /dev/null +++ b/docs/css/pygments.css @@ -0,0 +1,15 @@ +@import "pygments-friendly.css"; /* light theme */ +@import "pygments-nord.css"; /* dark theme */ + +[data-bs-theme="light"] { + --string-color: #4070a0; +} +[data-bs-theme="dark"] { + --string-color: #a3be8c; +} + +.codehilite .s { color: var(--string-color) } /* Literal.String */ + +.codehilite .gp { + user-select: none; /* prompts are non-selectable */ +} diff --git a/docs/img/favicon.ico b/docs/img/favicon.ico new file mode 100644 index 00000000..9d694282 Binary files /dev/null and b/docs/img/favicon.ico differ diff --git a/docs/img/favicon.svg b/docs/img/favicon.svg new file mode 100644 index 00000000..8fd02fdc --- /dev/null +++ b/docs/img/favicon.svg @@ -0,0 +1,332 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..abe29104 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,197 @@ +--- +title: Freezeyt +... + +# Freezeyt + +Freezeyt turns Python web applications into static websites. + + +## What this does + +Freezeyt is a static webpage *freezer*. +It takes a Python web application and turns it into a set of files +that can be served by a simple server like [GitHub Pages] or +Python's [http.server]. + +[GitHub Pages]: https://docs.github.com/en/free-pro-team@latest/github/working-with-github-pages/about-github-pages +[http.server]: https://docs.python.org/3/library/http.server.html + +Freezeyt is compatible with all Python web frameworks that use the common +[Web Server Gateway Interface] (WSGI). + +[Web Server Gateway Interface]: https://www.python.org/dev/peps/pep-3333/ + + +## Installation + +Freezeyt requires Python 3.6 or above. + +It is highly recommended to create and activate a separate virtual +environment for this project. +You can use [`venv`][venv], `virtualenv`, Conda, containers or any other kind +of virtual environment. + +[venv]: https://docs.python.org/3/library/venv.html?highlight=venv#module-venv + +The tool can be installed using: + +```console +$ python -m pip install freezeyt +``` + +To install a development version of Freezeyt, +see [Contributing documentation]. + +[Contributing documentation]: ./contrib.md + + +## Quick usage + +For a Flask app in `hello.py`, run: + +```console +$ python -m freezeyt hello _build +``` + +For detailed instructions, read on. + + +## Usage + +To use Freezeyt, you need a Python web application. +You can use the [example Flask app] to start. + +[example Flask app]: https://flask.palletsprojects.com/en/2.3.x/quickstart/ + +Specifically, Freezeyt needs a WSGI application, +ideally one named `app` which is the default in [Flask] and [Falcon]. +For other frameworks, search the documentation on how to export a WSGI +application. + +Both the application and Freezeyt need to be importable (installed) in your +envuronment. + +Run Freezeyt with two arguments: the Python module with your `app`, +and an output directory. +Note that Freezeyt wants a *module* name (as used in an `import` statement). +Don't use a *file* name with a `.py` suffix. + +For example, if your `app` is defined in the file `my_app.py`, run: + +```console +$ python -m freezeyt my_app _build +``` + +If your application is not named `app`, give its name after a colon. +For example, a [Django WSGI application] is usually in +the `wsgi` submodule and named `application`, so you should run: + +```console +$ python -m freezeyt my_project.wsgi:application _build +``` + +The output directory (here, `_build`), should either not exist yet +or contain output from a previous run of Freezeyt. +Any existing files in it will be removed. +(Freezeyt tries to avoid deleting data it didn't create itself, +but do not rely on this.) + +[WSGI application]: https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ + + +### More examples of CLI usage + +You can tell Freezeyt where the application will be hosted, +so it can generate correct URLs: + +```console +$ python -m freezeyt my_app _build/ --prefix https://pyladies.cz/ +``` + +You can save options like the *prefix* in a file (see [Configuration]), +and then use the `--config` (`-c`) option to use the file: + +```console +$ python -m freezeyt my_app _build/ --config config.yaml +``` + +If you use both a configuration file and CLI options like `--prefix`, +the options override settings from the file: + +```console +$ python -m freezeyt my_app _build/ --prefix https://pyladies.cz/ --config path/to/config.yaml +``` + + +### Python API + +Freezeyt also has a Python API: the `freeze` function +that takes an application to freeze and a configuration dict. +For example: + +```python +from freezeyt import freeze + +config = {'prefix': 'https://pyladies.cz/'} + +freeze(app, config) +``` + +The `config` should be a dict as if read from a YAML configuration +file (see [Configuration]). + +From asynchronous code running in an [`asyncio`][asyncio] event loop, +you can call `freeze_async` instead of `freeze`. + +[asyncio]: https://docs.python.org/3/library/asyncio.html + + +### Middleware {: #middleware } + +Some of Freezeyt's functionality is available as a WSGI middleware. +To use it, wrap your application in `freezeyt.Middeleware`. For example: + +```python +from freezeyt import Middleware + +config = {'prefix': 'https://pyladies.cz/'} + +app = Middleware(app, config) +``` + + +[Configuration]: config.md + +## Project info + +### History + +The Czech Python community uses a lot of static web pages that +are generated from a web application for community purposes. +For example, organizing and announcing workshops, courses, +or meetups. + +The community has been so far relying on [Frozen Flask] and [elsa] +in order to generate the static web content. +The new freezer ought to be used with any arbitrary Python Web +application framework ([Flask], [Django], [Falcon], [Tornado], etc.). +So the community won't be limited by one technology anymore. + +[Frozen Flask]: https://frozen-flask.readthedocs.io/en/latest/ +[elsa]: https://github.com/pyvec/elsa/ +[freezer]: https://github.com/encukou/freezeyt +[Django]: https://www.djangoproject.com/ +[Tornado]: https://www.tornadoweb.org/en/stable/ +[Flask]: https://flask.palletsprojects.com/en/3.0.x/ +[Falcon]: https://falconframework.org/ + + +### Authors +See GitHub history for all [contributors](https://github.com/encukou/freezeyt/graphs/contributors). + + +### License + +This project is licensed under an [MIT License](licence.md). +May it serve you well. diff --git a/docs/licence.md b/docs/licence.md new file mode 100644 index 00000000..1dfd8391 --- /dev/null +++ b/docs/licence.md @@ -0,0 +1,4 @@ + +Freezeyt is available under the following licence. + +# {!../LICENCE.MIT!} diff --git a/docs/pyapi.md b/docs/pyapi.md new file mode 100644 index 00000000..8491144b --- /dev/null +++ b/docs/pyapi.md @@ -0,0 +1,81 @@ +# Freezeyt's Python API + +## Functions + + +::: freezeyt.freeze + +::: freezeyt.freeze_async + + +## WSGI Middleware + +::: freezeyt.Middleware + + +## Hook Arguments + +Objects of these classes are passed to custom [hooks][conf-hooks] and +[plugins][conf-plugins]. + +::: freezeyt.FreezeInfo + +::: freezeyt.TaskInfo + + + +## Exceptions + +::: freezeyt.VersionMismatch +::: freezeyt.InfiniteRedirection +::: freezeyt.ExternalURLError +::: freezeyt.RelativeURLError +::: freezeyt.UnexpectedStatus +::: freezeyt.MultiError +::: freezeyt.DirectoryExistsError + + +## Types + +::: freezeyt.Config + +A dictionary that holds [configuration][configuration] for Freezeyt. + +## Actions + +Built-in [actions][freeze-actions] are available as functions in +the ``freezeyt.actions`` module. +[Custom actions][custom-actions] should call one of these +functions and return its result. + +::: freezeyt.actions.warn +::: freezeyt.actions.ignore +::: freezeyt.actions.follow +::: freezeyt.actions.save +::: freezeyt.actions.error + +## URL finders + +Built-in [URL finders][conf-url_finders] are available as +functions in the ``freezeyt.url_finders`` module: + +::: freezeyt.url_finders.get_html_links +::: freezeyt.url_finders.get_html_links_async + +::: freezeyt.url_finders.get_css_links +::: freezeyt.url_finders.get_css_links_async + +::: freezeyt.url_finders.none + +## Plugins + +[Built-in plugins][built-in-plugins] are available in the +``freezeyt.plugins`` module: + +::: freezeyt.plugins.ProgressBarPlugin +::: freezeyt.plugins.LogPlugin +::: freezeyt.plugins.GHPagesPlugin + +## Utilities + +::: freezeyt.url_to_path diff --git a/docs/why.md b/docs/why.md new file mode 100644 index 00000000..eb508f9f --- /dev/null +++ b/docs/why.md @@ -0,0 +1,125 @@ +# Why Freezeyt? + +Freezeyt helps you build *static websites*. + + +## Static sites + +According to [Wikipedia](https://en.wikipedia.org/wiki/Static_web_page): + +> A **static web page**, sometimes called a **flat page** or a +> **stationary page**, is a web page that is delivered to a web browser +> exactly as stored, in contrast to *dynamic web pages* which are generated +> by a web application. + +Compared to dynamic websites: +- Static sites can be hosted on any platform that allows Web hosting. +- There is no data or database that would need backups. +- Static sites are easily archived, for example by the Wayback machine. +- Static sites are usually more secure: they need only need a Web server, + not processes like Python or a database. + +However, static sites also have significant limitations. +In particular, users cannot make any changes to a static site. +Adding comments, publishing posts, even adding “likes” is not possible. + +With every change -- for example, adding a blog post -- a static site needs +to be *rebuilt* and re-*uploaded* to a Web server. + + +## Static first + +With Freezeyt, your website *is* generated by an application, +but then stored on disk exactly as it will be delivered to the web browser. + +This means that you can write your app in the traditional way, using a Python +framework like [Flask], [Falcon] or [Django], and then *freeze* it to produce +an easy-to-host, read-only version. + +This works great in the early phases of a project, when you're setting up +the structure of the app and don't *yet* have users that would want to make +changes. Or you want to focus on other things than enabling comments. + +It also works great at the late stages of a project: for example, +after a conference is over, you'll want to keep a read-only archive +of its site as long, and as cheaply, as possible. + +We call this approach *static-first*, and think of it as part of a family of +*graceful degradation* principles, where design starts with simpler, lower-tech +solutions, adding extras -- JavaScript, animations, images, CSS styles -- only +when the basics are in place, and making sure that the site works as well as +possible without these extras. + +In other words, a *static-first* website has a *read-only* mode, which is +cheap and easy to host but can only be updated by the admin. +It can *also* have dynamic features on top, but if those features are not +available, it degrades gracefully. + + +[Django]: https://www.djangoproject.com/ +[Flask]: https://flask.palletsprojects.com/en/3.0.x/ +[Falcon]: https://falconframework.org/ + + +## Considerations for static sites + +Not every website can be converted (“frozen”) to a set of files that can be +served statically. +There are two main concepts to keep in mind when designing a static site: +MIME type mapping, limited number of pages, and known URLs. + +### MIME type mapping {: #mime-type-mapping } + +When a Web server sends a Web page, it sends *headers* in addition to +the content. +These headers are lost when a site is converted to simple files. +(There are ways to preserve headers in a static site, but that requires custom +server configuration, making the static site less portable than it could be.) + +The most important lost header is [Content-Type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type), +which encodes the type of a page: `text/html` for a Web page, `image/png` +for a picture, `text/css` for a stylesheet. + +In a static site, this information is typically stored in *file extensions*: +a Web page is named `index.html`; an image is `smile.png`; a stylesheet +is `body.css`. +When the static server serves the file, it will generate a Content-Type header +based on the extension. + +The extension is part of the file name, and thus part of the URL of +the resource. +Authors of static websites must be careful to keep the extension in sync with +the Content-Type. +Otherwise the static server will not guess the type correctly, leading to +broken sites. + +Freezeyt is designed to help you keep Content-Type and file extensions in sync, +whether your app is currently not static-only or not. + + +### Limited number of pages {: #limited-num-pages } + +For example, a calendar app might have a page for *any* month of any year, past +or future. That many pages wouldn't fit on any reasonable disk. +The static version of such a page would need to be limited, perhaps to one +decade or century. + +This needs to be done in the application itself. +To hide a page from Freezeyt only, the app can set the +[`Freezeyt-Action`][freeze-actions] HTTP header to `ignore`. + + +### Known URLs {: #known-urls } + +Most pages of a typical websites are reachable via hyperlinks from the home +page. +Freezeyt will follow such links (in HTML and CSS documents) to find pages +it needs to save. +However, some types of pages aren't linked this way. For example: + +- Pages redirecting from old, obsolete URLs to new locations. +- Data or script files loaded with JavaScript. + +You will need to tell Freezeyt about such pages using the +[`extra_pages`][conf-extra_pages] mechanism. + diff --git a/freezeyt/__init__.py b/freezeyt/__init__.py index de38b686..190486c0 100644 --- a/freezeyt/__init__.py +++ b/freezeyt/__init__.py @@ -4,6 +4,7 @@ from freezeyt.freezer import default_url_to_path as url_to_path from freezeyt.middleware import Middleware from freezeyt.types import Config +from freezeyt.hooks import FreezeInfo, TaskInfo __version__ = '1.1.1' @@ -21,4 +22,6 @@ 'MultiError', 'VersionMismatch', 'Config', + "FreezeInfo", + "TaskInfo", ] diff --git a/freezeyt/actions.py b/freezeyt/actions.py index a3168560..aa737ae1 100644 --- a/freezeyt/actions.py +++ b/freezeyt/actions.py @@ -7,6 +7,7 @@ def warn(task: TaskInfo) -> str: + """Save the content, but send warn message to stdout.""" url = task.get_a_url() response_status = task._task.response_status if response_status is None: @@ -21,6 +22,11 @@ def warn(task: TaskInfo) -> str: def follow(task: TaskInfo) -> str: + """Save content from the redirected location. + + This requires a Location header, which is usually added for redirects. + Redirects to external pages are not supported. + """ url = task._task.get_a_url() response_headers = task._task.response_headers if response_headers is None: @@ -41,14 +47,17 @@ def follow(task: TaskInfo) -> str: def ignore(task: TaskInfo) -> str: + """Do not save any content for the page.""" return 'ignore' def save(task: TaskInfo) -> str: + """Save the body of the page.""" return 'save' def error(task: TaskInfo) -> str: + """Raise an exception.""" return 'error' diff --git a/freezeyt/filesaver.py b/freezeyt/filesaver.py index a2aa9857..46504cba 100644 --- a/freezeyt/filesaver.py +++ b/freezeyt/filesaver.py @@ -10,7 +10,7 @@ class DirectoryExistsError(Exception): - """Attempt to overwrite directory that doesn't contain freezeyt output""" + """Attempt to overwrite directory that doesn't contain freezeyt output.""" class FileSaver(Saver): diff --git a/freezeyt/freezer.py b/freezeyt/freezer.py index 1d8647cd..8b703bdf 100644 --- a/freezeyt/freezer.py +++ b/freezeyt/freezer.py @@ -44,6 +44,15 @@ def freeze(app: Optional[WSGIApplication], config: Config) -> SaverResult: + """Freeze the given *app*. + + + Args: + app: The application to freeze. If `None`, the application + is taken from *config*. + config: The configuration dict. See [Configuration][configuration] + for what goes in. + """ return asyncio_run(freeze_async(app, config)) @@ -51,6 +60,10 @@ async def freeze_async( app: Optional[WSGIApplication], config: Config, ) -> SaverResult: + """Asynchronous version of [freeze][freezeyt.freeze]. + + If an asyncio event loop is active, call (and `await`) this function + rather than [freeze][freezeyt.freeze].""" freezer = Freezer(app, config) try: await freezer.prepare() @@ -102,6 +115,13 @@ def parse_handlers( def default_url_to_path(path: str) -> str: + """Return the filesystem path corresponding to the given URL path. + + Note that the input should only contain the path part of an URL; not, + for example, a hostname. + + This function adds `index.html` to paths ending with a slash. + """ if path.endswith('/') or not path: path = path + 'index.html' return encode_file_path(path) @@ -175,13 +195,13 @@ def update_status(self, old_status, new_status): new_collection[self.path] = self class IsARedirect(BaseException): - """Raised when a page redirects and freezing it should be postponed""" + """Raised when a page redirects and freezing it should be postponed.""" class IgnorePage(BaseException): - """Raised when freezing a page should be ignored""" + """Raised when freezing a page should be ignored.""" class VersionMismatch(ValueError): - """Raised when major version in config is not correct""" + """Raised when major version in config is not correct.""" def needs_semaphore(func): """Decorator for a "task" method that holds self.semaphore when running""" diff --git a/freezeyt/hooks.py b/freezeyt/hooks.py index e4aa9143..9a8d6373 100644 --- a/freezeyt/hooks.py +++ b/freezeyt/hooks.py @@ -14,7 +14,11 @@ def __init__(self, task: 'Task'): self._freezer = task.freezer def get_a_url(self) -> str: - """Return a URL of this page""" + """Return a URL of this page + + Note that a page may be reachable via several URLs; this function + returns an arbitrary one. + """ return urllib.parse.urlunsplit(self._task.get_a_url()) @property @@ -24,10 +28,16 @@ def path(self) -> str: @property def freeze_info(self) -> 'FreezeInfo': + """ + A [`FreezeInfo`][freezeyt.FreezeInfo] object corresponding to the + entire freeze process. + """ + return self._freezer.freeze_info @property def exception(self) -> Optional[BaseException]: + """For failed tasks, the exception raised.""" aio_task = self._task.asyncio_task if aio_task is None: return None @@ -40,7 +50,8 @@ def exception(self) -> Optional[BaseException]: def reasons(self) -> Iterable[str]: """A list of strings explaining why the given page was visited. - New entries may be added as the freezing goes on. + Note that as the freezing progresses, new reasons may be added to + existing tasks. """ return sorted(self._task.reasons) @@ -52,9 +63,27 @@ def __init__(self, freezer: 'Freezer'): self._freezer = freezer def add_url(self, url: str, reason: Optional[str] = None) -> None: + """Add the URL to the set of pages to be frozen. + + Args: + url: The URL to add. + + If that URL was frozen already or is external + (that is, outside the [prefix][conf-prefix]), + `add_url` does nothing. + + reason: A note that will be used in error messages as + the reason why the added URL is being handled. + """ self._freezer.add_task(parse_absolute_url(url), reason=reason) def add_hook(self, hook_name: str, func: Callable) -> None: + """Register an additional hook function. + + Args: + hook_name: Hook name. See [hook docs][conf-hooks] for a list. + func: Function to call. + """ self._freezer.add_hook(hook_name, func) @property @@ -63,12 +92,22 @@ def fail_fast(self) -> bool: @property def total_task_count(self) -> int: + """Number of pages Freezeyt currently “knows about”. + + This includes pages that are already done plus ones that are + scheduled to be frozen. + """ return sum( len(tasks) for tasks in self._freezer.task_collections.values() ) @property def done_task_count(self) -> int: + """The number of pages that are done. + + This includes both pages that were successfully frozen and failed ones. + """ + # Import TaskStatus here to avoid a circular import # (since freezer imports hooks) from freezeyt.freezer import TaskStatus @@ -79,6 +118,7 @@ def done_task_count(self) -> int: @property def failed_task_count(self) -> int: + """The number of pages that failed to freeze.""" # Import TaskStatus here, see done_task_count from freezeyt.freezer import TaskStatus return len(self._freezer.task_collections[TaskStatus.FAILED]) diff --git a/freezeyt/middleware.py b/freezeyt/middleware.py index 04fa8a6c..068635a5 100644 --- a/freezeyt/middleware.py +++ b/freezeyt/middleware.py @@ -14,6 +14,24 @@ class Middleware: + """WSGI middleware. + + By default, the middleware: + + - serves [extra pages][conf-extra_pages] and [extra files][conf-extra_files] + - checks that [mime types correspond to extensions][mime-type-checking] + + If [static mode][conf-static_mode] is enabled, the middleware gives a + preview of how the application would work when frozen. Specifically: + + - HTTP requests other than `GET` (and `OPTIONS`) are blocked + - Non-essential HTTP headers are removed + - Query strings and request bodies are removed + + Args: + app: The application to wrap + config: The configuration dict, as for [freeze][freezeyt.freeze] + """ def __init__(self, app: WSGIApplication, config: Config): self.app = app self.mimetype_checker = MimetypeChecker(config) diff --git a/freezeyt/plugins.py b/freezeyt/plugins.py index 525653c8..34041b2b 100644 --- a/freezeyt/plugins.py +++ b/freezeyt/plugins.py @@ -14,6 +14,7 @@ class GitCommandError(ValueError): """An exception occurred while executing git commands.""" class ProgressBarPlugin: + """Plugin to fill a CLI progress par as a site is being frozen.""" bar_format = '{percentage:3.0f}%▕{bar}▏{elapsed}, {rate:.2f} pg/s' def __init__(self, freeze_info: FreezeInfo): self.manager = enlighten.get_manager() @@ -34,6 +35,7 @@ def update_bar(self, task_info: TaskInfo) -> None: self.counter.update(0) class LogPlugin: + """Plugin to log progress messages to stderr.""" def __init__(self, freeze_info: FreezeInfo): freeze_info.add_hook('page_frozen', self.page_frozen) freeze_info.add_hook('page_failed', self.page_failed) @@ -64,6 +66,11 @@ def page_failed(self, task_info: TaskInfo) -> None: traceback.print_exception(type(exc), exc, exc.__traceback__) class GHPagesPlugin: + """Plugin for GitHub Pages integration. + + Saves the output to a Git repository, and adds extra files necessary + for GitHub Pages (`CNAME` and `.nojekyll`). + """ def __init__(self, freeze_info: FreezeInfo): if freeze_info._freezer.prefix.path != "/": raise ValueError("When using the Github Pages plugin, you can't specify a path in the prefix, so github can't handle it.") diff --git a/freezeyt/url_finders.py b/freezeyt/url_finders.py index 67930a2d..4e6ced97 100644 --- a/freezeyt/url_finders.py +++ b/freezeyt/url_finders.py @@ -63,6 +63,7 @@ def get_links_from_node( def get_html_links( html_file: BinaryIO, base_url: str, headers: _Headers=None, ) -> Iterable[str]: + """Yield URLs linked from a HTML page.""" content = html_file.read() return _get_html_links(content, base_url, headers) @@ -70,6 +71,7 @@ def get_html_links( def get_css_links( css_file: BinaryIO, base_url: str, headers: _Headers=None, ) -> Iterable[str]: + """Yield URLs linked from a CSS file.""" content = css_file.read() return _get_css_links(content, base_url, headers) @@ -77,6 +79,7 @@ def get_css_links( async def get_css_links_async( css_file: BinaryIO, base_url: str, headers: _Headers=None, ) -> Iterable[str]: + """Yield URLs linked from a HTML page, asynchronously.""" loop = compat.get_running_loop() content = css_file.read() return await loop.run_in_executor( @@ -87,6 +90,7 @@ async def get_css_links_async( async def get_html_links_async( html_file: BinaryIO, base_url: str, headers: _Headers=None, ) -> Iterable[str]: + """Yield URLs linked from a CSS file, asynchronously.""" loop = compat.get_running_loop() content = html_file.read() return await loop.run_in_executor( @@ -96,6 +100,11 @@ async def get_html_links_async( def none( html_file: BinaryIO, base_url: str, headers: _Headers=None, ) -> Iterable[str]: + """Return an empty sequence. + + Useful for text-based configuration, where you can specify "none" to + disable finding links. + """ return [] if TYPE_CHECKING: diff --git a/freezeyt/util.py b/freezeyt/util.py index d822b840..d49975b7 100644 --- a/freezeyt/util.py +++ b/freezeyt/util.py @@ -20,7 +20,7 @@ class InfiniteRedirection(Exception): - """Infinite redirection was detected with redirect_policy='follow'""" + """Infinite redirection was detected in the `'follow'` [action][freeze-actions].""" def __init__(self, task: 'Task'): redirects_to = task.redirects_to assert redirects_to is not None @@ -30,16 +30,16 @@ def __init__(self, task: 'Task'): ) class ExternalURLError(ValueError): - """Unexpected external URL specified""" + """Unexpected external URL specified.""" class RelativeURLError(ValueError): - """Absolute URL was expected""" + """Unexpected relative URL was expected.""" class UnsupportedSchemeError(ValueError): - """Raised for URLs with unsupported schemes""" + """Raised for URLs with unsupported schemes.""" class UnexpectedStatus(ValueError): - """The application returned an unexpected status code for a page""" + """The application returned an unexpected status code for a page.""" def __init__(self, url: AbsoluteURL, status: str): self.url = urllib.parse.urlunsplit(url) self.status = status @@ -55,7 +55,11 @@ def __init__(self, expected: List[str], got: str, url_path: str): ) class MultiError(_MultiErrorBase): - """Contains multiple errors""" + """Contains multiple errors. + + On Python 3.11 and above, this is a subclass of the built-in + `ExceptionGroup`. + """ tasks: 'Sequence[TaskInfo]' if not HAVE_EXCEPTION_GROUP: diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..c2af471f --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,47 @@ +site_name: Freezeyt +# site_url: ... +repo_url: https://github.com/encukou/freezeyt +edit_uri: edit/main/docs/ +# site_description: +nav: + - 'Home': 'index.md' + - 'Why?': 'why.md' + - 'Configuration': 'config.md' + - 'API': 'pyapi.md' + - 'Contributing': 'contrib.md' +not_in_nav: + licence.md +theme: + name: mkdocs + color_mode: auto + user_color_mode_toggle: true + navigation_depth: 3 + # highlightjs: false # disables user_color_mode_toggle?! +watch: + - "freezeyt/" +plugins: + - mkdocstrings: + handlers: + python: + options: + paths: ["./"] # actually not needed, default + show_source: false + show_root_heading: true + docstring_section_style: "list" + modernize_annotations: true + heading_level: 3 + members_order: source + group_by_category: false + show_bases: false + - autorefs +markdown_extensions: + - toc: + permalink: true + toc_depth: 3 + - codehilite + - tables + - attr_list + - markdown_include.include: + base_path: docs +extra_css: + - css/pygments.css diff --git a/setup.cfg b/setup.cfg index 2aca4918..266d817e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,6 +40,11 @@ dev = falcon freezegun packaging +docs = + mkdocs + pygments + markdown-include + mkdocstrings[python] blog = flask markdown-it-py