diff --git a/.github/workflows/automatic-doc-checks.yml b/.github/workflows/automatic-doc-checks.yml new file mode 100644 index 0000000000..66c917d51d --- /dev/null +++ b/.github/workflows/automatic-doc-checks.yml @@ -0,0 +1,23 @@ +# +name: Automatic doc checks + +on: + push: + branches: [ main ] + pull_request: + paths: + - 'docs/**' # Only run on changes to the docs directory + + workflow_dispatch: + # Manual trigger + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + documentation-checks: + uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main + with: + working-directory: "docs" + fetch-depth: 0 diff --git a/.github/workflows/check_libs.yaml b/.github/workflows/check_libs.yaml index 6ba39d8f32..b46880e64c 100644 --- a/.github/workflows/check_libs.yaml +++ b/.github/workflows/check_libs.yaml @@ -14,6 +14,7 @@ on: - 'LICENSE' - '**.md' - 'renovate.json' + - 'docs/**' jobs: lib-check: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 334cc49a18..ab6017b5a9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ on: paths-ignore: - '**.md' - '.github/renovate.json5' - - '.github/workflows/sync_docs.yaml' + - 'docs/**' schedule: - cron: '53 0 * * *' # Daily at 00:53 UTC # Triggered on push to branch "main" by .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1297ccd9bd..c055ae6d98 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,7 +10,6 @@ on: - 'docs/**' - '.github/renovate.json5' - '.github/workflows/check_libs.yaml' - - '.github/workflows/sync_docs.yaml' jobs: ci-tests: diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml deleted file mode 100644 index b1ff70bacd..0000000000 --- a/.github/workflows/sync_docs.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. -name: Sync docs from Discourse - -on: - workflow_dispatch: - schedule: - - cron: '53 0 * * *' # Daily at 00:53 UTC - -jobs: - sync-docs: - name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v32.0.0 - with: - reviewers: a-velasco - permissions: - contents: write # Needed to create git tags - pull-requests: write # Needed to create PR diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..7469fec387 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,39 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + jobs: + post_checkout: + - git fetch --unshallow || true + # Cancel building pull requests when there aren't changed in the docs directory. + # If there are no changes (git diff exits with 0) we force the command to return with 183. + # This is a special exit code on Read the Docs that will cancel the build immediately. + # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition + - | + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/docs-dev -- 'docs/' '.readthedocs.yaml'; + then + exit 183; + fi + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: dirhtml + configuration: docs/conf.py + fail_on_warning: true + +# If using Sphinx, optionally build your docs in additional formats such as PDF +formats: +- pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/README.md b/README.md index 81b3b37212..c1af7a22cf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![CharmHub Badge](https://charmhub.io/mysql/badge.svg)](https://charmhub.io/mysql) [![Release](https://github.com/canonical/mysql-operator/actions/workflows/release.yaml/badge.svg)](https://github.com/canonical/mysql-operator/actions/workflows/release.yaml) [![Tests](https://github.com/canonical/mysql-operator/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/canonical/mysql-operator/actions/workflows/ci.yaml?query=branch%3Amain) -[![Docs](https://github.com/canonical/mysql-operator/actions/workflows/sync_docs.yaml/badge.svg)](https://github.com/canonical/mysql-operator/actions/workflows/sync_docs.yaml) ## Description diff --git a/docs/.custom_wordlist.txt b/docs/.custom_wordlist.txt new file mode 100644 index 0000000000..50a6100232 --- /dev/null +++ b/docs/.custom_wordlist.txt @@ -0,0 +1,80 @@ +# Leave a blank line at the end of this file to support concatenation +airgap +async +backend +backends +backport +Charmcraft +cjk +cryptographically +databag +dvipng +fonts +freefont +Furo +github +GitHub +GPG +gyre +html +https +InnoDB +Intersphinx +io +ip +Jira +landscape +lang +lastmod +LaTeX +latexmk +MinIO +Multipass +mydumper +mysql +MyST +nameserver +nameservers +Open Graph +otf +PDF +Percona +plantuml +PNG +PR +Pygments +pymarkdown +QEMU +Read the Docs +readthedocs +reStructuredText +Rockcraft +rst +sitemapindex +snap_daemon +Sphinx +Spread +spread_test_example +subproject +subprojects +SVG +tex +texlive +TOC +toctree +txt +uncommenting +URL +utils +VMs +WCAG +whitespace +whitespaces +wordlist +xbstream +xetex +xindy +xml +xtrabackup +yaml +YouTube diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000..aad78f1f7c --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,26 @@ +# Environment +*env*/ +.sphinx/venv/ + +# Sphinx +.sphinx/warnings.txt +.sphinx/.wordlist.dic +.sphinx/.doctrees/ +.sphinx/update/ +.sphinx/node_modules/ + +# Vale +.sphinx/styles/* +.sphinx/vale.ini + +# Build outputs +_build + +# Node.js +package*.json + +# Unrelated cache and config files +.DS_Store +__pycache__ +.idea/ +.vscode/ diff --git a/docs/.sphinx/.pre-commit-config.yaml b/docs/.sphinx/.pre-commit-config.yaml new file mode 100644 index 0000000000..07e0b48cbd --- /dev/null +++ b/docs/.sphinx/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + - repo: local + hooks: + - id: make-spelling + name: Run make spelling + entry: make -C docs spelling + language: system + pass_filenames: false + files: ^docs/.*\.(rst|md|txt)$ + + - id: make-linkcheck + name: Run make linkcheck + entry: make -C docs linkcheck + language: system + pass_filenames: false + files: ^docs/.*\.(rst|md|txt)$ + + - id: make-woke + name: Run make woke + entry: make -C docs woke + language: system + pass_filenames: false + files: ^docs/.*\.(rst|md|txt)$ diff --git a/docs/.sphinx/.pymarkdown.json b/docs/.sphinx/.pymarkdown.json new file mode 100644 index 0000000000..2c4c669dbf --- /dev/null +++ b/docs/.sphinx/.pymarkdown.json @@ -0,0 +1,46 @@ +{ + "plugins": { + "selectively_enable_rules": true, + "heading-style": { + "enabled": true, + "style": "atx" + }, + "commands-show-output": { + "enabled": true + }, + "no-missing-space-atx": { + "enabled": true + }, + "blanks-around-headings": { + "enabled": true + }, + "heading-start-left": { + "enabled": true + }, + "no-trailing-punctuation": { + "enabled": true, + "punctuation": ".,;。,;" + }, + "blanks-around-fences": { + "enabled": true, + "list_items": false + }, + "blanks-around-lists": { + "enabled": true + }, + "hr-style": { + "enabled": true + }, + "no-empty-links": { + "enabled": true + }, + "no-alt-text": { + "enabled": true + } + }, + "extensions": { + "front-matter" : { + "enabled" : true + } + } +} diff --git a/docs/.sphinx/get_vale_conf.py b/docs/.sphinx/get_vale_conf.py new file mode 100644 index 0000000000..e2a81c088a --- /dev/null +++ b/docs/.sphinx/get_vale_conf.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python + +import os +import shutil +import subprocess +import tempfile +import sys +import logging +import argparse + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) + +SPHINX_DIR = os.path.join(os.getcwd(), ".sphinx") + +GITHUB_REPO = "canonical/documentation-style-guide" +GITHUB_CLONE_URL = f"https://github.com/{GITHUB_REPO}.git" + +# Source paths to copy from repo +VALE_FILE_LIST = [ + "styles/Canonical", + "styles/config/vocabularies/Canonical", + "styles/config/dictionaries", + "vale.ini" +] + +def clone_repo_and_copy_paths(file_source_dest, overwrite=False): + """ + Clone the repository to a temporary directory and copy required files + + Args: + file_source_dest: dictionary of file paths to copy from the repository, + and their destination paths + overwrite: boolean flag to overwrite existing files in the destination + + Returns: + bool: True if all files were copied successfully, False otherwise + """ + + if not file_source_dest: + logging.error("No files to copy") + return False + + # Create temporary directory on disk for cloning + temp_dir = tempfile.mkdtemp() + logging.info("Cloning repository <%s> to temporary directory: %s", GITHUB_REPO, temp_dir) + clone_cmd = ["git", "clone", "--depth", "1", GITHUB_CLONE_URL, temp_dir] + + try: + result = subprocess.run( + clone_cmd, + capture_output=True, + text=True, + check=True + ) + logging.debug("Git clone output: %s", result.stdout) + except subprocess.CalledProcessError as e: + logging.error("Git clone failed: %s", e.stderr) + return False + + # Copy files from the cloned repository to the destination paths + is_copy_success = True + for source, dest in file_source_dest.items(): + source_path = os.path.join(temp_dir, source) + + if not os.path.exists(source_path): + is_copy_success = False + logging.error("Source path not found: %s", source_path) + continue + + if not copy_files_to_path(source_path, dest, overwrite): + is_copy_success = False + logging.error("Failed to copy %s to %s", source_path, dest) + + # Clean up temporary directory + logging.info("Cleaning up temporary directory: %s", temp_dir) + shutil.rmtree(temp_dir) + + return is_copy_success + +def copy_files_to_path(source_path, dest_path, overwrite=False): + """ + Copy a file or directory from source to destination + + Args: + source_path: Path to the source file or directory + dest_path: Path to the destination + overwrite: Boolean flag to overwrite existing files in the destination + + Returns: + bool: True if copy was successful, False otherwise + """ + # Skip if source file doesn't exist + if not os.path.exists(source_path): + logging.warning("Source path not found: %s", source_path) + return False + + logging.info("Copying %s to %s", source_path, dest_path) + # Handle existing files + if os.path.exists(dest_path): + if overwrite: + logging.info(" Destination exists, overwriting: %s", dest_path) + if os.path.isdir(dest_path): + shutil.rmtree(dest_path) + else: + os.remove(dest_path) + else: + logging.info(" Destination exists, skip copying (use overwrite=True to replace): %s", + dest_path) + return True # Skip copying + + # Copy the source to destination + try: + if os.path.isdir(source_path): + # entire directory + shutil.copytree(source_path, dest_path) + else: + # individual files + shutil.copy2(source_path, dest_path) + return True + except (shutil.Error, OSError) as e: + logging.error("Copy failed: %s", e) + return False + +def parse_arguments(): + parser = argparse.ArgumentParser(description="Download Vale configuration files") + parser.add_argument("--no-overwrite", action="store_true", help="Don't overwrite existing files") + return parser.parse_args() + +def main(): + # Define local directory paths + vale_files_dict = {file: os.path.join(SPHINX_DIR, file) for file in VALE_FILE_LIST} + + # Parse command line arguments, default to overwrite_enabled = True + overwrite_enabled = not parse_arguments().no_overwrite + + # Download into /tmp through git clone + if not clone_repo_and_copy_paths(vale_files_dict, overwrite=overwrite_enabled): + logging.error("Failed to download files from repository") + return 1 + + logging.info("Download complete") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) # Keep return code diff --git a/docs/.sphinx/metrics/build_metrics.py b/docs/.sphinx/metrics/build_metrics.py new file mode 100644 index 0000000000..529fb85b9b --- /dev/null +++ b/docs/.sphinx/metrics/build_metrics.py @@ -0,0 +1,94 @@ +#!/usr/bin/python3 + +import sys +import argparse +from pathlib import Path +from html.parser import HTMLParser +from urllib.parse import urlsplit + + +class MetricsParser(HTMLParser): + def __init__(self): + super().__init__() + self.int_link_count = 0 + self.ext_link_count = 0 + self.fragment_count = 0 + self.image_count = 0 + self.in_object = 0 + + @property + def link_count(self): + return self.fragment_count + self.int_link_count + self.ext_link_count + + def read(self, file): + """ + Read *file* (a file-like object with a ``read`` method returning + strings) a chunk at a time, feeding each chunk to the parser. + """ + # Ensure the parser state is reset before each file (just in case + # there's an erroneous dangling ) + self.reset() + self.in_object = 0 + buf = '' + while True: + # Parse 1MB chunks at a time + buf = file.read(1024**2) + if not buf: + break + self.feed(buf) + + def handle_starttag(self, tag, attrs): + """ + Count , , and tags to determine the number of internal + and external links, and the number of images. + """ + attrs = dict(attrs) + if tag == 'a' and 'href' in attrs: + # If there's no href, it's an anchor; if there's no hostname + # (netloc) or path, it's just a fragment link within the page + url = urlsplit(attrs['href']) + if url.netloc: + self.ext_link_count += 1 + elif url.path: + self.int_link_count += 1 + else: + self.fragment_count += 1 + elif tag == 'object': + # tags are a bit complex as they nest to offer fallbacks + # and may contain an fallback. We only want to count the + # outer-most in this case + if self.in_object == 0: + self.image_count += 1 + self.in_object += 1 + elif tag == 'img' and self.in_object == 0: + self.image_count += 1 + + def handle_endtag(self, tag): + if tag == 'object': + # Never let in_object be negative + self.in_object = max(0, self.in_object - 1) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument( + 'build_dir', metavar='build-dir', nargs='?', default='.', + help="The directory to scan for HTML files") + config = parser.parse_args(args) + + parser = MetricsParser() + for path in Path(config.build_dir).rglob('*.html'): + with path.open('r', encoding='utf-8', errors='replace') as f: + parser.read(f) + + print('Summarising metrics for build files (.html)...') + print(f'\tlinks: {parser.link_count} (' + f'{parser.fragment_count} #frag…, ' + f'{parser.int_link_count} /int…, ' + f'{parser.ext_link_count} https://ext…' + ')') + print(f'\timages: {parser.image_count}') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs/.sphinx/metrics/source_metrics.sh b/docs/.sphinx/metrics/source_metrics.sh new file mode 100755 index 0000000000..07147d6361 --- /dev/null +++ b/docs/.sphinx/metrics/source_metrics.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# shellcheck disable=all + +VENV=".sphinx/venv/bin/activate" + +files=0 +words=0 +readabilityWords=0 +readabilitySentences=0 +readabilitySyllables=0 +readabilityAverage=0 +readable=true + +# measure number of files (.rst and .md), excluding those in .sphinx dir +files=$(find . -type d -path './.sphinx' -prune -o -type f \( -name '*.md' -o -name '*.rst' \) -print | wc -l) + +# calculate metrics only if source files are present +if [ "$files" -eq 0 ]; then + echo "There are no source files to calculate metrics" +else + # measure raw total number of words, excluding those in .sphinx dir + words=$(find . -type d -path './.sphinx' -prune -o \( -name '*.md' -o -name '*.rst' \) -exec cat {} + | wc -w) + + # calculate readability for markdown source files + echo "Activating virtual environment to run vale..." + source "${VENV}" + + for file in *.md *.rst; do + if [ -f "$file" ]; then + readabilityWords=$(vale ls-metrics "$file" | grep '"words"' | sed 's/[^0-9]*//g') + readabilitySentences=$(vale ls-metrics "$file" | grep '"sentences"' | sed 's/[^0-9]*//g') + readabilitySyllables=$(vale ls-metrics "$file" | grep '"syllables"' | sed 's/[^0-9]*//g') + fi + done + + echo "Deactivating virtual environment..." + deactivate + + # calculate mean number of words + if [ "$files" -ge 1 ]; then + meanval=$((readabilityWords / files)) + else + meanval=$readabilityWords + fi + + readabilityAverage=$(echo "scale=2; 0.39 * ($readabilityWords / $readabilitySentences) + (11.8 * ($readabilitySyllables / $readabilityWords)) - 15.59" | bc) + + # cast average to int for comparison + readabilityAverageInt=$(echo "$readabilityAverage / 1" | bc) + + # value below 8 is considered readable + if [ "$readabilityAverageInt" -lt 8 ]; then + readable=true + else + readable=false + fi + + # summarise latest metrics + echo "Summarising metrics for source files (.md, .rst)..." + echo -e "\ttotal files: $files" + echo -e "\ttotal words (raw): $words" + echo -e "\ttotal words (prose): $readabilityWords" + echo -e "\taverage word count: $meanval" + echo -e "\treadability: $readabilityAverage" + echo -e "\treadable: $readable" +fi diff --git a/docs/.sphinx/pa11y.json b/docs/.sphinx/pa11y.json new file mode 100644 index 0000000000..04dc1e1c5e --- /dev/null +++ b/docs/.sphinx/pa11y.json @@ -0,0 +1,9 @@ +{ + "chromeLaunchConfig": { + "args": [ + "--no-sandbox" + ] + }, + "reporter": "cli", + "standard": "WCAG2AA" +} diff --git a/docs/.sphinx/version b/docs/.sphinx/version new file mode 100644 index 0000000000..26aaba0e86 --- /dev/null +++ b/docs/.sphinx/version @@ -0,0 +1 @@ +1.2.0 diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000..0b498e12f6 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,185 @@ +# Minimal makefile for Sphinx documentation +# +# Add your customisation to `Makefile` instead. + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXDIR = .sphinx +SPHINXOPTS ?= -c . -d $(SPHINXDIR)/.doctrees -j auto +SPHINXBUILD ?= $(VENVDIR)/bin/sphinx-build +SOURCEDIR = . +BUILDDIR = _build +VENVDIR = $(SPHINXDIR)/venv +PA11Y = $(SPHINXDIR)/node_modules/pa11y/bin/pa11y.js --config $(SPHINXDIR)/pa11y.json +VENV = $(VENVDIR)/bin/activate +TARGET = * +ALLFILES = *.rst **/*.rst +METRICSDIR = $(SOURCEDIR)/.sphinx/metrics +REQPDFPACKS = latexmk fonts-freefont-otf texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-font-utils texlive-lang-cjk texlive-xetex plantuml xindy tex-gyre dvipng +CONFIRM_SUDO ?= N +VALE_CONFIG = $(SPHINXDIR)/vale.ini +SPHINX_HOST ?= 127.0.0.1 +SPHINX_PORT ?= 8000 + +# Put it first so that "make" without argument is like "make help". +help: + @echo + @echo "-------------------------------------------------------------" + @echo "* watch, build and serve the documentation: make run" + @echo "* only build: make html" + @echo "* only serve: make serve" + @echo "* clean built doc files: make clean-doc" + @echo "* clean full environment: make clean" + @echo "* check links: make linkcheck" + @echo "* check markdown: make lint-md" + @echo "* check spelling: make spelling" + @echo "* check spelling (without building again): make spellcheck" + @echo "* check inclusive language: make woke" + @echo "* check accessibility: make pa11y" + @echo "* check style guide compliance: make vale" + @echo "* check style guide compliance on target: make vale TARGET=*" + @echo "* check metrics for documentation: make allmetrics" + @echo "* other possible targets: make " + @echo "-------------------------------------------------------------" + @echo + +.PHONY: help full‑help html epub pdf linkcheck spelling spellcheck woke \ + vale pa11y run serve install pa11y‑install \ + vale‑install pdf‑prep pdf‑prep‑force clean clean‑doc allmetrics \ + update lint-md + +full-help: $(VENVDIR) + @. $(VENV); $(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @echo "\n\033[1;31mNOTE: This help texts shows unsupported targets!\033[0m" + @echo "Run 'make help' to see supported targets." + +# If requirements are updated, venv should be rebuilt and timestamped. +$(VENVDIR): + @echo "... setting up virtualenv" + python3 -m venv $(VENVDIR) || { echo "You must install python3-venv before you can build the documentation."; exit 1; } + . $(VENV); pip install $(PIPOPTS) --require-virtualenv \ + --upgrade -r requirements.txt \ + --log $(VENVDIR)/pip_install.log + @test ! -f $(VENVDIR)/pip_list.txt || \ + mv $(VENVDIR)/pip_list.txt $(VENVDIR)/pip_list.txt.bak + @. $(VENV); pip list --local --format=freeze > $(VENVDIR)/pip_list.txt + @touch $(VENVDIR) + +pa11y-install: + @command -v $(PA11Y) >/dev/null || { \ + echo "Installing \"pa11y\" from npm..."; echo; \ + mkdir -p $(SPHINXDIR)/node_modules/ ; \ + npm install --prefix $(SPHINXDIR) pa11y; \ + } + +pymarkdownlnt-install: + @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/pymarkdown || pip install pymarkdownlnt + +install: $(VENVDIR) + +run: install + . $(VENV); $(VENVDIR)/bin/sphinx-autobuild -b dirhtml --host $(SPHINX_HOST) --port $(SPHINX_PORT) "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + +# Does not depend on $(BUILDDIR) to rebuild properly at every run. +html: install + . $(VENV); $(SPHINXBUILD) -W --keep-going -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + +epub: install + . $(VENV); $(SPHINXBUILD) -b epub "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + +serve: html + cd "$(BUILDDIR)"; python3 -m http.server --bind 127.0.0.1 8000 + +clean: clean-doc + @test ! -e "$(VENVDIR)" -o -d "$(VENVDIR)" -a "$(abspath $(VENVDIR))" != "$(VENVDIR)" + rm -rf $(VENVDIR) + rm -rf $(SPHINXDIR)/node_modules/ + rm -rf $(SPHINXDIR)/styles + rm -rf $(VALE_CONFIG) + +clean-doc: + git clean -fx "$(BUILDDIR)" + rm -rf $(SPHINXDIR)/.doctrees + +linkcheck: install + . $(VENV) ; $(SPHINXBUILD) -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) || { grep --color -F "[broken]" "$(BUILDDIR)/output.txt"; exit 1; } + exit 0 + +pa11y: pa11y-install html + find $(BUILDDIR) -name *.html -print0 | xargs -n 1 -0 $(PA11Y) + +lint-md: pymarkdownlnt-install + @. $(VENV); pymarkdownlnt --config $(SPHINXDIR)/.pymarkdown.json scan --recurse --exclude=./$(SPHINXDIR)/** $(SOURCEDIR) + +vale-install: install + @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/vale || pip install rst2html vale + @. $(VENV); test -f $(VALE_CONFIG) || python3 $(SPHINXDIR)/get_vale_conf.py + @echo '.Name=="Canonical.400-Enforce-inclusive-terms"' > $(SPHINXDIR)/styles/woke.filter + @echo '.Level=="error" and .Name!="Canonical.500-Repeated-words" and .Name!="Canonical.000-US-spellcheck"' > $(SPHINXDIR)/styles/error.filter + @echo '.Name=="Canonical.000-US-spellcheck"' > $(SPHINXDIR)/styles/spelling.filter + @. $(VENV); find $(SPHINXDIR)/venv/lib/python*/site-packages/vale/vale_bin -size 195c -exec vale --version \; + +woke: vale-install + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @echo "Running Vale acceptable term check against $(TARGET). To change target set TARGET= with make command" + @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/woke.filter' --glob='*.{md,rst}' $(TARGET) + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + +vale: vale-install + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" + @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/error.filter' --glob='*.{md,rst}' $(TARGET) + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + +spelling: vale-install + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" + @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/spelling.filter' --glob='*.{md,rst}' $(TARGET) + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + +spellcheck: spelling + @echo "Please note that the \`make spellcheck\` command is being deprecated in favor of \`make spelling\`" + +pdf-prep: install + @for packageName in $(REQPDFPACKS); do (dpkg-query -W -f='$${Status}' $$packageName 2>/dev/null | \ + grep -c "ok installed" >/dev/null && echo "Package $$packageName is installed") && continue || \ + (echo; echo "PDF generation requires the installation of the following packages: $(REQPDFPACKS)" && \ + echo "" && echo "Run 'sudo make pdf-prep-force' to install these packages" && echo "" && echo \ + "Please be aware these packages will be installed to your system") && exit 1 ; done + +pdf-prep-force: + apt-get update + apt-get upgrade -y + apt-get install --no-install-recommends -y $(REQPDFPACKS) \ + +pdf: pdf-prep + @. $(VENV); sphinx-build -M latexpdf "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + @rm ./$(BUILDDIR)/latex/front-page-light.pdf || true + @rm ./$(BUILDDIR)/latex/normal-page-footer.pdf || true + @find ./$(BUILDDIR)/latex -name "*.pdf" -exec mv -t ./$(BUILDDIR) {} + + @rm -r $(BUILDDIR)/latex + @echo + @echo "Output can be found in ./$(BUILDDIR)" + @echo + +allmetrics: html + @echo "Recording documentation metrics..." + @echo "Checking for existence of vale..." + . $(VENV) + @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/vale || pip install vale + @. $(VENV); test -f $(VALE_CONFIG) || python3 $(SPHINXDIR)/get_vale_conf.py + @. $(VENV); find $(SPHINXDIR)/venv/lib/python*/site-packages/vale/vale_bin -size 195c -exec vale --config "$(VALE_CONFIG)" $(TARGET) > /dev/null \; + @eval '$(METRICSDIR)/source_metrics.sh $(PWD)' + @$(METRICSDIR)/build_metrics.py $(BUILDDIR) + +update: install + @. $(VENV); .sphinx/update_sp.py + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: + $(MAKE) —no-print-directory install + . $(VENV); $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000..93b40ee9ff --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,358 @@ +import datetime +import os +import yaml + +# Configuration for the Sphinx documentation builder. +# All configuration specific to your project should be done in this file. +# +# If you're new to Sphinx and don't want any advanced or custom features, +# just go through the items marked 'TODO'. +# +# A complete list of built-in Sphinx configuration values: +# https://www.sphinx-doc.org/en/master/usage/configuration.html +# +# Our starter pack uses the custom Canonical Sphinx extension +# to keep all documentation based on it consistent and on brand: +# https://github.com/canonical/canonical-sphinx + + +####################### +# Project information # +####################### + +# Project name +# +# TODO: Update with the official name of your project or product + +project = "Charmed MySQL" +author = "Canonical Ltd." + + +# Sidebar documentation title; best kept reasonably short +# +# TODO: To include a version number, add it here (hardcoded or automated). +# +# TODO: To disable the title, set to an empty string. + +html_title = project + " documentation" + + +# Copyright string; shown at the bottom of the page +# +# Now, the starter pack uses CC-BY-SA as the license +# and the current year as the copyright year. +# +# TODO: If your docs need another license, specify it instead of 'CC-BY-SA'. +# +# TODO: If your documentation is a part of the code repository of your project, +# it inherits the code license instead; specify it instead of 'CC-BY-SA'. +# +# NOTE: For static works, it is common to provide the first publication year. +# Another option is to provide both the first year of publication +# and the current year, especially for docs that frequently change, +# e.g. 2022–2023 (note the en-dash). +# +# A way to check a repo's creation date is to get a classic GitHub token +# with 'repo' permissions; see https://github.com/settings/tokens +# Next, use 'curl' and 'jq' to extract the date from the API's output: +# +# curl -H 'Authorization: token ' \ +# -H 'Accept: application/vnd.github.v3.raw' \ +# https://api.github.com/repos/canonical/ | jq '.created_at' + +copyright = "%s CC-BY-SA, %s" % (datetime.date.today().year, author) + + +# Documentation website URL +# +# TODO: Update with the official URL of your docs or leave empty if unsure. +# +# NOTE: The Open Graph Protocol (OGP) enhances page display in a social graph +# and is used by social media platforms; see https://ogp.me/ + +# ogp_site_url = "https://canonical-starter-pack.readthedocs-hosted.com/" + + +# Preview name of the documentation website +# +# TODO: To use a different name for the project in previews, update as needed. + +ogp_site_name = project + + +# Preview image URL +# +# TODO: To customise the preview image, update as needed. + +ogp_image = "https://assets.ubuntu.com/v1/253da317-image-document-ubuntudocs.svg" + + +# Product favicon; shown in bookmarks, browser tabs, etc. + +# TODO: To customise the favicon, uncomment and update as needed. + +# html_favicon = '.sphinx/_static/favicon.png' + + +# Dictionary of values to pass into the Sphinx context for all pages: +# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_context + +html_context = { + # Product page URL; can be different from product docs URL + # + # TODO: Change to your product website URL, + # dropping the 'https://' prefix, e.g. 'ubuntu.com/lxd'. + # + # TODO: If there's no such website, + # remove the {{ product_page }} link from the page header template + # (usually .sphinx/_templates/header.html; also, see README.rst). + "product_page": "canonical.com/data/mysql", + # Product tag image; the orange part of your logo, shown in the page header + # + # TODO: To add a tag image, uncomment and update as needed. + # 'product_tag': '_static/tag.png', + # Your Discourse instance URL + # + # TODO: Change to your Discourse instance URL or leave empty. + # + # NOTE: If set, adding ':discourse: 123' to an .rst file + # will add a link to Discourse topic 123 at the bottom of the page. + "discourse": "https://discourse.charmhub.io", + # Your Mattermost channel URL + # + # TODO: Change to your Mattermost channel URL or leave empty. + "mattermost": "", + # Your Matrix channel URL + # + # TODO: Change to your Matrix channel URL or leave empty. + "matrix": "https://matrix.to/#/#charmhub-data-platform:ubuntu.com", + # Your documentation GitHub repository URL + # + # TODO: Change to your documentation GitHub repository URL or leave empty. + # + # NOTE: If set, links for viewing the documentation source files + # and creating GitHub issues are added at the bottom of each page. + "github_url": "https://github.com/canonical/mysql-operator", + # Docs branch in the repo; used in links for viewing the source files + # + # TODO: To customise the branch, uncomment and update as needed. + 'repo_default_branch': 'main', + # Docs location in the repo; used in links for viewing the source files + "repo_folder": "/docs/", + + # Docs location in the repo; used in links for viewing the source files + "repo_folder": "/docs/", + + # TODO: To enable or disable the Previous / Next buttons at the bottom of pages + # Valid options: none, prev, next, both + "sequential_nav": "none", + + # TODO: To enable listing contributors on individual pages, set to True + "display_contributors": True, + + # Required for feedback button + 'github_issues': 'enabled', +} + +# TODO: To enable the edit button on pages, uncomment and change the link to a +# public repository on GitHub or Launchpad. Any of the following link domains +# are accepted: +# - https://github.com/example-org/example" +# - https://launchpad.net/example +# - https://git.launchpad.net/example +# +html_theme_options = { +'source_edit_link': 'https://github.com/canonical/mysql-operator', +} + +# Project slug; see https://meta.discourse.org/t/what-is-category-slug/87897 +# +# TODO: If your documentation is hosted on https://docs.ubuntu.com/, +# uncomment and update as needed. + +# slug = '' + +####################### +# Sitemap configuration: https://sphinx-sitemap.readthedocs.io/ +####################### + +# Base URL of RTD hosted project + +html_baseurl = 'https://canonical-charmed-mysql.readthedocs-hosted.com/' + +# URL scheme. Add language and version scheme elements. +# When configured with RTD variables, check for RTD environment so manual runs succeed: + +if 'READTHEDOCS_VERSION' in os.environ: + version = os.environ["READTHEDOCS_VERSION"] + sitemap_url_scheme = '{version}{link}' +else: + sitemap_url_scheme = 'MANUAL/{link}' + +# Include `lastmod` dates in the sitemap: + +sitemap_show_lastmod = True + +####################### +# Template and asset locations +####################### + +#html_static_path = ["_static"] +#templates_path = ["_templates"] + + +############# +# Redirects # +############# + +# To set up redirects: https://documatt.gitlab.io/sphinx-reredirects/usage.html +# For example: 'explanation/old-name.html': '../how-to/prettify.html', + +# To set up redirects in the Read the Docs project dashboard: +# https://docs.readthedocs.io/en/stable/guides/redirects.html + +# NOTE: If undefined, set to None, or empty, +# the sphinx_reredirects extension will be disabled. + +# redirects = {} + +# rediraffe_redirects = "redirects.txt" + +########################### +# Link checker exceptions # +########################### + +# A regex list of URLs that are ignored by 'make linkcheck' +# +# TODO: Remove or adjust the ACME entry after you update the contributing guide + +linkcheck_ignore = [ + "http://127.0.0.1:8000", + "https://github.com/canonical/ACME/*", + "https://matrix.to/*", + "https://portal.azure.com/*", + "https://dev.mysql.com/*", + "https://www.mysql.com/*" + ] + + +# A regex list of URLs where anchors are ignored by 'make linkcheck' + +linkcheck_anchors_ignore_for_url = [r"https://github\.com/.*"] + +# give linkcheck multiple tries on failure +# linkcheck_timeout = 30 +linkcheck_retries = 3 + +######################## +# Configuration extras # +######################## + +# Custom MyST syntax extensions; see +# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html +# +# NOTE: By default, the following MyST extensions are enabled: +# substitution, deflist, linkify + +myst_enable_extensions = set() + +# Enables MyST references via standard anchors +myst_heading_anchors = 3 + +# Custom Sphinx extensions; see +# https://www.sphinx-doc.org/en/master/usage/extensions/index.html + +# NOTE: The canonical_sphinx extension is required for the starter pack. +# It automatically enables the following extensions: +# - custom-rst-roles +# - myst_parser +# - notfound.extension +# - related-links +# - sphinx_copybutton +# - sphinx_design +# - sphinx_reredirects +# - sphinx_tabs.tabs +# - sphinxcontrib.jquery +# - sphinxext.opengraph +# - terminal-output +# - youtube-links + +extensions = [ + "canonical_sphinx", + "sphinxcontrib.cairosvgconverter", + "sphinx_last_updated_by_git", + "sphinx.ext.intersphinx", + "sphinx_sitemap", +] + +# Excludes files or directories from processing + +exclude_patterns = [ +] + +# Adds custom CSS files, located under 'html_static_path' + +# html_css_files = [] + + +# Adds custom JavaScript files, located under 'html_static_path' + +# html_js_files = [] + + +# Specifies a reST snippet to be appended to each .rst file + +rst_epilog = """ +.. include:: /reuse/links.txt +.. include:: /reuse/substitutions.txt +""" + +# Feedback button at the top; enabled by default +# +# TODO: To disable the button, uncomment this. + +# disable_feedback_button = True + + +# Your manpage URL +# +# TODO: To enable manpage links, uncomment and replace {codename} with required +# release, preferably an LTS release (e.g. noble). Do *not* substitute +# {section} or {page}; these will be replaced by sphinx at build time +# +# NOTE: If set, adding ':manpage:' to an .rst file +# adds a link to the corresponding man section at the bottom of the page. + +# manpages_url = 'https://manpages.ubuntu.com/manpages/{codename}/en/' + \ +# 'man{section}/{page}.{section}.html' + + +# Specifies a reST snippet to be prepended to each .rst file +# This defines a :center: role that centers table cell content. +# This defines a :h2: role that styles content for use with PDF generation. + +rst_prolog = """ +.. role:: center + :class: align-center +.. role:: h2 + :class: hclass2 +.. role:: woke-ignore + :class: woke-ignore +.. role:: vale-ignore + :class: vale-ignore +""" + +# Workaround for https://github.com/canonical/canonical-sphinx/issues/34 + +if "discourse_prefix" not in html_context and "discourse" in html_context: + html_context["discourse_prefix"] = html_context["discourse"] + "/t/" + +# Workaround for substitutions.yaml + +if os.path.exists('./reuse/substitutions.yaml'): + with open('./reuse/substitutions.yaml', 'r') as fd: + myst_substitutions = yaml.safe_load(fd.read()) + +# Add configuration for intersphinx mapping + +# intersphinx_mapping = {} diff --git a/docs/explanation.md b/docs/explanation.md deleted file mode 100644 index 3a8eb478c8..0000000000 --- a/docs/explanation.md +++ /dev/null @@ -1,32 +0,0 @@ -# Explanation - -Additional context about key concepts behind the MySQL charm. - -## Core concepts and design -* [Architecture] -* [Interfaces and endpoints] -* [Juju] -* [Legacy charm] - -## Operational concepts -* [Users] -* [Logs] - * [Audit logs] - -## Security and hardening -* [Security hardening guide][Security] - * [Cryptography] - - - -[Architecture]: /t/11756 -[Interfaces and endpoints]: /t/10250 -[Juju]: /t/11959 -[Legacy charm]: /t/10788 - -[Users]: /t/10789 -[Logs]: /t/11993 -[Audit logs]: /t/15424 - -[Security]: /t/16784 -[Cryptography]: /t/16785 \ No newline at end of file diff --git a/docs/explanation/e-architecture.md b/docs/explanation/architecture.md similarity index 74% rename from docs/explanation/e-architecture.md rename to docs/explanation/architecture.md index a6ab02d031..81e444f664 100644 --- a/docs/explanation/e-architecture.md +++ b/docs/explanation/architecture.md @@ -2,10 +2,10 @@ [MySQL](https://www.mysql.com/) is the world’s most popular open source database. The "[Charmed MySQL](https://charmhub.io/mysql)" is a Juju-based operator to deploy and support MySQL from [day 0 to day 2](https://codilime.com/blog/day-0-day-1-day-2-the-software-lifecycle-in-the-cloud-age/), it is based on the [MySQL Community Edition](https://www.mysql.com/products/community/) using the built-in cluster functionality: [MySQL InnoDB ClusterSet](https://dev.mysql.com/doc/mysql-shell/8.0/en/innodb-clusterset.html). - ## HLD (High Level Design) The charm design leverages on the SNAP “[charmed-mysql](https://snapcraft.io/charmed-mysql)” which is deployed by Juju on the specified VM/MAAS/bare-metal machine based on Ubuntu Jammy/22.04. SNAP allows to run MySQL service(s) in a secure and isolated environment ([strict confinement](https://ubuntu.com/blog/demystifying-snap-confinement)). The installed SNAP: + ``` > juju ssh mysql/0 > snap list charmed-mysql @@ -40,11 +40,11 @@ The `mysqld` snap service is a main MySQL instance which is normally up and runn The `mysql-router` snap service used in [Charmed MySQL Router](https://charmhub.io/mysql-router?channel=dpe/edge) only and should be stopped on [Charmed MySQL](https://charmhub.io/mysql) deployments. -All `exporter` services are activated after the relation with [COS Monitoring](/t/9900) only. +All `exporter` services are activated after the relation with [COS Monitoring](/how-to/monitoring-cos/enable-monitoring) only. -> **:information_source: Note:** it is possible to star/stop/restart snap services manually but it is NOT recommended to avoid a split brain with a charm state machine! Do it with a caution!!! +> **Note:** it is possible to star/stop/restart snap services manually but it is NOT recommended to avoid a split brain with a charm state machine! Do it with a caution!!! -> **:warning: Important:** all snap resources must be executed under the special user `snapd_daemon` only! +> **Important:** all snap resources must be executed under the special user `snapd_daemon` only! The snap "charmed-mysql" also ships list of tools used by charm: * `charmed-mysql.mysql` (alias `mysql`) - mysql client to connect `mysqld`. @@ -54,9 +54,8 @@ The snap "charmed-mysql" also ships list of tools used by charm: * `charmed-mysql.xtrabackup` - a tool to backup/restore MySQL DB. The `mysql` and `mysqlsh` are well known and popular tools to manage MySQL. -The `xtrabackup (xbcloud+xbstream)` used for [MySQL Backups](/t/9896) only to store backups on S3 compatible storage. +The `xtrabackup (xbcloud+xbstream)` used for [MySQL Backups](/how-to/back-up-and-restore/create-a-backup) only to store backups on S3 compatible storage. - ## Integrations ### MySQL Router @@ -81,48 +80,48 @@ The charm "[MySQL Test App](https://charmhub.io/mysql-test-app)" is a Canonical ### Grafana -Grafana is an open-source visualization tools that allows to query, visualize, alert on, and visualize metrics from mixed datasources in configurable dashboards for observability. This charms is shipped with its own Grafana dashboard and supports integration with the [Grafana Operator](https://charmhub.io/grafana-k8s) to simplify observability. Please follow [COS Monitoring](/t/9900) setup. +Grafana is an open-source visualization tools that allows to query, visualize, alert on, and visualize metrics from mixed datasources in configurable dashboards for observability. This charms is shipped with its own Grafana dashboard and supports integration with the [Grafana Operator](https://charmhub.io/grafana-k8s) to simplify observability. Please follow [COS Monitoring](/how-to/monitoring-cos/enable-monitoring) setup. ### Loki -Loki is an open-source fully-featured logging system. This charms is shipped with support for the [Loki Operator](https://charmhub.io/loki-k8s) to collect the generated logs. Please follow [COS Monitoring](/t/9900) setup. +Loki is an open-source fully-featured logging system. This charms is shipped with support for the [Loki Operator](https://charmhub.io/loki-k8s) to collect the generated logs. Please follow [COS Monitoring](/how-to/monitoring-cos/enable-monitoring) setup. ### Prometheus -Prometheus is an open-source systems monitoring and alerting toolkit with a dimensional data model, flexible query language, efficient time series database and modern alerting approach. This charm is shipped with a Prometheus exporters, alerts and support for integrating with the [Prometheus Operator](https://charmhub.io/prometheus-k8s) to automatically scrape the targets. Please follow [COS Monitoring](/t/9900) setup. +Prometheus is an open-source systems monitoring and alerting toolkit with a dimensional data model, flexible query language, efficient time series database and modern alerting approach. This charm is shipped with a Prometheus exporters, alerts and support for integrating with the [Prometheus Operator](https://charmhub.io/prometheus-k8s) to automatically scrape the targets. Please follow [COS Monitoring](/how-to/monitoring-cos/enable-monitoring) setup. - ## LLD (Low Level Design) -Please check the charm state machines displayed on [workflow diagrams](/t/10031). The low-level logic is mostly common for both VM and K8s charms. +Please check the charm state machines displayed on [workflow diagrams](https://charmhub.io/mysql-k8s/docs/e-flowcharts). The low-level logic is mostly common for both VM and K8s charms. -### Juju Events +### Juju events Accordingly to the [Juju SDK](https://juju.is/docs/sdk/event): “an event is a data structure that encapsulates part of the execution context of a charm”. For this charm, the following events are observed: -1. [on_install](https://juju.is/docs/sdk/install-event): install the snap "charmed-mysql" and perform basic preparations to bootstrap the cluster on the first leader (or join the already configured cluster). -2. [leader-elected](https://juju.is/docs/sdk/leader-elected-event): generate all the secrets to bootstrap the cluster. -3. [leader-settings-changed](https://juju.is/docs/sdk/leader-settings-changed-event): Handle the leader settings changed event. -4. [start](https://juju.is/docs/sdk/start-event): Init/setting up the cluster node. -5. [config_changed](https://juju.is/docs/sdk/config-changed-event): usually fired in response to a configuration change using the GUI or CLI. Create and set default cluster and cluster-set names in the peer relation databag (on the leader only). -6. [update-status](https://juju.is/docs/sdk/update-status-event): Takes care of workload health checks. +1. [`on_install`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#install): install the snap "charmed-mysql" and perform basic preparations to bootstrap the cluster on the first leader (or join the already configured cluster). +2. [`leader-elected`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#leader-elected): generate all the secrets to bootstrap the cluster. +3. [`leader-settings-changed`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#leader-settings-changed): Handle the leader settings changed event. +4. [`start`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#start): Init/setting up the cluster node. +5. [`config_changed`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#config-changed): usually fired in response to a configuration change using the GUI or CLI. Create and set default cluster and cluster-set names in the peer relation databag (on the leader only). +6. [`update-status`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#update-status): Takes care of workload health checks. -### Charm Code Overview +### Charm code overview -The "[src/charm.py](https://github.com/canonical/mysql-operator/blob/main/src/charm.py)" is the default entry point for a charm and has the [MySQLCharmBase](https://github.com/canonical/mysql-operator/blob/main/lib/charms/mysql/v0/mysql.py) Python class which inherits from CharmBase. +[`src/charm.py`](https://github.com/canonical/mysql-operator/blob/main/src/charm.py) is the default entry point for a charm and has the [`MySQLCharmBase`](https://github.com/canonical/mysql-operator/blob/main/lib/charms/mysql/v0/mysql.py) Python class which inherits from `CharmBase`. -CharmBase is the base class from which all Charms are formed, defined by [Ops](https://juju.is/docs/sdk/ops) (Python framework for developing charms). See more information in [Charm](https://juju.is/docs/sdk/constructs#heading--charm). +`CharmBase` is the base class from which all Charms are formed, defined by [Ops](https://ops.readthedocs.io/en/latest/) (Python framework for developing charms). See more information in the [Ops documentation for `CharmBase`](https://ops.readthedocs.io/en/latest/reference/ops.html#ops.CharmBase). The `__init__` method guarantees that the charm observes all events relevant to its operation and handles them. -The VM and K8s charm flavors shares the codebase via [charm libraries](https://juju.is/docs/sdk/libraries) in '[lib/charms/mysql/v0/](https://github.com/canonical/mysql-operator/blob/main/lib/charms/mysql/v0/)': +The VM and K8s charm flavors shares the codebase via charm libraries in [`lib/charms/mysql/v0/`](https://github.com/canonical/mysql-operator/blob/main/lib/charms/mysql/v0/): + ``` charmcraft list-lib mysql Library name API Patch @@ -130,4 +129,5 @@ backups 0 7 mysql 0 45 s3_helpers 0 4 tls 0 2 -``` \ No newline at end of file +``` + diff --git a/docs/explanation/e-users.md b/docs/explanation/e-users.md deleted file mode 100644 index 6ba1a96fd3..0000000000 --- a/docs/explanation/e-users.md +++ /dev/null @@ -1,141 +0,0 @@ -# Charm Users explanations - -There are two types of users in MySQL: -* Internal users (used by charm operator) -* Relation/integration users (used by related applications) - * Extra user roles (if default permissions are not enough) - - -## Internal users explanations: - -The operator uses the following internal DB users: - -* `root` - the [initial/default](https://charmhub.io/mysql/docs/t-manage-passwords) MySQL user. Used for very initial bootstrap and restricted to local access. -* `clusteradmin` - the user to manage replication in the MySQL InnoDB ClusterSet. -* `serverconfig` - the user that operates MySQL instances. -* `monitoring` - the user for [COS integration](https://charmhub.io/mysql/docs/h-enable-monitoring). -* `backups` - the user to [perform/list/restore backups](https://charmhub.io/mysql/docs/h-create-and-list-backups). -* `mysql_innodb_cluster_#######` - the [internal recovery users](https://dev.mysql.com/doc/mysql-shell/8.0/en/innodb-cluster-user-accounts.html#mysql-innodb-cluster-users-created) which enable connections between the servers in the cluster. Dedicated user created for each Juju unit/InnoDB Cluster member. -* `mysql_innodb_cs_#######` - the internal recovery user which enable connections between MySQl InnoDB Clusters in ClusterSet. One user is created for entire MySQL ClusterSet. - -The full list of internal users is available in charm [source code](https://github.com/canonical/mysql-operator/blob/main/src/constants.py). The full dump of internal `mysql.user` table (on newly installed charm): - -```shell -mysql> select Host,User,account_locked from mysql.user; -+-----------+---------------------------------+----------------+ -| Host | User | account_locked | -+-----------+---------------------------------+----------------+ -| % | backups | N | -| % | clusteradmin | N | -| % | monitoring | N | -| % | mysql_innodb_cluster_2277159043 | N | -| % | mysql_innodb_cluster_2277159122 | N | -| % | mysql_innodb_cluster_2277159949 | N | -| % | mysql_innodb_cs_f8ead780 | N | -| % | serverconfig | N | -| localhost | mysql.infoschema | Y | -| localhost | mysql.session | Y | -| localhost | mysql.sys | Y | -| localhost | root | N | -+-----------+---------------------------------+----------------+ -10 rows in set (0.00 sec) -``` -**Note**: it is forbidden to use/manage described above users! They are dedicated to the operators logic! -Please use [data-integrator](https://charmhub.io/mysql/docs/t-integrations) charm to generate/manage/remove an external credentials. - -It is allowed to rotate passwords for *internal* users using action 'set-password' -```shell -> juju show-action mysql set-password -Change the system user's password, which is used by charm. It is for internal charm users and SHOULD NOT be used by applications. - -Arguments -password: - type: string - description: The password will be auto-generated if this option is not specified. -username: - type: string - description: The username, the default value 'root'. Possible values - root, - serverconfig, clusteradmin. -``` - -For example, to generate a new random password for *internal* user: - -```shell -> juju run-action --wait mysql/leader set-password username=clusteradmin -unit-mysql-3: - ... - results: {} - status: completed - -> juju run-action --wait mysql/leader get-password username=clusteradmin -unit-mysql-3: - ... - results: - password: PFLIwiwy0Pn7n7xgYtXKw39H - username: clusteradmin -```from ..helpers import get_leader_unit, get_primary_unit_wrapper, retrieve_database_variable_value - - -To set a predefined password for the specific user, run: -```shell -> juju run-action --wait mysql/leader set-password username=clusteradmin password=newpassword -unit-mysql-0: - ... - results: {} - status: completed - -> juju run-action --wait mysql/leader get-password username=clusteradmin -unit-mysql-3: - UnitId: mysql/3 - id: "14" - results: - password: newpassword - username: clusteradmin -``` -**Note**: the action `set-password` must be executed on juju leader unit (to update peer relation data with new value). - - -## Relation/integration users explanations: - -The operator created a dedicated user for every application related/integrated with database. -The username is composed by the relation ID and truncated uuid for the model, to ensure there is no -username clash in cross model relations. Usernames are limited to 32 chars as per [MySQL limit](https://dev.mysql.com/doc/refman/8.0/en/user-names.html). -Those users are removed on the juju relation/integration removal request. -However, DB data stays in place and can be reused on re-created relations (using new user credentials): - -```shell -mysql> select Host,User,account_locked from mysql.user where User like 'relation%'; -+------+----------------------------+----------------+ -| Host | User | account_locked | -+------+----------------------------+----------------+ -| % | relation-8_99200344b67b4e9 | N | -| % | relation-9_99200344b67b4e9 | N | -+------+----------------------------+----------------+ -2 row in set (0.00 sec) -``` - -The extra user(s) will be created for relation with [mysql-router](https://charmhub.io/mysql-router) charm to provide necessary users for applications related via mysql-router app: -```shell -mysql> select Host,User,account_locked from mysql.user where User like 'mysql_router%'; -+------+----------------------------+----------------+ -| Host | User | account_locked | -+------+----------------------------+----------------+ -| % | mysql_router1_gwa0oy6xnp8l | N | -+------+----------------------------+----------------+ -1 row in set (0.00 sec) -``` - -**Note**: If password rotation is needed for users used in relations, it is needed to remove the relation and create it again: -```shell -> juju remove-relation mysql myclientapp -> juju wait-for application mysql -> juju relate mysql myclientapp -``` - - - - -### Admin Port User Access - -The charm mainly uses the `serverconfig` user for internal operations. For connections with this user, a special admin port is used (port `33062`), which enables the charm to operate MySQL even when users connections are saturated. -For further information on the administrative connection, refer to [MySQL docs](https://dev.mysql.com/doc/refman/8.0/en/administrative-connection-interface.html) on the topic. \ No newline at end of file diff --git a/docs/explanation/index.md b/docs/explanation/index.md new file mode 100644 index 0000000000..0bff3ed82c --- /dev/null +++ b/docs/explanation/index.md @@ -0,0 +1,34 @@ +# Explanation + +Additional context about key concepts behind the MySQL charm. + +## Core concepts and design + +```{toctree} +:titlesonly: +:maxdepth: 2 + +Architecture +Interfaces and endpoints +Juju +Legacy charm +``` + +## Operational concepts + +```{toctree} +:titlesonly: +:maxdepth: 2 + +Users +Logs +``` + +## Security and hardening + +```{toctree} +:titlesonly: +:maxdepth: 2 + +Security +``` diff --git a/docs/explanation/e-interfaces-endpoints.md b/docs/explanation/interfaces-and-endpoints.md similarity index 82% rename from docs/explanation/e-interfaces-endpoints.md rename to docs/explanation/interfaces-and-endpoints.md index f436c9a8df..8af0c03143 100644 --- a/docs/explanation/e-interfaces-endpoints.md +++ b/docs/explanation/interfaces-and-endpoints.md @@ -1,5 +1,4 @@ # Interfaces/endpoints -> **:information_source: Hint**: Use [Juju 3](/t/5064). Otherwise replace `juju integrate` with `juju relate` for Juju 2.9. Charmed MySQL VM supports modern `mysql_client` and legacy `mysql`, `mysql-shared`, `mysql-router` interfaces (in a backward compatible mode). @@ -11,7 +10,7 @@ This charm provides modern [‘mysql_client’ ](https://github.com/canonical/ch ### Modern `mysql_client` interface (`database` endpoint): -Adding a relation is accomplished with `juju integrate` via endpoint `database`. Read more about [Juju relations (integrations)](https://juju.is/docs/olm/relations). Example: +Adding a relation is accomplished with `juju integrate` via endpoint `database`. Read more about [Juju relations (integrations)](https://documentation.ubuntu.com/juju/3.6/reference/relation/). Example: ```shell # Deploy Charmed MySQL cluster with 3 nodes @@ -31,19 +30,19 @@ juju status --relations # > mysql:database mysql-test-app:database mysql_client regular ``` -Find all details about default and extra DB user roles in “[Charm Users explanations](/t/10789)”. +Find all details about default and extra DB user roles in “[Charm Users explanations](/explanation/users)”. **Note:** In order to integrate with this charm, every table created by the integrated application must have a primary key. This is required by the [group replication plugin](https://dev.mysql.com/doc/refman/8.0/en/group-replication-requirements.html) enabled in this charm. ## Legacy relations -**Note:** Legacy relations are deprecated and will be discontinued on future releases. Usage should be avoided. Check the legacy interface implementation limitations in the "[Legacy charm](/t/10788)" document. +**Note:** Legacy relations are deprecated and will be discontinued on future releases. Usage should be avoided. Check the legacy interface implementation limitations in the "[Legacy charm](/explanation/legacy-charm)" document. -This charm supports several legacy interfaces, e.g. `mysql`, `mysql-shared`, `mysql-router`. They were used in some legacy charms in [cross-model relations](https://juju.is/docs/olm/cross-model-integration). +This charm supports several legacy interfaces, e.g. `mysql`, `mysql-shared`, `mysql-router`. They were used in some legacy charms in [cross-model relations](https://documentation.ubuntu.com/juju/3.6/reference/relation/#cross-model). ### `mysql` interface (`mysql` endpoint) -It was a popular interface used by some legacy charms (e.g. "[MariaDB](https://charmhub.io/mariadb)", "[OSM MariaDB](https://charmhub.io/charmed-osm-mariadb-k8s)", "[Percona Cluster](https://charmhub.io/percona-cluster)" and "[Mysql Innodb Cluster](https://charmhub.io/mysql-innodb-cluster)"), often in [cross-model relations](https://juju.is/docs/olm/cross-model-integration): +It was a popular interface used by some legacy charms (e.g. "[MariaDB](https://charmhub.io/mariadb)", "[OSM MariaDB](https://charmhub.io/charmed-osm-mariadb-k8s)", "[Percona Cluster](https://charmhub.io/percona-cluster)" and "[Mysql Innodb Cluster](https://charmhub.io/mysql-innodb-cluster)"), often in [cross-model relations](https://documentation.ubuntu.com/juju/3.6/reference/relation/#cross-model): ```shell juju deploy mysql --channel 8.0 @@ -64,7 +63,7 @@ juju integrate mysql-router keystone juju integrate mysql:db-router mysql-router:db-router ``` -**Note:** pay attention to deploy identical [series](https://juju.is/docs/olm/deploy-an-application-with-a-specific-series) for `keystone` and `mysql-router` applications (due to the [subordinate](https://juju.is/docs/sdk/charm-types#heading--subordinate-charms) charm nature of `mysql-router`). +**Note:** pay attention to deploy identical [series/base](https://documentation.ubuntu.com/juju/3.6/reference/machine/#machine-base) for `keystone` and `mysql-router` applications (due to the [subordinate](https://documentation.ubuntu.com/juju/3.6/reference/charm/#subordinate) charm nature of `mysql-router`). ### `mysql-shared` interface (`shared-db` endpoint) @@ -74,4 +73,5 @@ It is a relation that one uses when the application needs to connect directly to juju deploy mysql --channel 8.0 juju deploy keystone --series focal juju integrate keystone:shared-db mysql:shared-db -``` \ No newline at end of file +``` + diff --git a/docs/explanation/e-juju.md b/docs/explanation/juju.md similarity index 70% rename from docs/explanation/e-juju.md rename to docs/explanation/juju.md index 47c1d117ba..16c0f50773 100644 --- a/docs/explanation/e-juju.md +++ b/docs/explanation/juju.md @@ -11,12 +11,7 @@ See also: This page aims to provide some context on some of the inner workings of Juju that affect this charm. -## Summary -* [Breaking changes between Juju 2.8.x and 3.x](#heading--breaking-changes) -* [Juju upgrades](#heading--upgrades) - ---- -

Breaking changes between Juju 2.9.x and 3.x

+## Breaking changes between Juju 2.9.x and 3.x As this charm documentation is written for Juju 3.x, users of 2.9.x will encounter noteworthy changes when following the instructions. This section explains those changes. @@ -31,7 +26,7 @@ In the context of this guide, the pertinent changes are shown here: |`run`|`exec`| |`run-action --wait`|`run`| -See the [Juju 3.0 release notes](https://juju.is/docs/juju/roadmap#heading--juju-3-0-0---22-oct-2022) for the comprehensive list of changes. +See the [Juju 3.0 release notes](https://documentation.ubuntu.com/juju/3.6/reference/juju/juju-roadmap-and-releases/#juju-3-0-0-22-oct-2022) for the comprehensive list of changes. The response is to therefore substitute the documented command with the equivalent 2.9.x command. For example: @@ -47,14 +42,14 @@ juju relate mysql:database mysql-test-app juju run-action --wait mysql/leader get-password ``` -[note] +```{note} This section is based on the [OpenStack guide.](https://docs.openstack.org/charm-guide/latest/project/support-notes.html#breaking-changes-between-juju-2-9-x-and-3-x) -[/note] +``` -

Juju upgrades

-Newly released charm revisions might require a new [Juju version](/t/11421). This is usually because the new revision requires new Juju features, e.g. [Juju secrets](https://juju.is/docs/juju/secret). +## Juju upgrades +Newly released charm revisions might require a new Juju version. This is usually because the new revision requires new Juju features, e.g. [Juju secrets](https://juju.is/docs/juju/secret). -Information about Juju requirements will be clearly indicated in the charm's [release notes](/t/11878) and in the repository's [metadata.yaml](https://github.com/canonical/mysql-operator/blob/14c06ff88c4e564cd6d098aa213bd03e78e84b52/metadata.yaml#L72-L80) file. +Information about Juju requirements will be clearly indicated in the charm's [release notes](/reference/releases) and in the repository's [metadata.yaml](https://github.com/canonical/mysql-operator/blob/14c06ff88c4e564cd6d098aa213bd03e78e84b52/metadata.yaml#L72-L80) file. When upgrading your database charm with juju refresh, Juju checks that its version is compatible with the target revision. If not, it stops the upgrade and prevents further changes to keep the installation safe. @@ -67,4 +62,5 @@ ERROR Charm feature requirements cannot be met: - charm requires feature "juju" (version >= 3.1.5) but model currently supports version 3.1.4 ``` -You must then [upgrade to the required Juju version](/t/14325) before proceeding with the charm upgrade. \ No newline at end of file +You must then [upgrade to the required Juju version](/how-to/upgrade/upgrade-juju) before proceeding with the charm upgrade. + diff --git a/docs/explanation/e-legacy-charm.md b/docs/explanation/legacy-charm.md similarity index 58% rename from docs/explanation/e-legacy-charm.md rename to docs/explanation/legacy-charm.md index 2eb7208eb6..2b27aa9441 100644 --- a/docs/explanation/e-legacy-charm.md +++ b/docs/explanation/legacy-charm.md @@ -1,10 +1,13 @@ + +# Legacy charm + ## Charm types "legacy" vs "modern" -Historically, there were [several](https://juju.is/docs/sdk/charm-taxonomy#heading--charm-types-by-generation) operators/charms to provide MySQL/MariaDB functionality: "[MariaDB](https://charmhub.io/mariadb)", "[OSM MariaDB](https://charmhub.io/charmed-osm-mariadb-k8s)", "[Percona Cluster](https://charmhub.io/percona-cluster)" and "[Mysql Innodb Cluster](https://charmhub.io/mysql-innodb-cluster)". All of them are named "legacy" charms. +Historically, there were [several](https://documentation.ubuntu.com/juju/3.6/reference/charm/#by-generation) operators/charms to provide MySQL/MariaDB functionality: "[MariaDB](https://charmhub.io/mariadb)", "[OSM MariaDB](https://charmhub.io/charmed-osm-mariadb-k8s)", "[Percona Cluster](https://charmhub.io/percona-cluster)" and "[Mysql Innodb Cluster](https://charmhub.io/mysql-innodb-cluster)". All of them are named "legacy" charms. -This "Charmed MySQL" operator is a modern "[Charmed Operator SDK](https://juju.is/docs/sdk)"-based charm to replace all legacy operators [providing](/t/charmed-mysql-k8s-explanations-interfaces-endpoints/10250) all juju-interfaces of legacy charms. +This "Charmed MySQL" operator is a modern "[Charmed Operator SDK](https://juju.is/docs/sdk)"-based charm to replace all legacy operators providing all [interfaces/endpoints](/explanation/interfaces-and-endpoints) of legacy charms. -The legacy charm provided endpoints `mysql` and `mysql-root` (for the interface `mysql`). The modern charm provides old endpoints as well + new endpoint `database` (for the interface `mysql_client`). Read more detail about the available endpoints/interfaces for [VM](https://charmhub.io/mysql/docs/e-interfaces) and [K8s](https://charmhub.io/mysql-k8s/docs/e-interfaces) charms. +The legacy charm provided endpoints `mysql` and `mysql-root` (for the interface `mysql`). The modern charm provides old endpoints as well + new endpoint `database` (for the interface `mysql_client`). Read more detail about the available endpoints/interfaces for [VM](https://charmhub.io/mysql/integrations) and [K8s](https://charmhub.io/mysql-k8s/integrations) charms. **Note**: Please choose one endpoint to use. No need to relate all of them simultaneously! @@ -25,9 +28,14 @@ The "modern" charm provides temporary support for the legacy interfaces: trust: true ``` -* **proper migration**: migrate the application to the new interface "[mysql_client](https://github.com/canonical/charm-relation-interfaces)". The application will connect MySQL using "[data_interfaces](https://charmhub.io/data-platform-libs/libraries/data_interfaces)" library from "[data-platform-libs](https://github.com/canonical/data-platform-libs/)" via endpoint `database`. +* **proper migration**: migrate the application to the new interface "[`mysql_client`](https://github.com/canonical/charm-relation-interfaces)". The application will connect MySQL using "[`data_interfaces`](https://charmhub.io/data-platform-libs/libraries/data_interfaces)" library from "[`data-platform-libs`](https://github.com/canonical/data-platform-libs/)" via endpoint `database`. + +```{caution} +In-place upgrades from the legacy charm to the modern, Ops-based charm are **not supported**. -**Warning**: NO in-place upgrades possible! Legacy charm cannot be upgraded to Operator-framework-based one. To move DB data, the second/modern DB application must be launched nearby and data should be copied from "legacy" application to the "modern" one. Canonical Data Platform team will prepare the copy&paste guide. Please [contact us](https://chat.charmhub.io/charmhub/channels/data-platform) if you need migration instructions. + +To migrate database data, the legacy and modern MySQL charms must both be running simultaneously. The data can then be copied from the legacy application to the modern one. Please [contact us](/reference/contacts) if you need help with data migration. +``` ## How to deploy old "legacy" mysql charm @@ -46,14 +54,15 @@ Deploy the charm using the proper charm/channel `latest/stable`: ## Supported MySQL versions by modern charm At the moment, both K8s and VM modern charms support MySQL 8.0 (based on Jammy/22.04 series) only. -Please [contact us](https://chat.charmhub.io/charmhub/channels/data-platform) if you need different versions/series. +Please [contact us](/reference/contacts) if you need different versions/series. ## Supported architectures: amd64, arm64, ... -Currently, the charm supports architecture `amd64` only. See the technical details in “[Supported architectures](/t/11742)”. +Currently, the charm supports architecture `amd64` only. See the technical details in “[Supported architectures](/reference/system-requirements)”. ## How to report issues and contact authors The "modern charm" (from `8.0/stable`) is stored on [GitHub](https://github.com/canonical/mysql-operator), here is the link to report [modern charm issues](https://github.com/canonical/mysql-operator/issues/new/choose). -Do you have questions? [Contact us](https://chat.charmhub.io/charmhub/channels/data-platform)! \ No newline at end of file +Do you have questions? [Contact us](/reference/contacts)! + diff --git a/docs/explanation/e-audit-logs.md b/docs/explanation/logs/audit-logs.md similarity index 90% rename from docs/explanation/e-audit-logs.md rename to docs/explanation/logs/audit-logs.md index 0fae7323cb..3db149ee8e 100644 --- a/docs/explanation/e-audit-logs.md +++ b/docs/explanation/logs/audit-logs.md @@ -13,7 +13,7 @@ The following is a sample of the audit logs, with format json with only logins r The logs are stored in the `/var/snap/charmed-mysql/common/var/log/mysql` directory, and it's rotated every minute to the `/var/snap/charmed-mysql/common/var/log/mysql/archive_audit` directory. -It's recommended to integrate the charm with [COS](/t/9900), from where the logs can be easily persisted and queried using Loki/Grafana. +It's recommended to integrate the charm with [COS](/how-to/monitoring-cos/enable-monitoring), from where the logs can be easily persisted and queried using Loki/Grafana. ## Configurations @@ -37,4 +37,5 @@ It's recommended to integrate the charm with [COS](/t/9900), from where the logs ```bash juju config mysql plugin-audit-strategy=semi-async ``` - Valid values are `async` and `semi-async`. \ No newline at end of file + Valid values are `async` and `semi-async`. + diff --git a/docs/explanation/e-logs.md b/docs/explanation/logs/index.md similarity index 98% rename from docs/explanation/e-logs.md rename to docs/explanation/logs/index.md index 41c64de6b6..434c25b73f 100644 --- a/docs/explanation/e-logs.md +++ b/docs/explanation/logs/index.md @@ -7,16 +7,6 @@ The charm currently has audit and error logs enabled by default. All of these fi present into a separate dedicated archive folder under the logs directory. We do not yet support the rotation of binary logs (binlog, relay log, undo log, redo log, etc). -## Summary - -* [Log types](#log-types) -* [Audit logs](#audit-logs) -* [Error logs](#error-logs) -* [Log rotation configuration](#log-rotation-configuration) -* [High Level Design](#high-level-design) - ---- - ## Log types The charm stores its logs in `/var/snap/charmed-mysql/common/var/log/mysql`. @@ -175,5 +165,13 @@ for the duration of the event handler. -[COS integration]: /t/9900 -[Audit Logs explanation]: /t/15424 \ No newline at end of file +[COS integration]: /how-to/monitoring-cos/enable-monitoring +[Audit Logs explanation]: /explanation/logs/audit-logs + + +```{toctree} +:titlesonly: +:maxdepth: 2 + +Audit logs +``` diff --git a/docs/explanation/e-cryptography.md b/docs/explanation/security/cryptography.md similarity index 99% rename from docs/explanation/e-cryptography.md rename to docs/explanation/security/cryptography.md index 3ee5b2884e..3fefdbb45b 100644 --- a/docs/explanation/e-cryptography.md +++ b/docs/explanation/security/cryptography.md @@ -1,3 +1,4 @@ + # Cryptography This document describes the cryptography used by Charmed MySQL. @@ -58,4 +59,5 @@ An internal user is used for this authentication with its hashed password stored Authentication to MySQL Router is based on the [caching_sha2_password auth plugin](https://dev.mysql.com/doc/refman/8.0/en/caching-sha2-pluggable-authentication.html). -Credentials are exchanged via [Juju secrets](https://canonical-juju.readthedocs-hosted.com/en/latest/user/howto/manage-secrets/). \ No newline at end of file +Credentials are exchanged via [Juju secrets](https://canonical-juju.readthedocs-hosted.com/en/latest/user/howto/manage-secrets/). + diff --git a/docs/explanation/e-security.md b/docs/explanation/security/index.md similarity index 94% rename from docs/explanation/e-security.md rename to docs/explanation/security/index.md index 90ddb508b3..1e10e0741a 100644 --- a/docs/explanation/e-security.md +++ b/docs/explanation/security/index.md @@ -1,3 +1,4 @@ + # Security hardening guide This document provides an overview of security features and guidance for hardening the security of [Charmed MySQL](https://charmhub.io/mysql) deployments, including setting up and managing a secure environment. @@ -16,14 +17,14 @@ Charmed MySQL can be deployed on top of several clouds and virtualisation layers | Cloud | Security guides | |--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | OpenStack | [OpenStack Security Guide](https://docs.openstack.org/security-guide/) | -| AWS | [Best Practices for Security, Identity and Compliance](https://aws.amazon.com/architecture/security-identity-compliance), [AWS security credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html#access-keys-and-secret-access-keys) | +| AWS | [Best Practices for Security, Identity and Compliance](https://aws.amazon.com/architecture/security-identity-compliance), [AWS security credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html) | | Azure | [Azure security best practices and patterns](https://learn.microsoft.com/en-us/azure/security/fundamentals/best-practices-and-patterns), [Managed identities for Azure resource](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/) | | GCP | [Google security overview](https://cloud.google.com/kubernetes-engine/docs/concepts/security-overview), [Harden your cluster's security](https://cloud.google.com/kubernetes-engine/docs/concepts/security-overview) | ### Juju Juju is the component responsible for orchestrating the entire lifecycle, from deployment to Day 2 operations. For more information on Juju security hardening, see the -[Juju security page](/t/juju-security/15684) and the [How to harden your deployment](https://juju.is/docs/juju/harden-your-deployment) guide. +[Juju security page](https://documentation.ubuntu.com/juju/latest/explanation/juju-security/index.html) and the [How to harden your deployment](https://documentation.ubuntu.com/juju/3.6/howto/manage-your-deployment/#harden-your-deployment) guide. #### Cloud credentials @@ -62,7 +63,7 @@ Charmed MySQL operator and Charmed MySQL Router operator install a pinned revisi New versions (revisions) of charmed operators can be released to upgrade workloads, the operator's code, or both. It is important to refresh the charm regularly to make sure the workload is as secure as possible. -For more information on upgrading the charm, see the [How to upgrade MySQL](https://canonical.com/data/docs/mysql/iaas/h-upgrade) and [How to upgrade MySQL Router](https://charmhub.io/mysql-router/docs/h-upgrade-intro?channel=dpe/edge) guides, as well as the [Release notes](https://canonical.com/data/docs/mysql/iaas/r-releases). +For more information on upgrading the charm, see the [How to upgrade MySQL](https://canonical.com/data/docs/mysql/iaas/h-upgrade) and [How to upgrade MySQL Router](https://charmhub.io/mysql-router/docs/h-upgrade?channel=dpe/edge) guides, as well as the [Release notes](https://canonical.com/data/docs/mysql/iaas/r-releases). ### Encryption @@ -86,4 +87,13 @@ The Audit log plugin is enabled by default and produces login/logout logs. See t ## Additional Resources -For details on the cryptography used by Charmed MySQL, see the [Cryptography](https://discourse.charmhub.io/t/charmed-mysql-explanations-cryptography/16785) explanation page. \ No newline at end of file +For details on the cryptography used by Charmed MySQL, see the [Cryptography](https://discourse.charmhub.io/t/charmed-mysql-explanations-cryptography/16785) explanation page. + + +```{toctree} +:titlesonly: +:maxdepth: 2 +:hidden: + +cryptography +``` diff --git a/docs/explanation/users.md b/docs/explanation/users.md new file mode 100644 index 0000000000..3dc949a90c --- /dev/null +++ b/docs/explanation/users.md @@ -0,0 +1,100 @@ +# Users + +There are two main types of users in MySQL: + +* **Internal users**, used by the charm operator +* **Relation users**, used by related (integrated) applications + * **Extra user roles** if the default permissions are not enough + +## Internal users + +The operator uses the following internal database users: + +* `root` - the initial/default MySQL user. Used for very initial bootstrap and restricted to local access. +* `clusteradmin` - the user to manage replication in the MySQL InnoDB ClusterSet. +* `serverconfig` - the user that operates MySQL instances. +* `monitoring` - the user for [COS integration](/how-to/monitoring-cos/enable-monitoring). +* `backups` - the user to [perform/list/restore backups](/how-to/back-up-and-restore/create-a-backup). +* `mysql_innodb_cluster_#######` - the [internal recovery users](https://dev.mysql.com/doc/mysql-shell/8.0/en/innodb-cluster-user-accounts.html#mysql-innodb-cluster-users-created) which enable connections between the servers in the cluster. Dedicated user created for each Juju unit/InnoDB Cluster member. +* `mysql_innodb_cs_#######` - the internal recovery user which enable connections between MySQl InnoDB Clusters in ClusterSet. One user is created for entire MySQL ClusterSet. + +The full list of internal users is available in charm [source code](https://github.com/canonical/mysql-operator/blob/main/src/constants.py). + +```{caution} +It is forbidden to manage internal users, as they are dedicated to the operator’s logic. + +Use the [data-integrator](https://charmhub.io/data-integrator) charm to generate, manage, and remove external credentials. +``` + +Example of internal `mysql.user` table on a newly installed charm: + +```shell +mysql> select Host,User,account_locked from mysql.user; ++-----------+---------------------------------+----------------+ +| Host | User | account_locked | ++-----------+---------------------------------+----------------+ +| % | backups | N | +| % | clusteradmin | N | +| % | monitoring | N | +| % | mysql_innodb_cluster_2277159043 | N | +| % | mysql_innodb_cluster_2277159122 | N | +| % | mysql_innodb_cluster_2277159949 | N | +| % | mysql_innodb_cs_f8ead780 | N | +| % | serverconfig | N | +| localhost | mysql.infoschema | Y | +| localhost | mysql.session | Y | +| localhost | mysql.sys | Y | +| localhost | root | N | ++-----------+---------------------------------+----------------+ +10 rows in set (0.00 sec) +``` + +Passwords for *internal* users can be rotated using the action `set-password` on the juju leader unit. + +```{seealso} +[How to manage passwords](/how-to/manage-passwords) +``` + +## Relation users + +The operator created a dedicated user for every application related/integrated with database. The username is composed by the relation ID and truncated uuid for the model, to ensure there is no username clash in cross model relations. Usernames are limited to 32 chars as per [MySQL limit](https://dev.mysql.com/doc/refman/8.0/en/user-names.html). + +Relation users are removed on the juju relation/integration removal request. However, database data stays in place and can be reused on re-created relations (using new user credentials): + +```shell +mysql> select Host,User,account_locked from mysql.user where User like 'relation%'; ++------+----------------------------+----------------+ +| Host | User | account_locked | ++------+----------------------------+----------------+ +| % | relation-8_99200344b67b4e9 | N | +| % | relation-9_99200344b67b4e9 | N | ++------+----------------------------+----------------+ +2 row in set (0.00 sec) +``` + +The extra user(s) will be created for relation with [mysql-router](https://charmhub.io/mysql-router) charm to provide necessary users for applications related via the `mysql-router` app: + +```shell +mysql> select Host,User,account_locked from mysql.user where User like 'mysql_router%'; ++------+----------------------------+----------------+ +| Host | User | account_locked | ++------+----------------------------+----------------+ +| % | mysql_router1_gwa0oy6xnp8l | N | ++------+----------------------------+----------------+ +1 row in set (0.00 sec) +``` + +To rotate passwords for relation users, remove the relation and re-relate: + +```shell +juju remove-relation mysql myclientapp +juju wait-for application mysql +juju relate mysql myclientapp +``` + +### Admin port user access + +The charm mainly uses the `serverconfig` user for internal operations. For connections with this user, a special admin port is used (port `33062`), which enables the charm to operate MySQL even when users connections are saturated. + +For further information on the administrative connection, refer to [MySQL docs](https://dev.mysql.com/doc/refman/8.0/en/administrative-connection-interface.html) on the topic. + diff --git a/docs/how-to.md b/docs/how-to.md deleted file mode 100644 index 7fed0469b6..0000000000 --- a/docs/how-to.md +++ /dev/null @@ -1,102 +0,0 @@ -# How-to guides - -Key processes and common tasks for managing and using Charmed MySQL on machines. - -## Deployment and setup - -Guidance for different cloud services: -* [Sunbeam] -* [LXD] -* [MAAS] -* [AWS EC2] -* [GCE] -* [Azure] -* [Multi-AZ] - -Specific deployment scenarios and architectures: - -* [Terraform] -* [Air-gapped] - -## Usage and maintenance -* [Integrate with another application] -* [External network access] -* [Scale replicas] -* [Enable TLS] - -## Back up and restore -* [Configure S3 AWS] -* [Configure S3 RadosGW] -* [Create a backup] -* [Restore a backup] -* [Migrate a cluster] - -## Monitoring (COS) -* [Enable monitoring] -* [Enable alert rules] -* [Enable tracing] - -## Upgrades -See the [Upgrades landing page] for more details. -* [Upgrade Juju] -* [Perform a minor upgrade] -* [Perform a minor rollback] - -## Cross-regional (cluster-cluster) async replication -* [Deploy] -* [Clients] -* [Switchover / Failover] -* [Recovery] -* [Removal] - -## Development -* [Integrate with your charm] -* [Migrate data via mysqldump] -* [Migrate data via mydumper] -* [Migrate data via backup/restore] -* [Troubleshooting] - - - - -[Sunbeam]: /t/15986 -[LXD]: /t/11870 -[MAAS]: /t/13900 -[AWS EC2]: /t/15718 -[GCE]: /t/15723 -[Azure]: /t/15859 -[Multi-AZ]: /t/15823 -[Terraform]: /t/14925 -[Air-gapped]: /t/15747 - -[Integrate with another application]: /t/9902 -[External network access]: /t/15801 -[Scale replicas]: /t/9904 -[Enable TLS]: /t/9898 - -[Configure S3 AWS]: /t/9894 -[Configure S3 RadosGW]: /t/10318 -[Create a backup]: /t/9896 -[Restore a backup]: /t/9908 -[Migrate a cluster]: /t/9906 - -[Enable monitoring]: /t/9900 -[Enable alert rules]: /t/15486 -[Enable tracing]: /t/14350 - -[Upgrades landing page]: /t/11745 -[Upgrade Juju]: /t/14325 -[Perform a minor upgrade]: /t/11748 -[Perform a minor rollback]: /t/11749 - -[Integrate with your charm]: /t/11890 -[Migrate data via mysqldump]: /t/11958 -[Migrate data via mydumper]: /t/11988 -[Migrate data via backup/restore]: /t/12008 -[Troubleshooting]: /t/11891 - -[Deploy]: /t/14169 -[Clients]: /t/14170 -[Switchover / Failover]: /t/14171 -[Recovery]: /t/14172 -[Removal]: /t/14174 \ No newline at end of file diff --git a/docs/how-to/h-configure-s3-aws.md b/docs/how-to/back-up-and-restore/configure-s3-aws.md similarity index 62% rename from docs/how-to/h-configure-s3-aws.md rename to docs/how-to/back-up-and-restore/configure-s3-aws.md index b04317caa3..f3b0d5e52a 100644 --- a/docs/how-to/h-configure-s3-aws.md +++ b/docs/how-to/back-up-and-restore/configure-s3-aws.md @@ -1,12 +1,17 @@ -[note] -**Note**: All commands are written for `juju >= v.3.0` +# Configure S3 for AWS -If you are using an earlier version, check the [Juju 3.0 Release Notes](https://juju.is/docs/juju/roadmap#heading--juju-3-0-0---22-oct-2022). -[/note] +Charmed MySQL backups can be stored on any S3 compatible storage. S3 access and configurations are managed with the [`s3-integrator` charm](https://charmhub.io/s3-integrator). -# Configure S3 for AWS +This guide will teach you how to deploy and configure the s3-integrator charm for [AWS S3](https://aws.amazon.com/s3/), send the configuration to a Charmed MySQL application, and update it. + +```{seealso} +[](/how-to/back-up-and-restore/configure-s3-radosgw) +``` + +## Set up `s3-integrator` + +Deploy and configure the `s3-integrator` charm for AWS S3: -Charmed MySQL backup can be stored on any S3 compatible storage. The S3 access and configurations are managed with the [s3-integrator charm](https://charmhub.io/s3-integrator). Deploy and configure the s3-integrator charm for **[AWS S3](https://aws.amazon.com/s3/)** (click [here](/t/charmed-mysql-how-to-configure-s3-for-radosgw/10318) to backup on Ceph via RadosGW): ```shell juju deploy s3-integrator juju run s3-integrator/leader sync-s3-credentials access-key= secret-key= @@ -16,18 +21,20 @@ juju config s3-integrator \ path="/mysql-test" \ region="us-west-2" ``` -[note] + +```{note} The amazon S3 endpoint must be specified as `s3..amazonaws.com ` within the first 24 hours of creating the bucket. For older buckets, the endpoint `s3.amazonaws.com` can be used. See [this post](https://repost.aws/knowledge-center/s3-http-307-response) for more information. -[/note] +``` To pass these configurations to Charmed MySQL, relate the two applications: + ```shell juju integrate s3-integrator mysql ``` -You can create/list/restore backups now: +You can create, list, and restore backups now: ```shell juju run mysql/leader list-backups @@ -36,9 +43,11 @@ juju run mysql/leader list-backups juju run mysql/leader restore backup-id= ``` -You can also update your S3 configuration options after relating, using: +You can also update your S3 configuration options after relating: + ```shell juju config s3-integrator