From 1abc02e18e33cbaf52fe1cff8ef6122f90b4a278 Mon Sep 17 00:00:00 2001 From: Jannik Clausen <12862103+masseelch@users.noreply.github.com> Date: Thu, 8 Sep 2022 16:56:25 +0200 Subject: [PATCH] doc/blog: announce and introduce cli execution (#1138) * doc/blog: announce and introduce cli execution * apply CR * apply CR --- ...2-09-05-announcing-migration-execution.mdx | 375 ++++++++++++++++++ doc/website/blog/authors.yml | 7 +- doc/website/sidebars.js | 2 +- 3 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 doc/website/blog/2022-09-05-announcing-migration-execution.mdx diff --git a/doc/website/blog/2022-09-05-announcing-migration-execution.mdx b/doc/website/blog/2022-09-05-announcing-migration-execution.mdx new file mode 100644 index 00000000000..d407cd565c3 --- /dev/null +++ b/doc/website/blog/2022-09-05-announcing-migration-execution.mdx @@ -0,0 +1,375 @@ +--- +title: The Atlas Migration Execution Engine +authors: masseelch +tags: [atlas, migrations, versioned, announcement] +image: https://blog.ariga.io/uploads/images/posts/v0.6.0/atlas-migrate-diff.png +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +With the release of [v0.6.0](https://github.com/ariga/atlas/releases/tag/v0.6.0), we +[introduced](https://atlasgo.io/blog/2022/08/11/announcing-versioned-migration-authoring) a workflow for managing +changes to database schemas that we have called: _Versioned Migration Authoring_. + +Today, we released the first version of the Atlas migration execution engine, that can apply migration files on +your database. In this post, we will give a brief overview of the features and what to expect in the future. + +### Migration File Format + +The Atlas migration filename format follows a very simple structure: `version_[name].sql`, with the `name` being +optional. `version` can be an arbitrary string. Migration files are lexicographically sorted by filename. + + + + +```shell +↪ tree . +. +├── 1_initial.sql +├── 2_second.sql +├── 3_third.sql +└── atlas.sum + +0 directories, 4 files +``` + + + + + +```sql +-- add new schema named "my_schema" +CREATE DATABASE `my_schema`; +-- create "tbl" table +CREATE TABLE `my_schema`.`tbl` (`col` int NOT NULL); +``` + + + + + +```sql +ALTER TABLE `my_schema`.`tbl` ADD `col_2` TEXT; +``` + + + + + +```sql +CREATE TABLE `tbl_2` (`col` int NOT NULL); +``` + + + + + +```text +h1:cD9kOv5VDRLrKVZ0pM4CxAlhH6mgE8PQLpUeuIMKDcs= +1_initial.sql h1:SrFyOe0eg5WnE96GH3TtAt6046sfrOK4YKZYBlYr1SA= +2_second.sql h1:FPzwV+MzwyCss7SASZtyafXiYc9Un5bzlcc3u7MxLJU= +3_third.sql h1:uD+xDcA3Q+gHqwca2ZBDAHWYvC2eiUcwr1IgMsN0Q6c= + +``` + + + + + +If you want to follow along, you can simply copy and paste the above files in a folder on your system. Make sure you +have a database ready to work on. You can start an ephemeral docker container with the following command: + +```shell +# Run a local mysql container listening on port 3306. +docker run --rm --name atlas-apply --detach --env MYSQL_ROOT_PASSWORD=pass -p 3306:3306 mysql:8 +``` + +### Apply Migrations + +In order to apply migrations you need to have the Atlas CLI in version v0.7.0 or above. Follow the +[installation instructions](https://atlasgo.io/getting-started#installation) if you don't have Atlas installed yet. + +Now, to apply the first migration of our migration directory, we call `atlas migrate apply` and pass in some +configuration parameters. + +```shell +atlas migrate apply 1 \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/" +``` + +```shell +Migrating to version 1 (1 migrations in total): + + -- migrating version 1 + -> CREATE DATABASE `my_schema`; + -> CREATE TABLE `my_schema`.`tbl` (`col` int NOT NULL); + -- ok (17.247319ms) + + ------------------------- + -- 18.784204ms + -- 1 migrations + -- 2 sql statements + +``` + +### Migration Status + +Atlas saves information about the database schema revisions (applied migration versions) in a special table called +`atlas_schema_revisions`. In the example above we connected to the database without specifying which schema to operate +against. For this reason, Atlas created the revision table in a new schema called `atlas_schema_revisions`. For a +schema-bound connection Atlas will put the table into the connected schema. We will see that in a bit. + +Go ahead and call `atlas migrate status` to gather information about the database migration state: + +```shell +atlas migrate status \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/" +``` + +```shell +Migration Status: PENDING + -- Current Version: 1 + -- Next Version: 2 + -- Executed Files: 1 + -- Pending Files: 2 + +``` + +This output tells us that the last applied version is `1`, the next one is called `2` and that we still have two +migrations pending. Let's apply the pending migrations: + +Note, that we do not pass an argument to the `apply`, in which case Atlas will attempt to apply all pending migrations. + +```shell +atlas migrate apply \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/" +``` + +```shell +Migrating to version 3 from 1 (2 migrations in total): + + -- migrating version 2 + -> ALTER TABLE `my_schema`.`tbl` ADD `col_2` TEXT; + -- ok (13.98847ms) + + -- migrating version 3 + -> CREATE TABLE `tbl_2` (`col` int NOT NULL); + Error 1046: No database selected + + ------------------------- + -- 15.604338ms + -- 1 migrations ok (1 with errors) + -- 1 sql statements ok (1 with errors) + +Error: Execution had errors: +Error 1046: No database selected + +Error: sql/migrate: execute: executing statement "CREATE TABLE `tbl_2` (`col` int NOT NULL);" from version "3": Error 1046: No database selected +exit status 1 + +``` + +What happened here? After further investigation, you will find that our connection URL is bound to the entire database, +not to a schema. The third migration file however does not contain a schema qualifier for the `CREATE TABLE` statement. + +By default, Atlas wraps the execution of each migration file into one transaction. This transaction gets rolled back +if any error occurs withing execution. Be aware though, that some databases, such as MySQL and MariaDB, don't support +transactional DDL. If you want to learn how to configure the way Atlas uses transactions, have a look at the +[docs](/versioned/apply#transaction-configuration). + +### Migration Retry + +To resolve this edit the migration file and add a qualifier to the statement: + +```sql +CREATE TABLE `my_schema`.`tbl_2` (`col` int NOT NULL); +``` + +Since you changed the contents of a migration file, we have to re-calculate the directory integrity hash-sum by calling: + +```shell +atlas migrate hash --force \ + --dir "file://migrations" +``` + +Then we can proceed and simply attempt to execute the migration file again. + +```shell +atlas migrate apply \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/" +``` + +```shell +Migrating to version 3 from 2 (1 migrations in total): + + -- migrating version 3 + -> CREATE TABLE `my_schema`.`tbl_2` (`col` int NOT NULL); + -- ok (15.168892ms) + + ------------------------- + -- 16.741173ms + -- 1 migrations + -- 1 sql statements + +``` + +Attempting to migrate again or calling `atlas migrate status` will tell us that all migrations have been applied onto +the database and there is nothing to do at the moment. + + + + +```shell +atlas migrate apply \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/" +``` + +```shell +The migration directory is synced with the database, no migration files to execute +``` + + + + + +```shell +atlas migrate status \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/" +``` + +```shell +Migration Status: OK + -- Current Version: 3 + -- Next Version: Already at latest version + -- Executed Files: 3 + -- Pending Files: 0 + +``` + + + + + +### Moving an existing project to Atlas with Baseline Migrations + +Another common scenario is when you need to move an existing project to Atlas. To do so, create an initial migration +file reflecting the current state of a database schema by using `atlas migrate diff`. A very simple way to do so would +be by heading over to the database from before, deleting the `atlas_schema_revisions` schema, emptying your migration +directory and running the `atlas migrate diff` command. + +```shell +rm -rf migrations +docker exec atlas-apply mysql -ppass -e "CREATE SCHEMA `my_schema_dev`;" # create a dev-db +docker exec atlas-apply mysql -ppass -e "DROP SCHEMA `atlas_schema_revisions`;" +atlas migrate diff \ + --dir "file://migrations" \ + --to "mysql://root:pass@localhost:3306/my_schema" \ + --dev-url "mysql://root:pass@localhost:3306/my_schema_dev" +``` + +To demonstrate that Atlas can also work on a schema level instead of a realm connection, we are running on a connection +bound to the `my_schema` schema this time. + +You should end up with the following migration directory: + + + + +```sql +-- create "tbl" table +CREATE TABLE `tbl` (`col` int NOT NULL, `col_2` text NULL) CHARSET utf8mb4 COLLATE utf8mb4_0900_ai_ci; +-- create "tbl_2" table +CREATE TABLE `tbl_2` (`col` int NOT NULL) CHARSET utf8mb4 COLLATE utf8mb4_0900_ai_ci; + +``` + + + + + +```text +h1:5zAQ8J0qziTKWg09fRNdbUf3rnLdvA1KHayh8l1SgM0= +20220908105652.sql h1:4WEB39tqALVYNQJTfULKizxEuUC37sgFs0LN5dKJpOw= + +``` + + + + + +Now, let's create a new migration file to create a table `tbl_3` and update the directory integrity file. + +```shell +atlas migrate new add_table --dir "file://migrations" +echo "CREATE TABLE `tbl_3` (`col` text NULL);" >> migrations/$(ls -t migrations | head -n1) +atlas migrate hash --force --dir "file://migrations" +``` + +Since we now have both a migration file representing our current database state and the new migration file to apply, +we can make use of the `--baseline` flag: + +```shell +atlas migrate apply \ + --dir "file://migrations" \ + --url "mysql://root:pass@localhost:3306/my_schema" \ + --baseline "20220908110527" # replace the version with the one generated by you +``` + +```shell +Migrating to version 20220908110847 from 20220908110527 (1 migrations in total): + + -- migrating version 20220908110847 + -> CREATE TABLE `tbl_3` (`col` text NULL); + -- ok (14.325493ms) + + ------------------------- + -- 15.786455ms + -- 1 migrations + -- 1 sql statements +``` + +### Outlook + +The Atlas migration engine is powering [Ent](https://entgo.io) and the execution engine is already being used within +Ariga for several months. We will continue working on improving it, releasing cool features, such as assisted +troubleshooting for failed migrations, a more intelligent, dialect-aware execution planning for things like MySQLs +implicits commits and more. + +### Wrapping up + +In this post we learned about the new migration execution engine of Atlas and some information about its internals. + +### Further reading + +To learn more about Versioned Migration Authoring: +* Read the [docs](/versioned/diff) +* [CLI Command Reference](/cli-reference#atlas-migrate-diff) + +Have questions? Feedback? Find our team [on our Discord server](https://discord.gg/zZ6sWVg6NT). + diff --git a/doc/website/blog/authors.yml b/doc/website/blog/authors.yml index 2e9f3daf17a..1a595a88d21 100644 --- a/doc/website/blog/authors.yml +++ b/doc/website/blog/authors.yml @@ -15,4 +15,9 @@ a8m: title: Building Atlas url: "https://github.com/a8m" image_url: "https://avatars.githubusercontent.com/u/7413593?v=4" - twitter: arielmashraki \ No newline at end of file + twitter: arielmashraki +masseelch: + name: Jannik Clausen + title: Building Atlas + url: "https://github.com/masseelch" + image_url: "https://avatars.githubusercontent.com/u/12862103?v=4" \ No newline at end of file diff --git a/doc/website/sidebars.js b/doc/website/sidebars.js index 679219e4767..490ecaf1eac 100644 --- a/doc/website/sidebars.js +++ b/doc/website/sidebars.js @@ -37,7 +37,7 @@ module.exports = { {type: 'doc', id: 'versioned/diff', label: 'Migration Authoring'}, {type: 'doc', id: 'versioned/lint', label: 'Migration Linting'}, {type: 'doc', id: 'versioned/new', label: 'Manual Migrations'}, - // {type: 'doc', id: 'versioned/apply', label: 'Migration Applying'}, + {type: 'doc', id: 'versioned/apply', label: 'Migration Applying'}, ] }, {