From f2987374b88331442792cf4d8d63c5de195c21fa Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 5 Mar 2024 12:50:20 +0700 Subject: [PATCH 01/45] Add production site to readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4428509..0a204ff 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ The build system for polar is called [icepak](https://github.com/upmaru/icepak). ## Demo -+ [Sandbox Site](https://images.opsmaru.dev) ++ [Production](https://images.opsmaru.com) ++ [Sandbox](https://images.opsmaru.dev) ## Basic Architecture From 6815db177fbac3fd2c103cbddae0488152892c9f Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 7 Mar 2024 16:15:25 +0700 Subject: [PATCH 02/45] Update release task --- lib/polar/{release.ex => release/tasks.ex} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename lib/polar/{release.ex => release/tasks.ex} (93%) diff --git a/lib/polar/release.ex b/lib/polar/release/tasks.ex similarity index 93% rename from lib/polar/release.ex rename to lib/polar/release/tasks.ex index 050fc00..e60a004 100644 --- a/lib/polar/release.ex +++ b/lib/polar/release/tasks.ex @@ -1,4 +1,4 @@ -defmodule Polar.Release do +defmodule Polar.Release.Tasks do @app :polar def migrate do From bc153710261604f5fd0f73ce15655689a3fe1e42 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 7 Mar 2024 16:16:25 +0700 Subject: [PATCH 03/45] Update migration tassk --- instellar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instellar.yml b/instellar.yml index d5772de..2066d39 100644 --- a/instellar.yml +++ b/instellar.yml @@ -33,7 +33,7 @@ build: run: commands: - binary: polar - call: eval 'Polar.Release.migrate' + call: eval 'Polar.Release.Tasks.migrate' name: migrate - binary: polar call: remote From 9214d40a224bd36f399c98463f44585882b35b17 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 7 Mar 2024 17:01:31 +0700 Subject: [PATCH 04/45] Use variable for instellar endpoint --- .github/workflows/deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index b6a4e7d..8f70e6c 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -75,6 +75,6 @@ jobs: env: WORKFLOW_REF: ${{ github.event.workflow_run.head_branch }} WORKFLOW_SHA: ${{ github.event.workflow_run.head_sha }} - INSTELLAR_ENDPOINT: https://opsmaru.com + INSTELLAR_ENDPOINT: ${{vars.INSTELLAR_ENDPOINT}} INSTELLAR_PACKAGE_TOKEN: ${{secrets.INSTELLAR_PACKAGE_TOKEN}} INSTELLAR_AUTH_TOKEN: ${{secrets.INSTELLAR_AUTH_TOKEN}} \ No newline at end of file From b2071dd3d0656f7bafd8fc36c72d6c4296cf9ea0 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 7 Mar 2024 17:29:51 +0700 Subject: [PATCH 05/45] Update config for s3 --- instellar.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/instellar.yml b/instellar.yml index 2066d39..4191970 100644 --- a/instellar.yml +++ b/instellar.yml @@ -81,4 +81,6 @@ kits: driver: database/postgresql key: DATABASE - driver: bucket/aws-s3 + driver_options: + acl: public key: AWS_S3 From 34796cd44aafae0984ea4995d3cbdcdee1cd5f34 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 09:09:04 +0700 Subject: [PATCH 06/45] Setup new states for version Resolves ops-713 Resolves ops-712 Resolves ops-711 --- lib/polar/streams.ex | 2 +- lib/polar/streams/version.ex | 2 +- lib/polar/streams/version/manager.ex | 16 +++++----------- lib/polar/streams/version/transitions.ex | 12 ++++++++++++ lib/polar/streams/version/triggers.ex | 9 +++++++++ test/polar/streams/product/manager_test.exs | 14 ++++++++++---- test/polar/streams/version/manager_test.exs | 6 +++--- 7 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 lib/polar/streams/version/triggers.ex diff --git a/lib/polar/streams.ex b/lib/polar/streams.ex index 39410fe..5f68124 100644 --- a/lib/polar/streams.ex +++ b/lib/polar/streams.ex @@ -23,7 +23,7 @@ defmodule Polar.Streams do to: Version.Manager, as: :create - defdelegate deactivate_previous_versions(version), + defdelegate deactivate_previous_versions(event, version), to: Version.Manager, as: :deactivate_previous diff --git a/lib/polar/streams/version.ex b/lib/polar/streams/version.ex index 56faa1b..cf9dfe2 100644 --- a/lib/polar/streams/version.ex +++ b/lib/polar/streams/version.ex @@ -16,7 +16,7 @@ defmodule Polar.Streams.Version do import Ecto.Query, only: [from: 2] schema "versions" do - field :current_state, :string, default: "active" + field :current_state, :string, default: "created" field :serial, :string belongs_to :product, Product diff --git a/lib/polar/streams/version/manager.ex b/lib/polar/streams/version/manager.ex index 25813ad..0df7d2a 100644 --- a/lib/polar/streams/version/manager.ex +++ b/lib/polar/streams/version/manager.ex @@ -9,18 +9,9 @@ defmodule Polar.Streams.Version.Manager do %Version{product_id: product.id} |> Version.changeset(attrs) |> Repo.insert() - |> case do - {:ok, version} = result -> - deactivate_previous(version) - - result - - error -> - error - end end - def deactivate_previous(version) do + def deactivate_previous(event, version) do basic_setting = Globals.get("basic") bot = Polar.Accounts.Automation.get_bot!() @@ -34,7 +25,10 @@ defmodule Polar.Streams.Version.Manager do ) |> Repo.all() |> Enum.map(fn v -> - Eventful.Transit.perform(v, bot, "deactivate") + Eventful.Transit.perform(v, bot, "deactivate", + comment: "New version activated.", + parameters: %{"event_id" => event.id} + ) end) end end diff --git a/lib/polar/streams/version/transitions.ex b/lib/polar/streams/version/transitions.ex index 30e0c18..41c5ab8 100644 --- a/lib/polar/streams/version/transitions.ex +++ b/lib/polar/streams/version/transitions.ex @@ -4,6 +4,18 @@ defmodule Polar.Streams.Version.Transitions do alias Polar.Streams.Version + Version + |> transition( + [from: "created", to: "testing", via: "test"], + fn changes -> transit(changes) end + ) + + Version + |> transition( + [from: "testing", to: "active", via: "activate"], + fn changes -> transit(changes, Version.Triggers) end + ) + Version |> transition( [from: "active", to: "inactive", via: "deactivate"], diff --git a/lib/polar/streams/version/triggers.ex b/lib/polar/streams/version/triggers.ex new file mode 100644 index 0000000..b5f62ac --- /dev/null +++ b/lib/polar/streams/version/triggers.ex @@ -0,0 +1,9 @@ +defmodule Polar.Streams.Version.Triggers do + use Eventful.Trigger + + alias Polar.Streams + alias Polar.Streams.Version + + Version + |> trigger([currently: "active"], &Streams.deactivate_previous_versions/2) +end diff --git a/test/polar/streams/product/manager_test.exs b/test/polar/streams/product/manager_test.exs index da1d6a9..6cee99d 100644 --- a/test/polar/streams/product/manager_test.exs +++ b/test/polar/streams/product/manager_test.exs @@ -11,13 +11,13 @@ defmodule Polar.Streams.Product.ManagerTest do setup do password = Accounts.generate_automation_password() - _bot = bot_fixture(%{password: password}) + bot = bot_fixture(%{password: password}) - :ok + {:ok, bot: bot} end describe "filter" do - setup do + setup %{bot: bot} do {:ok, %Product{} = without_active_versions} = Streams.create_product(%{ aliases: ["alpine/3.19", "alpine/3.19/default"], @@ -44,7 +44,7 @@ defmodule Polar.Streams.Product.ManagerTest do } }) - {:ok, _version} = + {:ok, version} = Streams.create_version(with_active_versions, %{ serial: "20240209_13:00", items: [ @@ -65,6 +65,12 @@ defmodule Polar.Streams.Product.ManagerTest do ] }) + {:ok, %{resource: testing_version}} = + Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: _active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + {:ok, without_active_versions: without_active_versions, with_active_versions: with_active_versions} diff --git a/test/polar/streams/version/manager_test.exs b/test/polar/streams/version/manager_test.exs index bd06003..d127911 100644 --- a/test/polar/streams/version/manager_test.exs +++ b/test/polar/streams/version/manager_test.exs @@ -10,7 +10,7 @@ defmodule Polar.Streams.Version.ManagerTest do setup do password = Accounts.generate_automation_password() - _bot = bot_fixture(%{password: password}) + bot = bot_fixture(%{password: password}) {:ok, product} = Streams.create_product(%{ @@ -25,7 +25,7 @@ defmodule Polar.Streams.Version.ManagerTest do } }) - {:ok, product: product} + {:ok, product: product, bot: bot} end describe "create_version" do @@ -43,7 +43,7 @@ defmodule Polar.Streams.Version.ManagerTest do %{existing_version: version} end - test "creating a new version deactivates old versions", %{ + test "activating a new version deactivates old versions", %{ product: product, existing_version: existing_version } do From b3dd153fbeaf4a559d8982cf09ee5b14aa7b0254 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 09:24:32 +0700 Subject: [PATCH 07/45] Clean up all old tests --- lib/polar/streams/version/manager.ex | 31 +++++++------ test/polar/streams/version/manager_test.exs | 46 ++++++++++++++++--- .../streams/version/transitions_test.exs | 9 +++- .../controllers/stream_controller_test.exs | 9 +++- test/polar_web/live/root_live_test.exs | 10 +++- 5 files changed, 78 insertions(+), 27 deletions(-) diff --git a/lib/polar/streams/version/manager.ex b/lib/polar/streams/version/manager.ex index 0df7d2a..a11f396 100644 --- a/lib/polar/streams/version/manager.ex +++ b/lib/polar/streams/version/manager.ex @@ -15,20 +15,23 @@ defmodule Polar.Streams.Version.Manager do basic_setting = Globals.get("basic") bot = Polar.Accounts.Automation.get_bot!() - from( - v in Version, - where: - v.product_id == ^version.product_id and - v.current_state == ^"active", - offset: ^basic_setting.versions_per_product, - order_by: [desc: :inserted_at] - ) - |> Repo.all() - |> Enum.map(fn v -> - Eventful.Transit.perform(v, bot, "deactivate", - comment: "New version activated.", - parameters: %{"event_id" => event.id} + results = + from( + v in Version, + where: + v.product_id == ^version.product_id and + v.current_state == ^"active", + offset: ^basic_setting.versions_per_product, + order_by: [desc: :inserted_at] ) - end) + |> Repo.all() + |> Enum.map(fn v -> + Eventful.Transit.perform(v, bot, "deactivate", + comment: "New version activated.", + parameters: %{"event_id" => event.id} + ) + end) + + {:ok, results} end end diff --git a/test/polar/streams/version/manager_test.exs b/test/polar/streams/version/manager_test.exs index d127911..4aefe1e 100644 --- a/test/polar/streams/version/manager_test.exs +++ b/test/polar/streams/version/manager_test.exs @@ -36,20 +36,33 @@ defmodule Polar.Streams.Version.ManagerTest do end describe "deactivate old version on create" do - setup %{product: product} do + setup %{product: product, bot: bot} do {:ok, version} = Streams.create_version(product, valid_version_attributes(2)) - %{existing_version: version} + {:ok, %{resource: testing_version}} = + Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + + %{existing_version: active_version} end test "activating a new version deactivates old versions", %{ product: product, + bot: bot, existing_version: existing_version } do - assert {:ok, _version} = + assert {:ok, version} = Streams.create_version(product, valid_version_attributes(3)) + {:ok, %{resource: testing_version}} = + Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: _active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + existing_version = Repo.reload(existing_version) assert existing_version.current_state == "inactive" @@ -57,26 +70,45 @@ defmodule Polar.Streams.Version.ManagerTest do end describe "keep 2 previous version" do - setup %{product: product} do + setup %{product: product, bot: bot} do Polar.Globals.save("basic", %{versions_per_product: 2}) {:ok, version3} = Streams.create_version(product, valid_version_attributes(3)) + {:ok, %{resource: testing_version3}} = + Eventful.Transit.perform(version3, bot, "test") + + {:ok, %{resource: active_version3}} = + Eventful.Transit.perform(testing_version3, bot, "activate") + {:ok, version4} = Streams.create_version(product, valid_version_attributes(4)) - %{existing_version: version4, to_be_inactive: version3} + {:ok, %{resource: testing_version4}} = + Eventful.Transit.perform(version4, bot, "test") + + {:ok, %{resource: active_version4}} = + Eventful.Transit.perform(testing_version4, bot, "activate") + + %{existing_version: active_version4, to_be_inactive: active_version3} end - test "create new version deactivate version 3", %{ + test "create and activate new version deactivate version 3", %{ product: product, + bot: bot, existing_version: existing_version, to_be_inactive: to_be_inactive } do - assert {:ok, _version} = + assert {:ok, version} = Streams.create_version(product, valid_version_attributes(5)) + {:ok, %{resource: testing_version}} = + Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: _active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + to_be_inactive = Repo.reload(to_be_inactive) assert to_be_inactive.current_state == "inactive" diff --git a/test/polar/streams/version/transitions_test.exs b/test/polar/streams/version/transitions_test.exs index e973675..4aa5a43 100644 --- a/test/polar/streams/version/transitions_test.exs +++ b/test/polar/streams/version/transitions_test.exs @@ -11,7 +11,7 @@ defmodule Polar.Streams.Version.TransitionsTest do password = Accounts.generate_automation_password() - _bot = bot_fixture(%{password: password}) + bot = bot_fixture(%{password: password}) {:ok, product} = Streams.create_product(%{ @@ -47,7 +47,12 @@ defmodule Polar.Streams.Version.TransitionsTest do ] }) - {:ok, user: user, version: version} + {:ok, %{resource: testing_version}} = Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + + {:ok, user: user, version: active_version} end describe "deactivate" do diff --git a/test/polar_web/controllers/stream_controller_test.exs b/test/polar_web/controllers/stream_controller_test.exs index d141fb3..bfbd282 100644 --- a/test/polar_web/controllers/stream_controller_test.exs +++ b/test/polar_web/controllers/stream_controller_test.exs @@ -12,7 +12,7 @@ defmodule PolarWeb.StreamControllerTest do password = Accounts.generate_automation_password() - _bot = bot_fixture(%{password: password}) + bot = bot_fixture(%{password: password}) {:ok, space} = Accounts.create_space(user, %{name: "some-test-123"}) @@ -36,7 +36,7 @@ defmodule PolarWeb.StreamControllerTest do } }) - {:ok, _version} = + {:ok, version} = Streams.create_version(product, %{ serial: "20240209_13:00", items: [ @@ -57,6 +57,11 @@ defmodule PolarWeb.StreamControllerTest do ] }) + {:ok, %{resource: testing_version}} = Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: _active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + {:ok, product: product, credential: credential} end diff --git a/test/polar_web/live/root_live_test.exs b/test/polar_web/live/root_live_test.exs index b8e16f9..4db8960 100644 --- a/test/polar_web/live/root_live_test.exs +++ b/test/polar_web/live/root_live_test.exs @@ -13,7 +13,7 @@ defmodule PolarWeb.RootLiveTest do password = Accounts.generate_automation_password() - _bot = bot_fixture(%{password: password}) + bot = bot_fixture(%{password: password}) {:ok, space} = Accounts.create_space(user, %{name: "some-test-123"}) @@ -58,7 +58,7 @@ defmodule PolarWeb.RootLiveTest do ] }) - {:ok, _version} = + {:ok, version} = Streams.create_version(product, %{ serial: "20240209-13", items: [ @@ -79,6 +79,12 @@ defmodule PolarWeb.RootLiveTest do ] }) + {:ok, %{resource: testing_version}} = + Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: _active_version}} = + Eventful.Transit.perform(testing_version, bot, "activate") + {:ok, product: product, credential: credential} end From f24b81d4366e904ecb8f6d503acbb90d4b63eb19 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 09:55:11 +0700 Subject: [PATCH 08/45] Add test for create version event controller --- .../publish/version/event_controller.ex | 20 ++++++++ .../controllers/publish/version/event_json.ex | 5 ++ lib/polar_web/router.ex | 4 ++ .../publish/version/event_controller_test.exs | 50 +++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 lib/polar_web/controllers/publish/version/event_controller.ex create mode 100644 lib/polar_web/controllers/publish/version/event_json.ex create mode 100644 test/polar_web/controllers/publish/version/event_controller_test.exs diff --git a/lib/polar_web/controllers/publish/version/event_controller.ex b/lib/polar_web/controllers/publish/version/event_controller.ex new file mode 100644 index 0000000..5c43bb8 --- /dev/null +++ b/lib/polar_web/controllers/publish/version/event_controller.ex @@ -0,0 +1,20 @@ +defmodule PolarWeb.Publish.Version.EventController do + use PolarWeb, :controller + + alias Polar.Repo + alias Polar.Streams.Version + + action_fallback PolarWeb.FallbackController + + def create(%{assigns: %{current_user: current_user}} = conn, %{ + "version_id" => version_id, + "event" => %{"name" => event_name} + }) do + with %Version{} = version <- Repo.get(Version, version_id), + {:ok, %{event: event}} <- Eventful.Transit.perform(version, current_user, event_name) do + conn + |> put_status(:created) + |> render(:create, %{event: event}) + end + end +end diff --git a/lib/polar_web/controllers/publish/version/event_json.ex b/lib/polar_web/controllers/publish/version/event_json.ex new file mode 100644 index 0000000..2e82970 --- /dev/null +++ b/lib/polar_web/controllers/publish/version/event_json.ex @@ -0,0 +1,5 @@ +defmodule PolarWeb.Publish.Version.EventJSON do + def create(%{event: event}) do + %{data: %{id: event.id, name: event.name}} + end +end diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index 6337b1f..b84e5b8 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -104,6 +104,10 @@ defmodule PolarWeb.Router do resources "/products", ProductController, only: [:show] do resources "/versions", VersionController, only: [:create] end + + scope "/versions/:version_id", Version, as: :version do + resources "/events", EventController, only: [:create] + end end end diff --git a/test/polar_web/controllers/publish/version/event_controller_test.exs b/test/polar_web/controllers/publish/version/event_controller_test.exs new file mode 100644 index 0000000..c5cc2a1 --- /dev/null +++ b/test/polar_web/controllers/publish/version/event_controller_test.exs @@ -0,0 +1,50 @@ +defmodule PolarWeb.Publish.Version.EventControllerTest do + use PolarWeb.ConnCase + + import Polar.AccountsFixtures + import Polar.StreamsFixtures + + alias Polar.Accounts + alias Polar.Streams + + setup do + password = Accounts.generate_automation_password() + + bot = bot_fixture(%{password: password}) + + user = Accounts.get_user_by_email_and_password(bot.email, password) + + session_token = + Accounts.generate_user_session_token(user) + |> Base.encode64() + + conn = + build_conn() + |> put_req_header("authorization", session_token) + |> put_req_header("content-type", "application/json") + + product_attributes = valid_product_attributes("alpine:3.19:amd64:default") + + {:ok, product} = Streams.create_product(product_attributes) + + {:ok, version} = + Streams.create_version(product, valid_version_attributes(2)) + + {:ok, conn: conn, version: version} + end + + describe "POST /publish/versions/:version_id/events" do + test "can create transition event", %{conn: conn, version: version} do + conn = + post(conn, "/publish/versions/#{version.id}/events", %{ + event: %{ + name: "test" + } + }) + + assert %{"data" => data} = json_response(conn, 201) + + assert %{"id" => _id, "name" => "test"} = data + end + end +end From 26d4fbd1a33765ffcffe903411722af534ff4e8b Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 14:43:43 +0700 Subject: [PATCH 09/45] Add ability to select release channel via credential --- lib/polar/accounts/space/credential.ex | 6 +- lib/polar/streams/product.ex | 10 ++ lib/polar/streams/release_channel.ex | 17 ++ .../controllers/stream_controller.ex | 8 +- .../controllers/streams/image_controller.ex | 10 +- .../controllers/streams/image_json.ex | 10 +- ...d_release_channel_to_space_credentials.exs | 9 + .../streams/image_controller_test.exs | 159 ++++++++++++++++++ 8 files changed, 223 insertions(+), 6 deletions(-) create mode 100644 lib/polar/streams/release_channel.ex create mode 100644 priv/repo/migrations/20240619064429_add_release_channel_to_space_credentials.exs create mode 100644 test/polar_web/controllers/streams/image_controller_test.exs diff --git a/lib/polar/accounts/space/credential.ex b/lib/polar/accounts/space/credential.ex index 421022c..176fe14 100644 --- a/lib/polar/accounts/space/credential.ex +++ b/lib/polar/accounts/space/credential.ex @@ -3,6 +3,7 @@ defmodule Polar.Accounts.Space.Credential do import Ecto.Changeset alias Polar.Accounts.Space + alias Polar.Streams.ReleaseChannel alias __MODULE__.Transitions alias __MODULE__.Event @@ -32,6 +33,8 @@ defmodule Polar.Accounts.Space.Credential do field :expires_in, :integer, virtual: true field :expires_at, :utc_datetime + field :release_channel, :string, default: "active" + belongs_to :space, Space timestamps(type: :utc_datetime_usec) @@ -46,12 +49,13 @@ defmodule Polar.Accounts.Space.Credential do expires_in_range_values = Enum.map(@expires_in_range, fn r -> r.value end) credential - |> cast(attrs, [:name, :expires_in, :type]) + |> cast(attrs, [:name, :expires_in, :type, :release_channel]) |> generate_token() |> validate_inclusion(:expires_in, expires_in_range_values) |> validate_inclusion(:type, types()) |> maybe_set_expires_at() |> validate_required([:token, :type, :name]) + |> validate_inclusion(:release_channel, ReleaseChannel.valid_names()) |> unique_constraint(:name, name: :space_credentials_space_id_name_index) end diff --git a/lib/polar/streams/product.ex b/lib/polar/streams/product.ex index 34237d4..3df2605 100644 --- a/lib/polar/streams/product.ex +++ b/lib/polar/streams/product.ex @@ -37,6 +37,7 @@ defmodule Polar.Streams.Product do has_one :latest_version, Version, where: [current_state: "active"] has_many :active_versions, Version, where: [current_state: "active"] + has_many :testing_versions, Version, where: [current_state: "testing"] timestamps(type: :utc_datetime_usec) end @@ -54,6 +55,15 @@ defmodule Polar.Streams.Product do |> validate_inclusion(:arch, ["arm64", "amd64"]) end + def scope(:testing, queryable) do + from( + p in queryable, + join: v in assoc(p, :testing_versions), + where: not is_nil(v.product_id), + group_by: [:id] + ) + end + def scope(:active, queryable) do from( p in queryable, diff --git a/lib/polar/streams/release_channel.ex b/lib/polar/streams/release_channel.ex new file mode 100644 index 0000000..36aa3ea --- /dev/null +++ b/lib/polar/streams/release_channel.ex @@ -0,0 +1,17 @@ +defmodule Polar.Streams.ReleaseChannel do + def entries, + do: %{ + "active" => %{ + scope: [:active], + preload: [active_versions: [:items]] + }, + "testing" => %{ + scope: [:testing], + preload: [testing_versions: [:items]] + } + } + + def valid_names do + Map.keys(entries()) + end +end diff --git a/lib/polar_web/controllers/stream_controller.ex b/lib/polar_web/controllers/stream_controller.ex index 2227cfe..7e48741 100644 --- a/lib/polar_web/controllers/stream_controller.ex +++ b/lib/polar_web/controllers/stream_controller.ex @@ -4,13 +4,19 @@ defmodule PolarWeb.StreamController do alias Polar.Accounts alias Polar.Streams + alias Polar.Streams.ReleaseChannel + action_fallback PolarWeb.FallbackController def index(conn, %{"space_token" => space_token}) do credential = Accounts.get_space_credential(token: space_token) if credential do - products = Streams.list_products([:active]) + release_channel = + ReleaseChannel.entries() + |> Map.fetch!(credential.release_channel) + + products = Streams.list_products(release_channel.scope) render(conn, :index, %{products: products}) end diff --git a/lib/polar_web/controllers/streams/image_controller.ex b/lib/polar_web/controllers/streams/image_controller.ex index 0501d60..71f1f49 100644 --- a/lib/polar_web/controllers/streams/image_controller.ex +++ b/lib/polar_web/controllers/streams/image_controller.ex @@ -5,15 +5,21 @@ defmodule PolarWeb.Streams.ImageController do alias Polar.Accounts alias Polar.Streams + alias Polar.Streams.ReleaseChannel + action_fallback PolarWeb.FallbackController def index(conn, %{"space_token" => space_token}) do credential = Accounts.get_space_credential(token: space_token) if credential do + release_channel = + ReleaseChannel.entries() + |> Map.fetch!(credential.release_channel) + products = - Streams.list_products([:active]) - |> Repo.preload(active_versions: [:items]) + Streams.list_products(release_channel.scope) + |> Repo.preload(release_channel.preload) render(conn, :index, %{products: products, credential: credential}) end diff --git a/lib/polar_web/controllers/streams/image_json.ex b/lib/polar_web/controllers/streams/image_json.ex index 6edacbd..096d64d 100644 --- a/lib/polar_web/controllers/streams/image_json.ex +++ b/lib/polar_web/controllers/streams/image_json.ex @@ -20,7 +20,13 @@ defmodule PolarWeb.Streams.ImageJSON do {Product.key(product), product_attributes(product, params)} end - defp product_attributes(product, params) do + defp product_attributes(product, %{credential: credential} = params) do + versions_key = + "#{credential.release_channel}_versions" + |> String.to_existing_atom() + + versions = get_in(product, [Access.key!(versions_key)]) + %{ aliases: Enum.join(product.aliases, ","), arch: product.arch, @@ -30,7 +36,7 @@ defmodule PolarWeb.Streams.ImageJSON do requirements: product.requirements, variant: product.variant, versions: - Enum.map(product.active_versions, &render_version(&1, params)) + Enum.map(versions, &render_version(&1, params)) |> Enum.into(%{}) } end diff --git a/priv/repo/migrations/20240619064429_add_release_channel_to_space_credentials.exs b/priv/repo/migrations/20240619064429_add_release_channel_to_space_credentials.exs new file mode 100644 index 0000000..4d517f6 --- /dev/null +++ b/priv/repo/migrations/20240619064429_add_release_channel_to_space_credentials.exs @@ -0,0 +1,9 @@ +defmodule Polar.Repo.Migrations.AddReleaseChannelToSpaceCredentials do + use Ecto.Migration + + def change do + alter table(:space_credentials) do + add :release_channel, :string, default: "active" + end + end +end diff --git a/test/polar_web/controllers/streams/image_controller_test.exs b/test/polar_web/controllers/streams/image_controller_test.exs new file mode 100644 index 0000000..ab53692 --- /dev/null +++ b/test/polar_web/controllers/streams/image_controller_test.exs @@ -0,0 +1,159 @@ +defmodule PolarWeb.Streams.ImageControllerTest do + use PolarWeb.ConnCase + + alias Polar.Accounts + alias Polar.Streams + + alias Polar.Streams.Product + + import Polar.AccountsFixtures + + setup do + user = user_fixture() + + password = Accounts.generate_automation_password() + + _bot = bot_fixture(%{password: password}) + + {:ok, space} = Accounts.create_space(user, %{name: "some-test-item"}) + + {:ok, %Product{} = product_with_testing} = + Streams.create_product(%{ + aliases: ["alpine/3.19", "alpine/3.19/default"], + arch: "amd64", + os: "Alpine", + release: "3.19", + release_title: "3.19", + variant: "default", + requirements: %{ + secureboot: false + } + }) + + {:ok, version} = + Streams.create_version(product_with_testing, %{ + serial: "20240209_13:00", + items: [ + %{ + name: "lxd.tar.gz", + file_type: "lxd.tar.gz", + hash: "35363f3d086271ed5402d61ab18ec03987bed51758c00079b8c9d372ff6d62dd", + size: 876, + is_metadata: true, + path: "images/alpine/edge/amd64/default/20240209_13:00/incus.tar.xz" + }, + %{ + name: "root.squashfs", + file_type: "squashfs", + hash: "47cc4070da1bf17d8364c390…3603f4ed7e9e46582e690d2", + size: 2_982_800, + path: "images/alpine/edge/amd64/default/20240209_13:00/rootfs.tar.xz" + } + ] + }) + + {:ok, %{resource: _testing_version}} = + Eventful.Transit.perform(version, user, "test") + + {:ok, %Product{} = product_without_testing} = + Streams.create_product(%{ + aliases: ["alpine/3.20", "alpine/3.20/default"], + arch: "amd64", + os: "Alpine", + release: "3.20", + release_title: "3.20", + variant: "default", + requirements: %{ + secureboot: false + } + }) + + {:ok, version} = + Streams.create_version(product_without_testing, %{ + serial: "20240209_13:00", + items: [ + %{ + name: "lxd.tar.gz", + file_type: "lxd.tar.gz", + hash: "35363f3d086271ed5402d61ab18ec03987bed51758c00079b8c9d372ff6d62aa", + size: 876, + is_metadata: true, + path: "images/alpine/edge/amd64/default/20240209_13:00/incus.tar.xz" + }, + %{ + name: "root.squashfs", + file_type: "squashfs", + hash: "67cc4070da1bf17d8364c390…3603f4ed7e9e46582e690d1", + size: 2_982_800, + path: "images/alpine/edge/amd64/default/20240209_13:00/rootfs.tar.xz" + } + ] + }) + + {:ok, %{resource: testing_version}} = + Eventful.Transit.perform(version, user, "test") + + {:ok, %{resource: _active_version}} = + Eventful.Transit.perform(testing_version, user, "activate") + + {:ok, + space: space, + user: user, + product_with_testing: product_with_testing, + product_without_testing: product_without_testing} + end + + describe "product with testing version" do + setup %{space: space, user: user} do + {:ok, credential} = + Accounts.create_space_credential(space, user, %{ + expires_in: 1_296_000, + name: "test-02", + type: "lxd", + release_channel: "testing" + }) + + {:ok, credential: credential} + end + + test "GET /spaces/:space_token/images.json", %{ + credential: credential, + product_without_testing: product_without_testing + } do + conn = get(build_conn(), ~s"/spaces/#{credential.token}/streams/v1/images.json") + + assert %{"products" => products} = json_response(conn, 200) + + product_keys = Map.keys(products) + + assert Product.key(product_without_testing) not in product_keys + end + end + + describe "product with active version" do + setup %{space: space, user: user} do + {:ok, credential} = + Accounts.create_space_credential(space, user, %{ + expires_in: 1_296_000, + name: "test-02", + type: "lxd", + release_channel: "active" + }) + + {:ok, credential: credential} + end + + test "GET /spaces/:space_token/images.json", %{ + credential: credential, + product_with_testing: product_with_testing + } do + conn = get(build_conn(), ~s"/spaces/#{credential.token}/streams/v1/images.json") + + assert %{"products" => products} = json_response(conn, 200) + + product_keys = Map.keys(products) + + assert Product.key(product_with_testing) not in product_keys + end + end +end From 1497e32a874ce8a0e521fa7e9987948db084017b Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 15:19:39 +0700 Subject: [PATCH 10/45] Add radio select for release channel --- lib/polar/accounts/space/credential.ex | 2 +- lib/polar/streams/version/transitions.ex | 6 +++ .../live/dashboard/credential/new_live.ex | 40 ++++++++++++++++++- .../live/dashboard/credential_live.ex | 6 +++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/lib/polar/accounts/space/credential.ex b/lib/polar/accounts/space/credential.ex index 176fe14..9884cdc 100644 --- a/lib/polar/accounts/space/credential.ex +++ b/lib/polar/accounts/space/credential.ex @@ -28,7 +28,7 @@ defmodule Polar.Accounts.Space.Credential do field :name, :string field :token, :binary - field :type, :string + field :type, :string, default: "lxd" field :expires_in, :integer, virtual: true field :expires_at, :utc_datetime diff --git a/lib/polar/streams/version/transitions.ex b/lib/polar/streams/version/transitions.ex index 41c5ab8..1aa8ef9 100644 --- a/lib/polar/streams/version/transitions.ex +++ b/lib/polar/streams/version/transitions.ex @@ -10,6 +10,12 @@ defmodule Polar.Streams.Version.Transitions do fn changes -> transit(changes) end ) + Version + |> transition( + [from: "testing", to: "inactive", via: "deactivate"], + fn changes -> transit(changes) end + ) + Version |> transition( [from: "testing", to: "active", via: "activate"], diff --git a/lib/polar_web/live/dashboard/credential/new_live.ex b/lib/polar_web/live/dashboard/credential/new_live.ex index daf8d9b..b0e5706 100644 --- a/lib/polar_web/live/dashboard/credential/new_live.ex +++ b/lib/polar_web/live/dashboard/credential/new_live.ex @@ -6,6 +6,8 @@ defmodule PolarWeb.Dashboard.Credential.NewLive do alias Polar.Accounts alias Polar.Accounts.Space + alias Polar.Streams.ReleaseChannel + def render(assigns) do ~H"""
@@ -64,6 +66,40 @@ defmodule PolarWeb.Dashboard.Credential.NewLive do
+
+
+

+ <%= gettext("Release channel") %> +

+
+
+ <%= gettext("Choose release channel") %> + +
+ +
+
+

@@ -83,7 +119,7 @@ defmodule PolarWeb.Dashboard.Credential.NewLive do for={Phoenix.HTML.Form.input_id(:credential, :type, type)} class={[ "flex items-center justify-center rounded-md py-3 px-3 text-sm font-semibold sm:flex-1 cursor-pointer focus:outline-none", - "#{if @credential_form.source.changes[:type] == type, do: "bg-indigo-600 text-white hover:bg-indigo-500", else: "ring-1 ring-inset ring-slate-300 bg-white text-slate-900 hover:bg-slate-50"}" + "#{if (@credential_form.source.changes[:type] || @credential_form.data.type) == type, do: "bg-indigo-600 text-white hover:bg-indigo-500", else: "ring-1 ring-inset ring-slate-300 bg-white text-slate-900 hover:bg-slate-50"}" ]} > <.input @@ -93,7 +129,7 @@ defmodule PolarWeb.Dashboard.Credential.NewLive do value={type} class="sr-only" /> - <%= type %> + <%= Phoenix.Naming.humanize(type) %>

diff --git a/lib/polar_web/live/dashboard/credential_live.ex b/lib/polar_web/live/dashboard/credential_live.ex index 61f2bec..f09a6ac 100644 --- a/lib/polar_web/live/dashboard/credential_live.ex +++ b/lib/polar_web/live/dashboard/credential_live.ex @@ -34,6 +34,12 @@ defmodule PolarWeb.Dashboard.CredentialLive do <%= @credential.type %>
+
+
<%= gettext("Release Channel") %>
+
+ <%= @credential.release_channel %> +
+
<%= gettext("Expires At") %>
From 51339b44ff93888d06212d3540143bb2b25dfbb3 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 15:38:42 +0700 Subject: [PATCH 11/45] Add release channel creation testing --- .../dashboard/credential/new_live_test.exs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/polar_web/live/dashboard/credential/new_live_test.exs b/test/polar_web/live/dashboard/credential/new_live_test.exs index 6361780..88e58a0 100644 --- a/test/polar_web/live/dashboard/credential/new_live_test.exs +++ b/test/polar_web/live/dashboard/credential/new_live_test.exs @@ -44,5 +44,29 @@ defmodule PolarWeb.Dashboard.Credential.NewLiveTest do assert render(lv) =~ "can't be blank" end + + test "create new credential with testing release channel", %{conn: conn, space: space} do + {:ok, lv, _html} = live(conn, ~p"/dashboard/spaces/#{space.id}/credentials/new") + + lv + |> form("#new-credential-form", %{ + "credential" => %{ + "name" => "new-cred-test-testing", + "type" => "lxd", + "release_channel" => "testing", + "expires_in" => "2592000" + } + }) + |> render_submit() + + credential = Repo.get_by!(Space.Credential, name: "new-cred-test-testing") + + assert credential.release_channel == "testing" + + {:ok, lv, _html} = + live(conn, ~p"/dashboard/spaces/#{space.id}/credentials/#{credential.id}") + + assert render(lv) =~ "testing" + end end end From 0745d36f7ebfa08d1715b51641671a7227e424da Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 17:49:21 +0700 Subject: [PATCH 12/45] Setup ability to manage lxd cluster pool for testing images --- config/config.exs | 2 ++ config/runtime.exs | 15 ++++++++++++ lib/polar/encrypted/map.ex | 3 +++ lib/polar/machines.ex | 2 ++ lib/polar/machines/cluster.ex | 23 +++++++++++++++++++ lib/polar/machines/cluster/manager.ex | 0 lib/polar/vault.ex | 3 +++ mix.exs | 3 +++ mix.lock | 2 ++ .../20240619102744_create_clusters.exs | 15 ++++++++++++ 10 files changed, 68 insertions(+) create mode 100644 lib/polar/encrypted/map.ex create mode 100644 lib/polar/machines.ex create mode 100644 lib/polar/machines/cluster.ex create mode 100644 lib/polar/machines/cluster/manager.ex create mode 100644 lib/polar/vault.ex create mode 100644 priv/repo/migrations/20240619102744_create_clusters.exs diff --git a/config/config.exs b/config/config.exs index 6d1c697..43e94c7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -11,6 +11,8 @@ config :polar, ecto_repos: [Polar.Repo], generators: [timestamp_type: :utc_datetime] +config :polar, Polar.Vault, json_library: Jason + # Configures the endpoint config :polar, PolarWeb.Endpoint, url: [host: "localhost"], diff --git a/config/runtime.exs b/config/runtime.exs index 266a8ca..f0d2140 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -34,6 +34,21 @@ config :polar, Polar.Assets, endpoint: System.get_env("AWS_S3_ENDPOINT"), default_cdn_host: default_cdn_host +cloak_key = + System.get_env("CLOAK_KEY") || System.get_env("POLAR_CLOAK_KEY") || + raise """ + environment variable CLOAK_KEY or INSTELLAR_CLOAK_KEY is missing. + You can generate one using 32 |> :crypto.strong_rand_bytes() |> Base.encode64() + """ + +config :polar, Polar.Vault, + ciphers: [ + default: { + Cloak.Ciphers.AES.GCM, + tag: "AES.GCM.V1", key: Base.decode64!(cloak_key) + } + ] + if config_env() == :prod do database_url = System.get_env("DATABASE_URL") || diff --git a/lib/polar/encrypted/map.ex b/lib/polar/encrypted/map.ex new file mode 100644 index 0000000..d1a1299 --- /dev/null +++ b/lib/polar/encrypted/map.ex @@ -0,0 +1,3 @@ +defmodule Polar.Encrypted.Map do + use Cloak.Ecto.Map, vault: Polar.Vault +end diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex new file mode 100644 index 0000000..bd5a857 --- /dev/null +++ b/lib/polar/machines.ex @@ -0,0 +1,2 @@ +defmodule Polar.Machines do +end diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex new file mode 100644 index 0000000..ec4b803 --- /dev/null +++ b/lib/polar/machines/cluster.ex @@ -0,0 +1,23 @@ +defmodule Polar.Machines.Cluster do + use Ecto.Schema + import Ecto.Changeset + + schema "clusters" do + field :name, :string + field :current_state, :string, default: "created" + + field :type, :string + field :architecture, :string + + field :credential, Polar.Encrypted.Map + + timestamps(type: :utc_datetime) + end + + @doc false + def changeset(cluster, attrs) do + cluster + |> cast(attrs, [:name, :type, :architecture, :current_state, :credential]) + |> validate_required([:name, :type, :architecture, :current_state, :credential]) + end +end diff --git a/lib/polar/machines/cluster/manager.ex b/lib/polar/machines/cluster/manager.ex new file mode 100644 index 0000000..e69de29 diff --git a/lib/polar/vault.ex b/lib/polar/vault.ex new file mode 100644 index 0000000..2599083 --- /dev/null +++ b/lib/polar/vault.ex @@ -0,0 +1,3 @@ +defmodule Polar.Vault do + use Cloak.Vault, otp_app: :polar +end diff --git a/mix.exs b/mix.exs index 15d080f..723b485 100644 --- a/mix.exs +++ b/mix.exs @@ -71,6 +71,9 @@ defmodule Polar.MixProject do # Cert {:x509, "~> 0.8"}, + # Encryption + {:cloak_ecto, "~> 1.3"}, + # Dev / Test {:dialyxir, "~> 1.0", only: [:dev], runtime: false} ] diff --git a/mix.lock b/mix.lock index a712e81..5aafdc5 100644 --- a/mix.lock +++ b/mix.lock @@ -4,6 +4,8 @@ "bandit": {:hex, :bandit, "1.2.0", "2b5784909cc25b2514868055ff27458cdc63314514b90d86448ff91d18bece80", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "05688b883d87cc3b32991517a61e8c2ce8ee2dd6aa6eb73635426002a6661491"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.1.0", "0b110a9a6c619b19a7f73fa3004aa11d6e719a67e672d1633dc36b6b2290a0f7", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2ad2acb5a8bc049e8d5aa267802631912bb80d5f4110a178ae7999e69dca1bf7"}, "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, + "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, + "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"}, "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, diff --git a/priv/repo/migrations/20240619102744_create_clusters.exs b/priv/repo/migrations/20240619102744_create_clusters.exs new file mode 100644 index 0000000..8188eca --- /dev/null +++ b/priv/repo/migrations/20240619102744_create_clusters.exs @@ -0,0 +1,15 @@ +defmodule Polar.Repo.Migrations.CreateClusters do + use Ecto.Migration + + def change do + create table(:clusters) do + add :name, :citext, null: false + add :type, :citext, null: false + add :arch, :citext, null: false + add :credential, :binary, null: false + add :current_state, :citext, default: "created" + + timestamps(type: :utc_datetime_usec) + end + end +end From e1ef16f976e2afcb1d5a86e5b2fc5c93e4543e92 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 20:14:12 +0700 Subject: [PATCH 13/45] Setup data structures for tracking tests --- lib/polar/machines.ex | 5 ++++ lib/polar/machines/assessment.ex | 27 +++++++++++++++++++ lib/polar/machines/check.ex | 18 +++++++++++++ lib/polar/machines/cluster.ex | 13 ++++++--- lib/polar/machines/cluster/manager.ex | 10 +++++++ .../20240619130455_create_checks.exs | 14 ++++++++++ .../20240619130844_create_assessments.exs | 21 +++++++++++++++ 7 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 lib/polar/machines/assessment.ex create mode 100644 lib/polar/machines/check.ex create mode 100644 priv/repo/migrations/20240619130455_create_checks.exs create mode 100644 priv/repo/migrations/20240619130844_create_assessments.exs diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index bd5a857..67de716 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -1,2 +1,7 @@ defmodule Polar.Machines do + alias __MODULE__.Manager + + defdelegate create_cluster(params), + to: Manager, + as: :create end diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex new file mode 100644 index 0000000..69cef93 --- /dev/null +++ b/lib/polar/machines/assessment.ex @@ -0,0 +1,27 @@ +defmodule Polar.Machines.Assessment do + use Ecto.Schema + import Ecto.Changeset + + alias Polar.Streams + alias Polar.Machines.Check + alias Polar.Machines.Cluster + s + + schema "assessments" do + field :current_state, :string, default: "created" + + belongs_to :check, Check + belongs_to :cluster, Cluster + + belongs_to :version, Streams.Version + + timestamps(type: :utc_datetime) + end + + @doc false + def changeset(assessment, attrs) do + assessment + |> cast(attrs, [:current_state]) + |> validate_required([:current_state]) + end +end diff --git a/lib/polar/machines/check.ex b/lib/polar/machines/check.ex new file mode 100644 index 0000000..15e2c26 --- /dev/null +++ b/lib/polar/machines/check.ex @@ -0,0 +1,18 @@ +defmodule Polar.Machines.Check do + use Ecto.Schema + import Ecto.Changeset + + schema "checks" do + field :slug, :string + field :description, :string + + timestamps(type: :utc_datetime_usec) + end + + @doc false + def changeset(check, attrs) do + check + |> cast(attrs, [:slug, :description]) + |> validate_required([:slug, :description]) + end +end diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex index ec4b803..f125c99 100644 --- a/lib/polar/machines/cluster.ex +++ b/lib/polar/machines/cluster.ex @@ -2,12 +2,19 @@ defmodule Polar.Machines.Cluster do use Ecto.Schema import Ecto.Changeset + @valid_attrs ~w( + name + type + arch + credential + )a + schema "clusters" do field :name, :string field :current_state, :string, default: "created" field :type, :string - field :architecture, :string + field :arch, :string field :credential, Polar.Encrypted.Map @@ -17,7 +24,7 @@ defmodule Polar.Machines.Cluster do @doc false def changeset(cluster, attrs) do cluster - |> cast(attrs, [:name, :type, :architecture, :current_state, :credential]) - |> validate_required([:name, :type, :architecture, :current_state, :credential]) + |> cast(attrs, @valid_attrs) + |> validate_required(@valid_attrs) end end diff --git a/lib/polar/machines/cluster/manager.ex b/lib/polar/machines/cluster/manager.ex index e69de29..a9cc3d1 100644 --- a/lib/polar/machines/cluster/manager.ex +++ b/lib/polar/machines/cluster/manager.ex @@ -0,0 +1,10 @@ +defp Polar.Machines.Cluster do + alias Polar.Repo + alias Polar.Machines.Cluster + + def create(params) do + %Cluster{} + |> Cluster.changeset(params) + |> Repo.insert() + end +end diff --git a/priv/repo/migrations/20240619130455_create_checks.exs b/priv/repo/migrations/20240619130455_create_checks.exs new file mode 100644 index 0000000..c7c587d --- /dev/null +++ b/priv/repo/migrations/20240619130455_create_checks.exs @@ -0,0 +1,14 @@ +defmodule Polar.Repo.Migrations.CreateChecks do + use Ecto.Migration + + def change do + create table(:checks) do + add :slug, :citext, null: false + add :description, :citext, null: false + + timestamps(type: :utc_datetime_usec) + end + + create index(:checks, [:slug], unique: true) + end +end diff --git a/priv/repo/migrations/20240619130844_create_assessments.exs b/priv/repo/migrations/20240619130844_create_assessments.exs new file mode 100644 index 0000000..47b47dd --- /dev/null +++ b/priv/repo/migrations/20240619130844_create_assessments.exs @@ -0,0 +1,21 @@ +defmodule Polar.Repo.Migrations.CreateAssessments do + use Ecto.Migration + + def change do + create table(:assessments) do + add :current_state, :citext, default: "created", null: false + + add :check_id, references(:checks, on_delete: :restrict), null: false + add :version_id, references(:versions, on_delete: :restrict), null: false + add :cluster_id, references(:clusters, on_delete: :restrict), null: false + + timestamps(type: :utc_datetime_utc) + end + + create index(:assessments, [:check_id]) + create index(:assessments, [:version_id]) + create index(:assessments, [:cluster_id]) + + create index(:assessments, [:check_id, :version_id], unique: true) + end +end From 79f32ad8daea2b13bbb63035668bb3ed1b39c001 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 20:19:28 +0700 Subject: [PATCH 14/45] Clean up code and fix build errors --- lib/polar/machines.ex | 4 ++-- lib/polar/machines/assessment.ex | 3 +-- lib/polar/machines/cluster.ex | 2 +- lib/polar/machines/cluster/manager.ex | 2 +- priv/repo/migrations/20240619130844_create_assessments.exs | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index 67de716..8454c2a 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -1,7 +1,7 @@ defmodule Polar.Machines do - alias __MODULE__.Manager + alias __MODULE__.Cluster defdelegate create_cluster(params), - to: Manager, + to: Cluster.Manager, as: :create end diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index 69cef93..663275f 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -5,7 +5,6 @@ defmodule Polar.Machines.Assessment do alias Polar.Streams alias Polar.Machines.Check alias Polar.Machines.Cluster - s schema "assessments" do field :current_state, :string, default: "created" @@ -15,7 +14,7 @@ defmodule Polar.Machines.Assessment do belongs_to :version, Streams.Version - timestamps(type: :utc_datetime) + timestamps(type: :utc_datetime_usec) end @doc false diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex index f125c99..d663072 100644 --- a/lib/polar/machines/cluster.ex +++ b/lib/polar/machines/cluster.ex @@ -18,7 +18,7 @@ defmodule Polar.Machines.Cluster do field :credential, Polar.Encrypted.Map - timestamps(type: :utc_datetime) + timestamps(type: :utc_datetime_usec) end @doc false diff --git a/lib/polar/machines/cluster/manager.ex b/lib/polar/machines/cluster/manager.ex index a9cc3d1..28d86fd 100644 --- a/lib/polar/machines/cluster/manager.ex +++ b/lib/polar/machines/cluster/manager.ex @@ -1,4 +1,4 @@ -defp Polar.Machines.Cluster do +defmodule Polar.Machines.Cluster.Manager do alias Polar.Repo alias Polar.Machines.Cluster diff --git a/priv/repo/migrations/20240619130844_create_assessments.exs b/priv/repo/migrations/20240619130844_create_assessments.exs index 47b47dd..527afe6 100644 --- a/priv/repo/migrations/20240619130844_create_assessments.exs +++ b/priv/repo/migrations/20240619130844_create_assessments.exs @@ -9,7 +9,7 @@ defmodule Polar.Repo.Migrations.CreateAssessments do add :version_id, references(:versions, on_delete: :restrict), null: false add :cluster_id, references(:clusters, on_delete: :restrict), null: false - timestamps(type: :utc_datetime_utc) + timestamps(type: :utc_datetime_usec) end create index(:assessments, [:check_id]) From b1fdefe722720f0800572a17d3b1f656cb23cbcf Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Wed, 19 Jun 2024 20:24:10 +0700 Subject: [PATCH 15/45] Setup cloak key in ci --- .github/workflows/ci.yml | 1 + config/runtime.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b4e659..ac86d32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,3 +151,4 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} DEFAULT_CDN_HOST: some.domain.com + POLAR_CLOAK_KEY: ${{secrets.POLAR_CLOAK_KEY}} diff --git a/config/runtime.exs b/config/runtime.exs index f0d2140..20327da 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -37,7 +37,7 @@ config :polar, Polar.Assets, cloak_key = System.get_env("CLOAK_KEY") || System.get_env("POLAR_CLOAK_KEY") || raise """ - environment variable CLOAK_KEY or INSTELLAR_CLOAK_KEY is missing. + environment variable CLOAK_KEY or POLAR_CLOAK_KEY is missing. You can generate one using 32 |> :crypto.strong_rand_bytes() |> Base.encode64() """ From 3d34da9dc00b227d812e67a472b9df5e11010c02 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 16:00:39 +0700 Subject: [PATCH 16/45] Add credential module for cluster --- lib/polar/machines/cluster.ex | 60 +++++++++++++++++++- lib/polar/machines/cluster/credential.ex | 70 ++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 lib/polar/machines/cluster/credential.ex diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex index d663072..5373270 100644 --- a/lib/polar/machines/cluster.ex +++ b/lib/polar/machines/cluster.ex @@ -2,6 +2,8 @@ defmodule Polar.Machines.Cluster do use Ecto.Schema import Ecto.Changeset + alias __MODULE__.Credential + @valid_attrs ~w( name type @@ -9,13 +11,25 @@ defmodule Polar.Machines.Cluster do credential )a + @required_attrs ~w( + name + type + arch + credential_endpoint + credential_password + credential_password_confirmation + )a + schema "clusters" do field :name, :string field :current_state, :string, default: "created" - field :type, :string + field :type, :string, default: "lxd" field :arch, :string + field :credential_endpoint, :string, virtual: true + field :credential_password, :string, virtual: true + field :credential_password_confirmation, :string, virtual: true field :credential, Polar.Encrypted.Map timestamps(type: :utc_datetime_usec) @@ -26,5 +40,49 @@ defmodule Polar.Machines.Cluster do cluster |> cast(attrs, @valid_attrs) |> validate_required(@valid_attrs) + |> validate_inclusion(:type, ["lxd", "incus"]) + |> validate_inclusion(:arch, ["amd64", "arm64"]) + |> process_credential() + end + + def update_changeset(cluster, attrs) do + cluster + |> cast(attrs, [:credential_endpoint]) + |> maybe_update_credential() + end + + defp maybe_update_credential(%{data: %{credential: credential}} = changeset) do + if changeset.valid? do + endpoint = get_change(changeset, :credential_endpoint) + + credential = + %Credential{ + endpoint: credential["endpoint"], + private_key: credential["private_key"], + certificate: credential["certificate"] + } + |> Credential.update!(%{endpoint: "https://#{endpoint}"}) + + put_change(changeset, :credential, credential) + else + changeset + end + end + + defp process_credential(changeset) do + if changeset.valid? do + endpoint = get_change(changeset, :credential_endpoint) + + credential = + Credential.create!(%{ + endpoint: "https://#{endpoint}", + password: get_change(changeset, :credential_password), + password_confirmation: get_change(changeset, :credential_password_confirmation) + }) + + put_change(changeset, :credential, credential) + else + changeset + end end end diff --git a/lib/polar/machines/cluster/credential.ex b/lib/polar/machines/cluster/credential.ex new file mode 100644 index 0000000..a81e1e8 --- /dev/null +++ b/lib/polar/machines/cluster/credential.ex @@ -0,0 +1,70 @@ +defmodule Polar.Machines.Cluster.Credential do + use Ecto.Schema + import Ecto.Changeset + + @valid_attrs ~w(endpoint password password_confirmation)a + @required_attrs ~w(endpoint password private_key certificate)a + + @primary_key false + embedded_schema do + field :endpoint, :string + + field :password, :string + field :password_confirmation, :string, virtual: true + + field :private_key, :string + field :certificate, :string + end + + @spec create!(map) :: %__MODULE__{} + def create!(attrs) do + %__MODULE__{} + |> changeset(attrs) + |> apply_action!(:insert) + end + + def update!(credential, attrs) do + credential + |> cast(attrs, [:endpoint]) + |> apply_action!(:insert) + end + + def changeset(credential, attrs) do + credential + |> cast(attrs, @valid_attrs) + |> generate_certificate() + |> validate_password() + |> validate_required(@required_attrs) + end + + defp validate_password(changeset) do + validate_change(changeset, :password, fn :password, password -> + password_confirmation = get_change(changeset, :password_confirmation) + + if password == password_confirmation do + [] + else + [password: "do not match"] + end + end) + end + + defp generate_certificate(changeset) do + ca_key = X509.PrivateKey.new_ec(:secp256r1) + + ca = + X509.Certificate.self_signed( + ca_key, + "/C=US/ST=DE/L=Newark/O=Upmaru/CN=instellar.app", + template: :root_ca + ) + + ca_key = X509.PrivateKey.to_pem(ca_key) + + ca = X509.Certificate.to_pem(ca) + + changeset + |> put_change(:private_key, ca_key) + |> put_change(:certificate, ca) + end +end From 297bee4a126e560a8aebd311cb6ebb400933ad39 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 16:43:09 +0700 Subject: [PATCH 17/45] Setup oban for background jobs --- config/config.exs | 2 ++ config/test.exs | 2 ++ lib/polar/application.ex | 1 + lib/polar/machines/cluster.ex | 6 +++-- lib/polar/machines/cluster/connect.ex | 0 lib/polar/machines/cluster/credential.ex | 2 ++ lib/polar/machines/cluster/transitions.ex | 18 +++++++++++++++ lib/polar/machines/cluster/triggers.ex | 12 ++++++++++ mix.exs | 3 +++ mix.lock | 1 + .../20240619102744_create_clusters.exs | 2 ++ .../20240621094029_add_oban_jobs_table.exs | 13 +++++++++++ test/polar/machines/cluster/manager_test.exs | 22 +++++++++++++++++++ 13 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 lib/polar/machines/cluster/connect.ex create mode 100644 lib/polar/machines/cluster/transitions.ex create mode 100644 lib/polar/machines/cluster/triggers.ex create mode 100644 priv/repo/migrations/20240621094029_add_oban_jobs_table.exs create mode 100644 test/polar/machines/cluster/manager_test.exs diff --git a/config/config.exs b/config/config.exs index 43e94c7..bd0ab6e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,6 +13,8 @@ config :polar, config :polar, Polar.Vault, json_library: Jason +config :polar, Oban, engine: Oban.Engines.Basic, queues: [default: 3], repo: Polar.Repo + # Configures the endpoint config :polar, PolarWeb.Endpoint, url: [host: "localhost"], diff --git a/config/test.exs b/config/test.exs index 263374a..53a52d7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -23,6 +23,8 @@ config :polar, PolarWeb.Endpoint, secret_key_base: "6KFZ6zNwk0UXHww8AnEmReHHKixN5GmuKJLFVB+/YvfcvgVaKMwM3G4SvSNz5Z8s", server: false +config :polar, Oban, testing: :inline + # In test we don't send emails. config :polar, Polar.Mailer, adapter: Swoosh.Adapters.Test diff --git a/lib/polar/application.ex b/lib/polar/application.ex index 5b1c659..97d4afd 100644 --- a/lib/polar/application.ex +++ b/lib/polar/application.ex @@ -14,6 +14,7 @@ defmodule Polar.Application do {Phoenix.PubSub, name: Polar.PubSub}, # Start the Finch HTTP client for sending emails {Finch, name: Polar.Finch}, + Polar.Vault, # Start a worker by calling: Polar.Worker.start_link(arg) # {Polar.Worker, arg}, # Start to serve requests, typically the last entry diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex index 5373270..c6e3268 100644 --- a/lib/polar/machines/cluster.ex +++ b/lib/polar/machines/cluster.ex @@ -8,7 +8,9 @@ defmodule Polar.Machines.Cluster do name type arch - credential + credential_endpoint + credential_password + credential_password_confirmation )a @required_attrs ~w( @@ -39,7 +41,7 @@ defmodule Polar.Machines.Cluster do def changeset(cluster, attrs) do cluster |> cast(attrs, @valid_attrs) - |> validate_required(@valid_attrs) + |> validate_required(@required_attrs) |> validate_inclusion(:type, ["lxd", "incus"]) |> validate_inclusion(:arch, ["amd64", "arm64"]) |> process_credential() diff --git a/lib/polar/machines/cluster/connect.ex b/lib/polar/machines/cluster/connect.ex new file mode 100644 index 0000000..e69de29 diff --git a/lib/polar/machines/cluster/credential.ex b/lib/polar/machines/cluster/credential.ex index a81e1e8..306d1c4 100644 --- a/lib/polar/machines/cluster/credential.ex +++ b/lib/polar/machines/cluster/credential.ex @@ -5,6 +5,8 @@ defmodule Polar.Machines.Cluster.Credential do @valid_attrs ~w(endpoint password password_confirmation)a @required_attrs ~w(endpoint password private_key certificate)a + @derive Jason.Encoder + @primary_key false embedded_schema do field :endpoint, :string diff --git a/lib/polar/machines/cluster/transitions.ex b/lib/polar/machines/cluster/transitions.ex new file mode 100644 index 0000000..a285e91 --- /dev/null +++ b/lib/polar/machines/cluster/transitions.ex @@ -0,0 +1,18 @@ +defmodule Polar.Machines.Cluster.Transitions do + @behaviour Eventful.Handler + use Eventful.Transition, repo: Polar.Repo + + alias Polar.Machines.Cluster + + Cluster + |> transition( + [from: "created", to: "connecting", via: "connect"], + fn changes -> transit(changes) end + ) + + Cluster + |> transition( + [from: "connecting", to: "healthy", via: "healthy"], + fn changes -> transit(changes) end + ) +end diff --git a/lib/polar/machines/cluster/triggers.ex b/lib/polar/machines/cluster/triggers.ex new file mode 100644 index 0000000..65df0e9 --- /dev/null +++ b/lib/polar/machines/cluster/triggers.ex @@ -0,0 +1,12 @@ +defmodule Polar.Machines.Cluster.Triggers do + use Eventful.Trigger + + alias Polar.Machines.Cluster + + Cluster + |> trigger([currently: "connecting"], fn event, cluster -> + %{user_id: event.user_id, cluster_id: cluster.id} + |> Connect.new() + |> Oban.insert() + end) +end diff --git a/mix.exs b/mix.exs index 723b485..94f5fb1 100644 --- a/mix.exs +++ b/mix.exs @@ -68,6 +68,9 @@ defmodule Polar.MixProject do {:aws, "~> 0.14.0"}, {:aws_signature, "~> 0.3.1"}, + # Background processing + {:oban, "~> 2.17"}, + # Cert {:x509, "~> 0.8"}, diff --git a/mix.lock b/mix.lock index 5aafdc5..3da6ed8 100644 --- a/mix.lock +++ b/mix.lock @@ -29,6 +29,7 @@ "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"}, "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, + "oban": {:hex, :oban, "2.17.10", "c3e5bd739b5c3fdc38eba1d43ab270a8c6ca4463bb779b7705c69400b0d87678", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4afd027b8e2bc3c399b54318b4f46ee8c40251fb55a285cb4e38b5363f0ee7c4"}, "phoenix": {:hex, :phoenix, "1.7.11", "1d88fc6b05ab0c735b250932c4e6e33bfa1c186f76dcf623d8dd52f07d6379c7", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "b1ec57f2e40316b306708fe59b92a16b9f6f4bf50ccfa41aa8c7feb79e0ec02a"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"}, "phoenix_html": {:hex, :phoenix_html, "4.0.0", "4857ec2edaccd0934a923c2b0ba526c44a173c86b847e8db725172e9e51d11d6", [:mix], [], "hexpm", "cee794a052f243291d92fa3ccabcb4c29bb8d236f655fb03bcbdc3a8214b8d13"}, diff --git a/priv/repo/migrations/20240619102744_create_clusters.exs b/priv/repo/migrations/20240619102744_create_clusters.exs index 8188eca..c600cb4 100644 --- a/priv/repo/migrations/20240619102744_create_clusters.exs +++ b/priv/repo/migrations/20240619102744_create_clusters.exs @@ -11,5 +11,7 @@ defmodule Polar.Repo.Migrations.CreateClusters do timestamps(type: :utc_datetime_usec) end + + create index(:clusters, [:name], unique: true) end end diff --git a/priv/repo/migrations/20240621094029_add_oban_jobs_table.exs b/priv/repo/migrations/20240621094029_add_oban_jobs_table.exs new file mode 100644 index 0000000..b9caf71 --- /dev/null +++ b/priv/repo/migrations/20240621094029_add_oban_jobs_table.exs @@ -0,0 +1,13 @@ +defmodule Polar.Repo.Migrations.AddObanJobsTable do + use Ecto.Migration + + def up do + Oban.Migration.up(version: 12) + end + + # We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if + # necessary, regardless of which version we've migrated `up` to. + def down do + Oban.Migration.down(version: 1) + end +end diff --git a/test/polar/machines/cluster/manager_test.exs b/test/polar/machines/cluster/manager_test.exs new file mode 100644 index 0000000..2ee82e5 --- /dev/null +++ b/test/polar/machines/cluster/manager_test.exs @@ -0,0 +1,22 @@ +defmodule Polar.Machines.Cluster.ManagerTest do + use Polar.DataCase, async: true + + alias Polar.Machines + + describe "create cluster" do + test "successfully create cluster" do + assert {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + assert %Machines.Cluster.Credential{private_key: _private_key, certificate: _certificate} = + cluster.credential + end + end +end From 31191a20a5839a1a59d152234fc5ed64672977ff Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 18:19:25 +0700 Subject: [PATCH 18/45] Add connect worker --- lib/polar/machines/cluster/connect.ex | 42 +++++++++++++++++++++++++++ mix.exs | 3 ++ mix.lock | 11 ++++--- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/lib/polar/machines/cluster/connect.ex b/lib/polar/machines/cluster/connect.ex index e69de29..d51f8d0 100644 --- a/lib/polar/machines/cluster/connect.ex +++ b/lib/polar/machines/cluster/connect.ex @@ -0,0 +1,42 @@ +defmodule Polar.Machines.Cluster.Connect do + use Oban.Worker, queue: :default + + alias Polar.Repo + alias Polar.Accounts.User + alias Polar.Machines.Cluster + + @lexdee Application.compile_env(:instellar, :lexdee) || Lexdee + + def perform(%Job{args: %{"user_id" => user_id, "cluster_id" => cluster_id}}) do + user = Repo.get(User, user_id) + %{credential: credential} = cluster = Repo.get(Cluster, cluster_id) + + client = + Lexdee.create_client( + credential["endpoint"], + credential["certificiate"], + credential["private_key"] + ) + + params = %{ + "password" => credential["password"], + "certificate" => credential["certificate"] + } + + case @lexdee.create_certificate(client, params) do + {:ok, _response} -> + Eventful.Transit.perform(cluster, user, "healthy") + + {:error, %{"error" => "Certificate already in trust store"}} -> + Eventful.Transit.perform(cluster, user, "healthy") + + {:error, %{"error" => reason, "error_code" => 403}} -> + Eventful.Transit.perform(cluster, user, "revert", comment: reason) + + _ -> + Eventful.Transit.perform(cluster, user, "revert", + comment: "Create certificate failed, make sure port is open." + ) + end + end +end diff --git a/mix.exs b/mix.exs index 94f5fb1..ddeb3bf 100644 --- a/mix.exs +++ b/mix.exs @@ -71,6 +71,9 @@ defmodule Polar.MixProject do # Background processing {:oban, "~> 2.17"}, + # LXD client + {:lexdee, "~> 2.3"}, + # Cert {:x509, "~> 0.8"}, diff --git a/mix.lock b/mix.lock index 3da6ed8..dececbc 100644 --- a/mix.lock +++ b/mix.lock @@ -3,7 +3,7 @@ "aws_signature": {:hex, :aws_signature, "0.3.1", "67f369094cbd55ffa2bbd8cc713ede14b195fcfb45c86665cd7c5ad010276148", [:rebar3], [], "hexpm", "50fc4dc1d1f7c2d0a8c63f455b3c66ecd74c1cf4c915c768a636f9227704a674"}, "bandit": {:hex, :bandit, "1.2.0", "2b5784909cc25b2514868055ff27458cdc63314514b90d86448ff91d18bece80", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "05688b883d87cc3b32991517a61e8c2ce8ee2dd6aa6eb73635426002a6661491"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.1.0", "0b110a9a6c619b19a7f73fa3004aa11d6e719a67e672d1633dc36b6b2290a0f7", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2ad2acb5a8bc049e8d5aa267802631912bb80d5f4110a178ae7999e69dca1bf7"}, - "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, + "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"}, @@ -25,10 +25,12 @@ "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized"]}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "lexdee": {:hex, :lexdee, "2.3.9", "f0883bdf572fca7c3e1978f6ec388ea3832c73be1419457c2a452e218f688edc", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.5", [hex: :mint, repo: "hexpm", optional: false]}, {:mint_web_socket, "~> 1.0.2", [hex: :mint_web_socket, repo: "hexpm", optional: false]}, {:tesla, "~> 1.7.0", [hex: :tesla, repo: "hexpm", optional: false]}, {:x509, "~> 0.8.1", [hex: :x509, repo: "hexpm", optional: false]}], "hexpm", "9ac4cc802ac6810d1395769cea57124aa255df97af9f745237d0609a052d41ea"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, - "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"}, - "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"}, - "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, + "mint": {:hex, :mint, "1.6.1", "065e8a5bc9bbd46a41099dfea3e0656436c5cbcb6e741c80bd2bad5cd872446f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4fc518dcc191d02f433393a72a7ba3f6f94b101d094cb6bf532ea54c89423780"}, + "mint_web_socket": {:hex, :mint_web_socket, "1.0.4", "0b539116dbb3d3f861cdf5e15e269a933cb501c113a14db7001a3157d96ffafd", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "027d4c5529c45a4ba0ce27a01c0f35f284a5468519c045ca15f43decb360a991"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "oban": {:hex, :oban, "2.17.10", "c3e5bd739b5c3fdc38eba1d43ab270a8c6ca4463bb779b7705c69400b0d87678", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4afd027b8e2bc3c399b54318b4f46ee8c40251fb55a285cb4e38b5363f0ee7c4"}, "phoenix": {:hex, :phoenix, "1.7.11", "1d88fc6b05ab0c735b250932c4e6e33bfa1c186f76dcf623d8dd52f07d6379c7", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "b1ec57f2e40316b306708fe59b92a16b9f6f4bf50ccfa41aa8c7feb79e0ec02a"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"}, @@ -46,6 +48,7 @@ "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, + "tesla": {:hex, :tesla, "1.7.0", "a62dda2f80d4f8a925eb7b8c5b78c461e0eb996672719fe1a63b26321a5f8b4e", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "2e64f01ebfdb026209b47bc651a0e65203fcff4ae79c11efb73c4852b00dc313"}, "thousand_island": {:hex, :thousand_island, "1.3.2", "bc27f9afba6e1a676dd36507d42e429935a142cf5ee69b8e3f90bff1383943cd", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0e085b93012cd1057b378fce40cbfbf381ff6d957a382bfdd5eca1a98eec2535"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, From a3d7ca77fe043695b3ee6733abc663e444961abe Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 18:49:02 +0700 Subject: [PATCH 19/45] Implement transit for cluster --- lib/polar/machines/cluster/transit.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 lib/polar/machines/cluster/transit.ex diff --git a/lib/polar/machines/cluster/transit.ex b/lib/polar/machines/cluster/transit.ex new file mode 100644 index 0000000..5842269 --- /dev/null +++ b/lib/polar/machines/cluster/transit.ex @@ -0,0 +1,15 @@ +defimpl Eventful.Transit, for: Polar.Machines.Cluster do + def perform(cluster, user, event_name, options \\ []) do + comment = Keyword.get(options, :comment) + domain = Keyword.get(options, :domain, "transitions") + parameters = Keyword.get(options, :parameters, %{}) + + cluster + |> Event.handle(user, %{ + domain: domain, + name: event_name, + comment: comment, + parameters: parameters + }) + end +end From e0b25562a8813daa6d9d0c69d8cd67e77a046de3 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 18:53:49 +0700 Subject: [PATCH 20/45] implement transtions and state machine for cluster --- lib/polar/machines/cluster.ex | 7 +++++ lib/polar/machines/cluster/event.ex | 12 ++++++++ lib/polar/machines/cluster/transit.ex | 2 ++ lib/polar/machines/cluster/triggers.ex | 1 + .../20240621115038_create_cluster_events.exs | 28 +++++++++++++++++++ 5 files changed, 50 insertions(+) create mode 100644 lib/polar/machines/cluster/event.ex create mode 100644 priv/repo/migrations/20240621115038_create_cluster_events.exs diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex index c6e3268..5098d38 100644 --- a/lib/polar/machines/cluster.ex +++ b/lib/polar/machines/cluster.ex @@ -3,6 +3,13 @@ defmodule Polar.Machines.Cluster do import Ecto.Changeset alias __MODULE__.Credential + alias __MODULE__.Transitions + alias __MODULE__.Event + + use Eventful.Transitable + + Transitions + |> governs(:current_state, on: Event) @valid_attrs ~w( name diff --git a/lib/polar/machines/cluster/event.ex b/lib/polar/machines/cluster/event.ex new file mode 100644 index 0000000..b1d5dc7 --- /dev/null +++ b/lib/polar/machines/cluster/event.ex @@ -0,0 +1,12 @@ +defmodule Polar.Machines.Cluster.Event do + alias Polar.Machines.Cluster + alias Polar.Accounts.User + + use Eventful, + parent: {:cluster, Cluster}, + actor: {:user, User} + + alias Cluster.Transitions + + handle(:transitions, using: Transitions) +end diff --git a/lib/polar/machines/cluster/transit.ex b/lib/polar/machines/cluster/transit.ex index 5842269..d2ed892 100644 --- a/lib/polar/machines/cluster/transit.ex +++ b/lib/polar/machines/cluster/transit.ex @@ -1,4 +1,6 @@ defimpl Eventful.Transit, for: Polar.Machines.Cluster do + alias Polar.Machines.Cluster.Event + def perform(cluster, user, event_name, options \\ []) do comment = Keyword.get(options, :comment) domain = Keyword.get(options, :domain, "transitions") diff --git a/lib/polar/machines/cluster/triggers.ex b/lib/polar/machines/cluster/triggers.ex index 65df0e9..bab4450 100644 --- a/lib/polar/machines/cluster/triggers.ex +++ b/lib/polar/machines/cluster/triggers.ex @@ -2,6 +2,7 @@ defmodule Polar.Machines.Cluster.Triggers do use Eventful.Trigger alias Polar.Machines.Cluster + alias Polar.Machines.Cluster.Connect Cluster |> trigger([currently: "connecting"], fn event, cluster -> diff --git a/priv/repo/migrations/20240621115038_create_cluster_events.exs b/priv/repo/migrations/20240621115038_create_cluster_events.exs new file mode 100644 index 0000000..6d35608 --- /dev/null +++ b/priv/repo/migrations/20240621115038_create_cluster_events.exs @@ -0,0 +1,28 @@ +defmodule Polar.Repo.Migrations.CreateClusterEvents do + use Ecto.Migration + + def change do + create table(:cluster_events) do + add(:name, :citext, null: false) + add(:domain, :citext, null: false) + add(:metadata, :map, default: "{}") + + add( + :cluster_id, + references(:clusters, on_delete: :restrict), + null: false + ) + + add( + :user_id, + references(:users, on_delete: :restrict), + null: false + ) + + timestamps(type: :utc_datetime_usec) + end + + create index(:cluster_events, [:cluster_id]) + create index(:cluster_events, [:user_id]) + end +end From 98f74f29f474969a3e830a8b98467e84a0e24710 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 19:03:14 +0700 Subject: [PATCH 21/45] start oban in application --- lib/polar/application.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/polar/application.ex b/lib/polar/application.ex index 97d4afd..ee87981 100644 --- a/lib/polar/application.ex +++ b/lib/polar/application.ex @@ -10,6 +10,7 @@ defmodule Polar.Application do children = [ PolarWeb.Telemetry, Polar.Repo, + {Oban, Application.fetch_env!(:polar, Oban)}, {DNSCluster, query: Application.get_env(:polar, :dns_cluster_query) || :ignore}, {Phoenix.PubSub, name: Polar.PubSub}, # Start the Finch HTTP client for sending emails From 83be7290c0aa56c14d850ca3e6045c394a05daee Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 22:35:14 +0700 Subject: [PATCH 22/45] Add test for transitions and connect cluster --- config/test.exs | 4 +- lib/polar/machines/cluster/connect.ex | 2 +- lib/polar/machines/cluster/transitions.ex | 8 ++- mix.exs | 5 +- mix.lock | 1 + test/polar/machines/cluster/connect_test.exs | 50 +++++++++++++++++++ .../machines/cluster/transitions_test.exs | 34 +++++++++++++ test/support/mocks.ex | 1 + 8 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 test/polar/machines/cluster/connect_test.exs create mode 100644 test/polar/machines/cluster/transitions_test.exs create mode 100644 test/support/mocks.ex diff --git a/config/test.exs b/config/test.exs index 53a52d7..dc38920 100644 --- a/config/test.exs +++ b/config/test.exs @@ -23,7 +23,9 @@ config :polar, PolarWeb.Endpoint, secret_key_base: "6KFZ6zNwk0UXHww8AnEmReHHKixN5GmuKJLFVB+/YvfcvgVaKMwM3G4SvSNz5Z8s", server: false -config :polar, Oban, testing: :inline +config :polar, Oban, testing: :manual + +config :polar, :lexdee, Polar.LexdeeMock # In test we don't send emails. config :polar, Polar.Mailer, adapter: Swoosh.Adapters.Test diff --git a/lib/polar/machines/cluster/connect.ex b/lib/polar/machines/cluster/connect.ex index d51f8d0..b089127 100644 --- a/lib/polar/machines/cluster/connect.ex +++ b/lib/polar/machines/cluster/connect.ex @@ -5,7 +5,7 @@ defmodule Polar.Machines.Cluster.Connect do alias Polar.Accounts.User alias Polar.Machines.Cluster - @lexdee Application.compile_env(:instellar, :lexdee) || Lexdee + @lexdee Application.compile_env(:polar, :lexdee) || Lexdee def perform(%Job{args: %{"user_id" => user_id, "cluster_id" => cluster_id}}) do user = Repo.get(User, user_id) diff --git a/lib/polar/machines/cluster/transitions.ex b/lib/polar/machines/cluster/transitions.ex index a285e91..c1f10d8 100644 --- a/lib/polar/machines/cluster/transitions.ex +++ b/lib/polar/machines/cluster/transitions.ex @@ -7,7 +7,7 @@ defmodule Polar.Machines.Cluster.Transitions do Cluster |> transition( [from: "created", to: "connecting", via: "connect"], - fn changes -> transit(changes) end + fn changes -> transit(changes, Cluster.Triggers) end ) Cluster @@ -15,4 +15,10 @@ defmodule Polar.Machines.Cluster.Transitions do [from: "connecting", to: "healthy", via: "healthy"], fn changes -> transit(changes) end ) + + Cluster + |> transition( + [from: "connecting", to: "created", via: "revert"], + fn changes -> transit(changes) end + ) end diff --git a/mix.exs b/mix.exs index ddeb3bf..6e76f03 100644 --- a/mix.exs +++ b/mix.exs @@ -73,7 +73,7 @@ defmodule Polar.MixProject do # LXD client {:lexdee, "~> 2.3"}, - + # Cert {:x509, "~> 0.8"}, @@ -81,7 +81,8 @@ defmodule Polar.MixProject do {:cloak_ecto, "~> 1.3"}, # Dev / Test - {:dialyxir, "~> 1.0", only: [:dev], runtime: false} + {:dialyxir, "~> 1.0", only: [:dev], runtime: false}, + {:mox, "~> 1.0", only: :test} ] end diff --git a/mix.lock b/mix.lock index dececbc..742c8fd 100644 --- a/mix.lock +++ b/mix.lock @@ -29,6 +29,7 @@ "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mint": {:hex, :mint, "1.6.1", "065e8a5bc9bbd46a41099dfea3e0656436c5cbcb6e741c80bd2bad5cd872446f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4fc518dcc191d02f433393a72a7ba3f6f94b101d094cb6bf532ea54c89423780"}, "mint_web_socket": {:hex, :mint_web_socket, "1.0.4", "0b539116dbb3d3f861cdf5e15e269a933cb501c113a14db7001a3157d96ffafd", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "027d4c5529c45a4ba0ce27a01c0f35f284a5468519c045ca15f43decb360a991"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "oban": {:hex, :oban, "2.17.10", "c3e5bd739b5c3fdc38eba1d43ab270a8c6ca4463bb779b7705c69400b0d87678", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4afd027b8e2bc3c399b54318b4f46ee8c40251fb55a285cb4e38b5363f0ee7c4"}, diff --git a/test/polar/machines/cluster/connect_test.exs b/test/polar/machines/cluster/connect_test.exs new file mode 100644 index 0000000..66d4834 --- /dev/null +++ b/test/polar/machines/cluster/connect_test.exs @@ -0,0 +1,50 @@ +defmodule Polar.Machines.Cluster.ConnectTest do + use Polar.DataCase, async: true + use Oban.Testing, repo: Polar.Repo + + alias Polar.Machines + alias Polar.Machines.Cluster.Connect + + import Polar.AccountsFixtures + + import Mox + + setup :verify_on_exit! + + setup do + user = user_fixture() + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, cluster: cluster, user: user} + end + + describe "perform" do + setup %{cluster: cluster, user: user} do + {:ok, %{resource: connecting_cluster}} = + Eventful.Transit.perform(cluster, user, "connect") + + {:ok, cluster: connecting_cluster} + end + + test "connect to the cluster", %{cluster: cluster, user: user} do + Polar.LexdeeMock + |> expect(:create_certificate, fn _, _ -> + {:ok, nil} + end) + + assert {:ok, %{resource: healthy_cluster}} = + perform_job(Connect, %{cluster_id: cluster.id, user_id: user.id}) + + assert healthy_cluster.current_state == "healthy" + end + end +end diff --git a/test/polar/machines/cluster/transitions_test.exs b/test/polar/machines/cluster/transitions_test.exs new file mode 100644 index 0000000..3c4638f --- /dev/null +++ b/test/polar/machines/cluster/transitions_test.exs @@ -0,0 +1,34 @@ +defmodule Polar.Machines.Cluster.TransitionsTest do + use Polar.DataCase, async: true + + alias Polar.Machines + + import Polar.AccountsFixtures + + setup do + user = user_fixture() + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, cluster: cluster, user: user} + end + + describe "connect" do + test "can transition", %{user: user, cluster: cluster} do + assert {:ok, %{resource: connecting_cluster, trigger: trigger}} = + Eventful.Transit.perform(cluster, user, "connect") + + assert %Oban.Job{worker: "Polar.Machines.Cluster.Connect"} = trigger + + assert connecting_cluster.current_state == "connecting" + end + end +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex new file mode 100644 index 0000000..575bd64 --- /dev/null +++ b/test/support/mocks.ex @@ -0,0 +1 @@ +Mox.defmock(Polar.LexdeeMock, for: Lexdee.Behaviour) From 7779de87d1275fd921c0690cd22e7dcd54f9026d Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 21 Jun 2024 22:47:10 +0700 Subject: [PATCH 23/45] fix typo in connect worker --- lib/polar/machines/cluster/connect.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/polar/machines/cluster/connect.ex b/lib/polar/machines/cluster/connect.ex index b089127..ceb4ff2 100644 --- a/lib/polar/machines/cluster/connect.ex +++ b/lib/polar/machines/cluster/connect.ex @@ -11,10 +11,12 @@ defmodule Polar.Machines.Cluster.Connect do user = Repo.get(User, user_id) %{credential: credential} = cluster = Repo.get(Cluster, cluster_id) + IO.inspect(credential) + client = Lexdee.create_client( credential["endpoint"], - credential["certificiate"], + credential["certificate"], credential["private_key"] ) From d5bb35b4991ff2b8a133fe86f78d7faebab9b952 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 11:51:41 +0700 Subject: [PATCH 24/45] Add cluster controller to publish namespace --- lib/polar/machines.ex | 4 ++ lib/polar/machines/cluster/connect.ex | 2 - lib/polar/machines/cluster/manager.ex | 8 +++ .../publish/testing/assessment_controller.ex | 0 .../publish/testing/assessment_json.ex | 0 .../publish/testing/cluster_controller.ex | 11 +++ .../publish/testing/cluster_json.ex | 17 +++++ lib/polar_web/router.ex | 4 ++ .../testing/cluster_controller_test.exs | 67 +++++++++++++++++++ 9 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 lib/polar_web/controllers/publish/testing/assessment_controller.ex create mode 100644 lib/polar_web/controllers/publish/testing/assessment_json.ex create mode 100644 lib/polar_web/controllers/publish/testing/cluster_controller.ex create mode 100644 lib/polar_web/controllers/publish/testing/cluster_json.ex create mode 100644 test/polar_web/controllers/publish/testing/cluster_controller_test.exs diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index 8454c2a..414532a 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -1,6 +1,10 @@ defmodule Polar.Machines do alias __MODULE__.Cluster + defdelegate list_clusters(scope), + to: Cluster.Manager, + as: :list + defdelegate create_cluster(params), to: Cluster.Manager, as: :create diff --git a/lib/polar/machines/cluster/connect.ex b/lib/polar/machines/cluster/connect.ex index ceb4ff2..6ba574f 100644 --- a/lib/polar/machines/cluster/connect.ex +++ b/lib/polar/machines/cluster/connect.ex @@ -11,8 +11,6 @@ defmodule Polar.Machines.Cluster.Connect do user = Repo.get(User, user_id) %{credential: credential} = cluster = Repo.get(Cluster, cluster_id) - IO.inspect(credential) - client = Lexdee.create_client( credential["endpoint"], diff --git a/lib/polar/machines/cluster/manager.ex b/lib/polar/machines/cluster/manager.ex index 28d86fd..ed8e0a9 100644 --- a/lib/polar/machines/cluster/manager.ex +++ b/lib/polar/machines/cluster/manager.ex @@ -2,6 +2,14 @@ defmodule Polar.Machines.Cluster.Manager do alias Polar.Repo alias Polar.Machines.Cluster + import Ecto.Query, only: [where: 3] + + def list(:for_testing) do + Cluster + |> where([c], c.current_state == "healthy") + |> Repo.all() + end + def create(params) do %Cluster{} |> Cluster.changeset(params) diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex new file mode 100644 index 0000000..e69de29 diff --git a/lib/polar_web/controllers/publish/testing/assessment_json.ex b/lib/polar_web/controllers/publish/testing/assessment_json.ex new file mode 100644 index 0000000..e69de29 diff --git a/lib/polar_web/controllers/publish/testing/cluster_controller.ex b/lib/polar_web/controllers/publish/testing/cluster_controller.ex new file mode 100644 index 0000000..c191370 --- /dev/null +++ b/lib/polar_web/controllers/publish/testing/cluster_controller.ex @@ -0,0 +1,11 @@ +defmodule PolarWeb.Publish.Testing.ClusterController do + use PolarWeb, :controller + + alias Polar.Machines + + def index(conn, _params) do + clusters = Machines.list_clusters(:for_testing) + + render(conn, :index, %{clusters: clusters}) + end +end diff --git a/lib/polar_web/controllers/publish/testing/cluster_json.ex b/lib/polar_web/controllers/publish/testing/cluster_json.ex new file mode 100644 index 0000000..0cea9c9 --- /dev/null +++ b/lib/polar_web/controllers/publish/testing/cluster_json.ex @@ -0,0 +1,17 @@ +defmodule PolarWeb.Publish.Testing.ClusterJSON do + alias Polar.Machines.Cluster + + def index(%{clusters: clusters}) do + %{ + data: Enum.map(clusters, &data/1) + } + end + + def data(%Cluster{} = cluster) do + %{ + id: cluster.id, + credential: cluster.credential, + current_state: cluster.current_state + } + end +end diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index b84e5b8..eae81b2 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -99,6 +99,10 @@ defmodule PolarWeb.Router do scope "/" do pipe_through :publish + scope "/testing", Testing, as: :testing do + resources "/clusters", ClusterController, only: [:index] + end + resources "/storage", StorageController, only: [:show], singleton: true resources "/products", ProductController, only: [:show] do diff --git a/test/polar_web/controllers/publish/testing/cluster_controller_test.exs b/test/polar_web/controllers/publish/testing/cluster_controller_test.exs new file mode 100644 index 0000000..58edc96 --- /dev/null +++ b/test/polar_web/controllers/publish/testing/cluster_controller_test.exs @@ -0,0 +1,67 @@ +defmodule PolarWeb.Publish.Testing.ClusterControllerTest do + use PolarWeb.ConnCase + + alias Polar.Accounts + alias Polar.Machines + + import Polar.AccountsFixtures + + setup do + password = Accounts.generate_automation_password() + + bot = bot_fixture(%{password: password}) + + user = Accounts.get_user_by_email_and_password(bot.email, password) + + session_token = + Accounts.generate_user_session_token(user) + |> Base.encode64() + + conn = + build_conn() + |> put_req_header("authorization", session_token) + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, _created_cluster} = + Machines.create_cluster(%{ + name: "example2", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, conn: conn, cluster: cluster, user: user} + end + + describe "GET /publish/testing/clusters" do + setup %{cluster: cluster, user: user} do + {:ok, %{resource: connecting_cluster}} = + Eventful.Transit.perform(cluster, user, "connect") + + {:ok, %{resource: _healthy_cluster}} = + Eventful.Transit.perform(connecting_cluster, user, "healthy") + + {:ok, cluster: cluster} + end + + test "list healthy clusters", %{conn: conn, cluster: cluster} do + conn = + get(conn, "/publish/testing/clusters") + + assert %{"data" => data} = json_response(conn, 200) + + assert cluster.id in Enum.map(data, & &1["id"]) + end + end +end From 0a1ff0e9c8fef5341deae45c8bdd77e2b2dc268c Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 13:34:05 +0700 Subject: [PATCH 25/45] Clean up event controller for publish api --- lib/polar/machines.ex | 6 +++ lib/polar/machines/assessment.ex | 6 ++- lib/polar/machines/assessment/manager.ex | 10 +++++ .../controllers/publish/event_controller.ex | 40 +++++++++++++++++++ .../publish/{version => }/event_json.ex | 2 +- .../publish/testing/assessment_controller.ex | 14 +++++++ .../publish/version/event_controller.ex | 20 ---------- lib/polar_web/router.ex | 16 +++++--- 8 files changed, 85 insertions(+), 29 deletions(-) create mode 100644 lib/polar/machines/assessment/manager.ex create mode 100644 lib/polar_web/controllers/publish/event_controller.ex rename lib/polar_web/controllers/publish/{version => }/event_json.ex (65%) delete mode 100644 lib/polar_web/controllers/publish/version/event_controller.ex diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index 414532a..99ea22b 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -8,4 +8,10 @@ defmodule Polar.Machines do defdelegate create_cluster(params), to: Cluster.Manager, as: :create + + alias __MODULE__.Assessment + + defdelegate create_assessment(version, params), + to: Assessment.Manager, + as: :create end diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index 663275f..cf13485 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -9,6 +9,8 @@ defmodule Polar.Machines.Assessment do schema "assessments" do field :current_state, :string, default: "created" + field :check_slug, :string, virtual: true + belongs_to :check, Check belongs_to :cluster, Cluster @@ -20,7 +22,7 @@ defmodule Polar.Machines.Assessment do @doc false def changeset(assessment, attrs) do assessment - |> cast(attrs, [:current_state]) - |> validate_required([:current_state]) + |> cast(attrs, [:cluster_id, :check_slug]) + |> validate_required([:cluster_id, :check_slug]) end end diff --git a/lib/polar/machines/assessment/manager.ex b/lib/polar/machines/assessment/manager.ex new file mode 100644 index 0000000..621f840 --- /dev/null +++ b/lib/polar/machines/assessment/manager.ex @@ -0,0 +1,10 @@ +defmodule Polar.Machines.Assessment.Manager do + alias Polar.Repo + alias Polar.Machines.Assessment + + def create(version, params) do + %Assessment{version_id: version.id} + |> Assessment.changeset(params) + |> Repo.insert() + end +end diff --git a/lib/polar_web/controllers/publish/event_controller.ex b/lib/polar_web/controllers/publish/event_controller.ex new file mode 100644 index 0000000..8f4e90f --- /dev/null +++ b/lib/polar_web/controllers/publish/event_controller.ex @@ -0,0 +1,40 @@ +defmodule PolarWeb.Publish.EventController do + use PolarWeb, :controller + + alias Polar.Repo + alias Polar.Streams.Version + + action_fallback PolarWeb.FallbackController + + def create(conn, %{ + "version_id" => version_id, + "event" => event_params + }) do + with %Version{} = version <- Repo.get(Version, version_id) do + transit_and_render(conn, version, event_params) + end + end + + alias Polar.Machines.Assessment + + def create(conn, %{ + "assessment_id" => assessment_id, + "event" => event_params + }) do + with %Assessment{} = assessment <- Repo.get(Assessment, assessment_id) do + transit_and_render(conn, assessment, event_params) + end + end + + defp transit_and_render( + %{assigns: %{current_user: user}} = conn, + resource, + %{"name" => event_name} + ) do + with {:ok, %{event: event}} <- Eventful.Transit.perform(resource, user, event_name) do + conn + |> put_status(:created) + |> render(:create, %{event: event}) + end + end +end diff --git a/lib/polar_web/controllers/publish/version/event_json.ex b/lib/polar_web/controllers/publish/event_json.ex similarity index 65% rename from lib/polar_web/controllers/publish/version/event_json.ex rename to lib/polar_web/controllers/publish/event_json.ex index 2e82970..2d7fe2a 100644 --- a/lib/polar_web/controllers/publish/version/event_json.ex +++ b/lib/polar_web/controllers/publish/event_json.ex @@ -1,4 +1,4 @@ -defmodule PolarWeb.Publish.Version.EventJSON do +defmodule PolarWeb.Publish.EventJSON do def create(%{event: event}) do %{data: %{id: event.id, name: event.name}} end diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex index e69de29..f13cd9c 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_controller.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_controller.ex @@ -0,0 +1,14 @@ +defmodule PolarWeb.Publish.Testing.AssessmentController do + use PolarWeb, :controller + + alias Polar.Repo + alias Polar.Machines + alias Polar.Streams.Version + + def create(conn, %{"version_id" => version_id, "assessment" => assessment_params}) do + with %Version{} = version <- Repo.get(Version, version_id), + {:ok, assessment} <- Machines.create_assessment(version, assessment_params) do + render(conn, :create, %{assessment: assessment}) + end + end +end diff --git a/lib/polar_web/controllers/publish/version/event_controller.ex b/lib/polar_web/controllers/publish/version/event_controller.ex deleted file mode 100644 index 5c43bb8..0000000 --- a/lib/polar_web/controllers/publish/version/event_controller.ex +++ /dev/null @@ -1,20 +0,0 @@ -defmodule PolarWeb.Publish.Version.EventController do - use PolarWeb, :controller - - alias Polar.Repo - alias Polar.Streams.Version - - action_fallback PolarWeb.FallbackController - - def create(%{assigns: %{current_user: current_user}} = conn, %{ - "version_id" => version_id, - "event" => %{"name" => event_name} - }) do - with %Version{} = version <- Repo.get(Version, version_id), - {:ok, %{event: event}} <- Eventful.Transit.perform(version, current_user, event_name) do - conn - |> put_status(:created) - |> render(:create, %{event: event}) - end - end -end diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index eae81b2..bd2bd92 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -99,18 +99,22 @@ defmodule PolarWeb.Router do scope "/" do pipe_through :publish - scope "/testing", Testing, as: :testing do - resources "/clusters", ClusterController, only: [:index] - end - resources "/storage", StorageController, only: [:show], singleton: true resources "/products", ProductController, only: [:show] do resources "/versions", VersionController, only: [:create] end - scope "/versions/:version_id", Version, as: :version do - resources "/events", EventController, only: [:create] + resources "/versions/:version_id/events", EventController, only: [:create] + + scope "/testing", as: :testing do + resources "/clusters", Testing.ClusterController, only: [:index] + + resources "/assessments/:assessment_id/events", EventController, only: [:create] + + scope "/versions/:version_id" do + resources "/assessments", Testing.AssessmentController, only: [:create] + end end end end From aa5d3a73f476a2b6f690f2ad87b3eb77654deda6 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 14:42:56 +0700 Subject: [PATCH 26/45] Add ability to create check --- lib/polar/machines.ex | 8 +++++++- lib/polar/machines/assessment.ex | 7 +++++-- lib/polar/machines/assessment/manager.ex | 4 ++-- lib/polar/machines/check.ex | 15 +++++++++++++-- lib/polar/machines/check/manager.ex | 10 ++++++++++ .../publish/testing/assessment_controller.ex | 14 ++++++++++---- .../publish/testing/assessment_json.ex | 17 +++++++++++++++++ .../publish/testing/check_controller.ex | 0 lib/polar_web/router.ex | 2 +- mix.exs | 3 +++ mix.lock | 1 + test/polar/machines/check/manager_test.exs | 17 +++++++++++++++++ 12 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 lib/polar/machines/check/manager.ex create mode 100644 lib/polar_web/controllers/publish/testing/check_controller.ex create mode 100644 test/polar/machines/check/manager_test.exs diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index 99ea22b..e8cfec7 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -9,9 +9,15 @@ defmodule Polar.Machines do to: Cluster.Manager, as: :create + alias __MODULE__.Check + + defdelegate create_check(params), + to: Check.Manager, + as: :create + alias __MODULE__.Assessment - defdelegate create_assessment(version, params), + defdelegate create_assessment(check, params), to: Assessment.Manager, as: :create end diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index cf13485..c44b481 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -6,6 +6,9 @@ defmodule Polar.Machines.Assessment do alias Polar.Machines.Check alias Polar.Machines.Cluster + @valid_attrs ~w(version_id cluster_id check_slug)a + @required_attrs ~w(version_id cluster_id check_slug)a + schema "assessments" do field :current_state, :string, default: "created" @@ -22,7 +25,7 @@ defmodule Polar.Machines.Assessment do @doc false def changeset(assessment, attrs) do assessment - |> cast(attrs, [:cluster_id, :check_slug]) - |> validate_required([:cluster_id, :check_slug]) + |> cast(attrs, @valid_attrs) + |> validate_required(@required_attrs) end end diff --git a/lib/polar/machines/assessment/manager.ex b/lib/polar/machines/assessment/manager.ex index 621f840..30bce08 100644 --- a/lib/polar/machines/assessment/manager.ex +++ b/lib/polar/machines/assessment/manager.ex @@ -2,8 +2,8 @@ defmodule Polar.Machines.Assessment.Manager do alias Polar.Repo alias Polar.Machines.Assessment - def create(version, params) do - %Assessment{version_id: version.id} + def create(check, params) do + %Assessment{check_id: check.id} |> Assessment.changeset(params) |> Repo.insert() end diff --git a/lib/polar/machines/check.ex b/lib/polar/machines/check.ex index 15e2c26..25dc4ac 100644 --- a/lib/polar/machines/check.ex +++ b/lib/polar/machines/check.ex @@ -3,6 +3,8 @@ defmodule Polar.Machines.Check do import Ecto.Changeset schema "checks" do + field :name, :string, virtual: true + field :slug, :string field :description, :string @@ -12,7 +14,16 @@ defmodule Polar.Machines.Check do @doc false def changeset(check, attrs) do check - |> cast(attrs, [:slug, :description]) - |> validate_required([:slug, :description]) + |> cast(attrs, [:name, :description]) + |> validate_required([:name, :description]) + |> generate_slug() + end + + defp generate_slug(changeset) do + if name = get_change(changeset, :name) do + put_change(changeset, :slug, Slug.slugify(name)) + else + changeset + end end end diff --git a/lib/polar/machines/check/manager.ex b/lib/polar/machines/check/manager.ex new file mode 100644 index 0000000..31e0d88 --- /dev/null +++ b/lib/polar/machines/check/manager.ex @@ -0,0 +1,10 @@ +defmodule Polar.Machines.Check.Manager do + alias Polar.Repo + alias Polar.Machines.Check + + def create(params) do + %Check{} + |> Check.changeset(params) + |> Repo.insert() + end +end diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex index f13cd9c..e3b903f 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_controller.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_controller.ex @@ -3,11 +3,17 @@ defmodule PolarWeb.Publish.Testing.AssessmentController do alias Polar.Repo alias Polar.Machines - alias Polar.Streams.Version - def create(conn, %{"version_id" => version_id, "assessment" => assessment_params}) do - with %Version{} = version <- Repo.get(Version, version_id), - {:ok, assessment} <- Machines.create_assessment(version, assessment_params) do + alias Polar.Machines.Check + + def create(conn, %{ + "check_id" => check_id, + "assessment" => assessment_params + }) do + with %Check{} = check <- Repo.get(Check, check_id), + {:ok, assessment} <- Machines.create_assessment(check, assessment_params) do + assessment = Repo.preload(assessment, [:check]) + render(conn, :create, %{assessment: assessment}) end end diff --git a/lib/polar_web/controllers/publish/testing/assessment_json.ex b/lib/polar_web/controllers/publish/testing/assessment_json.ex index e69de29..124bb3b 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_json.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_json.ex @@ -0,0 +1,17 @@ +defmodule PolarWeb.Publish.Testing.AssessmentJSON do + alias Polar.Machines.Assessment + + def create(%{assessment: assessment}) do + %{data: data(assessment)} + end + + defp data(%Assessment{} = assessment) do + %{ + id: assessment.id, + current_state: assessment.current_state, + check: %{ + slug: assessment.check.slug + } + } + end +end diff --git a/lib/polar_web/controllers/publish/testing/check_controller.ex b/lib/polar_web/controllers/publish/testing/check_controller.ex new file mode 100644 index 0000000..e69de29 diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index bd2bd92..3cb9acf 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -112,7 +112,7 @@ defmodule PolarWeb.Router do resources "/assessments/:assessment_id/events", EventController, only: [:create] - scope "/versions/:version_id" do + scope "/checks/:check_id" do resources "/assessments", Testing.AssessmentController, only: [:create] end end diff --git a/mix.exs b/mix.exs index 6e76f03..dc430e7 100644 --- a/mix.exs +++ b/mix.exs @@ -77,6 +77,9 @@ defmodule Polar.MixProject do # Cert {:x509, "~> 0.8"}, + # Slug + {:slugify, "~> 1.3"}, + # Encryption {:cloak_ecto, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 742c8fd..8d49703 100644 --- a/mix.lock +++ b/mix.lock @@ -44,6 +44,7 @@ "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"}, "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"}, + "slugify": {:hex, :slugify, "1.3.1", "0d3b8b7e5c1eeaa960e44dce94382bee34a39b3ea239293e457a9c5b47cc6fd3", [:mix], [], "hexpm", "cb090bbeb056b312da3125e681d98933a360a70d327820e4b7f91645c4d8be76"}, "swoosh": {:hex, :swoosh, "1.15.2", "490ea85a98e8fb5178c07039e0d8519839e38127724a58947a668c00db7574ee", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.4 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9f7739c02f6c7c0ca82ee397f3bfe0465dbe4c8a65372ac2a5584bf147dd5831"}, "tailwind": {:hex, :tailwind, "0.2.2", "9e27288b568ede1d88517e8c61259bc214a12d7eed271e102db4c93fcca9b2cd", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "ccfb5025179ea307f7f899d1bb3905cd0ac9f687ed77feebc8f67bdca78565c4"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, diff --git a/test/polar/machines/check/manager_test.exs b/test/polar/machines/check/manager_test.exs new file mode 100644 index 0000000..d536d9a --- /dev/null +++ b/test/polar/machines/check/manager_test.exs @@ -0,0 +1,17 @@ +defmodule Polar.Machines.Check.ManagerTest do + use Polar.DataCase, async: true + + alias Polar.Machines + + describe "create check" do + test "successfully create check" do + assert {:ok, check} = + Machines.create_check(%{ + name: "ipv4 issuing", + description: "checks that ipv4 can be issued." + }) + + assert check.slug == "ipv4-issuing" + end + end +end From 9746776bd78524b3e1180b5da9c6bccf0177ece1 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 15:07:48 +0700 Subject: [PATCH 27/45] Add ability to list checks --- lib/polar/machines.ex | 4 ++ lib/polar/machines/check/manager.ex | 4 ++ .../publish/testing/check_controller.ex | 11 +++++ .../controllers/publish/testing/check_json.ex | 11 +++++ lib/polar_web/router.ex | 1 + .../{version => }/event_controller_test.exs | 2 +- .../publish/testing/check_controller_test.exs | 42 +++++++++++++++++++ 7 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 lib/polar_web/controllers/publish/testing/check_json.ex rename test/polar_web/controllers/publish/{version => }/event_controller_test.exs (95%) create mode 100644 test/polar_web/controllers/publish/testing/check_controller_test.exs diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index e8cfec7..06a6fa1 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -11,6 +11,10 @@ defmodule Polar.Machines do alias __MODULE__.Check + defdelegate list_checks(), + to: Check.Manager, + as: :list + defdelegate create_check(params), to: Check.Manager, as: :create diff --git a/lib/polar/machines/check/manager.ex b/lib/polar/machines/check/manager.ex index 31e0d88..34cfcff 100644 --- a/lib/polar/machines/check/manager.ex +++ b/lib/polar/machines/check/manager.ex @@ -2,6 +2,10 @@ defmodule Polar.Machines.Check.Manager do alias Polar.Repo alias Polar.Machines.Check + def list() do + Repo.all(Check) + end + def create(params) do %Check{} |> Check.changeset(params) diff --git a/lib/polar_web/controllers/publish/testing/check_controller.ex b/lib/polar_web/controllers/publish/testing/check_controller.ex index e69de29..7707ca8 100644 --- a/lib/polar_web/controllers/publish/testing/check_controller.ex +++ b/lib/polar_web/controllers/publish/testing/check_controller.ex @@ -0,0 +1,11 @@ +defmodule PolarWeb.Publish.Testing.CheckController do + use PolarWeb, :controller + + alias Polar.Machines + + def index(conn, _params) do + checks = Machines.list_checks() + + render(conn, :index, %{checks: checks}) + end +end diff --git a/lib/polar_web/controllers/publish/testing/check_json.ex b/lib/polar_web/controllers/publish/testing/check_json.ex new file mode 100644 index 0000000..2f06e34 --- /dev/null +++ b/lib/polar_web/controllers/publish/testing/check_json.ex @@ -0,0 +1,11 @@ +defmodule PolarWeb.Publish.Testing.CheckJSON do + alias Polar.Machines.Check + + def index(%{checks: checks}) do + %{data: Enum.map(checks, &data/1)} + end + + def data(%Check{} = check) do + %{id: check.id, slug: check.slug} + end +end diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index 3cb9acf..33f6d0c 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -108,6 +108,7 @@ defmodule PolarWeb.Router do resources "/versions/:version_id/events", EventController, only: [:create] scope "/testing", as: :testing do + resources "/checks", Testing.CheckController, only: [:index] resources "/clusters", Testing.ClusterController, only: [:index] resources "/assessments/:assessment_id/events", EventController, only: [:create] diff --git a/test/polar_web/controllers/publish/version/event_controller_test.exs b/test/polar_web/controllers/publish/event_controller_test.exs similarity index 95% rename from test/polar_web/controllers/publish/version/event_controller_test.exs rename to test/polar_web/controllers/publish/event_controller_test.exs index c5cc2a1..8fd5ba8 100644 --- a/test/polar_web/controllers/publish/version/event_controller_test.exs +++ b/test/polar_web/controllers/publish/event_controller_test.exs @@ -1,4 +1,4 @@ -defmodule PolarWeb.Publish.Version.EventControllerTest do +defmodule PolarWeb.Publish.EventControllerTest do use PolarWeb.ConnCase import Polar.AccountsFixtures diff --git a/test/polar_web/controllers/publish/testing/check_controller_test.exs b/test/polar_web/controllers/publish/testing/check_controller_test.exs new file mode 100644 index 0000000..93c5ce7 --- /dev/null +++ b/test/polar_web/controllers/publish/testing/check_controller_test.exs @@ -0,0 +1,42 @@ +defmodule PolarWeb.Publish.Testing.CheckControllerTest do + use PolarWeb.ConnCase + + alias Polar.Accounts + alias Polar.Machines + + import Polar.AccountsFixtures + + setup do + password = Accounts.generate_automation_password() + + bot = bot_fixture(%{password: password}) + + user = Accounts.get_user_by_email_and_password(bot.email, password) + + session_token = + Accounts.generate_user_session_token(user) + |> Base.encode64() + + conn = + build_conn() + |> put_req_header("authorization", session_token) + + {:ok, _check} = + Machines.create_check(%{ + name: "ipv4-issuing", + description: "checks that ipv4 is correctly issued" + }) + + {:ok, conn: conn} + end + + describe "GET /publish/testing/checks" do + test "get list of available checks", %{conn: conn} do + conn = get(conn, "/publish/testing/checks") + + assert %{"data" => data} = json_response(conn, 200) + + assert Enum.count(data) == 1 + end + end +end From 0a8812aa06d54c9b4e6fb12f3378a4294729c3d2 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 16:23:20 +0700 Subject: [PATCH 28/45] Add test for transitioning assessments --- lib/polar/machines/assessment.ex | 20 ++++- lib/polar/machines/assessment/event.ex | 12 +++ lib/polar/machines/assessment/transit.ex | 17 ++++ lib/polar/machines/assessment/transitions.ex | 24 ++++++ ...0240624084728_create_assessment_events.exs | 28 +++++++ .../machines/assessment/manager_test.exs | 56 +++++++++++++ .../machines/assessment/transitions_test.exs | 79 +++++++++++++++++++ 7 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 lib/polar/machines/assessment/event.ex create mode 100644 lib/polar/machines/assessment/transit.ex create mode 100644 lib/polar/machines/assessment/transitions.ex create mode 100644 priv/repo/migrations/20240624084728_create_assessment_events.exs create mode 100644 test/polar/machines/assessment/manager_test.exs create mode 100644 test/polar/machines/assessment/transitions_test.exs diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index c44b481..760ff83 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -6,14 +6,28 @@ defmodule Polar.Machines.Assessment do alias Polar.Machines.Check alias Polar.Machines.Cluster - @valid_attrs ~w(version_id cluster_id check_slug)a - @required_attrs ~w(version_id cluster_id check_slug)a + alias __MODULE__.Event + alias __MODULE__.Transitions + + use Eventful.Transitable + + Transitions + |> governs(:current_state, on: Event) + + @valid_attrs ~w( + version_id + cluster_id + )a + + @required_attrs ~w( + version_id + cluster_id + )a schema "assessments" do field :current_state, :string, default: "created" field :check_slug, :string, virtual: true - belongs_to :check, Check belongs_to :cluster, Cluster diff --git a/lib/polar/machines/assessment/event.ex b/lib/polar/machines/assessment/event.ex new file mode 100644 index 0000000..8d192a6 --- /dev/null +++ b/lib/polar/machines/assessment/event.ex @@ -0,0 +1,12 @@ +defmodule Polar.Machines.Assessment.Event do + alias Polar.Machines.Assessment + alias Polar.Accounts.User + + use Eventful, + parent: {:assessment, Assessment}, + actor: {:user, User} + + alias Assessment.Transitions + + handle(:transitions, using: Transitions) +end diff --git a/lib/polar/machines/assessment/transit.ex b/lib/polar/machines/assessment/transit.ex new file mode 100644 index 0000000..6afe34d --- /dev/null +++ b/lib/polar/machines/assessment/transit.ex @@ -0,0 +1,17 @@ +defimpl Eventful.Transit, for: Polar.Machines.Assessment do + alias Polar.Machines.Assessment.Event + + def perform(assessment, user, event_name, options \\ []) do + comment = Keyword.get(options, :comment) + domain = Keyword.get(options, :domain, "transitions") + parameters = Keyword.get(options, :parameters, %{}) + + assessment + |> Event.handle(user, %{ + domain: domain, + name: event_name, + comment: comment, + parameters: parameters + }) + end +end diff --git a/lib/polar/machines/assessment/transitions.ex b/lib/polar/machines/assessment/transitions.ex new file mode 100644 index 0000000..0bed85e --- /dev/null +++ b/lib/polar/machines/assessment/transitions.ex @@ -0,0 +1,24 @@ +defmodule Polar.Machines.Assessment.Transitions do + @behaviour Eventful.Handler + use Eventful.Transition, repo: Polar.Repo + + alias Polar.Machines.Assessment + + Assessment + |> transition( + [from: "created", to: "running", via: "run"], + fn changes -> transit(changes) end + ) + + Assessment + |> transition( + [from: "running", to: "passed", via: "pass"], + fn changes -> transit(changes) end + ) + + Assessment + |> transition( + [from: "running", to: "failed", via: "fail"], + fn changes -> transit(changes) end + ) +end diff --git a/priv/repo/migrations/20240624084728_create_assessment_events.exs b/priv/repo/migrations/20240624084728_create_assessment_events.exs new file mode 100644 index 0000000..c69e4a1 --- /dev/null +++ b/priv/repo/migrations/20240624084728_create_assessment_events.exs @@ -0,0 +1,28 @@ +defmodule Polar.Repo.Migrations.CreateAssessmentEvents do + use Ecto.Migration + + def change do + create table(:assessment_events) do + add(:name, :citext, null: false) + add(:domain, :citext, null: false) + add(:metadata, :map, default: "{}") + + add( + :assessment_id, + references(:assessments, on_delete: :restrict), + null: false + ) + + add( + :user_id, + references(:users, on_delete: :restrict), + null: false + ) + + timestamps(type: :utc_datetime_usec) + end + + create index(:assessment_events, [:assessment_id]) + create index(:assessment_events, [:user_id]) + end +end diff --git a/test/polar/machines/assessment/manager_test.exs b/test/polar/machines/assessment/manager_test.exs new file mode 100644 index 0000000..0803032 --- /dev/null +++ b/test/polar/machines/assessment/manager_test.exs @@ -0,0 +1,56 @@ +defmodule Polar.Machines.Assessment.ManagerTest do + use Polar.DataCase, async: true + + alias Polar.Machines + alias Polar.Streams + + import Polar.StreamsFixtures + + setup do + {:ok, check} = + Machines.create_check(%{ + name: "ipv4-issuing", + description: "issue ipv4 correctly" + }) + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, product} = + Streams.create_product(%{ + aliases: ["alpine/3.19", "alpine/3.19/default"], + arch: "amd64", + os: "Alpine", + release: "3.19", + release_title: "3.19", + variant: "default", + requirements: %{ + secureboot: "false" + } + }) + + {:ok, version} = + Streams.create_version(product, valid_version_attributes(2)) + + {:ok, check: check, cluster: cluster, version: version} + end + + describe "create assessment" do + test "successfully create assessment", %{check: check, cluster: cluster, version: version} do + assert {:ok, assessment} = + Machines.create_assessment(check, %{ + version_id: version.id, + cluster_id: cluster.id + }) + + assert assessment.current_state == "created" + end + end +end diff --git a/test/polar/machines/assessment/transitions_test.exs b/test/polar/machines/assessment/transitions_test.exs new file mode 100644 index 0000000..30fc4c6 --- /dev/null +++ b/test/polar/machines/assessment/transitions_test.exs @@ -0,0 +1,79 @@ +defmodule Polar.Machines.Assessment.TransitionsTest do + use Polar.DataCase, async: true + + alias Polar.Machines + alias Polar.Streams + + import Polar.StreamsFixtures + import Polar.AccountsFixtures + + setup do + user = user_fixture() + + {:ok, check} = + Machines.create_check(%{ + name: "ipv4-issuing", + description: "issue ipv4 correctly" + }) + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, product} = + Streams.create_product(%{ + aliases: ["alpine/3.19", "alpine/3.19/default"], + arch: "amd64", + os: "Alpine", + release: "3.19", + release_title: "3.19", + variant: "default", + requirements: %{ + secureboot: "false" + } + }) + + {:ok, version} = + Streams.create_version(product, valid_version_attributes(2)) + + {:ok, assessment} = + Machines.create_assessment(check, %{ + version_id: version.id, + cluster_id: cluster.id + }) + + {:ok, assessment: assessment, user: user} + end + + describe "transitions" do + test "run", %{assessment: assessment, user: user} do + assert {:ok, %{resource: resource}} = Eventful.Transit.perform(assessment, user, "run") + + assert resource.current_state == "running" + end + + test "pass", %{assessment: assessment, user: user} do + {:ok, %{resource: running_assessment}} = Eventful.Transit.perform(assessment, user, "run") + + assert {:ok, %{resource: resource}} = + Eventful.Transit.perform(running_assessment, user, "pass") + + assert resource.current_state == "passed" + end + + test "fail", %{assessment: assessment, user: user} do + {:ok, %{resource: running_assessment}} = Eventful.Transit.perform(assessment, user, "run") + + assert {:ok, %{resource: resource}} = + Eventful.Transit.perform(running_assessment, user, "fail") + + assert resource.current_state == "failed" + end + end +end From df74f09de5e66e6a99f286ed3a8c9118570b12f3 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 17:09:52 +0700 Subject: [PATCH 29/45] Publish api can now create assessment --- lib/polar/machines.ex | 2 +- lib/polar/machines/assessment.ex | 5 +- lib/polar/machines/assessment/manager.ex | 4 +- .../publish/testing/assessment_controller.ex | 10 ++- lib/polar_web/router.ex | 2 +- .../machines/assessment/manager_test.exs | 4 +- .../machines/assessment/transitions_test.exs | 4 +- .../testing/assessment_controller_test.exs | 81 +++++++++++++++++++ 8 files changed, 97 insertions(+), 15 deletions(-) create mode 100644 test/polar_web/controllers/publish/testing/assessment_controller_test.exs diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index 06a6fa1..303fe5b 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -21,7 +21,7 @@ defmodule Polar.Machines do alias __MODULE__.Assessment - defdelegate create_assessment(check, params), + defdelegate create_assessment(version, params), to: Assessment.Manager, as: :create end diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index 760ff83..48a4fb0 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -15,19 +15,18 @@ defmodule Polar.Machines.Assessment do |> governs(:current_state, on: Event) @valid_attrs ~w( - version_id + check_id cluster_id )a @required_attrs ~w( - version_id + check_id cluster_id )a schema "assessments" do field :current_state, :string, default: "created" - field :check_slug, :string, virtual: true belongs_to :check, Check belongs_to :cluster, Cluster diff --git a/lib/polar/machines/assessment/manager.ex b/lib/polar/machines/assessment/manager.ex index 30bce08..621f840 100644 --- a/lib/polar/machines/assessment/manager.ex +++ b/lib/polar/machines/assessment/manager.ex @@ -2,8 +2,8 @@ defmodule Polar.Machines.Assessment.Manager do alias Polar.Repo alias Polar.Machines.Assessment - def create(check, params) do - %Assessment{check_id: check.id} + def create(version, params) do + %Assessment{version_id: version.id} |> Assessment.changeset(params) |> Repo.insert() end diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex index e3b903f..27694e1 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_controller.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_controller.ex @@ -4,17 +4,19 @@ defmodule PolarWeb.Publish.Testing.AssessmentController do alias Polar.Repo alias Polar.Machines - alias Polar.Machines.Check + alias Polar.Streams.Version def create(conn, %{ - "check_id" => check_id, + "version_id" => version_id, "assessment" => assessment_params }) do - with %Check{} = check <- Repo.get(Check, check_id), + with %Version{} = check <- Repo.get(Version, version_id), {:ok, assessment} <- Machines.create_assessment(check, assessment_params) do assessment = Repo.preload(assessment, [:check]) - render(conn, :create, %{assessment: assessment}) + conn + |> put_status(:created) + |> render(:create, %{assessment: assessment}) end end end diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index 33f6d0c..81ce85f 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -113,7 +113,7 @@ defmodule PolarWeb.Router do resources "/assessments/:assessment_id/events", EventController, only: [:create] - scope "/checks/:check_id" do + scope "/versions/:version_id" do resources "/assessments", Testing.AssessmentController, only: [:create] end end diff --git a/test/polar/machines/assessment/manager_test.exs b/test/polar/machines/assessment/manager_test.exs index 0803032..c6f872d 100644 --- a/test/polar/machines/assessment/manager_test.exs +++ b/test/polar/machines/assessment/manager_test.exs @@ -45,8 +45,8 @@ defmodule Polar.Machines.Assessment.ManagerTest do describe "create assessment" do test "successfully create assessment", %{check: check, cluster: cluster, version: version} do assert {:ok, assessment} = - Machines.create_assessment(check, %{ - version_id: version.id, + Machines.create_assessment(version, %{ + check_id: check.id, cluster_id: cluster.id }) diff --git a/test/polar/machines/assessment/transitions_test.exs b/test/polar/machines/assessment/transitions_test.exs index 30fc4c6..8dc081f 100644 --- a/test/polar/machines/assessment/transitions_test.exs +++ b/test/polar/machines/assessment/transitions_test.exs @@ -43,8 +43,8 @@ defmodule Polar.Machines.Assessment.TransitionsTest do Streams.create_version(product, valid_version_attributes(2)) {:ok, assessment} = - Machines.create_assessment(check, %{ - version_id: version.id, + Machines.create_assessment(version, %{ + check_id: check.id, cluster_id: cluster.id }) diff --git a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs new file mode 100644 index 0000000..eb412a8 --- /dev/null +++ b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs @@ -0,0 +1,81 @@ +defmodule PolarWeb.Publish.Testing.AssessmentControllerTest do + use PolarWeb.ConnCase + + alias Polar.Accounts + alias Polar.Machines + alias Polar.Streams + + import Polar.AccountsFixtures + import Polar.StreamsFixtures + + setup do + password = Accounts.generate_automation_password() + + bot = bot_fixture(%{password: password}) + + user = Accounts.get_user_by_email_and_password(bot.email, password) + + session_token = + Accounts.generate_user_session_token(user) + |> Base.encode64() + + conn = + build_conn() + |> put_req_header("authorization", session_token) + + {:ok, check} = + Machines.create_check(%{ + name: "ipv4-issuing", + description: "issue ipv4 correctly" + }) + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, product} = + Streams.create_product(%{ + aliases: ["alpine/3.19", "alpine/3.19/default"], + arch: "amd64", + os: "Alpine", + release: "3.19", + release_title: "3.19", + variant: "default", + requirements: %{ + secureboot: "false" + } + }) + + {:ok, version} = + Streams.create_version(product, valid_version_attributes(2)) + + {:ok, version: version, cluster: cluster, check: check, conn: conn} + end + + describe "POST /publish/testing/versions/:version_id/assessments" do + test "successfully create assessment", %{ + version: version, + conn: conn, + check: check, + cluster: cluster + } do + conn = + post(conn, ~p"/publish/testing/versions/#{version.id}/assessments", %{ + "assessment" => %{ + "check_id" => check.id, + "cluster_id" => cluster.id + } + }) + + assert %{"data" => data} = json_response(conn, 201) + + assert %{"id" => _id, "current_state" => "created", "check" => _check} = data + end + end +end From fad50e1e569e4ee7d77f2433b8c41aacccd62fee Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 24 Jun 2024 17:14:49 +0700 Subject: [PATCH 30/45] Add test for create event for assessment --- .../publish/event_controller_test.exs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/polar_web/controllers/publish/event_controller_test.exs b/test/polar_web/controllers/publish/event_controller_test.exs index 8fd5ba8..8e1c440 100644 --- a/test/polar_web/controllers/publish/event_controller_test.exs +++ b/test/polar_web/controllers/publish/event_controller_test.exs @@ -6,6 +6,7 @@ defmodule PolarWeb.Publish.EventControllerTest do alias Polar.Accounts alias Polar.Streams + alias Polar.Machines setup do password = Accounts.generate_automation_password() @@ -47,4 +48,40 @@ defmodule PolarWeb.Publish.EventControllerTest do assert %{"id" => _id, "name" => "test"} = data end end + + describe "POST /publish/testing/assessments/:assessment_id/events" do + setup %{version: version} do + {:ok, check} = + Machines.create_check(%{ + name: "ipv4-issuing", + description: "issue ipv4 correctly" + }) + + {:ok, cluster} = + Machines.create_cluster(%{ + name: "example", + type: "lxd", + arch: "amd64", + credential_endpoint: "some.cluster.com:8443", + credential_password: "sometoken", + credential_password_confirmation: "sometoken" + }) + + {:ok, assessment} = + Machines.create_assessment(version, %{check_id: check.id, cluster_id: cluster.id}) + + {:ok, assessment: assessment} + end + + test "can transition assessment to running", %{conn: conn, assessment: assessment} do + conn = + post(conn, ~p"/publish/testing/assessments/#{assessment.id}/events", %{ + "event" => %{"name" => "run"} + }) + + assert %{"data" => data} = json_response(conn, 201) + + assert %{"id" => _id, "name" => _name} = data + end + end end From 9d7d7367c02e446fb6368e0be1ec069d988605ea Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 27 Jun 2024 11:44:32 +0700 Subject: [PATCH 31/45] Add attributes needed by icepak --- lib/polar_web/controllers/publish/testing/cluster_json.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/polar_web/controllers/publish/testing/cluster_json.ex b/lib/polar_web/controllers/publish/testing/cluster_json.ex index 0cea9c9..016d847 100644 --- a/lib/polar_web/controllers/publish/testing/cluster_json.ex +++ b/lib/polar_web/controllers/publish/testing/cluster_json.ex @@ -10,6 +10,8 @@ defmodule PolarWeb.Publish.Testing.ClusterJSON do def data(%Cluster{} = cluster) do %{ id: cluster.id, + type: cluster.type, + arch: cluster.arch, credential: cluster.credential, current_state: cluster.current_state } From 3b0263959194084c528ab5cd9af1f3d315ca658a Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 28 Jun 2024 16:29:36 +0700 Subject: [PATCH 32/45] Add support for different assessment based on machine type --- lib/polar/machines/assessment.ex | 5 +++++ ...0240628092615_add_machine_type_to_assessments.exs | 12 ++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index 48a4fb0..aac62e8 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -17,16 +17,20 @@ defmodule Polar.Machines.Assessment do @valid_attrs ~w( check_id cluster_id + machine_type )a @required_attrs ~w( check_id cluster_id + machine_type )a schema "assessments" do field :current_state, :string, default: "created" + field :machine_type, :string + belongs_to :check, Check belongs_to :cluster, Cluster @@ -40,5 +44,6 @@ defmodule Polar.Machines.Assessment do assessment |> cast(attrs, @valid_attrs) |> validate_required(@required_attrs) + |> validate_inclusion(:machine_type, ["container", "vm"]) end end diff --git a/priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs b/priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs new file mode 100644 index 0000000..d6690f2 --- /dev/null +++ b/priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs @@ -0,0 +1,12 @@ +defmodule Polar.Repo.Migrations.AddMachineTypeToAssessments do + use Ecto.Migration + + def change do + alter table(:assessments) do + add :machine_type, :string, null: false + end + + drop index(:assessments, [:check_id, :version_id], unique: true) + create index(:assessments, [:check_id, :version_id, :machine_type], unique: true) + end +end From 21152f8161819f2186e36127f801eb2666a69161 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 28 Jun 2024 23:52:32 +0700 Subject: [PATCH 33/45] Add instance type to assessment --- lib/polar/machines.ex | 4 ++-- lib/polar/machines/assessment.ex | 8 ++++---- lib/polar/machines/assessment/manager.ex | 19 +++++++++++++++++++ .../publish/testing/assessment_controller.ex | 2 +- ...92615_add_instance_type_to_assessments.exs | 12 ++++++++++++ ...092615_add_machine_type_to_assessments.exs | 12 ------------ .../machines/assessment/manager_test.exs | 5 +++-- .../machines/assessment/transitions_test.exs | 5 +++-- .../publish/event_controller_test.exs | 6 +++++- .../testing/assessment_controller_test.exs | 3 ++- 10 files changed, 51 insertions(+), 25 deletions(-) create mode 100644 priv/repo/migrations/20240628092615_add_instance_type_to_assessments.exs delete mode 100644 priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs diff --git a/lib/polar/machines.ex b/lib/polar/machines.ex index 303fe5b..1e2ef74 100644 --- a/lib/polar/machines.ex +++ b/lib/polar/machines.ex @@ -21,7 +21,7 @@ defmodule Polar.Machines do alias __MODULE__.Assessment - defdelegate create_assessment(version, params), + defdelegate get_or_create_assessment(version, params), to: Assessment.Manager, - as: :create + as: :get_or_create end diff --git a/lib/polar/machines/assessment.ex b/lib/polar/machines/assessment.ex index aac62e8..a896bda 100644 --- a/lib/polar/machines/assessment.ex +++ b/lib/polar/machines/assessment.ex @@ -17,19 +17,19 @@ defmodule Polar.Machines.Assessment do @valid_attrs ~w( check_id cluster_id - machine_type + instance_type )a @required_attrs ~w( check_id cluster_id - machine_type + instance_type )a schema "assessments" do field :current_state, :string, default: "created" - field :machine_type, :string + field :instance_type, :string belongs_to :check, Check belongs_to :cluster, Cluster @@ -44,6 +44,6 @@ defmodule Polar.Machines.Assessment do assessment |> cast(attrs, @valid_attrs) |> validate_required(@required_attrs) - |> validate_inclusion(:machine_type, ["container", "vm"]) + |> validate_inclusion(:instance_type, ["container", "vm"]) end end diff --git a/lib/polar/machines/assessment/manager.ex b/lib/polar/machines/assessment/manager.ex index 621f840..e46e548 100644 --- a/lib/polar/machines/assessment/manager.ex +++ b/lib/polar/machines/assessment/manager.ex @@ -2,6 +2,25 @@ defmodule Polar.Machines.Assessment.Manager do alias Polar.Repo alias Polar.Machines.Assessment + def get_or_create(version, params) do + check_id = Map.get(params, "check_id") || params.check_id + instance_type = Map.get(params, "instance_type") || params.instance_type + + Assessment + |> Repo.get_by( + version_id: version.id, + check_id: check_id, + instance_type: instance_type + ) + |> case do + %Assessment{} = assessment -> + assessment + + nil -> + create(version, params) + end + end + def create(version, params) do %Assessment{version_id: version.id} |> Assessment.changeset(params) diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex index 27694e1..2cb4738 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_controller.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_controller.ex @@ -11,7 +11,7 @@ defmodule PolarWeb.Publish.Testing.AssessmentController do "assessment" => assessment_params }) do with %Version{} = check <- Repo.get(Version, version_id), - {:ok, assessment} <- Machines.create_assessment(check, assessment_params) do + {:ok, assessment} <- Machines.get_or_create_assessment(check, assessment_params) do assessment = Repo.preload(assessment, [:check]) conn diff --git a/priv/repo/migrations/20240628092615_add_instance_type_to_assessments.exs b/priv/repo/migrations/20240628092615_add_instance_type_to_assessments.exs new file mode 100644 index 0000000..dada87b --- /dev/null +++ b/priv/repo/migrations/20240628092615_add_instance_type_to_assessments.exs @@ -0,0 +1,12 @@ +defmodule Polar.Repo.Migrations.AddInstanceTypeToAssessments do + use Ecto.Migration + + def change do + alter table(:assessments) do + add :instance_type, :string, null: false + end + + drop index(:assessments, [:check_id, :version_id], unique: true) + create index(:assessments, [:check_id, :version_id, :instance_type], unique: true) + end +end diff --git a/priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs b/priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs deleted file mode 100644 index d6690f2..0000000 --- a/priv/repo/migrations/20240628092615_add_machine_type_to_assessments.exs +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Polar.Repo.Migrations.AddMachineTypeToAssessments do - use Ecto.Migration - - def change do - alter table(:assessments) do - add :machine_type, :string, null: false - end - - drop index(:assessments, [:check_id, :version_id], unique: true) - create index(:assessments, [:check_id, :version_id, :machine_type], unique: true) - end -end diff --git a/test/polar/machines/assessment/manager_test.exs b/test/polar/machines/assessment/manager_test.exs index c6f872d..a38ed69 100644 --- a/test/polar/machines/assessment/manager_test.exs +++ b/test/polar/machines/assessment/manager_test.exs @@ -45,9 +45,10 @@ defmodule Polar.Machines.Assessment.ManagerTest do describe "create assessment" do test "successfully create assessment", %{check: check, cluster: cluster, version: version} do assert {:ok, assessment} = - Machines.create_assessment(version, %{ + Machines.get_or_create_assessment(version, %{ check_id: check.id, - cluster_id: cluster.id + cluster_id: cluster.id, + instance_type: "container" }) assert assessment.current_state == "created" diff --git a/test/polar/machines/assessment/transitions_test.exs b/test/polar/machines/assessment/transitions_test.exs index 8dc081f..830998f 100644 --- a/test/polar/machines/assessment/transitions_test.exs +++ b/test/polar/machines/assessment/transitions_test.exs @@ -43,9 +43,10 @@ defmodule Polar.Machines.Assessment.TransitionsTest do Streams.create_version(product, valid_version_attributes(2)) {:ok, assessment} = - Machines.create_assessment(version, %{ + Machines.get_or_create_assessment(version, %{ check_id: check.id, - cluster_id: cluster.id + cluster_id: cluster.id, + instance_type: "container" }) {:ok, assessment: assessment, user: user} diff --git a/test/polar_web/controllers/publish/event_controller_test.exs b/test/polar_web/controllers/publish/event_controller_test.exs index 8e1c440..3f17d83 100644 --- a/test/polar_web/controllers/publish/event_controller_test.exs +++ b/test/polar_web/controllers/publish/event_controller_test.exs @@ -68,7 +68,11 @@ defmodule PolarWeb.Publish.EventControllerTest do }) {:ok, assessment} = - Machines.create_assessment(version, %{check_id: check.id, cluster_id: cluster.id}) + Machines.get_or_create_assessment(version, %{ + check_id: check.id, + cluster_id: cluster.id, + instance_type: "container" + }) {:ok, assessment: assessment} end diff --git a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs index eb412a8..cf9b304 100644 --- a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs +++ b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs @@ -69,7 +69,8 @@ defmodule PolarWeb.Publish.Testing.AssessmentControllerTest do post(conn, ~p"/publish/testing/versions/#{version.id}/assessments", %{ "assessment" => %{ "check_id" => check.id, - "cluster_id" => cluster.id + "cluster_id" => cluster.id, + "instance_type" => "container" } }) From c86f9c9dbe38668ac05d6d2ea41031aa60ff1623 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 1 Jul 2024 11:03:24 +0700 Subject: [PATCH 34/45] Add get version controller action --- .../controllers/publish/version_controller.ex | 9 ++++++++ .../controllers/publish/version_json.ex | 4 ++++ lib/polar_web/router.ex | 2 +- .../publish/version_controller_test.exs | 23 +++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/polar_web/controllers/publish/version_controller.ex b/lib/polar_web/controllers/publish/version_controller.ex index 69cafe2..59f5ca6 100644 --- a/lib/polar_web/controllers/publish/version_controller.ex +++ b/lib/polar_web/controllers/publish/version_controller.ex @@ -4,9 +4,18 @@ defmodule PolarWeb.Publish.VersionController do alias Polar.Repo alias Polar.Streams alias Polar.Streams.Product + alias Polar.Streams.Version action_fallback PolarWeb.FallbackController + def show(conn, %{"product_id" => product_id, "id" => serial}) do + version = Repo.get_by(Version, product_id: product_id, serial: serial) + + if version do + render(conn, :show, %{version: version}) + end + end + def create(conn, %{"product_id" => product_id, "version" => version_params}) do product = Repo.get(Product, product_id) diff --git a/lib/polar_web/controllers/publish/version_json.ex b/lib/polar_web/controllers/publish/version_json.ex index 73b07fc..937c014 100644 --- a/lib/polar_web/controllers/publish/version_json.ex +++ b/lib/polar_web/controllers/publish/version_json.ex @@ -1,4 +1,8 @@ defmodule PolarWeb.Publish.VersionJSON do + def show(%{version: version}) do + %{data: %{id: version.id}} + end + def create(%{version: version}) do %{data: %{id: version.id}} end diff --git a/lib/polar_web/router.ex b/lib/polar_web/router.ex index 81ce85f..72e710b 100644 --- a/lib/polar_web/router.ex +++ b/lib/polar_web/router.ex @@ -102,7 +102,7 @@ defmodule PolarWeb.Router do resources "/storage", StorageController, only: [:show], singleton: true resources "/products", ProductController, only: [:show] do - resources "/versions", VersionController, only: [:create] + resources "/versions", VersionController, only: [:show, :create] end resources "/versions/:version_id/events", EventController, only: [:create] diff --git a/test/polar_web/controllers/publish/version_controller_test.exs b/test/polar_web/controllers/publish/version_controller_test.exs index dbb237a..f598d12 100644 --- a/test/polar_web/controllers/publish/version_controller_test.exs +++ b/test/polar_web/controllers/publish/version_controller_test.exs @@ -7,6 +7,8 @@ defmodule PolarWeb.Publish.VersionControllerTest do alias Polar.Accounts alias Polar.Streams + import Polar.StreamsFixtures + setup do password = Accounts.generate_automation_password() @@ -25,6 +27,27 @@ defmodule PolarWeb.Publish.VersionControllerTest do {:ok, conn: conn} end + describe "GET /publish/products/:product_id/versions/:id" do + setup do + product_attributes = valid_product_attributes("alpine:3.19:amd64:default") + + {:ok, product} = Streams.create_product(product_attributes) + + {:ok, version} = + Streams.create_version(product, valid_version_attributes(2)) + + {:ok, product: product, version: version} + end + + test "can fetch existing version", %{conn: conn, product: product, version: version} do + conn = get(conn, "/publish/products/#{product.id}/versions/#{version.serial}") + + assert %{"data" => data} = json_response(conn, 200) + + assert %{"id" => _id} = data + end + end + describe "POST /publish/products/:product_id/versions" do setup do product_attributes = valid_product_attributes("alpine:3.19:amd64:default") From 354dd37e220cccbf5dcaee87f8603e263ada1cb1 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 1 Jul 2024 19:35:07 +0700 Subject: [PATCH 35/45] Add state transitions for assessment --- lib/polar/machines/assessment/transitions.ex | 12 ++++++++++++ .../publish/testing/assessment_controller.ex | 1 - .../publish/testing/assessment_controller_test.exs | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/polar/machines/assessment/transitions.ex b/lib/polar/machines/assessment/transitions.ex index 0bed85e..8775338 100644 --- a/lib/polar/machines/assessment/transitions.ex +++ b/lib/polar/machines/assessment/transitions.ex @@ -10,6 +10,18 @@ defmodule Polar.Machines.Assessment.Transitions do fn changes -> transit(changes) end ) + Assessment + |> transition( + [from: "failed", to: "running", via: "run"], + fn changes -> transit(changes) end + ) + + Assessment + |> transition( + [from: "running", to: "running", via: "run"], + fn changes -> transit(changes) end + ) + Assessment |> transition( [from: "running", to: "passed", via: "pass"], diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex index 2cb4738..b2eec0d 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_controller.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_controller.ex @@ -15,7 +15,6 @@ defmodule PolarWeb.Publish.Testing.AssessmentController do assessment = Repo.preload(assessment, [:check]) conn - |> put_status(:created) |> render(:create, %{assessment: assessment}) end end diff --git a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs index cf9b304..caa11ae 100644 --- a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs +++ b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs @@ -74,7 +74,7 @@ defmodule PolarWeb.Publish.Testing.AssessmentControllerTest do } }) - assert %{"data" => data} = json_response(conn, 201) + assert %{"data" => data} = json_response(conn, 200) assert %{"id" => _id, "current_state" => "created", "check" => _check} = data end From 3dc9a03a0271410bc17244bd7878181f4c254bc5 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 1 Jul 2024 20:08:17 +0700 Subject: [PATCH 36/45] Allow rerun of test --- lib/polar/streams/version/transitions.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/polar/streams/version/transitions.ex b/lib/polar/streams/version/transitions.ex index 1aa8ef9..2b79322 100644 --- a/lib/polar/streams/version/transitions.ex +++ b/lib/polar/streams/version/transitions.ex @@ -10,6 +10,12 @@ defmodule Polar.Streams.Version.Transitions do fn changes -> transit(changes) end ) + Version + |> transition( + [from: "inactive", to: "testing", via: "test"], + fn changes -> transit(changes) end + ) + Version |> transition( [from: "testing", to: "inactive", via: "deactivate"], From 9980b74f3c0d5b13dc2d8bce774fd934ba35d994 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 2 Jul 2024 10:59:21 +0700 Subject: [PATCH 37/45] Add parameter validation for assessment --- .../publish/testing/assessment_controller.ex | 8 ++++- .../publish/testing/check_controller.ex | 2 ++ .../publish/testing/cluster_controller.ex | 2 ++ lib/polar_web/params/assessment.ex | 29 +++++++++++++++++++ .../testing/assessment_controller_test.exs | 17 +++++++++++ 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 lib/polar_web/params/assessment.ex diff --git a/lib/polar_web/controllers/publish/testing/assessment_controller.ex b/lib/polar_web/controllers/publish/testing/assessment_controller.ex index b2eec0d..8067ed6 100644 --- a/lib/polar_web/controllers/publish/testing/assessment_controller.ex +++ b/lib/polar_web/controllers/publish/testing/assessment_controller.ex @@ -6,12 +6,18 @@ defmodule PolarWeb.Publish.Testing.AssessmentController do alias Polar.Streams.Version + alias PolarWeb.Params.Assessment + + action_fallback PolarWeb.FallbackController + def create(conn, %{ "version_id" => version_id, "assessment" => assessment_params }) do with %Version{} = check <- Repo.get(Version, version_id), - {:ok, assessment} <- Machines.get_or_create_assessment(check, assessment_params) do + {:ok, assessment_params} <- Assessment.parse(assessment_params), + {:ok, assessment} <- + Machines.get_or_create_assessment(check, Map.from_struct(assessment_params)) do assessment = Repo.preload(assessment, [:check]) conn diff --git a/lib/polar_web/controllers/publish/testing/check_controller.ex b/lib/polar_web/controllers/publish/testing/check_controller.ex index 7707ca8..6218756 100644 --- a/lib/polar_web/controllers/publish/testing/check_controller.ex +++ b/lib/polar_web/controllers/publish/testing/check_controller.ex @@ -3,6 +3,8 @@ defmodule PolarWeb.Publish.Testing.CheckController do alias Polar.Machines + action_fallback PolarWeb.FallbackController + def index(conn, _params) do checks = Machines.list_checks() diff --git a/lib/polar_web/controllers/publish/testing/cluster_controller.ex b/lib/polar_web/controllers/publish/testing/cluster_controller.ex index c191370..1851fd4 100644 --- a/lib/polar_web/controllers/publish/testing/cluster_controller.ex +++ b/lib/polar_web/controllers/publish/testing/cluster_controller.ex @@ -3,6 +3,8 @@ defmodule PolarWeb.Publish.Testing.ClusterController do alias Polar.Machines + action_fallback PolarWeb.FallbackController + def index(conn, _params) do clusters = Machines.list_clusters(:for_testing) diff --git a/lib/polar_web/params/assessment.ex b/lib/polar_web/params/assessment.ex new file mode 100644 index 0000000..d5b84e9 --- /dev/null +++ b/lib/polar_web/params/assessment.ex @@ -0,0 +1,29 @@ +defmodule PolarWeb.Params.Assessment do + use Ecto.Schema + import Ecto.Changeset + + @required_attrs ~w( + check_id + cluster_id + instance_type + )a + + @primary_key false + embedded_schema do + field :check_id, :integer + field :cluster_id, :integer + field :instance_type, :string + end + + def parse(params) do + %__MODULE__{} + |> changeset(params) + |> apply_action(:insert) + end + + def changeset(assessment, params) do + assessment + |> cast(params, @required_attrs) + |> validate_required(@required_attrs) + end +end diff --git a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs index caa11ae..4bdddf0 100644 --- a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs +++ b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs @@ -78,5 +78,22 @@ defmodule PolarWeb.Publish.Testing.AssessmentControllerTest do assert %{"id" => _id, "current_state" => "created", "check" => _check} = data end + + test "invalid parameter passed in", %{ + version: version, + conn: conn, + check: check, + cluster: cluster + } do + conn = + post(conn, ~p"/publish/testing/versions/#{version.id}/assessments", %{ + "assessment" => %{ + "check_id" => check.id, + "cluster_id" => cluster.id + } + }) + + assert %{"errors" => _errors} = json_response(conn, 422) + end end end From 320fe1f096e42417f45db8259c0f2e809dad86de Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 2 Jul 2024 14:59:49 +0700 Subject: [PATCH 38/45] Allow healthy cluster to connect --- lib/polar/machines/cluster/transitions.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/polar/machines/cluster/transitions.ex b/lib/polar/machines/cluster/transitions.ex index c1f10d8..ed91f65 100644 --- a/lib/polar/machines/cluster/transitions.ex +++ b/lib/polar/machines/cluster/transitions.ex @@ -10,6 +10,12 @@ defmodule Polar.Machines.Cluster.Transitions do fn changes -> transit(changes, Cluster.Triggers) end ) + Cluster + |> transition( + [from: "healthy", to: "connecting", via: "connect"], + fn changes -> transit(changes, Cluster.Triggers) end + ) + Cluster |> transition( [from: "connecting", to: "healthy", via: "healthy"], From 0fb3d3a50fbd05269fe2883bf336b731c710957e Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 2 Jul 2024 15:43:50 +0700 Subject: [PATCH 39/45] Allow from testing to testing on retry --- lib/polar/streams/version/transitions.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/polar/streams/version/transitions.ex b/lib/polar/streams/version/transitions.ex index 2b79322..b592e8a 100644 --- a/lib/polar/streams/version/transitions.ex +++ b/lib/polar/streams/version/transitions.ex @@ -10,6 +10,12 @@ defmodule Polar.Streams.Version.Transitions do fn changes -> transit(changes) end ) + Version + |> transition( + [from: "testing", to: "testing", via: "test"], + fn changes -> transit(changes) end + ) + Version |> transition( [from: "inactive", to: "testing", via: "test"], From 790b7522bbba96b40474f2649ba3a8fa0ed6b219 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 2 Jul 2024 16:46:03 +0700 Subject: [PATCH 40/45] fix issue with returning existing assessment --- lib/polar/machines/assessment/manager.ex | 2 +- .../testing/assessment_controller_test.exs | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/polar/machines/assessment/manager.ex b/lib/polar/machines/assessment/manager.ex index e46e548..2043d41 100644 --- a/lib/polar/machines/assessment/manager.ex +++ b/lib/polar/machines/assessment/manager.ex @@ -14,7 +14,7 @@ defmodule Polar.Machines.Assessment.Manager do ) |> case do %Assessment{} = assessment -> - assessment + {:ok, assessment} nil -> create(version, params) diff --git a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs index 4bdddf0..3b2b071 100644 --- a/test/polar_web/controllers/publish/testing/assessment_controller_test.exs +++ b/test/polar_web/controllers/publish/testing/assessment_controller_test.exs @@ -79,6 +79,31 @@ defmodule PolarWeb.Publish.Testing.AssessmentControllerTest do assert %{"id" => _id, "current_state" => "created", "check" => _check} = data end + test "when assessment already exists", %{ + version: version, + conn: conn, + check: check, + cluster: cluster + } do + {:ok, _assessment} = + Machines.get_or_create_assessment(version, %{ + check_id: check.id, + cluster_id: cluster.id, + instance_type: "container" + }) + + conn = + post(conn, ~p"/publish/testing/versions/#{version.id}/assessments", %{ + "assessment" => %{ + "check_id" => check.id, + "cluster_id" => cluster.id, + "instance_type" => "container" + } + }) + + assert %{"data" => _data} = json_response(conn, 200) + end + test "invalid parameter passed in", %{ version: version, conn: conn, From 472ab948a0d18c868db60870f6d1fa3c1dcbd4a4 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 2 Jul 2024 21:50:20 +0700 Subject: [PATCH 41/45] render requirements --- lib/polar_web/controllers/publish/product_json.ex | 3 ++- .../polar_web/controllers/publish/product_controller_test.exs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/polar_web/controllers/publish/product_json.ex b/lib/polar_web/controllers/publish/product_json.ex index fdcce2c..1712cda 100644 --- a/lib/polar_web/controllers/publish/product_json.ex +++ b/lib/polar_web/controllers/publish/product_json.ex @@ -5,7 +5,8 @@ defmodule PolarWeb.Publish.ProductJSON do %{ data: %{ id: product.id, - key: Product.key(product) + key: Product.key(product), + requirements: product.requirements } } end diff --git a/test/polar_web/controllers/publish/product_controller_test.exs b/test/polar_web/controllers/publish/product_controller_test.exs index 8133a8f..e7caac3 100644 --- a/test/polar_web/controllers/publish/product_controller_test.exs +++ b/test/polar_web/controllers/publish/product_controller_test.exs @@ -57,7 +57,9 @@ defmodule PolarWeb.Publish.ProductControllerTest do assert %{"data" => data} = json_response(conn, 200) - assert %{"id" => _id, "key" => _key} = data + assert %{"id" => _id, "key" => _key, "requirements" => requirements} = data + + assert %{"secureboot" => "false"} = requirements end end end From dd67e3595ba8f5d313930dcd81c680b1ca427656 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 2 Jul 2024 23:54:55 +0700 Subject: [PATCH 42/45] Add serial to version --- lib/polar_web/controllers/publish/version_json.ex | 4 ++-- .../polar_web/controllers/publish/version_controller_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/polar_web/controllers/publish/version_json.ex b/lib/polar_web/controllers/publish/version_json.ex index 937c014..341b707 100644 --- a/lib/polar_web/controllers/publish/version_json.ex +++ b/lib/polar_web/controllers/publish/version_json.ex @@ -1,9 +1,9 @@ defmodule PolarWeb.Publish.VersionJSON do def show(%{version: version}) do - %{data: %{id: version.id}} + %{data: %{id: version.id, serial: version.serial}} end def create(%{version: version}) do - %{data: %{id: version.id}} + %{data: %{id: version.id, serial: version.serial}} end end diff --git a/test/polar_web/controllers/publish/version_controller_test.exs b/test/polar_web/controllers/publish/version_controller_test.exs index f598d12..b091ffc 100644 --- a/test/polar_web/controllers/publish/version_controller_test.exs +++ b/test/polar_web/controllers/publish/version_controller_test.exs @@ -44,7 +44,7 @@ defmodule PolarWeb.Publish.VersionControllerTest do assert %{"data" => data} = json_response(conn, 200) - assert %{"id" => _id} = data + assert %{"id" => _id, "serial" => _serial} = data end end @@ -90,7 +90,7 @@ defmodule PolarWeb.Publish.VersionControllerTest do assert %{"data" => data} = json_response(conn, 201) - assert %{"id" => _id} = data + assert %{"id" => _id, "serial" => _serial} = data end end end From c932de3bb1362bba1a17fe954b0fd58c26d483ad Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 4 Jul 2024 12:16:55 +0700 Subject: [PATCH 43/45] Add instance wait times to cluster --- lib/polar/machines/cluster.ex | 4 ++++ lib/polar/machines/cluster/wait_time.ex | 18 ++++++++++++++++++ .../publish/testing/cluster_json.ex | 3 ++- ...3522_add_instance_wait_time_to_clusters.exs | 9 +++++++++ test/polar/machines/cluster/manager_test.exs | 8 +++++++- .../testing/cluster_controller_test.exs | 18 ++++++++++++++++-- 6 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 lib/polar/machines/cluster/wait_time.ex create mode 100644 priv/repo/migrations/20240704043522_add_instance_wait_time_to_clusters.exs diff --git a/lib/polar/machines/cluster.ex b/lib/polar/machines/cluster.ex index 5098d38..02edfeb 100644 --- a/lib/polar/machines/cluster.ex +++ b/lib/polar/machines/cluster.ex @@ -41,6 +41,8 @@ defmodule Polar.Machines.Cluster do field :credential_password_confirmation, :string, virtual: true field :credential, Polar.Encrypted.Map + embeds_many :instance_wait_times, __MODULE__.WaitTime, on_replace: :delete + timestamps(type: :utc_datetime_usec) end @@ -51,12 +53,14 @@ defmodule Polar.Machines.Cluster do |> validate_required(@required_attrs) |> validate_inclusion(:type, ["lxd", "incus"]) |> validate_inclusion(:arch, ["amd64", "arm64"]) + |> cast_embed(:instance_wait_times) |> process_credential() end def update_changeset(cluster, attrs) do cluster |> cast(attrs, [:credential_endpoint]) + |> cast_embed(:instance_wait_times) |> maybe_update_credential() end diff --git a/lib/polar/machines/cluster/wait_time.ex b/lib/polar/machines/cluster/wait_time.ex new file mode 100644 index 0000000..80eaaf5 --- /dev/null +++ b/lib/polar/machines/cluster/wait_time.ex @@ -0,0 +1,18 @@ +defmodule Polar.Machines.Cluster.WaitTime do + use Ecto.Schema + import Ecto.Changeset + + @derive Jason.Encoder + + @primary_key false + embedded_schema do + field :type, :string + field :duration, :integer + end + + def changeset(wait_time, params) do + wait_time + |> cast(params, [:type, :duration]) + |> validate_inclusion(:type, ["container", "vm"]) + end +end diff --git a/lib/polar_web/controllers/publish/testing/cluster_json.ex b/lib/polar_web/controllers/publish/testing/cluster_json.ex index 016d847..aa29e3a 100644 --- a/lib/polar_web/controllers/publish/testing/cluster_json.ex +++ b/lib/polar_web/controllers/publish/testing/cluster_json.ex @@ -13,7 +13,8 @@ defmodule PolarWeb.Publish.Testing.ClusterJSON do type: cluster.type, arch: cluster.arch, credential: cluster.credential, - current_state: cluster.current_state + current_state: cluster.current_state, + instance_wait_times: cluster.instance_wait_times } end end diff --git a/priv/repo/migrations/20240704043522_add_instance_wait_time_to_clusters.exs b/priv/repo/migrations/20240704043522_add_instance_wait_time_to_clusters.exs new file mode 100644 index 0000000..f25fe01 --- /dev/null +++ b/priv/repo/migrations/20240704043522_add_instance_wait_time_to_clusters.exs @@ -0,0 +1,9 @@ +defmodule Polar.Repo.Migrations.AddInstanceWaitTimeToClusters do + use Ecto.Migration + + def change do + alter table(:clusters) do + add :instance_wait_times, {:array, :map}, default: [] + end + end +end diff --git a/test/polar/machines/cluster/manager_test.exs b/test/polar/machines/cluster/manager_test.exs index 2ee82e5..24cd0bf 100644 --- a/test/polar/machines/cluster/manager_test.exs +++ b/test/polar/machines/cluster/manager_test.exs @@ -12,11 +12,17 @@ defmodule Polar.Machines.Cluster.ManagerTest do arch: "amd64", credential_endpoint: "some.cluster.com:8443", credential_password: "sometoken", - credential_password_confirmation: "sometoken" + credential_password_confirmation: "sometoken", + instance_wait_times: [ + %{type: "vm", duration: 10_000}, + %{type: "container", duration: 5_000} + ] }) assert %Machines.Cluster.Credential{private_key: _private_key, certificate: _certificate} = cluster.credential + + assert Enum.count(cluster.instance_wait_times) == 2 end end end diff --git a/test/polar_web/controllers/publish/testing/cluster_controller_test.exs b/test/polar_web/controllers/publish/testing/cluster_controller_test.exs index 58edc96..9c473eb 100644 --- a/test/polar_web/controllers/publish/testing/cluster_controller_test.exs +++ b/test/polar_web/controllers/publish/testing/cluster_controller_test.exs @@ -28,7 +28,11 @@ defmodule PolarWeb.Publish.Testing.ClusterControllerTest do arch: "amd64", credential_endpoint: "some.cluster.com:8443", credential_password: "sometoken", - credential_password_confirmation: "sometoken" + credential_password_confirmation: "sometoken", + instance_wait_times: [ + %{type: "vm", duration: 10_000}, + %{type: "container", duration: 5_000} + ] }) {:ok, _created_cluster} = @@ -38,7 +42,11 @@ defmodule PolarWeb.Publish.Testing.ClusterControllerTest do arch: "amd64", credential_endpoint: "some.cluster.com:8443", credential_password: "sometoken", - credential_password_confirmation: "sometoken" + credential_password_confirmation: "sometoken", + instance_wait_times: [ + %{type: "vm", duration: 10_000}, + %{type: "container", duration: 5_000} + ] }) {:ok, conn: conn, cluster: cluster, user: user} @@ -62,6 +70,12 @@ defmodule PolarWeb.Publish.Testing.ClusterControllerTest do assert %{"data" => data} = json_response(conn, 200) assert cluster.id in Enum.map(data, & &1["id"]) + + cluster = List.first(data) + + assert %{"instance_wait_times" => instance_wait_times} = cluster + + assert Enum.count(instance_wait_times) == 2 end end end From 712bc88be949b793fbab8a55ad5a82fe7fe50fa4 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 4 Jul 2024 14:04:48 +0700 Subject: [PATCH 44/45] add cron based version pruning --- config/config.exs | 8 +- lib/polar/streams/version/pruning.ex | 31 ++++++ test/polar/streams/version/pruning_test.exs | 111 ++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 lib/polar/streams/version/pruning.ex create mode 100644 test/polar/streams/version/pruning_test.exs diff --git a/config/config.exs b/config/config.exs index bd0ab6e..d0ea3b4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,7 +13,13 @@ config :polar, config :polar, Polar.Vault, json_library: Jason -config :polar, Oban, engine: Oban.Engines.Basic, queues: [default: 3], repo: Polar.Repo +config :polar, Oban, + engine: Oban.Engines.Basic, + queues: [default: 3], + repo: Polar.Repo, + plugins: [ + {Oban.Plugins.Cron, crontab: [{"@daily", Polar.Streams.Version.Pruning}]} + ] # Configures the endpoint config :polar, PolarWeb.Endpoint, diff --git a/lib/polar/streams/version/pruning.ex b/lib/polar/streams/version/pruning.ex new file mode 100644 index 0000000..1e95f1c --- /dev/null +++ b/lib/polar/streams/version/pruning.ex @@ -0,0 +1,31 @@ +defmodule Polar.Streams.Version.Pruning do + use Oban.Worker, queue: :default + + alias Polar.Repo + alias Polar.Accounts.Automation + alias Polar.Streams.Version + + import Ecto.Query, only: [from: 2] + + @age_days 2 + + def perform(%Job{}) do + age = + DateTime.utc_now() + |> DateTime.add(-@age_days, :day) + + versions_to_deactivate = + from(v in Version, + where: v.current_state == "testing" and v.inserted_at < ^age + ) + |> Repo.all() + + bot = Automation.get_bot!() + + Enum.each(versions_to_deactivate, fn version -> + Eventful.Transit.perform(version, bot, "deactivate", comment: "testing period expired") + end) + + :ok + end +end diff --git a/test/polar/streams/version/pruning_test.exs b/test/polar/streams/version/pruning_test.exs new file mode 100644 index 0000000..71e29d0 --- /dev/null +++ b/test/polar/streams/version/pruning_test.exs @@ -0,0 +1,111 @@ +defmodule Polar.Streams.Version.PruningTest do + use Polar.DataCase, async: true + use Oban.Testing, repo: Polar.Repo + + import Polar.AccountsFixtures + + alias Polar.Repo + alias Polar.Accounts + + alias Polar.Streams + alias Polar.Streams.Product + alias Polar.Streams.Version + alias Polar.Streams.Version.Pruning + + setup do + password = Accounts.generate_automation_password() + + bot = bot_fixture(%{password: password}) + + {:ok, bot: bot} + end + + setup %{bot: bot} do + {:ok, %Product{} = with_active_versions} = + Streams.create_product(%{ + aliases: ["alpine/3.18", "alpine/3.18/default"], + arch: "amd64", + os: "Alpine", + release: "3.18", + release_title: "3.18", + variant: "default", + requirements: %{ + secureboot: "false" + } + }) + + {:ok, version} = + Streams.create_version(with_active_versions, %{ + serial: "20240209-36", + items: [ + %{ + name: "lxd.tar.gz", + file_type: "lxd.tar.gz", + hash: "35363f3d086271ed5402d61ab18ec03987bed51758c00079b8c9d372ff6d62dd", + size: 876, + path: "images/alpine/edge/amd64/default/20240209_13:00/incus.tar.xz" + }, + %{ + name: "root.squashfs", + file_type: "squashfs", + hash: "47cc4070da1bf17d8364c390…3603f4ed7e9e46582e690d2", + size: 2_982_800, + path: "images/alpine/edge/amd64/default/20240209_13:00/rootfs.tar.xz" + } + ] + }) + + {:ok, another_version} = + Streams.create_version(with_active_versions, %{ + serial: "20240209-37", + items: [ + %{ + name: "lxd.tar.gz", + file_type: "lxd.tar.gz", + hash: "35363f3d086271ed5402d61ab18ec03987bed51758c00079b8c9d372ff6d62aa", + size: 876, + path: "images/alpine/edge/amd64/default/20240209_13:00/incus.tar.xz" + }, + %{ + name: "root.squashfs", + file_type: "squashfs", + hash: "47cc4070da1bf17d8364c390…3603f4ed7e9e46582e690aa", + size: 2_982_800, + path: "images/alpine/edge/amd64/default/20240209_13:00/rootfs.tar.xz" + } + ] + }) + + {:ok, %{resource: testing_version}} = Eventful.Transit.perform(version, bot, "test") + + {:ok, %{resource: another_testing_version}} = + Eventful.Transit.perform(another_version, bot, "test") + + expired = + DateTime.utc_now() + |> DateTime.add(-4, :day) + + query = from(v in Version, where: v.id == ^version.id) + + Repo.update_all(query, set: [inserted_at: expired]) + + {:ok, version: testing_version, another_version: another_testing_version} + end + + describe "perform" do + test "successfully deactivate expired version", %{ + version: version, + another_version: another_version + } do + assert version.current_state == "testing" + assert another_version.current_state == "testing" + + assert :ok = perform_job(Pruning, %{}) + + version = Repo.reload(version) + + assert version.current_state == "inactive" + assert another_version.current_state == "testing" + end + end +end From 33faf63a045b5ee4fbcf0ee3c9648b4298da7bd1 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 5 Jul 2024 11:27:41 +0700 Subject: [PATCH 45/45] Version bump --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index dc430e7..4e990bf 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Polar.MixProject do def project do [ app: :polar, - version: "0.1.2", + version: "0.2.0", elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod,