WIP: Try to run Create command for index creation

This will set it under test.
wireapp · Dec 19, 2024 · 0e76bd0 · 0e76bd0
1 parent 2dec66a
commit 0e76bd0
Showing 1 changed file with 111 additions and 93 deletions.
diff --git a/services/brig/test/integration/API/Search.hs b/services/brig/test/integration/API/Search.hs
@@ -628,102 +628,103 @@ testMigrationToNewIndex ::
   (Log.Logger -> Opt.Opts -> ES.IndexName -> IO ()) ->
   m ()
 testMigrationToNewIndex opts brig migrateIndexCommand = do
+  logger <- Log.create Log.StdOut
   -- running brig with `withSettingsOverride` to direct it to the expected index(es).  it's
   -- important to make both old and new index name/url explicit via `withESProxy`, or the
   -- calls to `refreshIndex` in this test will interfere with parallel test runs of this test.
-  withESProxy opts $ \oldESUrl oldESIndex -> withESProxy opts $ \newESUrl newESIndex ->
-    withESProxyOnly [oldESIndex, newESIndex] opts $ \bothIndexUrl -> do
-      let optsWithIndex :: Text -> Opt.Opts
-          optsWithIndex "old" =
-            opts
-              & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName oldESIndex)
-              & Opt.elasticsearchLens . Opt.urlLens .~ (ES.Server oldESUrl)
-          optsWithIndex "new" =
-            opts
-              & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName newESIndex)
-              & Opt.elasticsearchLens . Opt.urlLens .~ (ES.Server newESUrl)
-          optsWithIndex "both" =
-            optsWithIndex "old"
-              & Opt.elasticsearchLens . Opt.additionalWriteIndexLens ?~ (ES.IndexName newESIndex)
-              & Opt.elasticsearchLens . Opt.additionalWriteIndexUrlLens ?~ (ES.Server newESUrl)
-              -- 'additionalCaCertLens' needs to be added in order for brig to be able to reach both indices.
-              & Opt.elasticsearchLens . Opt.additionalCaCertLens .~ (opts ^. Opt.elasticsearchLens . Opt.caCertLens)
-              & Opt.elasticsearchLens . Opt.additionalInsecureSkipVerifyTlsLens .~ (opts ^. Opt.elasticsearchLens . Opt.insecureSkipVerifyTlsLens)
-          optsWithIndex "oldAccessToBoth" =
-            -- Configure only the old index. However, allow HTTP access to both
-            -- (such that jobs can create and fill the new one).
-            optsWithIndex "old" & Opt.elasticsearchLens . Opt.urlLens .~ (ES.Server bothIndexUrl)
-
-      -- Phase 1: Using old index only
-      (phase1NonTeamUser, teamOwner, phase1TeamUser1, phase1TeamUser2, tid) <- withSettingsOverrides (optsWithIndex "old") $ do
-        nonTeamUser <- randomUser brig
-        (tid, teamOwner, [teamUser1, teamUser2]) <- createPopulatedBindingTeam brig 2
-        pure (nonTeamUser, teamOwner, teamUser1, teamUser2, tid)
-
-      -- Phase 2: Using old index for search, writing to both indices, migrations have not run
-      (phase2NonTeamUser, phase2TeamUser) <- withSettingsOverrides (optsWithIndex "both") $ do
-        phase2NonTeamUser <- randomUser brig
-        phase2TeamUser <- inviteAndRegisterUser teamOwner tid brig
-        refreshIndex brig
-
-        -- searching phase1 users should work
-        assertCanFindByName brig phase1TeamUser1 phase1TeamUser2
-        assertCanFindByName brig phase1TeamUser1 phase1NonTeamUser
-
-        -- searching phase2 users should work
-        assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
-        assertCanFindByName brig phase1TeamUser1 phase2TeamUser
-
-        pure (phase2NonTeamUser, phase2TeamUser)
-
-      withSettingsOverrides (optsWithIndex "new") $ do
-        -- Before migration the phase1 users shouldn't be found in the new index
-        assertCan'tFindByName brig phase1TeamUser1 phase1TeamUser2
-        assertCan'tFindByName brig phase1TeamUser1 phase1NonTeamUser
-
-        -- Before migration the phase2 users should be found in the new index
-        assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
-        assertCanFindByName brig phase1TeamUser1 phase2TeamUser
-
-      -- Run Migrations
-      logger <- Log.create Log.StdOut
-      liftIO $ migrateIndexCommand logger (optsWithIndex "oldAccessToBoth") (ES.IndexName newESIndex)
-
-      -- Phase 3: Using old index for search, writing to both indices, migrations have run
-      (phase3NonTeamUser, phase3TeamUser) <- withSettingsOverrides (optsWithIndex "both") $ do
-        refreshIndex brig
-        phase3NonTeamUser <- randomUser brig
-        phase3TeamUser <- inviteAndRegisterUser teamOwner tid brig
-        refreshIndex brig
-
-        -- searching phase1/2 users should work
-        assertCanFindByName brig phase1TeamUser1 phase1TeamUser2
-        assertCanFindByName brig phase1TeamUser1 phase1NonTeamUser
-        assertCanFindByName brig phase1TeamUser1 phase2TeamUser
-        assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
-
-        -- searching new phase3 should also work
-        assertCanFindByName brig phase1TeamUser1 phase3NonTeamUser
-        assertCanFindByName brig phase1TeamUser1 phase3TeamUser
-        pure (phase3NonTeamUser, phase3TeamUser)
-
-      -- Phase 4: Using only new index
-      withSettingsOverrides (optsWithIndex "new") $ do
-        -- Searching should work for phase1 users
-        assertCanFindByName brig phase1TeamUser1 phase1TeamUser2
-        assertCanFindByName brig phase1TeamUser1 phase1NonTeamUser
-
-        -- Searching should work for phase2 users
-        assertCanFindByName brig phase1TeamUser1 phase2TeamUser
-        assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
-
-        -- Searching should work for phase3 users
-        assertCanFindByName brig phase1TeamUser1 phase3NonTeamUser
-        assertCanFindByName brig phase1TeamUser1 phase3TeamUser
+  withESProxy logger opts $ \oldESUrl oldESIndex ->
+    withESProxy logger opts $ \newESUrl newESIndex ->
+      withESProxyOnly [oldESIndex, newESIndex] opts $ \bothIndexUrl -> do
+        let optsWithIndex :: Text -> Opt.Opts
+            optsWithIndex "old" =
+              opts
+                & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName oldESIndex)
+                & Opt.elasticsearchLens . Opt.urlLens .~ (ES.Server oldESUrl)
+            optsWithIndex "new" =
+              opts
+                & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName newESIndex)
+                & Opt.elasticsearchLens . Opt.urlLens .~ (ES.Server newESUrl)
+            optsWithIndex "both" =
+              optsWithIndex "old"
+                & Opt.elasticsearchLens . Opt.additionalWriteIndexLens ?~ (ES.IndexName newESIndex)
+                & Opt.elasticsearchLens . Opt.additionalWriteIndexUrlLens ?~ (ES.Server newESUrl)
+                -- 'additionalCaCertLens' needs to be added in order for brig to be able to reach both indices.
+                & Opt.elasticsearchLens . Opt.additionalCaCertLens .~ (opts ^. Opt.elasticsearchLens . Opt.caCertLens)
+                & Opt.elasticsearchLens . Opt.additionalInsecureSkipVerifyTlsLens .~ (opts ^. Opt.elasticsearchLens . Opt.insecureSkipVerifyTlsLens)
+            optsWithIndex "oldAccessToBoth" =
+              -- Configure only the old index. However, allow HTTP access to both
+              -- (such that jobs can create and fill the new one).
+              optsWithIndex "old" & Opt.elasticsearchLens . Opt.urlLens .~ (ES.Server bothIndexUrl)
+
+        -- Phase 1: Using old index only
+        (phase1NonTeamUser, teamOwner, phase1TeamUser1, phase1TeamUser2, tid) <- withSettingsOverrides (optsWithIndex "old") $ do
+          nonTeamUser <- randomUser brig
+          (tid, teamOwner, [teamUser1, teamUser2]) <- createPopulatedBindingTeam brig 2
+          pure (nonTeamUser, teamOwner, teamUser1, teamUser2, tid)
+
+        -- Phase 2: Using old index for search, writing to both indices, migrations have not run
+        (phase2NonTeamUser, phase2TeamUser) <- withSettingsOverrides (optsWithIndex "both") $ do
+          phase2NonTeamUser <- randomUser brig
+          phase2TeamUser <- inviteAndRegisterUser teamOwner tid brig
+          refreshIndex brig
+
+          -- searching phase1 users should work
+          assertCanFindByName brig phase1TeamUser1 phase1TeamUser2
+          assertCanFindByName brig phase1TeamUser1 phase1NonTeamUser
+
+          -- searching phase2 users should work
+          assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
+          assertCanFindByName brig phase1TeamUser1 phase2TeamUser
+
+          pure (phase2NonTeamUser, phase2TeamUser)
+
+        withSettingsOverrides (optsWithIndex "new") $ do
+          -- Before migration the phase1 users shouldn't be found in the new index
+          assertCan'tFindByName brig phase1TeamUser1 phase1TeamUser2
+          assertCan'tFindByName brig phase1TeamUser1 phase1NonTeamUser
+
+          -- Before migration the phase2 users should be found in the new index
+          assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
+          assertCanFindByName brig phase1TeamUser1 phase2TeamUser
+
+        -- Run Migrations
+        liftIO $ migrateIndexCommand logger (optsWithIndex "oldAccessToBoth") (ES.IndexName newESIndex)
+
+        -- Phase 3: Using old index for search, writing to both indices, migrations have run
+        (phase3NonTeamUser, phase3TeamUser) <- withSettingsOverrides (optsWithIndex "both") $ do
+          refreshIndex brig
+          phase3NonTeamUser <- randomUser brig
+          phase3TeamUser <- inviteAndRegisterUser teamOwner tid brig
+          refreshIndex brig
+
+          -- searching phase1/2 users should work
+          assertCanFindByName brig phase1TeamUser1 phase1TeamUser2
+          assertCanFindByName brig phase1TeamUser1 phase1NonTeamUser
+          assertCanFindByName brig phase1TeamUser1 phase2TeamUser
+          assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
+
+          -- searching new phase3 should also work
+          assertCanFindByName brig phase1TeamUser1 phase3NonTeamUser
+          assertCanFindByName brig phase1TeamUser1 phase3TeamUser
+          pure (phase3NonTeamUser, phase3TeamUser)
+
+        -- Phase 4: Using only new index
+        withSettingsOverrides (optsWithIndex "new") $ do
+          -- Searching should work for phase1 users
+          assertCanFindByName brig phase1TeamUser1 phase1TeamUser2
+          assertCanFindByName brig phase1TeamUser1 phase1NonTeamUser
+
+          -- Searching should work for phase2 users
+          assertCanFindByName brig phase1TeamUser1 phase2TeamUser
+          assertCanFindByName brig phase1TeamUser1 phase2NonTeamUser
+
+          -- Searching should work for phase3 users
+          assertCanFindByName brig phase1TeamUser1 phase3NonTeamUser
+          assertCanFindByName brig phase1TeamUser1 phase3TeamUser
 
 runReindexFromAnotherIndex :: Log.Logger -> Opt.Opts -> ES.IndexName -> IO ()
 runReindexFromAnotherIndex logger opts newIndexName =
-  let esOldOpts :: Opt.ElasticSearchOpts = (opts ^. Opt.elasticsearchLens)
+  let esOldOpts :: Opt.ElasticSearchOpts = opts ^. Opt.elasticsearchLens
       esOldConnectionSettings :: ESConnectionSettings = toESConnectionSettings esOldOpts
       reindexSettings = ReindexFromAnotherIndexSettings esOldConnectionSettings newIndexName 5
    in do
@@ -767,13 +768,29 @@ toESConnectionSettings opts = ESConnectionSettings {..}
     esInsecureSkipVerifyTls = opts.insecureSkipVerifyTls
     esCredentials = opts.credentials
 
-withESProxy :: (TestConstraints m, MonadUnliftIO m, HasCallStack) => Opt.Opts -> (Text -> Text -> m a) -> m a
-withESProxy opts f = do
+withESProxy :: (TestConstraints m, MonadUnliftIO m, HasCallStack) => Log.Logger -> Opt.Opts -> (Text -> Text -> m a) -> m a
+withESProxy lg opts f = do
   indexName <- randomHandle
   -- TODO: Run create index command?
-  createIndexWithMapping opts indexName oldMapping
+  -- createIndexWithMapping opts indexName oldMapping
+  liftIO $ createCommand lg opts (ES.IndexName indexName)
+  -- TODO: Hand ES.IndexName around
   withESProxyOnly [indexName] opts $ flip f indexName
 
+createCommand :: Log.Logger -> Opt.Opts -> ES.IndexName -> IO ()
+createCommand logger opts newIndexName =
+  let esNewOpts = (opts ^. Opt.elasticsearchLens) & (Opt.indexLens .~ newIndexName)
+      replicas = 2
+      shards = 2
+      refreshInterval = 5
+      esSettings =
+        IndexOpts.localElasticSettings
+          & IndexOpts.esConnection .~ toESConnectionSettings esNewOpts
+          & IndexOpts.esIndexReplicas .~ ES.ReplicaCount replicas
+          & IndexOpts.esIndexShardCount .~ shards
+          & IndexOpts.esIndexRefreshInterval .~ refreshInterval
+   in runCommand logger $ Create esSettings opts.galley
+
 -- | Gives a URL to a HTTP proxy server to the continuation. The proxy is only
 -- configured for ES calls for the given @indexNames@ (and some other ES
 -- specific endpoints.)
@@ -836,6 +853,7 @@ optsForOldIndex opts = do
   createIndexWithMapping opts indexName oldMapping
   pure (opts & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName indexName), indexName)
 
+-- TODO: Replace in other tests. Or, write a comment and mention 'runCommand Create'
 createIndexWithMapping :: (MonadIO m, HasCallStack) => Opt.Opts -> Text -> Value -> m ()
 createIndexWithMapping opts name val = do
   let indexName = ES.IndexName name