Introduce embedding provider switcher and full switch to Voyage AI for CI/CD

test/DataStax.AstraDB.DataApi.IntegrationTests/EmbeddingProviderSwitcher.cs

@@ -0,0 +1,46 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace DataStax.AstraDB.DataApi.IntegrationTests;
+
+/// <summary>
+/// Static configuration on the embedding provider for all vectorize-related integration testing.
+/// </summary>
+public static class EmbeddingProviderSwitcher {
+
+    /// <summary>
+    /// Embedding provider to use.
+    /// </summary>
+    public const string Provider = "voyageAI";
+    /// <summary>
+    /// Embedding model to use.
+    /// </summary>
+    public const string ModelName = "voyage-2";
+
+    /// <summary>
+    /// A dimension value compatible with the provider/model choice.
+    /// </summary>
+    public const int Dimension = 1024;
+
+    /// <summary>
+    /// Name of the secret in Astra scoped to the DB being used to test.
+    /// </summary>
+    public const string KMSSecretName = "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI";
+    /// <summary>
+    /// Name of the environment variable containing the embedding API key.
+    /// </summary>
+    public const string SecretEnvironmentVariableName = "HEADER_EMBEDDING_API_KEY_VOYAGEAI";
+}

test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs

@@ -91,8 +91,8 @@ public class SimpleObjectWithVectorizeAttribute
 
 [CollectionName("coll_SimpleObjectWithVectorizeShSecret")]
 [CollectionVectorize(
-    Provider = "openai", ModelName = "text-embedding-3-small",
-    AuthenticationPairs = new string[] {"providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"}
+    Provider = EmbeddingProviderSwitcher.Provider, ModelName = EmbeddingProviderSwitcher.ModelName,
+    AuthenticationPairs = new string[] {"providerKey", EmbeddingProviderSwitcher.KMSSecretName}
 )]
 public class SimpleObjectWithVectorizeAttributeShSecret
 {
@@ -106,12 +106,12 @@ public class SimpleObjectWithVectorizeAttributeShSecret
 [CollectionName("coll_SimpleObjectWithVectorizeShSecret2A")]
 [CollectionVector(
     SimilarityMetric.Euclidean,
-    123,
+    EmbeddingProviderSwitcher.Dimension,
     SourceModel="bert"
 )]
 [CollectionVectorize(
-    Provider = "openai", ModelName = "text-embedding-3-small",
-    AuthenticationPairs = new string[] {"providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"}
+    Provider = EmbeddingProviderSwitcher.Provider, ModelName = EmbeddingProviderSwitcher.ModelName,
+    AuthenticationPairs = new string[] {"providerKey", EmbeddingProviderSwitcher.KMSSecretName}
 )]
 public class SimpleObjectWithVectorizeAttributeShSecret2A
 {
@@ -123,7 +123,7 @@ public class SimpleObjectWithVectorizeAttributeShSecret2A
 }
 
 [CollectionName("coll_SimpleObjectWithVectorizeHeader")]
-[CollectionVectorize(Provider = "openai", ModelName = "text-embedding-3-small")]
+[CollectionVectorize(Provider = EmbeddingProviderSwitcher.Provider, ModelName = EmbeddingProviderSwitcher.ModelName)]
 public class SimpleObjectWithVectorizeAttributeHeader
 {
     [DocumentId]
@@ -279,7 +279,7 @@ public class RowBookVectorizeHeaderBased
 {
     [ColumnPrimaryKey(1)]
     public string Title { get; set; }
-    [ColumnVectorize("openai", "text-embedding-3-small", dimension: 1536)]
+    [ColumnVectorize(EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName, dimension: EmbeddingProviderSwitcher.Dimension)]
     public object Author { get; set; }
     [ColumnPrimaryKey(2)]
     public int NumberOfPages { get; set; }
@@ -294,8 +294,8 @@ public class RowBookVectorizeSharedSecret
     [ColumnPrimaryKey(1)]
     public string Title { get; set; }
     [ColumnVectorize(
-        "openai", "text-embedding-3-small", dimension: 1536,
-        authenticationPairs: new string[] {"providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"}
+        EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName, dimension: EmbeddingProviderSwitcher.Dimension,
+        authenticationPairs: new string[] {"providerKey", EmbeddingProviderSwitcher.KMSSecretName}
     )]
     public object Author { get; set; }
     [ColumnPrimaryKey(2)]
@@ -311,8 +311,8 @@ public class RowBookVectorizeSharedSecretWithParameters
     [ColumnPrimaryKey(1)]
     public string Title { get; set; }
     [ColumnVectorize(
-        "voyageAI", "voyage-2",
-        authenticationPairs: new string[] { "providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI" },
+        EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName,
+        authenticationPairs: new string[] { "providerKey", EmbeddingProviderSwitcher.KMSSecretName },
         parameterPairs: new object[] { "autoTruncate", false }
     )]
     public object Author { get; set; }
@@ -842,8 +842,8 @@ public class FakeDocument
 
 [CollectionName("coll_one_embedding_header_test")]
 [CollectionVectorize(
-    "voyageAI",
-    "voyage-2",
+    EmbeddingProviderSwitcher.Provider,
+    EmbeddingProviderSwitcher.ModelName,
     parameterPairs: new object[] { "autoTruncate", false }
 )]
 public class DocumentForEmbeddingHeaderTest
@@ -876,7 +876,7 @@ public class RowForEmbeddingHeaderTest
     [ColumnPrimaryKey(1)]
     public string Id { get; set; }
     [ColumnVectorize(
-        "voyageAI", "voyage-2", dimension: 1024,
+        EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName, dimension: EmbeddingProviderSwitcher.Dimension,
         parameterPairs: new object[] { "autoTruncate", false }
     )]
     public object Name { get; set; }

test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs

@@ -552,11 +552,11 @@ await collection.InsertManyAsync(new List<SimpleObjectWithVector> {
         }
     }
 
-    // Requires VOYAGE embedding provider
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    // Requires a certain embedding provider (see EmbeddingProviderSwitcher)
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task Test_CollectionEmbeddingHeaders()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_VOYAGEAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         try
         {
             var collectionCr = await fixture.Database.CreateCollectionAsync<DocumentForEmbeddingHeaderTest>(

test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalTableTests.cs

@@ -1252,11 +1252,11 @@ await table.InsertManyAsync(new List<SimpleTwoColumnRow> {
         }
     }
 
-    // Requires VOYAGE embedding provider
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    // Requires a certain embedding provider (see EmbeddingProviderSwitcher)
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task Test_TableEmbeddingHeaders()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_VOYAGEAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         try
         {
             var tableCr = await fixture.Database.CreateTableAsync<RowForEmbeddingHeaderTest>(

test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs

@@ -476,7 +476,7 @@ await collection.InsertOneAsync(new SimpleObjectWithVectorizeAttribute
     }
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted below")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateCollection_WithVectorizeSharedSecret_Typed()
     {
         var collectionName = "coll_SimpleObjectWithVectorizeShSecret";
@@ -494,7 +494,7 @@ await collection.InsertOneAsync(new SimpleObjectWithVectorizeAttributeShSecret
     }
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted below")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateCollection_WithVectorizeSharedSecretDoubleAttribute_Typed()
     {
         var collectionName = "coll_SimpleObjectWithVectorizeShSecret2A";
@@ -511,10 +511,10 @@ await collection.InsertOneAsync(new SimpleObjectWithVectorizeAttributeShSecret2A
         await fixture.Database.DropCollectionAsync(collectionName);
     }
 
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task CreateGetCollection_WithVectorizeHeader_Typed()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_OPENAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         var headerOptions = new CreateCollectionOptions() { EmbeddingAPIKey = embeddingAPIKey };
         var collectionName = "coll_SimpleObjectWithVectorizeHeader";
         // Signature of overloads mandates that we supply the collection name here. Eeh, I think we can live with that.
@@ -571,23 +571,23 @@ await collection.InsertOneAsync(new Document
     }
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted below")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateCollection_WithVectorizeSharedSecret_Untyped()
     {
         var collectionName = "collectionVectorizesharedSecret_Untyped";
         var options = new CollectionDefinition
         {
             Vector = new VectorOptions
             {
-                Dimension = 1536,
+                Dimension = EmbeddingProviderSwitcher.Dimension,
                 Metric = SimilarityMetric.DotProduct,
                 Service = new VectorServiceOptions()
                 {
-                    Provider = "openai",
-                    ModelName = "text-embedding-3-small",
+                    Provider = EmbeddingProviderSwitcher.Provider,
+                    ModelName = EmbeddingProviderSwitcher.ModelName,
                     Authentication = new Dictionary<string, string>
                     {
-                        { "providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI" }
+                        { "providerKey", EmbeddingProviderSwitcher.KMSSecretName }
                     }
                 }
             }
@@ -604,22 +604,22 @@ await collection.InsertOneAsync(new Document
         await fixture.Database.DropCollectionAsync(collectionName);
     }
 
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task CreateGetCollection_WithVectorizeHeader_Untyped()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_OPENAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         var headerOptions = new CreateCollectionOptions() { EmbeddingAPIKey = embeddingAPIKey };
         var collectionName = "collection_WithVectorizeHeader_Untyped";
         var options = new CollectionDefinition
         {
             Vector = new VectorOptions
             {
-                Dimension = 1536,
+                Dimension = EmbeddingProviderSwitcher.Dimension,
                 Metric = SimilarityMetric.DotProduct,
                 Service = new VectorServiceOptions()
                 {
-                    Provider = "openai",
-                    ModelName = "text-embedding-3-small",
+                    Provider = EmbeddingProviderSwitcher.Provider,
+                    ModelName = EmbeddingProviderSwitcher.ModelName,
                 }
             }
         };
@@ -808,7 +808,7 @@ public async Task CreateTable_WithVectorizeNone_Typed()
     }
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted in RowBookVectorizeSharedSecret")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateTable_WithVectorizeSharedSecret_Typed()
     {
         try
@@ -825,7 +825,7 @@ public async Task CreateTable_WithVectorizeSharedSecret_Typed()
     }
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted in RowBookVectorizeSharedSecretWithParameters")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateTable_WithVectorizeSharedSecretWithParameters_Typed()
     {
         try
@@ -841,10 +841,10 @@ public async Task CreateTable_WithVectorizeSharedSecretWithParameters_Typed()
         }
     }
 
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task CreateGetTable_WithVectorizeHeader_Typed()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_OPENAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         var gtHeaderOptions = new GetTableOptions() { EmbeddingAPIKey = embeddingAPIKey };
         var ctHeaderOptions = new CreateTableOptions() { EmbeddingAPIKey = embeddingAPIKey };
         try
@@ -895,7 +895,7 @@ await table.InsertOneAsync(new Row() {
     }
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted below")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateTable_WithVectorizeSharedSecret_Untyped()
     {
         var tableName = "bookTestTableVectorizeSharedSecret_Untyped";
@@ -904,13 +904,13 @@ public async Task CreateTable_WithVectorizeSharedSecret_Untyped()
             var createDefinition = new TableDefinition()
                 .AddColumn("Title", DataAPIType.Text())
                 .AddColumn("NumberOfPages", DataAPIType.Int())
-                .AddColumn("Author", DataAPIType.Vectorize(1536, new VectorServiceOptions
+                .AddColumn("Author", DataAPIType.Vectorize(EmbeddingProviderSwitcher.Dimension, new VectorServiceOptions
                 {
-                    Provider = "openai",
-                    ModelName = "text-embedding-3-small",
+                    Provider = EmbeddingProviderSwitcher.Provider,
+                    ModelName = EmbeddingProviderSwitcher.ModelName,
                     Authentication = new Dictionary<string, string>
                     {
-                        { "providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI" }
+                        { "providerKey", EmbeddingProviderSwitcher.KMSSecretName }
                     }
                 }))
                 .AddCompositePrimaryKey(new [] {"Title", "NumberOfPages"});
@@ -932,7 +932,7 @@ await table.InsertOneAsync(new Row() {
 
 
     [SkipWhenNotAstra]
-    [Fact(Skip="Should be run manually after scoping a certain OpenAI key to the database with the name quoted below")]
+    [Fact(Skip="Should be run manually after scoping a certain embedding key to the database, see EmbeddingProviderSwitcher")]
     public async Task CreateTable_WithVectorizeSharedSecretWithParameters_Untyped()
     {
         var tableName = "bookTestTableVeczeShdSecretWParams_Untyped";
@@ -941,13 +941,13 @@ public async Task CreateTable_WithVectorizeSharedSecretWithParameters_Untyped()
             var createDefinition = new TableDefinition()
                 .AddColumn("Title", DataAPIType.Text())
                 .AddColumn("NumberOfPages", DataAPIType.Int())
-                .AddColumn("Author", DataAPIType.Vectorize(1024, new VectorServiceOptions
+                .AddColumn("Author", DataAPIType.Vectorize(EmbeddingProviderSwitcher.Dimension, new VectorServiceOptions
                 {
-                    Provider = "voyageAI",
-                    ModelName = "voyage-2",
+                    Provider = EmbeddingProviderSwitcher.Provider,
+                    ModelName = EmbeddingProviderSwitcher.ModelName,
                     Authentication = new Dictionary<string, string>
                     {
-                        { "providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI" }
+                        { "providerKey", EmbeddingProviderSwitcher.KMSSecretName }
                     },
                     Parameters = new Dictionary<string, object>
                     {
@@ -971,22 +971,22 @@ await table.InsertOneAsync(new Row() {
         }
     }
 
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task CreateGetTable_WithVectorizeHeader_Untyped()
     {
         var tableName = "bookTestTableVectorizeHeader_Untyped";
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_OPENAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         var gtHeaderOptions = new GetTableOptions() { EmbeddingAPIKey = embeddingAPIKey };
         var ctHeaderOptions = new CreateTableOptions() { EmbeddingAPIKey = embeddingAPIKey };
         try
         {
             var createDefinition = new TableDefinition()
                 .AddColumn("Title", DataAPIType.Text())
                 .AddColumn("NumberOfPages", DataAPIType.Int())
-                .AddColumn("Author", DataAPIType.Vectorize(1536, new VectorServiceOptions
+                .AddColumn("Author", DataAPIType.Vectorize(EmbeddingProviderSwitcher.Dimension, new VectorServiceOptions
                 {
-                    Provider = "openai",
-                    ModelName = "text-embedding-3-small",
+                    Provider = EmbeddingProviderSwitcher.Provider,
+                    ModelName = EmbeddingProviderSwitcher.ModelName,
                 }))
                 .AddCompositePrimaryKey(new [] {"Title", "NumberOfPages"});
 
@@ -1007,7 +1007,7 @@ await gottenTable.InsertOneAsync(new Row() {
         }
         finally
         {
-            await fixture.Database.DropTableAsync<RowBookVectorizeHeaderBased>();
+            await fixture.Database.DropTableAsync(tableName);
         }
     }