Mongo7 upgrade

.github/workflows/test.yml

@@ -22,16 +22,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # the current production setup
           - java: '8'
-            mongo: 'mongodb-linux-x86_64-3.6.13'
+            mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4'
             wired_tiger: 'false'
-            gradle_test: 'test'
-          # test all code w/ java 11
           - java: '11'
             mongo: 'mongodb-linux-x86_64-3.6.23'
             wired_tiger: 'true'
-            gradle_test: 'test'
     steps:
       - uses: actions/checkout@v3
 
@@ -64,8 +60,7 @@ jobs:
 
       - name: Run tests
         shell: bash
-        run: |
-          gradle ${{matrix.gradle_test}}
+        run: ./gradlew test
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

build.gradle

@@ -118,7 +118,10 @@ dependencies {
 		exclude group: 'javax.servlet', module: 'servlet-api'  // 2.5 vs 3.0.1 above
 	}
 	implementation 'org.ini4j:ini4j:0.5.2'
-	implementation 'org.mongodb:mongo-java-driver:3.8.2'
+	implementation 'org.mongodb:mongodb-driver-core:4.11.1'
+	implementation 'org.mongodb:mongodb-driver-sync:4.11.1'
+	implementation 'org.mongodb:bson-record-codec:4.11.1'
+	implementation 'org.mongodb:bson:4.11.1'
 	implementation 'org.apache.kafka:kafka-clients:2.1.0'
 	implementation 'com.google.guava:guava:18.0'
 	implementation 'org.slf4j:slf4j-api:1.7.25'

src/main/java/us/kbase/groups/build/GroupsBuilder.java

@@ -9,8 +9,9 @@
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.ImmutableMap;
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientOptions;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.MongoClientSettings;
 import com.mongodb.MongoCredential;
 import com.mongodb.MongoException;
 import com.mongodb.ServerAddress;
@@ -106,15 +107,16 @@
 
 	private MongoClient buildMongo(final GroupsConfig c) throws StorageInitException {
 		//TODO ZLATER MONGO handle shards & replica sets
+		final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder().applyToClusterSettings(
+				builder -> builder.hosts(Arrays.asList(new ServerAddress(c.getMongoHost()))));
 		try {
 			if (c.getMongoUser().isPresent()) {
 				final MongoCredential creds = MongoCredential.createCredential(
 						c.getMongoUser().get(), c.getMongoDatabase(), c.getMongoPwd().get());
 				// unclear if and when it's safe to clear the password
-				return new MongoClient(new ServerAddress(c.getMongoHost()), creds,
-						MongoClientOptions.builder().build());
+				return MongoClients.create(mongoBuilder.credential(creds).build());
 			} else {
-				return new MongoClient(new ServerAddress(c.getMongoHost()));
+				return MongoClients.create(mongoBuilder.build());
 			}
 		} catch (MongoException e) {
 			LoggerFactory.getLogger(getClass()).error(

src/main/java/us/kbase/groups/service/GroupsService.java

@@ -4,7 +4,7 @@
 import org.glassfish.jersey.server.ResourceConfig;
 import org.slf4j.LoggerFactory;
 
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;

src/test/java/us/kbase/test/groups/MongoStorageTestManager.java

@@ -12,7 +12,8 @@
 import org.bson.Document;
 
 import com.github.zafarkhaja.semver.Version;
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
 
 import us.kbase.groups.core.resource.ResourceType;
@@ -39,7 +40,7 @@ public MongoStorageTestManager(final String dbName) throws Exception {
 		wiredTiger = TestCommon.useWiredTigerEngine();
 		System.out.println(String.format("Testing against mongo executable %s on port %s",
 				TestCommon.getMongoExe(), mongo.getServerPort()));
-		mc = new MongoClient("localhost:" + mongo.getServerPort());
+		mc = MongoClients.create("mongodb://localhost:" + mongo.getServerPort());
 		db = mc.getDatabase(dbName);
 
 		final Document bi = db.runCommand(new Document("buildinfo", 1));

src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageDuplicateKeyCheckerTest.java

@@ -131,7 +131,9 @@ public void unparseable() throws Exception {
 			fail("expected exception");
 		} catch (InvocationTargetException ex) {
 			TestCommon.assertExceptionCorrect((Exception) ex.getTargetException(),
-					new GroupsStorageException("Unable to parse duplicate key error: some "));
+					new GroupsStorageException("Unable to parse duplicate key error: " +
+							"Write operation error on server 127.0.0.1:27017. " +
+							"Write error: WriteError{code=11000, message='some "));
 
 		}
 	}

src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageStartupTest.java

@@ -107,9 +107,10 @@ public void startUpWith2ConfigDocs() throws Exception {
 
 		final Pattern errorPattern = Pattern.compile(
 				"Failed to create index: Write failed with error code 11000 and error message " +
-				"'(exception: )?E11000 duplicate key error (index|collection): " +
+				"'(exception: )?(.*)E11000 duplicate key error (index|collection): " +
 				"startUpWith2ConfigDocs.config( index: |\\.\\$)schema_1\\s+dup key: " +
-				"\\{ : \"schema\" \\}'");
+				"(\\{ schema: \"schema\" \\}'|\\{ : \"schema\" \\}')");
+
 		try {
 			new MongoGroupsStorage(db, set());
 			fail("started mongo with bad config");
@@ -188,27 +189,21 @@ public void checkCollectionNames() throws Exception {
 
 	@Test
 	public void indexesConfig() {
-		final Set<Document> indexes = new HashSet<>();
-		manager.db.getCollection("config").listIndexes()
-				.forEach((Consumer<Document>) indexes::add);
+		final Set<Document> indexes = getAndNormalizeIndexes("config");
 		assertThat("incorrect indexes", indexes, is(set(
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("key", new Document("schema", 1))
-						.append("name", "schema_1")
-						.append("ns", "test_mongogroupsstorage.config"),
+						.append("name", "schema_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("_id", 1))
 						.append("name", "_id_")
-						.append("ns", "test_mongogroupsstorage.config")
 				)));
 	}
 
 	@Test
 	public void indexesGroups() {
-		final Set<Document> indexes = new HashSet<>();
-		manager.db.getCollection("groups").listIndexes()
-				.forEach((Consumer<Document>) indexes::add);
+		final Set<Document> indexes = getAndNormalizeIndexes("groups");
 		assertThat("incorrect indexes", indexes, is(getDefaultGroupsIndexes()));
 	}
 
@@ -218,28 +213,22 @@ private Set<Document> getDefaultGroupsIndexes() {
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("key", new Document("id", 1))
-						.append("name", "id_1")
-						.append("ns", col),
+						.append("name", "id_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("own", 1).append("id", 1))
-						.append("name", "own_1_id_1")
-						.append("ns", col),
+						.append("name", "own_1_id_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("admin", 1).append("id", 1))
-						.append("name", "admin_1_id_1")
-						.append("ns", col),
+						.append("name", "admin_1_id_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("_id", 1))
-						.append("name", "_id_")
-						.append("ns", col),
+						.append("name", "_id_"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("priv", 1).append("id", 1))
-						.append("name", "priv_1_id_1")
-						.append("ns", col),
+						.append("name", "priv_1_id_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("memb.user", 1).append("id", 1))
 						.append("name", "memb.user_1_id_1")
-						.append("ns", col)
 				);
 	}
 
@@ -250,10 +239,8 @@ public void indexesGroupsWithResourceTypes() throws Exception {
 		try {
 			new MongoGroupsStorage(
 					manager.db, Arrays.asList(new ResourceType("t1"), new ResourceType("t2")));
-			final Set<Document> indexes = new HashSet<>();
-			manager.db.getCollection("groups").listIndexes()
-					.forEach((Consumer<Document>) indexes::add);
-
+			final Set<Document> indexes = getAndNormalizeIndexes("groups");
+
 			final String col = "test_mongogroupsstorage.groups";
 			final Set<Document> expected = getDefaultGroupsIndexes();
 			for (final String t: set("t1", "t2")) {
@@ -263,19 +250,16 @@ public void indexesGroupsWithResourceTypes() throws Exception {
 								.append("key", new Document(resKey, 1)
 										.append("own", 1)
 										.append("id", 1))
-								.append("name", resKey + "_1_own_1_id_1")
-								.append("ns", col),
+								.append("name", resKey + "_1_own_1_id_1"),
 						new Document("v", manager.indexVer)
 								.append("key", new Document(resKey, 1)
 										.append("id", 1))
-								.append("name", resKey + "_1_id_1")
-								.append("ns", col),
+								.append("name", resKey + "_1_id_1"),
 						new Document("v", manager.indexVer)
 								.append("key", new Document(resKey, 1)
 										.append("priv", 1)
 										.append("id", 1))
 								.append("name", resKey + "_1_priv_1_id_1")
-								.append("ns", col)
 						));
 			}
 			assertThat("incorrect indexes", indexes, is(expected));
@@ -287,116 +271,113 @@ public void indexesGroupsWithResourceTypes() throws Exception {
 
 	@Test
 	public void indexesRequests() {
-		final Set<Document> indexes = new HashSet<>();
-		manager.db.getCollection("requests").listIndexes()
-				.forEach((Consumer<Document>) indexes::add);
+		final Set<Document> indexes = getAndNormalizeIndexes("requests");
 		final String col = "test_mongogroupsstorage.requests";
 		assertThat("incorrect indexes", indexes, is(set(
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("key", new Document("id", 1))
-						.append("name", "id_1")
-						.append("ns", col),
+						.append("name", "id_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("gid", 1).append("type", 1).append("mod", 1))
-						.append("name", "gid_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "gid_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("gid", 1)
 								.append("status", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "gid_1_status_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "gid_1_status_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("requester", 1).append("mod", 1))
-						.append("name", "requester_1_mod_1")
-						.append("ns", col),
+						.append("name", "requester_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("requester", 1)
 								.append("status", 1)
 								.append("mod", 1))
-						.append("name", "requester_1_status_1_mod_1")
-						.append("ns", col),
+						.append("name", "requester_1_status_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resaid", 1)
 								.append("restype", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resaid_1_restype_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resaid_1_restype_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resaid", 1)
 								.append("restype", 1)
 								.append("status", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resaid_1_restype_1_status_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resaid_1_restype_1_status_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resrid", 1)
 								.append("restype", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resrid_1_restype_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resrid_1_restype_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resrid", 1)
 								.append("restype", 1)
 								.append("status", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resrid_1_restype_1_status_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resrid_1_restype_1_status_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resrid", 1)
 								.append("requester", 1)
 								.append("restype", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resrid_1_requester_1_restype_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resrid_1_requester_1_restype_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resrid", 1)
 								.append("requester", 1)
 								.append("restype", 1)
 								.append("status", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resrid_1_requester_1_restype_1_status_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resrid_1_requester_1_restype_1_status_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resrid", 1)
 								.append("gid", 1)
 								.append("restype", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resrid_1_gid_1_restype_1_type_1_mod_1")
-						.append("ns", col),
+						.append("name", "resrid_1_gid_1_restype_1_type_1_mod_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("resrid", 1)
 								.append("gid", 1)
 								.append("restype", 1)
 								.append("status", 1)
 								.append("type", 1)
 								.append("mod", 1))
-						.append("name", "resrid_1_gid_1_restype_1_status_1_type_1_mod_1")
-						.append("ns", col),		
+						.append("name", "resrid_1_gid_1_restype_1_status_1_type_1_mod_1"),
 
 				new Document("v", manager.indexVer)
 						.append("key", new Document("expire", 1))
-						.append("name", "expire_1")
-						.append("ns", col),
+						.append("name", "expire_1"),
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("sparse", true)
 						.append("key", new Document("charstr", 1))
-						.append("name", "charstr_1")
-						.append("ns", col),
+						.append("name", "charstr_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("_id", 1))
 						.append("name", "_id_")
-						.append("ns", col)
 				)));
 	}
+
+	private Set<Document> getAndNormalizeIndexes(final String collectionName) {
+		final Set<Document> indexes = new HashSet<>();
+		for (Document index: manager.db.getCollection(collectionName).listIndexes()) {
+			// In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes()
+			// no longer returns the namespace ns field in the index specification documents.
+			index.remove("ns");
+			// some versions of Mongo return ints, some longs. Convert all to longs.
+			if (index.containsKey("expireAfterSeconds")) {
+				index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue());
+			}
+			indexes.add(index);
+		}
+		return indexes;
+	}
 }