From 3b4e7e7a7b95314bae9f7f1d387d4bd99bd00a6d Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 20 Aug 2024 15:11:21 -0700 Subject: [PATCH 1/4] add mongo7 into workflows matrix --- .github/workflows/test.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 079d5c0..06f4bee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,16 +22,12 @@ jobs: fail-fast: false matrix: include: - # the current production setup - java: '8' - mongo: 'mongodb-linux-x86_64-3.6.13' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' wired_tiger: 'false' - gradle_test: 'test' - # test all code w/ java 11 - java: '11' mongo: 'mongodb-linux-x86_64-3.6.23' wired_tiger: 'true' - gradle_test: 'test' steps: - uses: actions/checkout@v3 @@ -65,7 +61,7 @@ jobs: - name: Run tests shell: bash run: | - gradle ${{matrix.gradle_test}} + gradle test - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 02e6488a331f729ff0d92cea47dd0a04ea4adb20 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 22 Aug 2024 16:47:31 -0700 Subject: [PATCH 2/4] upgrade mongo dependencies --- build.gradle | 5 ++++- .../java/us/kbase/groups/build/GroupsBuilder.java | 12 +++++++----- .../java/us/kbase/groups/service/GroupsService.java | 2 +- .../kbase/test/groups/MongoStorageTestManager.java | 5 +++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 2d82ac9..15a0d3e 100644 --- a/build.gradle +++ b/build.gradle @@ -118,7 +118,10 @@ dependencies { exclude group: 'javax.servlet', module: 'servlet-api' // 2.5 vs 3.0.1 above } implementation 'org.ini4j:ini4j:0.5.2' - implementation 'org.mongodb:mongo-java-driver:3.8.2' + implementation 'org.mongodb:mongodb-driver-core:4.11.1' + implementation 'org.mongodb:mongodb-driver-sync:4.11.1' + implementation 'org.mongodb:bson-record-codec:4.11.1' + implementation 'org.mongodb:bson:4.11.1' implementation 'org.apache.kafka:kafka-clients:2.1.0' implementation 'com.google.guava:guava:18.0' implementation 'org.slf4j:slf4j-api:1.7.25' diff --git a/src/main/java/us/kbase/groups/build/GroupsBuilder.java b/src/main/java/us/kbase/groups/build/GroupsBuilder.java index 34c6441..ca05883 100644 --- a/src/main/java/us/kbase/groups/build/GroupsBuilder.java +++ b/src/main/java/us/kbase/groups/build/GroupsBuilder.java @@ -9,8 +9,9 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.MongoClientSettings; import com.mongodb.MongoCredential; import com.mongodb.MongoException; import com.mongodb.ServerAddress; @@ -106,15 +107,16 @@ public GroupsBuilder(final GroupsConfig cfg, final MongoClient mc) private MongoClient buildMongo(final GroupsConfig c) throws StorageInitException { //TODO ZLATER MONGO handle shards & replica sets + final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder().applyToClusterSettings( + builder -> builder.hosts(Arrays.asList(new ServerAddress(c.getMongoHost())))); try { if (c.getMongoUser().isPresent()) { final MongoCredential creds = MongoCredential.createCredential( c.getMongoUser().get(), c.getMongoDatabase(), c.getMongoPwd().get()); // unclear if and when it's safe to clear the password - return new MongoClient(new ServerAddress(c.getMongoHost()), creds, - MongoClientOptions.builder().build()); + return MongoClients.create(mongoBuilder.credential(creds).build()); } else { - return new MongoClient(new ServerAddress(c.getMongoHost())); + return MongoClients.create(mongoBuilder.build()); } } catch (MongoException e) { LoggerFactory.getLogger(getClass()).error( diff --git a/src/main/java/us/kbase/groups/service/GroupsService.java b/src/main/java/us/kbase/groups/service/GroupsService.java index 2b4f605..ef6acb2 100644 --- a/src/main/java/us/kbase/groups/service/GroupsService.java +++ b/src/main/java/us/kbase/groups/service/GroupsService.java @@ -4,7 +4,7 @@ import org.glassfish.jersey.server.ResourceConfig; import org.slf4j.LoggerFactory; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; diff --git a/src/test/java/us/kbase/test/groups/MongoStorageTestManager.java b/src/test/java/us/kbase/test/groups/MongoStorageTestManager.java index 14b9d0a..6345280 100644 --- a/src/test/java/us/kbase/test/groups/MongoStorageTestManager.java +++ b/src/test/java/us/kbase/test/groups/MongoStorageTestManager.java @@ -12,7 +12,8 @@ import org.bson.Document; import com.github.zafarkhaja.semver.Version; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.groups.core.resource.ResourceType; @@ -39,7 +40,7 @@ public MongoStorageTestManager(final String dbName) throws Exception { wiredTiger = TestCommon.useWiredTigerEngine(); System.out.println(String.format("Testing against mongo executable %s on port %s", TestCommon.getMongoExe(), mongo.getServerPort())); - mc = new MongoClient("localhost:" + mongo.getServerPort()); + mc = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); db = mc.getDatabase(dbName); final Document bi = db.runCommand(new Document("buildinfo", 1)); From 421b45a348ff1171b2d174a82e7b1f5775085268 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 22 Aug 2024 21:43:47 -0700 Subject: [PATCH 3/4] remove ns --- ...oGroupsStorageDuplicateKeyCheckerTest.java | 4 +- .../mongo/MongoGroupsStorageStartupTest.java | 111 ++++++++---------- 2 files changed, 49 insertions(+), 66 deletions(-) diff --git a/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageDuplicateKeyCheckerTest.java b/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageDuplicateKeyCheckerTest.java index 54594c8..6b1c2c4 100644 --- a/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageDuplicateKeyCheckerTest.java +++ b/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageDuplicateKeyCheckerTest.java @@ -131,7 +131,9 @@ public void unparseable() throws Exception { fail("expected exception"); } catch (InvocationTargetException ex) { TestCommon.assertExceptionCorrect((Exception) ex.getTargetException(), - new GroupsStorageException("Unable to parse duplicate key error: some ")); + new GroupsStorageException("Unable to parse duplicate key error: " + + "Write operation error on server 127.0.0.1:27017. " + + "Write error: WriteError{code=11000, message='some ")); } } diff --git a/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageStartupTest.java b/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageStartupTest.java index bee6fb9..536c7f3 100644 --- a/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageStartupTest.java +++ b/src/test/java/us/kbase/test/groups/storage/mongo/MongoGroupsStorageStartupTest.java @@ -107,9 +107,10 @@ public void startUpWith2ConfigDocs() throws Exception { final Pattern errorPattern = Pattern.compile( "Failed to create index: Write failed with error code 11000 and error message " + - "'(exception: )?E11000 duplicate key error (index|collection): " + + "'(exception: )?(.*)E11000 duplicate key error (index|collection): " + "startUpWith2ConfigDocs.config( index: |\\.\\$)schema_1\\s+dup key: " + - "\\{ : \"schema\" \\}'"); + "(\\{ schema: \"schema\" \\}'|\\{ : \"schema\" \\}')"); + try { new MongoGroupsStorage(db, set()); fail("started mongo with bad config"); @@ -188,27 +189,21 @@ public void checkCollectionNames() throws Exception { @Test public void indexesConfig() { - final Set indexes = new HashSet<>(); - manager.db.getCollection("config").listIndexes() - .forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("config"); assertThat("incorrect indexes", indexes, is(set( new Document("v", manager.indexVer) .append("unique", true) .append("key", new Document("schema", 1)) - .append("name", "schema_1") - .append("ns", "test_mongogroupsstorage.config"), + .append("name", "schema_1"), new Document("v", manager.indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "test_mongogroupsstorage.config") ))); } @Test public void indexesGroups() { - final Set indexes = new HashSet<>(); - manager.db.getCollection("groups").listIndexes() - .forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("groups"); assertThat("incorrect indexes", indexes, is(getDefaultGroupsIndexes())); } @@ -218,28 +213,22 @@ private Set getDefaultGroupsIndexes() { new Document("v", manager.indexVer) .append("unique", true) .append("key", new Document("id", 1)) - .append("name", "id_1") - .append("ns", col), + .append("name", "id_1"), new Document("v", manager.indexVer) .append("key", new Document("own", 1).append("id", 1)) - .append("name", "own_1_id_1") - .append("ns", col), + .append("name", "own_1_id_1"), new Document("v", manager.indexVer) .append("key", new Document("admin", 1).append("id", 1)) - .append("name", "admin_1_id_1") - .append("ns", col), + .append("name", "admin_1_id_1"), new Document("v", manager.indexVer) .append("key", new Document("_id", 1)) - .append("name", "_id_") - .append("ns", col), + .append("name", "_id_"), new Document("v", manager.indexVer) .append("key", new Document("priv", 1).append("id", 1)) - .append("name", "priv_1_id_1") - .append("ns", col), + .append("name", "priv_1_id_1"), new Document("v", manager.indexVer) .append("key", new Document("memb.user", 1).append("id", 1)) .append("name", "memb.user_1_id_1") - .append("ns", col) ); } @@ -250,10 +239,8 @@ public void indexesGroupsWithResourceTypes() throws Exception { try { new MongoGroupsStorage( manager.db, Arrays.asList(new ResourceType("t1"), new ResourceType("t2"))); - final Set indexes = new HashSet<>(); - manager.db.getCollection("groups").listIndexes() - .forEach((Consumer) indexes::add); - + final Set indexes = getAndNormalizeIndexes("groups"); + final String col = "test_mongogroupsstorage.groups"; final Set expected = getDefaultGroupsIndexes(); for (final String t: set("t1", "t2")) { @@ -263,19 +250,16 @@ public void indexesGroupsWithResourceTypes() throws Exception { .append("key", new Document(resKey, 1) .append("own", 1) .append("id", 1)) - .append("name", resKey + "_1_own_1_id_1") - .append("ns", col), + .append("name", resKey + "_1_own_1_id_1"), new Document("v", manager.indexVer) .append("key", new Document(resKey, 1) .append("id", 1)) - .append("name", resKey + "_1_id_1") - .append("ns", col), + .append("name", resKey + "_1_id_1"), new Document("v", manager.indexVer) .append("key", new Document(resKey, 1) .append("priv", 1) .append("id", 1)) .append("name", resKey + "_1_priv_1_id_1") - .append("ns", col) )); } assertThat("incorrect indexes", indexes, is(expected)); @@ -287,75 +271,63 @@ public void indexesGroupsWithResourceTypes() throws Exception { @Test public void indexesRequests() { - final Set indexes = new HashSet<>(); - manager.db.getCollection("requests").listIndexes() - .forEach((Consumer) indexes::add); + final Set indexes = getAndNormalizeIndexes("requests"); final String col = "test_mongogroupsstorage.requests"; assertThat("incorrect indexes", indexes, is(set( new Document("v", manager.indexVer) .append("unique", true) .append("key", new Document("id", 1)) - .append("name", "id_1") - .append("ns", col), + .append("name", "id_1"), new Document("v", manager.indexVer) .append("key", new Document("gid", 1).append("type", 1).append("mod", 1)) - .append("name", "gid_1_type_1_mod_1") - .append("ns", col), + .append("name", "gid_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("gid", 1) .append("status", 1) .append("type", 1) .append("mod", 1)) - .append("name", "gid_1_status_1_type_1_mod_1") - .append("ns", col), + .append("name", "gid_1_status_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("requester", 1).append("mod", 1)) - .append("name", "requester_1_mod_1") - .append("ns", col), + .append("name", "requester_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("requester", 1) .append("status", 1) .append("mod", 1)) - .append("name", "requester_1_status_1_mod_1") - .append("ns", col), + .append("name", "requester_1_status_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resaid", 1) .append("restype", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resaid_1_restype_1_type_1_mod_1") - .append("ns", col), + .append("name", "resaid_1_restype_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resaid", 1) .append("restype", 1) .append("status", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resaid_1_restype_1_status_1_type_1_mod_1") - .append("ns", col), + .append("name", "resaid_1_restype_1_status_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resrid", 1) .append("restype", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resrid_1_restype_1_type_1_mod_1") - .append("ns", col), + .append("name", "resrid_1_restype_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resrid", 1) .append("restype", 1) .append("status", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resrid_1_restype_1_status_1_type_1_mod_1") - .append("ns", col), + .append("name", "resrid_1_restype_1_status_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resrid", 1) .append("requester", 1) .append("restype", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resrid_1_requester_1_restype_1_type_1_mod_1") - .append("ns", col), + .append("name", "resrid_1_requester_1_restype_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resrid", 1) .append("requester", 1) @@ -363,16 +335,14 @@ public void indexesRequests() { .append("status", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resrid_1_requester_1_restype_1_status_1_type_1_mod_1") - .append("ns", col), + .append("name", "resrid_1_requester_1_restype_1_status_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resrid", 1) .append("gid", 1) .append("restype", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resrid_1_gid_1_restype_1_type_1_mod_1") - .append("ns", col), + .append("name", "resrid_1_gid_1_restype_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("resrid", 1) .append("gid", 1) @@ -380,23 +350,34 @@ public void indexesRequests() { .append("status", 1) .append("type", 1) .append("mod", 1)) - .append("name", "resrid_1_gid_1_restype_1_status_1_type_1_mod_1") - .append("ns", col), + .append("name", "resrid_1_gid_1_restype_1_status_1_type_1_mod_1"), new Document("v", manager.indexVer) .append("key", new Document("expire", 1)) - .append("name", "expire_1") - .append("ns", col), + .append("name", "expire_1"), new Document("v", manager.indexVer) .append("unique", true) .append("sparse", true) .append("key", new Document("charstr", 1)) - .append("name", "charstr_1") - .append("ns", col), + .append("name", "charstr_1"), new Document("v", manager.indexVer) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", col) ))); } + + private Set getAndNormalizeIndexes(final String collectionName) { + final Set indexes = new HashSet<>(); + for (Document index: manager.db.getCollection(collectionName).listIndexes()) { + // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes() + // no longer returns the namespace ns field in the index specification documents. + index.remove("ns"); + // some versions of Mongo return ints, some longs. Convert all to longs. + if (index.containsKey("expireAfterSeconds")) { + index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue()); + } + indexes.add(index); + } + return indexes; + } } From 15608b5df90632e15cdde85e7dfb22399d3ae020 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 22 Aug 2024 21:50:44 -0700 Subject: [PATCH 4/4] clean up test file --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06f4bee..83beaa9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,8 +60,7 @@ jobs: - name: Run tests shell: bash - run: | - gradle test + run: ./gradlew test - name: Upload coverage to Codecov uses: codecov/codecov-action@v3