-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: multithreaded bulk import (#162)
* feat: Add BulkImport APIs and cron * fix: PR changes * fix: PR changes * fix: Update version and changelog * fix: PR changes * fix: PR changes * fix: fixing transaction rolled back issues with multithreaded bulk import * fix: changelog * fix: reusing gson object * feat: bulk inserting the bulk migration data * fix: fixes and error handling changes * fix: fixing tests * chore: update build version and changelog * fix: adding exeption thrown declaration for bulk import and mysql --------- Co-authored-by: Ankit Tiwari <[email protected]>
- Loading branch information
1 parent
aa11e40
commit 435504f
Showing
27 changed files
with
615 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,7 +2,7 @@ plugins { | |
| id 'java-library' | ||
| } | ||
|
|
||
| version = "6.3.0" | ||
| version = "6.4.0" | ||
|
|
||
| repositories { | ||
| mavenCentral() | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
src/main/java/io/supertokens/pluginInterface/bulkimport/BulkImportStorage.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. | ||
| * | ||
| * This software is licensed under the Apache License, Version 2.0 (the | ||
| * "License") as published by the Apache Software Foundation. | ||
| * | ||
| * You may not use this file except in compliance with the License. You may | ||
| * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | ||
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| * License for the specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package io.supertokens.pluginInterface.bulkimport; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| import javax.annotation.Nonnull; | ||
| import javax.annotation.Nullable; | ||
|
|
||
| import io.supertokens.pluginInterface.exceptions.StorageQueryException; | ||
| import io.supertokens.pluginInterface.multitenancy.AppIdentifier; | ||
| import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; | ||
| import io.supertokens.pluginInterface.nonAuthRecipe.NonAuthRecipeStorage; | ||
|
|
||
| public interface BulkImportStorage extends NonAuthRecipeStorage { | ||
| /** | ||
| * Add users to the bulk_import_users table | ||
| */ | ||
| void addBulkImportUsers(AppIdentifier appIdentifier, List<BulkImportUser> users) | ||
| throws StorageQueryException, | ||
| TenantOrAppNotFoundException, | ||
| io.supertokens.pluginInterface.bulkimport.exceptions.DuplicateUserIdException; | ||
|
|
||
| /** | ||
| * Get users from the bulk_import_users table | ||
| */ | ||
| List<BulkImportUser> getBulkImportUsers(AppIdentifier appIdentifier, @Nonnull Integer limit, @Nullable BULK_IMPORT_USER_STATUS status, | ||
| @Nullable String bulkImportUserId, @Nullable Long createdAt) throws StorageQueryException; | ||
|
|
||
| /** | ||
| * Delete users by id from the bulk_import_users table | ||
| */ | ||
| List<String> deleteBulkImportUsers(AppIdentifier appIdentifier, @Nonnull String[] bulkImportUserIds) throws StorageQueryException; | ||
|
|
||
| /** | ||
| * Returns the users from the bulk_import_users table for processing | ||
| */ | ||
| List<BulkImportUser> getBulkImportUsersAndChangeStatusToProcessing(AppIdentifier appIdentifier, @Nonnull Integer limit) throws StorageQueryException; | ||
|
|
||
|
|
||
| /** | ||
| * Update the bulk_import_user's primary_user_id by bulk_import_user_id | ||
| */ | ||
| void updateBulkImportUserPrimaryUserId(AppIdentifier appIdentifier, @Nonnull String bulkImportUserId, @Nonnull String primaryUserId) throws StorageQueryException; | ||
|
|
||
| /** | ||
| * Returns the count of users from the bulk_import_users table | ||
| */ | ||
| long getBulkImportUsersCount(AppIdentifier appIdentifier, @Nullable BULK_IMPORT_USER_STATUS status) throws StorageQueryException; | ||
|
|
||
| public enum BULK_IMPORT_USER_STATUS { | ||
| NEW, PROCESSING, FAILED | ||
| } | ||
| } |
151 changes: 151 additions & 0 deletions
151
src/main/java/io/supertokens/pluginInterface/bulkimport/BulkImportUser.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| /* | ||
| * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. | ||
| * | ||
| * This software is licensed under the Apache License, Version 2.0 (the | ||
| * "License") as published by the Apache Software Foundation. | ||
| * | ||
| * You may not use this file except in compliance with the License. You may | ||
| * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | ||
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| * License for the specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package io.supertokens.pluginInterface.bulkimport; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| import com.google.gson.Gson; | ||
| import com.google.gson.JsonObject; | ||
|
|
||
| import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; | ||
|
|
||
| public class BulkImportUser { | ||
| public String id; | ||
| public String externalUserId; | ||
| public JsonObject userMetadata; | ||
| public List<UserRole> userRoles; | ||
| public List<TotpDevice> totpDevices; | ||
| public List<LoginMethod> loginMethods; | ||
|
|
||
| // Following fields come from the DB Record. | ||
| public BULK_IMPORT_USER_STATUS status; | ||
| public String primaryUserId; | ||
| public String errorMessage; | ||
| public Long createdAt; | ||
| public Long updatedAt; | ||
|
|
||
| private static final Gson gson = new Gson(); | ||
|
|
||
| public BulkImportUser(String id, String externalUserId, JsonObject userMetadata, List<UserRole> userRoles, | ||
| List<TotpDevice> totpDevices, List<LoginMethod> loginMethods) { | ||
| this.id = id; | ||
| this.externalUserId = externalUserId; | ||
| this.userMetadata = userMetadata; | ||
| this.userRoles = userRoles; | ||
| this.totpDevices = totpDevices; | ||
| this.loginMethods = loginMethods; | ||
| } | ||
|
|
||
| public static BulkImportUser forTesting_fromJson(JsonObject jsonObject) { | ||
| return gson.fromJson(jsonObject, BulkImportUser.class); | ||
| } | ||
|
|
||
| // The bulk_import_users table stores users to be imported via a Cron Job. | ||
| // It has a `raw_data` column containing user data in JSON format. | ||
|
|
||
| // The BulkImportUser class represents this `raw_data`, including additional fields like `status`, `createdAt`, and `updatedAt`. | ||
| // First, we validate all fields of `raw_data` using the BulkImportUser class, then store this data in the bulk_import_users table. | ||
| // This function retrieves the `raw_data` after removing the additional fields. | ||
| public String toRawDataForDbStorage() { | ||
| JsonObject jsonObject = gson.fromJson(new Gson().toJson(this), JsonObject.class); | ||
| jsonObject.remove("status"); | ||
| jsonObject.remove("createdAt"); | ||
| jsonObject.remove("updatedAt"); | ||
| return jsonObject.toString(); | ||
| } | ||
|
|
||
| // The bulk_import_users table contains a `raw_data` column with user data in JSON format, along with other columns such as `id`, `status`, `primary_user_id`, and `error_msg` etc. | ||
|
|
||
| // When creating an instance of the BulkImportUser class, the extra fields must be passed separately as they are not part of the `raw_data`. | ||
| // This function creates a BulkImportUser instance from a stored bulk_import_user entry. | ||
| public static BulkImportUser fromRawDataFromDbStorage(String id, String rawData, BULK_IMPORT_USER_STATUS status, String primaryUserId, String errorMessage, long createdAt, long updatedAt) { | ||
| BulkImportUser user = gson.fromJson(rawData, BulkImportUser.class); | ||
| user.id = id; | ||
| user.status = status; | ||
| user.primaryUserId = primaryUserId; | ||
| user.errorMessage = errorMessage; | ||
| user.createdAt = createdAt; | ||
| user.updatedAt = updatedAt; | ||
| return user; | ||
| } | ||
|
|
||
| public JsonObject toJsonObject() { | ||
| return gson.fromJson(gson.toJson(this), JsonObject.class); | ||
| } | ||
|
|
||
| public static class UserRole { | ||
| public String role; | ||
| public List<String> tenantIds; | ||
|
|
||
| public UserRole(String role, List<String> tenantIds) { | ||
| this.role = role; | ||
| this.tenantIds = tenantIds; | ||
| } | ||
| } | ||
|
|
||
| public static class TotpDevice { | ||
| public String secretKey; | ||
| public int period; | ||
| public int skew; | ||
| public String deviceName; | ||
|
|
||
| public TotpDevice(String secretKey, int period, int skew, String deviceName) { | ||
| this.secretKey = secretKey; | ||
| this.period = period; | ||
| this.skew = skew; | ||
| this.deviceName = deviceName; | ||
| } | ||
| } | ||
|
|
||
| public static class LoginMethod { | ||
| public List<String> tenantIds; | ||
| public boolean isVerified; | ||
| public boolean isPrimary; | ||
| public long timeJoinedInMSSinceEpoch; | ||
| public String recipeId; | ||
| public String email; | ||
| public String passwordHash; | ||
| public String hashingAlgorithm; | ||
| public String plainTextPassword; | ||
| public String thirdPartyId; | ||
| public String thirdPartyUserId; | ||
| public String phoneNumber; | ||
| public String superTokensUserId; | ||
| public String externalUserId; | ||
|
|
||
| public String getSuperTokenOrExternalUserId() { | ||
| return this.externalUserId != null ? this.externalUserId : this.superTokensUserId; | ||
| } | ||
|
|
||
| public LoginMethod(List<String> tenantIds, String recipeId, boolean isVerified, boolean isPrimary, | ||
| long timeJoinedInMSSinceEpoch, String email, String passwordHash, String hashingAlgorithm, String plainTextPassword, | ||
| String thirdPartyId, String thirdPartyUserId, String phoneNumber) { | ||
| this.tenantIds = tenantIds; | ||
| this.recipeId = recipeId; | ||
| this.isVerified = isVerified; | ||
| this.isPrimary = isPrimary; | ||
| this.timeJoinedInMSSinceEpoch = timeJoinedInMSSinceEpoch; | ||
| this.email = email; | ||
| this.passwordHash = passwordHash; | ||
| this.hashingAlgorithm = hashingAlgorithm; | ||
| this.plainTextPassword = plainTextPassword; | ||
| this.thirdPartyId = thirdPartyId; | ||
| this.thirdPartyUserId = thirdPartyUserId; | ||
| this.phoneNumber = phoneNumber; | ||
| } | ||
| } | ||
| } |
34 changes: 34 additions & 0 deletions
34
src/main/java/io/supertokens/pluginInterface/bulkimport/ImportUserBase.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* | ||
| * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. | ||
| * | ||
| * This software is licensed under the Apache License, Version 2.0 (the | ||
| * "License") as published by the Apache Software Foundation. | ||
| * | ||
| * You may not use this file except in compliance with the License. You may | ||
| * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | ||
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| * License for the specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package io.supertokens.pluginInterface.bulkimport; | ||
|
|
||
| import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; | ||
|
|
||
| public class ImportUserBase { | ||
|
|
||
| public String userId; | ||
| public String email; | ||
| public TenantIdentifier tenantIdentifier; | ||
| public long timeJoinedMSSinceEpoch; | ||
|
|
||
| public ImportUserBase(String userId, String email, TenantIdentifier tenantIdentifier, long timeJoinedMSSinceEpoch) { | ||
| this.userId = userId; //this will be the supertokens userId. | ||
| this.email = email; | ||
| this.tenantIdentifier = tenantIdentifier; | ||
| this.timeJoinedMSSinceEpoch = timeJoinedMSSinceEpoch; | ||
| } | ||
| } |
Oops, something went wrong.