diff --git a/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java b/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java index 29a709de..0e88127b 100644 --- a/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java +++ b/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java @@ -40,6 +40,7 @@ import org.keycloak.models.GroupModel; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; +import org.keycloak.models.KeycloakSessionTask; import org.keycloak.models.KeycloakUriInfo; import org.keycloak.models.PasswordPolicy; import org.keycloak.models.RealmModel; @@ -63,14 +64,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Optional; import java.util.Random; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.IntStream; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS; +import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_GROUPS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_USERS; @@ -179,7 +179,7 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i // create each 100 groups per transaction as default case // (to avoid transaction timeouts when creating too many groups in one transaction) - createGroupsInMultipleTransactions(config, context, task); + createGroupsInMultipleTransactions(config, context, task, config.getGroupsPerRealm()); // Step 2 - create clients (Using single executor for now... For multiple executors run separate create-clients endpoint) for (int i = 0; i < config.getClientsPerRealm(); i += config.getClientsPerTransaction()) { @@ -212,26 +212,26 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i } } - private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task) { - int groupsPerRealm = config.getGroupsPerRealm(); + private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task, int topLevelCount) { boolean hierarchicalGroups = Boolean.parseBoolean(config.getGroupsWithHierarchy()); int hierarchyDepth = config.getGroupsHierarchyDepth(); - int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : groupsPerRealm; - int totalNumberOfGroups = hierarchicalGroups ? (int) Math.pow(countGroupsAtEachLevel, hierarchyDepth) : groupsPerRealm; + int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : 0; - for (int i = 0; i < totalNumberOfGroups; i += config.getGroupsPerTransaction()) { - int groupsStartIndex = i; - int groupEndIndex = hierarchicalGroups ? Math.min(groupsStartIndex + config.getGroupsPerTransaction(), totalNumberOfGroups) - : Math.min(groupsStartIndex + config.getGroupsPerTransaction(), config.getGroupsPerRealm()); - - logger.tracef("groupsStartIndex: %d, groupsEndIndex: %d", groupsStartIndex, groupEndIndex); - - KeycloakModelUtils.runJobInTransactionWithTimeout(baseSession.getKeycloakSessionFactory(), - session -> createGroups(context, groupsStartIndex, groupEndIndex, hierarchicalGroups, hierarchyDepth, countGroupsAtEachLevel, session), - config.getTransactionTimeoutInSeconds()); - - task.debug(logger, "Created %d groups in realm %s", context.getGroups().size(), context.getRealm().getName()); + RealmModel realm = context.getRealm(); + for (Integer i = 0; i < topLevelCount; i++) { + Integer finalI = i; + KeycloakModelUtils.runJobInTransaction(baseSession.getKeycloakSessionFactory(), baseSession.getContext(), new KeycloakSessionTask() { + @Override + public void run(KeycloakSession session) { + String groupName = config.getGroupPrefix() + finalI; + session.groups().createGroup(realm, groupName); + logger.infof("Creating top-level group %s in realm %s", groupName, context.getRealm().getName()); + createGroupLevel(session, realm, countGroupsAtEachLevel, hierarchyDepth, groupName); + logger.infof("Created top-level group %s in realm %s", groupName, context.getRealm().getName()); + } + }); } + task.info(logger, "Created all %d groups in realm %s", context.getGroups().size(), context.getRealm().getName()); } @@ -899,6 +899,65 @@ public Response takeDCUp() { return Response.ok(TaskResponse.statusMessage("Site " + siteName + " was marked as up.")).build(); } + @GET + @Path("/create-groups") + @NoCache + @Produces(MediaType.APPLICATION_JSON) + public Response createGroups() { + boolean started = false; + boolean taskAdded = false; + try { + DatasetConfig config = ConfigUtil.createConfigFromQueryParams(httpRequest, CREATE_GROUPS); + + Task task = Task.start("Creation of " + config.getCount() + " groups in the realm " + config.getRealmName()); + TaskManager taskManager = new TaskManager(baseSession); + Task existingTask = taskManager.addTaskIfNotInProgress(task, config.getTaskTimeout()); + if (existingTask != null) { + return Response.status(400).entity(TaskResponse.errorSomeTaskInProgress(existingTask, getStatusUrl())).build(); + } else { + taskAdded = true; + } + + logger.infof("Trigger creating groups with the configuration: %s", config); + + // Use the cache + RealmModel realm = baseSession.realms().getRealmByName(config.getRealmName()); + if (realm == null) { + throw new DatasetException("Realm '" + config.getRealmName() + "' not found"); + } + + int startIndex = ConfigUtil.findFreeEntityIndex(index -> { + String name = config.getGroupPrefix() + index; + return baseSession.groups().getGroupByName(realm, null, name) != null; + }); + config.setStart(startIndex); + + // Run this in separate thread to not block HTTP request + RealmContext context = new RealmContext(config); + + context.setRealm(realm); + + new Thread(() -> { + try { + createGroupsInMultipleTransactions(config, context, task, config.getCount()); + success(); + } catch (Exception e) { + KeycloakModelUtils.runJobInTransaction(baseSession.getKeycloakSessionFactory(), session + -> new TaskManager(session).removeExistingTask(false)); + } + }).start(); + started = true; + + return Response.ok(TaskResponse.taskStarted(task, getStatusUrl())).build(); + } catch (DatasetException de) { + return handleDatasetException(de); + } finally { + if (taskAdded && !started) { + new TaskManager(baseSession).removeExistingTask(false); + } + } + } + @Override public void close() { } @@ -1017,59 +1076,6 @@ private void createClients(RealmContext context, Task task, KeycloakSession sess task.debug(logger, "Created %d clients in realm %s", context.getClientCount(), context.getRealm().getName()); } - private String getGroupName(boolean hierarchicalGroups, int countGroupsAtEachLevel, String prefix, int currentCount) { - - if (!hierarchicalGroups) { - return prefix + currentCount; - } - if(currentCount == 0) { - return prefix + "0"; - } - // we are using "." separated paths in the group names, this is basically a number system with countGroupsAtEachLevel being the basis - // this allows us to find the parent group by trimming the group name even if the parent was created in previous transaction - StringBuilder groupName = new StringBuilder(); - if(countGroupsAtEachLevel == 1) { - // numbering system does not work for base 1 - groupName.append("0"); - IntStream.range(0, currentCount).forEach(i -> groupName.append(GROUP_NAME_SEPARATOR).append("0")); - return prefix + groupName; - } - - int leftover = currentCount; - while (leftover > 0) { - int digit = leftover % countGroupsAtEachLevel; - groupName.insert(0, digit + GROUP_NAME_SEPARATOR); - leftover = (leftover - digit) / countGroupsAtEachLevel; - } - return prefix + groupName.substring(0, groupName.length() - 1); - } - - private String getParentGroupName(String groupName) { - if (groupName == null || groupName.lastIndexOf(GROUP_NAME_SEPARATOR) < 0) { - return null; - } - return groupName.substring(0, groupName.lastIndexOf(GROUP_NAME_SEPARATOR)); - } - - private void createGroups(RealmContext context, int startIndex, int endIndex, boolean hierarchicalGroups, int hierarchyDepth, int countGroupsAtEachLevel, KeycloakSession session) { - RealmModel realm = context.getRealm(); - for (int i = startIndex; i < endIndex; i++) { - String groupName = getGroupName(hierarchicalGroups, countGroupsAtEachLevel, context.getConfig().getGroupPrefix(), i); - String parentGroupName = getParentGroupName(groupName); - - if (parentGroupName != null) { - Optional maybeParent = session.groups().searchForGroupByNameStream(realm, parentGroupName, true, -1, -1).findFirst(); - maybeParent.ifPresent(parent -> { - GroupModel groupModel = session.groups().createGroup(realm, groupName, parent); - context.groupCreated(groupModel); - }); - } else { - GroupModel groupModel = session.groups().createGroup(realm, groupName); - context.groupCreated(groupModel); - } - } - } - // Worker task to be triggered by single executor thread private void createUsers(RealmContext context, KeycloakSession session, int startIndex, int endIndex) { // Refresh the realm @@ -1231,4 +1237,17 @@ protected void success() { -> new TaskManager(session).removeExistingTask(true)); } + private void createGroupLevel(KeycloakSession session, RealmModel realm, int count, int depth, String parent) { + GroupModel parentGroup = session.groups().searchForGroupByNameStream(realm, parent, true, -1, -1).findAny().get(); + for (int i = 0; i < count; i++) { + String groupName = parent + GROUP_NAME_SEPARATOR + i; + if (session.groups().searchForGroupByNameStream(realm, groupName, true, -1, -1).findAny().isPresent()) { + continue; + } + session.groups().createGroup(realm, groupName, parentGroup); + for (int i1 = 0; i1 < depth; i1++) { + createGroupLevel(session, realm, count, depth - 1, groupName); + } + } + } } diff --git a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java index d18e4a65..dffcbb38 100644 --- a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java +++ b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java @@ -21,6 +21,7 @@ import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_AUTHZ_CLIENT; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS; +import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_GROUPS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_ORGS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS; @@ -57,14 +58,14 @@ public class DatasetConfig { private Integer lastToRemove; // Realm-name is required when creating many clients or users. The realm where clients/users will be created must already exists - @QueryParamFill(paramName = "realm-name", required = true, operations = { CREATE_CLIENTS, CREATE_USERS, LAST_CLIENT, LAST_USER, CREATE_AUTHZ_CLIENT }) + @QueryParamFill(paramName = "realm-name", required = true, operations = { CREATE_CLIENTS, CREATE_USERS, CREATE_GROUPS, LAST_CLIENT, LAST_USER, CREATE_AUTHZ_CLIENT }) private String realmName; // NOTE: Start index is not available as parameter as it will be "auto-detected" based on already created realms (clients, users) private Integer start; // Count of entities to be created. Entity is realm, client or user based on the operation - @QueryParamIntFill(paramName = "count", required = true, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) + @QueryParamIntFill(paramName = "count", required = true, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_EVENTS, CREATE_GROUPS, CREATE_OFFLINE_SESSIONS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) private Integer count; // Prefix for realm roles to create in every realm (in case of CREATE_REALMS) or to assign to users (in case of CREATE_USERS) @@ -107,27 +108,27 @@ public class DatasetConfig { private Integer clientRolesPerClient; // Prefix of groups to be created (in case of CREATE_REALMS operation) or assigned to the users (In case of CREATE_USERS and CREATE_REALMS operations) - @QueryParamFill(paramName = "group-prefix", defaultValue = "group-", operations = { CREATE_REALMS, CREATE_USERS }) + @QueryParamFill(paramName = "group-prefix", defaultValue = "group-", operations = { CREATE_REALMS, CREATE_USERS, CREATE_GROUPS }) private String groupPrefix; // Count of groups to be created in every created realm - @QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer groupsPerRealm; // Number of groups to be created in one transaction - @QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer groupsPerTransaction; // When this parameter is false only top level groups are created, groups and subgroups are created - @QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS }) + @QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS, CREATE_GROUPS }) private String groupsWithHierarchy; // Depth of the group hierarchy tree. Active if groups-with-hierarchy = true - @QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer groupsHierarchyDepth; // Number of at each level of hierarchy. Each group will have this many subgroups. Active if groups-with-hierarchy = true - @QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer countGroupsAtEachLevel; @@ -165,7 +166,7 @@ public class DatasetConfig { // Transaction timeout used for transactions for creating objects @QueryParamIntFill(paramName = "transaction-timeout", defaultValue = 300, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, - CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) + CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS, CREATE_GROUPS }) private Integer transactionTimeoutInSeconds; // Count of users created in every transaction @@ -173,14 +174,14 @@ public class DatasetConfig { private Integer usersPerTransaction; // Count of worker threads concurrently creating entities - @QueryParamIntFill(paramName = "threads-count", operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, + @QueryParamIntFill(paramName = "threads-count", operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_GROUPS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) private Integer threadsCount; // Timeout for the whole task. If timeout expires, then the existing task may not be terminated immediatelly. However it will be permitted to start another task // (EG. Send another HTTP request for creating realms), which can cause conflicts @QueryParamIntFill(paramName = "task-timeout", defaultValue = 3600, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, - CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) + CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS, CREATE_GROUPS }) private Integer taskTimeout; // The client id of a client to which data is going to be provisioned diff --git a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java index e6b440e4..7f612832 100644 --- a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java +++ b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java @@ -26,6 +26,7 @@ public enum DatasetOperation { CREATE_CLIENTS, CREATE_AUTHZ_CLIENT, CREATE_USERS, + CREATE_GROUPS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, CREATE_ORGS,