Skip to content

Commit

Permalink
Improvements on how groups are provisioned
Browse files Browse the repository at this point in the history 
Closes #880

Signed-off-by: Pedro Igor <[email protected]>
  • Loading branch information
@pedroigor
pedroigor committed Jul 4, 2024
1 parent 18d84ab commit 9691feb
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 86 deletions.
167 changes: 95 additions & 72 deletions dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.keycloak.models.GroupModel;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
import org.keycloak.models.KeycloakUriInfo;
import org.keycloak.models.PasswordPolicy;
import org.keycloak.models.RealmModel;
Expand All @@ -63,14 +64,13 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_GROUPS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_USERS;
Expand Down Expand Up @@ -179,7 +179,7 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i

// create each 100 groups per transaction as default case
// (to avoid transaction timeouts when creating too many groups in one transaction)
createGroupsInMultipleTransactions(config, context, task);
createGroupsInMultipleTransactions(config, context, task, config.getGroupsPerRealm());

// Step 2 - create clients (Using single executor for now... For multiple executors run separate create-clients endpoint)
for (int i = 0; i < config.getClientsPerRealm(); i += config.getClientsPerTransaction()) {
Expand Down Expand Up @@ -212,26 +212,26 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i
}
}

private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task) {
int groupsPerRealm = config.getGroupsPerRealm();
private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task, int topLevelCount) {
boolean hierarchicalGroups = Boolean.parseBoolean(config.getGroupsWithHierarchy());
int hierarchyDepth = config.getGroupsHierarchyDepth();
int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : groupsPerRealm;
int totalNumberOfGroups = hierarchicalGroups ? (int) Math.pow(countGroupsAtEachLevel, hierarchyDepth) : groupsPerRealm;

for (int i = 0; i < totalNumberOfGroups; i += config.getGroupsPerTransaction()) {
int groupsStartIndex = i;
int groupEndIndex = hierarchicalGroups ? Math.min(groupsStartIndex + config.getGroupsPerTransaction(), totalNumberOfGroups)
: Math.min(groupsStartIndex + config.getGroupsPerTransaction(), config.getGroupsPerRealm());

logger.tracef("groupsStartIndex: %d, groupsEndIndex: %d", groupsStartIndex, groupEndIndex);

KeycloakModelUtils.runJobInTransactionWithTimeout(baseSession.getKeycloakSessionFactory(),
session -> createGroups(context, groupsStartIndex, groupEndIndex, hierarchicalGroups, hierarchyDepth, countGroupsAtEachLevel, session),
config.getTransactionTimeoutInSeconds());
int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : 0;
RealmModel realm = context.getRealm();

task.debug(logger, "Created %d groups in realm %s", context.getGroups().size(), context.getRealm().getName());
for (int index = 0; index < topLevelCount; index++) {
Integer finalIndex = index;
KeycloakModelUtils.runJobInTransaction(baseSession.getKeycloakSessionFactory(), baseSession.getContext(), new KeycloakSessionTask() {
@Override
public void run(KeycloakSession session) {
String groupName = config.getGroupPrefix() + finalIndex;
session.groups().createGroup(realm, groupName);
logger.infof("Creating top-level group %s in realm %s", groupName, context.getRealm().getName());
createGroupLevel(session, realm, countGroupsAtEachLevel, hierarchyDepth, groupName);
logger.infof("Created top-level group %s in realm %s", groupName, context.getRealm().getName());
}
});
}

task.info(logger, "Created all %d groups in realm %s", context.getGroups().size(), context.getRealm().getName());
}

Expand Down Expand Up @@ -899,6 +899,65 @@ public Response takeDCUp() {
return Response.ok(TaskResponse.statusMessage("Site " + siteName + " was marked as up.")).build();
}

@GET
@Path("/create-groups")
@NoCache
@Produces(MediaType.APPLICATION_JSON)
public Response createGroups() {
boolean started = false;
boolean taskAdded = false;
try {
DatasetConfig config = ConfigUtil.createConfigFromQueryParams(httpRequest, CREATE_GROUPS);

Task task = Task.start("Creation of " + config.getCount() + " groups in the realm " + config.getRealmName());
TaskManager taskManager = new TaskManager(baseSession);
Task existingTask = taskManager.addTaskIfNotInProgress(task, config.getTaskTimeout());
if (existingTask != null) {
return Response.status(400).entity(TaskResponse.errorSomeTaskInProgress(existingTask, getStatusUrl())).build();
} else {
taskAdded = true;
}

logger.infof("Trigger creating groups with the configuration: %s", config);

// Use the cache
RealmModel realm = baseSession.realms().getRealmByName(config.getRealmName());
if (realm == null) {
throw new DatasetException("Realm '" + config.getRealmName() + "' not found");
}

int startIndex = ConfigUtil.findFreeEntityIndex(index -> {
String name = config.getGroupPrefix() + index;
return baseSession.groups().getGroupByName(realm, null, name) != null;
});
config.setStart(startIndex);

// Run this in separate thread to not block HTTP request
RealmContext context = new RealmContext(config);

context.setRealm(realm);

new Thread(() -> {
try {
createGroupsInMultipleTransactions(config, context, task, config.getCount());
success();
} catch (Exception e) {
KeycloakModelUtils.runJobInTransaction(baseSession.getKeycloakSessionFactory(), session
-> new TaskManager(session).removeExistingTask(false));
}
}).start();
started = true;

return Response.ok(TaskResponse.taskStarted(task, getStatusUrl())).build();
} catch (DatasetException de) {
return handleDatasetException(de);
} finally {
if (taskAdded && !started) {
new TaskManager(baseSession).removeExistingTask(false);
}
}
}

@Override
public void close() {
}
Expand Down Expand Up @@ -1017,59 +1076,6 @@ private void createClients(RealmContext context, Task task, KeycloakSession sess
task.debug(logger, "Created %d clients in realm %s", context.getClientCount(), context.getRealm().getName());
}

private String getGroupName(boolean hierarchicalGroups, int countGroupsAtEachLevel, String prefix, int currentCount) {

if (!hierarchicalGroups) {
return prefix + currentCount;
}
if(currentCount == 0) {
return prefix + "0";
}
// we are using "." separated paths in the group names, this is basically a number system with countGroupsAtEachLevel being the basis
// this allows us to find the parent group by trimming the group name even if the parent was created in previous transaction
StringBuilder groupName = new StringBuilder();
if(countGroupsAtEachLevel == 1) {
// numbering system does not work for base 1
groupName.append("0");
IntStream.range(0, currentCount).forEach(i -> groupName.append(GROUP_NAME_SEPARATOR).append("0"));
return prefix + groupName;
}

int leftover = currentCount;
while (leftover > 0) {
int digit = leftover % countGroupsAtEachLevel;
groupName.insert(0, digit + GROUP_NAME_SEPARATOR);
leftover = (leftover - digit) / countGroupsAtEachLevel;
}
return prefix + groupName.substring(0, groupName.length() - 1);
}

private String getParentGroupName(String groupName) {
if (groupName == null || groupName.lastIndexOf(GROUP_NAME_SEPARATOR) < 0) {
return null;
}
return groupName.substring(0, groupName.lastIndexOf(GROUP_NAME_SEPARATOR));
}

private void createGroups(RealmContext context, int startIndex, int endIndex, boolean hierarchicalGroups, int hierarchyDepth, int countGroupsAtEachLevel, KeycloakSession session) {
RealmModel realm = context.getRealm();
for (int i = startIndex; i < endIndex; i++) {
String groupName = getGroupName(hierarchicalGroups, countGroupsAtEachLevel, context.getConfig().getGroupPrefix(), i);
String parentGroupName = getParentGroupName(groupName);

if (parentGroupName != null) {
Optional<GroupModel> maybeParent = session.groups().searchForGroupByNameStream(realm, parentGroupName, true, -1, -1).findFirst();
maybeParent.ifPresent(parent -> {
GroupModel groupModel = session.groups().createGroup(realm, groupName, parent);
context.groupCreated(groupModel);
});
} else {
GroupModel groupModel = session.groups().createGroup(realm, groupName);
context.groupCreated(groupModel);
}
}
}

// Worker task to be triggered by single executor thread
private void createUsers(RealmContext context, KeycloakSession session, int startIndex, int endIndex) {
// Refresh the realm
Expand Down Expand Up @@ -1231,4 +1237,21 @@ protected void success() {
-> new TaskManager(session).removeExistingTask(true));
}

private void createGroupLevel(KeycloakSession session, RealmModel realm, int count, int depth, String parent) {
GroupModel parentGroup = session.groups().searchForGroupByNameStream(realm, parent, true, -1, -1).findAny().get();

for (int index = 0; index < count; index++) {
String groupName = parent + GROUP_NAME_SEPARATOR + index;

if (session.groups().searchForGroupByNameStream(realm, groupName, true, -1, -1).findAny().isPresent()) {
continue;
}

session.groups().createGroup(realm, groupName, parentGroup);

for (int depthIndex = 0; depthIndex < depth; depthIndex++) {
createGroupLevel(session, realm, count, depth - 1, groupName);
}
}
}
}
23 changes: 12 additions & 11 deletions dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_AUTHZ_CLIENT;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_GROUPS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_ORGS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS;
Expand Down Expand Up @@ -57,14 +58,14 @@ public class DatasetConfig {
private Integer lastToRemove;

// Realm-name is required when creating many clients or users. The realm where clients/users will be created must already exists
@QueryParamFill(paramName = "realm-name", required = true, operations = { CREATE_CLIENTS, CREATE_USERS, LAST_CLIENT, LAST_USER, CREATE_AUTHZ_CLIENT })
@QueryParamFill(paramName = "realm-name", required = true, operations = { CREATE_CLIENTS, CREATE_USERS, CREATE_GROUPS, LAST_CLIENT, LAST_USER, CREATE_AUTHZ_CLIENT })
private String realmName;

// NOTE: Start index is not available as parameter as it will be "auto-detected" based on already created realms (clients, users)
private Integer start;

// Count of entities to be created. Entity is realm, client or user based on the operation
@QueryParamIntFill(paramName = "count", required = true, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, CREATE_AUTHZ_CLIENT, CREATE_ORGS })
@QueryParamIntFill(paramName = "count", required = true, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_EVENTS, CREATE_GROUPS, CREATE_OFFLINE_SESSIONS, CREATE_AUTHZ_CLIENT, CREATE_ORGS })
private Integer count;

// Prefix for realm roles to create in every realm (in case of CREATE_REALMS) or to assign to users (in case of CREATE_USERS)
Expand Down Expand Up @@ -107,27 +108,27 @@ public class DatasetConfig {
private Integer clientRolesPerClient;

// Prefix of groups to be created (in case of CREATE_REALMS operation) or assigned to the users (In case of CREATE_USERS and CREATE_REALMS operations)
@QueryParamFill(paramName = "group-prefix", defaultValue = "group-", operations = { CREATE_REALMS, CREATE_USERS })
@QueryParamFill(paramName = "group-prefix", defaultValue = "group-", operations = { CREATE_REALMS, CREATE_USERS, CREATE_GROUPS })
private String groupPrefix;

// Count of groups to be created in every created realm
@QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS })
@QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS, CREATE_GROUPS })
private Integer groupsPerRealm;

// Number of groups to be created in one transaction
@QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS })
@QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS, CREATE_GROUPS })
private Integer groupsPerTransaction;

// When this parameter is false only top level groups are created, groups and subgroups are created
@QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS })
@QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS, CREATE_GROUPS })
private String groupsWithHierarchy;

// Depth of the group hierarchy tree. Active if groups-with-hierarchy = true
@QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS })
@QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS, CREATE_GROUPS })
private Integer groupsHierarchyDepth;

// Number of at each level of hierarchy. Each group will have this many subgroups. Active if groups-with-hierarchy = true
@QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS })
@QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS, CREATE_GROUPS })
private Integer countGroupsAtEachLevel;


Expand Down Expand Up @@ -165,22 +166,22 @@ public class DatasetConfig {

// Transaction timeout used for transactions for creating objects
@QueryParamIntFill(paramName = "transaction-timeout", defaultValue = 300, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS,
CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS })
CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS, CREATE_GROUPS })
private Integer transactionTimeoutInSeconds;

// Count of users created in every transaction
@QueryParamIntFill(paramName = "users-per-transaction", defaultValue = 10, operations = { CREATE_REALMS, CREATE_USERS })
private Integer usersPerTransaction;

// Count of worker threads concurrently creating entities
@QueryParamIntFill(paramName = "threads-count", operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS,
@QueryParamIntFill(paramName = "threads-count", operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_GROUPS,
CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS })
private Integer threadsCount;

// Timeout for the whole task. If timeout expires, then the existing task may not be terminated immediatelly. However it will be permitted to start another task
// (EG. Send another HTTP request for creating realms), which can cause conflicts
@QueryParamIntFill(paramName = "task-timeout", defaultValue = 3600, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS,
CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS })
CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS, CREATE_GROUPS })
private Integer taskTimeout;

// The client id of a client to which data is going to be provisioned
Expand Down
1 change: 1 addition & 0 deletions dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public enum DatasetOperation {
CREATE_CLIENTS,
CREATE_AUTHZ_CLIENT,
CREATE_USERS,
CREATE_GROUPS,
CREATE_EVENTS,
CREATE_OFFLINE_SESSIONS,
CREATE_ORGS,
Expand Down
10 changes: 7 additions & 3 deletions doc/dataset/modules/ROOT/pages/using-provider.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,6 @@ The number of groups and the structure of the created groups can be managed by u
`groups-per-realm`:: Total number of groups per realm.
The default value is `20`.

`groups-per-transaction`:: Number of groups to be created in one transaction.
The default value is `100`.

`groups-with-hierarchy`:: `true` or `false`, the default value is `false`.
With the default value, only top-level groups are created.
With groups-with-hierarchy set to `true` a tree structure of groups is created; the depth of the tree is defined by the parameter `groups-hierarchy-depth` and `groups-count-each-level` defines how many subgroups each created group will have.
Expand All @@ -131,6 +128,13 @@ The adopted subgroup naming convention uses a dot (`.`) in the group names which
.../realms/master/dataset/create-realms?count=1&groups-with-hierarchy=true&groups-hierarchy-depth=3&groups-count-each-level=50
----

You can also create groups in an existing realm by invoking the `create-groups` endpoint and setting the `realm-name` parameter:

.Example parameters
----
.../realms/master/dataset/create-groups?realm-name=realm-0&count=10&groups-with-hierarchy=true&groups-hierarchy-depth=3&groups-count-each-level=5
----

=== Create many events

This is request to create 10M new events in the available realms with prefix `realm-`.
Expand Down

0 comments on commit 9691feb

Please sign in to comment.