Skip to content

Commit

Permalink
Adding group hierarchy (#472)
Browse files Browse the repository at this point in the history 
Closes #471

Co-authored-by: Alexander Schwartz <[email protected]>
  • Loading branch information
@eatikrh @ahus1
eatikrh and ahus1 authored Aug 16, 2023
1 parent aaa4654 commit db3d7c1
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 28 deletions.
114 changes: 87 additions & 27 deletions dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,12 @@

package org.keycloak.benchmark.dataset;

import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_USERS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.LAST_CLIENT;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.LAST_REALM;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.LAST_USER;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.REMOVE_REALMS;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Random;
import java.util.Collections;
import java.util.stream.Collectors;

import jakarta.ws.rs.DELETE;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.UriInfo;

import org.jboss.logging.Logger;
import org.jboss.resteasy.annotations.cache.NoCache;
import org.keycloak.benchmark.dataset.config.ConfigUtil;
Expand Down Expand Up @@ -76,12 +57,33 @@
import org.keycloak.services.managers.UserSessionManager;
import org.keycloak.services.resource.RealmResourceProvider;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_USERS;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.LAST_CLIENT;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.LAST_REALM;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.LAST_USER;
import static org.keycloak.benchmark.dataset.config.DatasetOperation.REMOVE_REALMS;

/**
* @author <a href="mailto:[email protected]">Marek Posolda</a>
*/
public class DatasetResourceProvider implements RealmResourceProvider {

protected static final Logger logger = Logger.getLogger(DatasetResourceProvider.class);
public static final String GROUP_NAME_SEPARATOR = ".";

// Ideally don't use this session to run any DB transactions
protected final KeycloakSession baseSession;
Expand Down Expand Up @@ -209,14 +211,21 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i
}

private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task) {
int groupsPerRealm = config.getGroupsPerRealm();
boolean hierarchicalGroups = Boolean.parseBoolean(config.getGroupsWithHierarchy());
int hierarchyDepth = config.getGroupsHierarchyDepth();
int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : groupsPerRealm;
int totalNumberOfGroups = hierarchicalGroups ? (int) Math.pow(countGroupsAtEachLevel, hierarchyDepth) : groupsPerRealm;

for (int i = 0; i < config.getGroupsPerRealm(); i += config.getGroupsPerTransaction()) {
for (int i = 0; i < totalNumberOfGroups; i += config.getGroupsPerTransaction()) {
int groupsStartIndex = i;
int groupEndIndex = Math.min(groupsStartIndex + config.getGroupsPerTransaction(), config.getGroupsPerRealm());
int groupEndIndex = hierarchicalGroups ? Math.min(groupsStartIndex + config.getGroupsPerTransaction(), totalNumberOfGroups)
: Math.min(groupsStartIndex + config.getGroupsPerTransaction(), config.getGroupsPerRealm());

logger.tracef("groupsStartIndex: %d, groupsEndIndex: %d", groupsStartIndex, groupEndIndex);

KeycloakModelUtils.runJobInTransactionWithTimeout(baseSession.getKeycloakSessionFactory(),
session -> createGroups(context, groupsStartIndex, groupEndIndex, session),
session -> createGroups(context, groupsStartIndex, groupEndIndex, hierarchicalGroups, hierarchyDepth, countGroupsAtEachLevel, session),
config.getTransactionTimeoutInSeconds());

task.debug(logger, "Created %d groups in realm %s", context.getGroups().size(), context.getRealm().getName());
Expand Down Expand Up @@ -972,12 +981,56 @@ private void createClients(RealmContext context, Task task, KeycloakSession sess
task.debug(logger, "Created %d clients in realm %s", context.getClientCount(), context.getRealm().getName());
}

private void createGroups(RealmContext context, int startIndex, int endIndex, KeycloakSession session) {
private String getGroupName(boolean hierarchicalGroups, int countGroupsAtEachLevel, String prefix, int currentCount) {

if (!hierarchicalGroups) {
return prefix + currentCount;
}
if(currentCount == 0) {
return prefix + "0";
}
// we are using "." separated paths in the group names, this is basically a number system with countGroupsAtEachLevel being the basis
// this allows us to find the parent group by trimming the group name even if the parent was created in previous transaction
StringBuilder groupName = new StringBuilder();
if(countGroupsAtEachLevel == 1) {
// numbering system does not work for base 1
groupName.append("0");
IntStream.range(0, currentCount).forEach(i -> groupName.append(GROUP_NAME_SEPARATOR).append("0"));
return prefix + groupName;
}

int leftover = currentCount;
while (leftover > 0) {
int digit = leftover % countGroupsAtEachLevel;
groupName.insert(0, digit + GROUP_NAME_SEPARATOR);
leftover = (leftover - digit) / countGroupsAtEachLevel;
}
return prefix + groupName.substring(0, groupName.length() - 1);
}

private String getParentGroupName(String groupName) {
if (groupName == null || groupName.lastIndexOf(GROUP_NAME_SEPARATOR) < 0) {
return null;
}
return groupName.substring(0, groupName.lastIndexOf(GROUP_NAME_SEPARATOR));
}

private void createGroups(RealmContext context, int startIndex, int endIndex, boolean hierarchicalGroups, int hierarchyDepth, int countGroupsAtEachLevel, KeycloakSession session) {
RealmModel realm = context.getRealm();
for (int i = startIndex; i < endIndex; i++) {
String groupName = context.getConfig().getGroupPrefix() + i;
GroupModel groupModel = session.groups().createGroup(realm, groupName);
context.groupCreated(groupModel);
String groupName = getGroupName(hierarchicalGroups, countGroupsAtEachLevel, context.getConfig().getGroupPrefix(), i);
String parentGroupName = getParentGroupName(groupName);

if (parentGroupName != null) {
Optional<GroupModel> maybeParent = session.groups().searchForGroupByNameStream(realm, parentGroupName, true, 1, 1).findFirst();
maybeParent.ifPresent(parent -> {
GroupModel groupModel = session.groups().createGroup(realm, groupName, parent);
context.groupCreated(groupModel);
});
} else {
GroupModel groupModel = session.groups().createGroup(realm, groupName);
context.groupCreated(groupModel);
}
}
}

Expand Down Expand Up @@ -1055,7 +1108,14 @@ private void cacheRealmAndPopulateContext(RealmContext context) {
.sorted((group1, group2) -> {
String name1 = group1.getName().substring(config.getGroupPrefix().length());
String name2 = group2.getName().substring(config.getGroupPrefix().length());
return Integer.parseInt(name1) - Integer.parseInt(name2);
String [] name1Exploded = name1.split(Pattern.quote(GROUP_NAME_SEPARATOR));
String [] name2Exploded = name2.split(Pattern.quote(GROUP_NAME_SEPARATOR));
for(int i = 0; i< Math.min(name1Exploded.length, name2Exploded.length); i++) {
if (name1Exploded[i].compareTo(name2Exploded[i]) != 0) {
return name1Exploded[i].compareTo(name2Exploded[i]);
}
}
return name1.compareTo(name2);
})
.collect(Collectors.toList());
context.setGroups(sortedGroups);
Expand Down
26 changes: 26 additions & 0 deletions dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,23 @@ public class DatasetConfig {
@QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS })
private Integer groupsPerRealm;

// Number of groups to be created in one transaction
@QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS })
private Integer groupsPerTransaction;

// When this parameter is false only top level groups are created, groups and subgroups are created
@QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS })
private String groupsWithHierarchy;

// Depth of the group hierarchy tree. Active if groups-with-hierarchy = true
@QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS })
private Integer groupsHierarchyDepth;

// Number of at each level of hierarchy. Each group will have this many subgroups. Active if groups-with-hierarchy = true
@QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS })
private Integer countGroupsAtEachLevel;


// Prefix for newly created users
@QueryParamFill(paramName = "user-prefix", defaultValue = "user-", operations = { CREATE_REALMS, CREATE_USERS, CREATE_OFFLINE_SESSIONS, LAST_USER, CREATE_AUTHZ_CLIENT })
private String userPrefix;
Expand Down Expand Up @@ -270,6 +284,18 @@ public Integer getGroupsPerUser() {
return groupsPerUser;
}

public String getGroupsWithHierarchy() {
return groupsWithHierarchy;
}

public Integer getGroupsHierarchyDepth() {
return groupsHierarchyDepth;
}

public Integer getCountGroupsAtEachLevel() {
return countGroupsAtEachLevel;
}

public Integer getRealmRolesPerUser() {
return realmRolesPerUser;
}
Expand Down
34 changes: 33 additions & 1 deletion doc/dataset/modules/ROOT/pages/using-provider.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ For the Wildfly distribution, the URL starts with `/auth/realms/master`.

This script contains a subset of the operations described in the next sections around realms, users and clients.

The script assumes that the dataset provider is installed as described in xref:kubernetes-guide::installation.adoc[] setup and that it is available at +
The script assumes that the dataset provider is installed as described in xref:kubernetes-guide::installation-minikube.adoc[] setup and that it is available at +
`++https://keycloak-keycloak.$(minikube ip).nip.io/realms/master/dataset/++`.

Run the following command to receive a help description:
Expand Down Expand Up @@ -99,6 +99,38 @@ Each user will have password like «Username»-password . For example `user-156`
.../realms/master/dataset/create-users?count=1000&realm-name=realm-5
----

=== Create many groups

Groups are created as part of the realm creation.
The number of groups and the structure of the created groups can be managed by using the following parameters:

`groups-per-realm`:: Total number of groups per realm.
The default value is `20`.

`groups-per-transaction`:: Number of groups to be created in one transaction.
The default value is `100`.

`groups-with-hierarchy`:: `true` or `false`, the default value is `false`.
With the default value, only top-level groups are created.
With groups-with-hierarchy set to `true` a tree structure of groups is created; the depth of the tree is defined by the parameter `groups-hierarchy-depth` and `groups-count-each-level` defines how many subgroups each created group will have.

`groups-hierarchy-depth`:: The depth of the groups tree structure.
The default value is 3. With the default value, top level groups will have `groups-count-each-level` subgroups and each subgroup will have `groups-count-each-level` themselves.
This parameter is active only when `groups-with-hierarchy` is `true`.

`groups-count-each-level`:: Number of subgroups each created group will have.
This parameter is active only when `groups-with-hierarchy` is `true`.

With the default values, only top-level groups are created.
With `groups-with-hierarchy` set to `true`, the `groups-per-realm` parameter is ignored and the group tree structure is created as defined by the other parameters. `groups-count-each-level`^`groups-hierarchy-depth`^ will be the total number of groups created.
The hierarchical groups implementation honors groups-per-transaction.
The adopted subgroup naming convention uses a dot (`.`) in the group names which allows finding the parent group even if it was created in a previous transaction.

.Example parameters
----
.../realms/master/dataset/create-realms?count=1&groups-with-hierarchy=true&groups-hierarchy-depth=3&groups-count-each-level=50
----

=== Create many events

This is request to create 10M new events in the available realms with prefix `realm-`.
Expand Down

0 comments on commit db3d7c1

Please sign in to comment.