Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow overriding cluster configs in quickstarts #14919

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Allow overriding cluster configs in quickstarts #14919

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,13 @@ protected Map<String, Object> getConfigOverrides() {
return configOverrides;
}

@Override
protected Map<String, String> getClusterConfigOverrides() {
return Map.of(
CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_MAX_SERVER_QUERY_THREADS, "50"
);
}

@Override
protected int getNumQuickstartRunnerServers() {
return 3;
Expand Down
4 changes: 4 additions & 0 deletions pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,10 @@ protected Map<String, Object> getConfigOverrides() {
}
}

protected Map<String, String> getClusterConfigOverrides() {
return Map.of();
}

protected String[] getDefaultBatchTableDirectories() {
return DEFAULT_OFFLINE_TABLE_DIRECTORIES;
}
Expand Down
2 changes: 1 addition & 1 deletion pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void execute()

QuickstartRunner runner =
new QuickstartRunner(quickstartTableRequests, 1, 1, getNumQuickstartRunnerServers(), 1, quickstartRunnerDir,
true, getAuthProvider(), getConfigOverrides(), _zkExternalAddress, true);
true, getAuthProvider(), getConfigOverrides(), _zkExternalAddress, true, getClusterConfigOverrides());

printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker, server and minion *****");
runner.startAll();
Expand Down
23 changes: 21 additions & 2 deletions pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.pinot.spi.auth.AuthProvider;
import org.apache.pinot.spi.config.tenant.TenantRole;
import org.apache.pinot.spi.env.PinotConfiguration;
Expand Down Expand Up @@ -69,6 +74,7 @@ public class QuickstartRunner {
private final boolean _enableTenantIsolation;
private final AuthProvider _authProvider;
private final Map<String, Object> _configOverrides;
private final Map<String, String> _clusterConfigOverrides;
private final boolean _deleteExistingData;

// If this field is non-null, an embedded Zookeeper instance will not be launched
Expand All @@ -82,12 +88,13 @@ public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int numContr
int numServers, int numMinions, File tempDir, Map<String, Object> configOverrides)
throws Exception {
this(tableRequests, numControllers, numBrokers, numServers, numMinions, tempDir, true, null, configOverrides, null,
true);
true, Map.of());
}

public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int numControllers, int numBrokers,
int numServers, int numMinions, File tempDir, boolean enableIsolation, AuthProvider authProvider,
Map<String, Object> configOverrides, String zkExternalAddress, boolean deleteExistingData)
Map<String, Object> configOverrides, String zkExternalAddress, boolean deleteExistingData,
Map<String, String> clusterConfigOverrides)
throws Exception {
_tableRequests = tableRequests;
_numControllers = numControllers;
Expand All @@ -98,6 +105,7 @@ public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int numContr
_enableTenantIsolation = enableIsolation;
_authProvider = authProvider;
_configOverrides = new HashMap<>(configOverrides);
_clusterConfigOverrides = clusterConfigOverrides;
if (numMinions > 0) {
// configure the controller to schedule tasks when minion is enabled
_configOverrides.put("controller.task.scheduler.enabled", true);
Expand Down Expand Up @@ -127,11 +135,22 @@ private void startControllers()
.setTenantIsolation(_enableTenantIsolation)
.setDataDir(new File(_tempDir, DEFAULT_CONTROLLER_DIR + i).getAbsolutePath())
.setConfigOverrides(_configOverrides);

if (!controllerStarter.execute()) {
throw new RuntimeException("Failed to start Controller");
}
_controllerPorts.add(DEFAULT_CONTROLLER_PORT + i);
}

HelixManager helixManager =
HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "localhost_" + _controllerPorts.get(0),
InstanceType.CONTROLLER, _zkExternalAddress != null ? _zkExternalAddress : ZK_ADDRESS);
helixManager.connect();
HelixConfigScope scope =
new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(CLUSTER_NAME).build();
for (Map.Entry<String, String> entry : _clusterConfigOverrides.entrySet()) {
helixManager.getConfigAccessor().set(scope, entry.getKey(), entry.getValue());
}
}

private void startBrokers()
Expand Down
Loading