Skip to content

Commit

Permalink
Parameterize worker thread count in CrawlAccountsCommand
Browse files Browse the repository at this point in the history
  • Loading branch information
eager-signal committed Jun 16, 2023
1 parent cb26bfd commit f1962a0
Showing 1 changed file with 11 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import io.dropwizard.cli.EnvironmentCommand;
import io.dropwizard.setup.Environment;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import net.sourceforge.argparse4j.inf.Argument;
import net.sourceforge.argparse4j.inf.ArgumentParser;
Expand All @@ -36,6 +37,7 @@
public class CrawlAccountsCommand extends EnvironmentCommand<WhisperServerConfiguration> {

private static final String CRAWL_TYPE = "crawlType";
private static final String WORKER_COUNT = "workers";

public enum CrawlType implements ArgumentType<CrawlType> {
GENERAL_PURPOSE,
Expand Down Expand Up @@ -67,6 +69,12 @@ public void configure(final Subparser subparser) {
.dest(CRAWL_TYPE)
.required(true)
.help("The type of crawl to perform");

subparser.addArgument("--workers")
.type(Integer.class)
.dest(WORKER_COUNT)
.required(true)
.help("The number of worker threads");
}

@Override
Expand Down Expand Up @@ -99,6 +107,8 @@ protected void run(final Environment environment, final Namespace namespace,
dynamicConfigurationManager.start();
MetricsUtil.registerSystemResourceMetrics(environment);

final int workers = Objects.requireNonNull(namespace.getInt(WORKER_COUNT));

final AccountDatabaseCrawler crawler = switch ((CrawlType) namespace.get(CRAWL_TYPE)) {
case GENERAL_PURPOSE -> {
final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
Expand All @@ -114,7 +124,7 @@ yield new AccountDatabaseCrawler("General-purpose account crawler",
}
case ACCOUNT_CLEANER -> {
final ExecutorService accountDeletionExecutor = environment.lifecycle()
.executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
.executorService(name(getClass(), "accountCleaner-%d")).maxThreads(workers).minThreads(workers).build();

final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
Expand Down

0 comments on commit f1962a0

Please sign in to comment.