Issue 2228: (Segment Store) Admin Tools (pravega#2256)

* Builds a shell for a command-line interface that can
support an arbitrary set of commands a user wants to run.

* Is able to load Pravega config from a config file and set
custom configs on it

* BookKeeper commands:
   . List all BookKeeperLog summaries
   . List details for a BookKeeperLog
   . Cleanup orphan ledgers (issue pravega#1165)
   . Enabling a BookKeeperLog that is currently disabled.
   . Disabling a BookKeeperLog that is currently enabled.

* Container commands:
   . Executing a non-invasive DurableLog recovery for a
   specified container

* Updates the SelfTester to pause for user input
towards the end of the test - this allows a quick
debugging of the admin tools, by pointing them
to a local cluster that already has data and/or is active

Signed-off-by: Andrei Paduroiu <[email protected]>

.codecov.yml

@@ -20,3 +20,4 @@ coverage:
     - "**/generated/**"
     - "standalone"
     - "test"
+    - "**/host/admin"

build.gradle

@@ -274,6 +274,13 @@ project('segmentstore:server:host') {
         outputDir = startScripts.outputDir
     }
 
+    task admin(type: JavaExec) {
+        main = "io.pravega.segmentstore.server.host.admin.AdminRunner"
+        classpath = sourceSets.main.runtimeClasspath
+        standardInput = System.in
+        systemProperties System.getProperties()
+    }
+
     applicationDistribution.into("bin") {
         from(createAppWithGCLogging)
     }
@@ -370,6 +377,7 @@ project('test:integration') {
     task selftest(type: JavaExec) {
         main = "io.pravega.test.integration.selftest.SelfTestRunner"
         classpath = sourceSets.main.runtimeClasspath
+        standardInput = System.in
         systemProperties System.getProperties()
     }
 }

segmentstore/server/host/src/main/java/io/pravega/segmentstore/server/host/admin/AdminRunner.java

@@ -0,0 +1,159 @@
+/**
+ * Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.pravega.segmentstore.server.host.admin;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.LoggerContext;
+import com.google.common.base.Strings;
+import io.pravega.segmentstore.server.host.admin.commands.AdminCommandState;
+import io.pravega.segmentstore.server.host.admin.commands.Command;
+import io.pravega.segmentstore.server.host.admin.commands.CommandArgs;
+import io.pravega.segmentstore.server.host.admin.commands.ConfigListCommand;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Scanner;
+import java.util.stream.Collectors;
+import lombok.Cleanup;
+import lombok.val;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Main entry point for the Admin tools.
+ */
+public final class AdminRunner {
+    private static final String CMD_HELP = "help";
+    private static final String CMD_EXIT = "exit";
+
+    /**
+     * Main entry point for the Admin Tools Runner.
+     * <p>
+     * To speed up setup, create a config.properties file and put the following properties (at a minimum):
+     * <p>
+     * pravegaservice.containerCount={number of containers}
+     * pravegaservice.zkURL={host:port for ZooKeeper}
+     * bookkeeper.bkLedgerPath={path in ZooKeeper where BookKeeper stores Ledger metadata}
+     * bookkeeper.zkMetadataPath={path in ZooKeeper where Pravega stores BookKeeperLog metadata}
+     * <p>
+     * Then invoke this program with:
+     * -Dpravega.configurationFile=config.properties
+     *
+     * @param args Arguments.
+     * @throws Exception If one occurred.
+     */
+    public static void main(String[] args) throws Exception {
+        LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+        context.getLoggerList().get(0).setLevel(Level.ERROR);
+
+        System.out.println("Pravega Admin Tools.\n");
+        @Cleanup
+        AdminCommandState state = new AdminCommandState();
+
+        // Output loaded config.
+        System.out.println("Initial configuration:");
+        val initialConfigCmd = new ConfigListCommand(new CommandArgs(Collections.emptyList(), state));
+        initialConfigCmd.execute();
+
+        // Continuously accept new commands as long as the user entered one.
+        System.out.println(String.format("%nType \"%s\" for list of commands, or \"%s\" to exit.", CMD_HELP, CMD_EXIT));
+        Scanner input = new Scanner(System.in);
+        while (true) {
+            System.out.print(System.lineSeparator() + "> ");
+            String line = input.nextLine();
+            if (Strings.isNullOrEmpty(line.trim())) {
+                continue;
+            }
+
+            Parser.Command pc = Parser.parse(line);
+            switch (pc.getComponent()) {
+                case CMD_HELP:
+                    printHelp(null);
+                    break;
+                case CMD_EXIT:
+                    System.exit(0);
+                    break;
+                default:
+                    execCommand(pc, state);
+                    break;
+            }
+        }
+    }
+
+    private static void execCommand(Parser.Command pc, AdminCommandState state) {
+        CommandArgs cmdArgs = new CommandArgs(pc.getArgs(), state);
+        try {
+            Command cmd = Command.Factory.get(pc.getComponent(), pc.getName(), cmdArgs);
+            if (cmd == null) {
+                // No command was found.
+                printHelp(pc);
+            } else {
+                cmd.execute();
+            }
+        } catch (IllegalArgumentException ex) {
+            // We found a command, but had the wrong arguments to it.
+            System.out.println("Bad command syntax: " + ex.getMessage());
+            printCommandDetails(pc);
+        } catch (Exception ex) {
+            ex.printStackTrace(System.out);
+        }
+    }
+
+    private static void printCommandSummary(Command.CommandDescriptor d) {
+        System.out.println(String.format("\t%s %s %s: %s",
+                d.getComponent(),
+                d.getName(),
+                Arrays.stream(d.getArgs()).map(AdminRunner::formatArgName).collect(Collectors.joining(" ")),
+                d.getDescription()));
+    }
+
+    private static void printCommandDetails(Parser.Command command) {
+        Command.CommandDescriptor d = Command.Factory.getDescriptor(command.getComponent(), command.getName());
+        if (d == null) {
+            printHelp(command);
+            return;
+        }
+
+        printCommandSummary(d);
+        for (Command.ArgDescriptor ad : d.getArgs()) {
+            System.out.println(String.format("\t\t%s: %s", formatArgName(ad), ad.getDescription()));
+        }
+    }
+
+    private static void printHelp(Parser.Command command) {
+        Collection<Command.CommandDescriptor> commands;
+        if (command == null) {
+            // All commands.
+            commands = Command.Factory.getDescriptors();
+            System.out.println("All available commands:");
+        } else {
+            // Commands specific to a component.
+            commands = Command.Factory.getDescriptors(command.getComponent());
+            if (commands.isEmpty()) {
+                System.out.println(String.format("No commands are available for component '%s'.", command.getComponent()));
+            } else {
+                System.out.println(String.format("All commands for component '%s':", command.getComponent()));
+            }
+        }
+
+        commands.stream()
+                .sorted((d1, d2) -> {
+                    int c = d1.getComponent().compareTo(d2.getComponent());
+                    if (c == 0) {
+                        c = d1.getName().compareTo(d2.getName());
+                    }
+                    return c;
+                })
+                .forEach(AdminRunner::printCommandSummary);
+    }
+
+    private static String formatArgName(Command.ArgDescriptor ad) {
+        return String.format("<%s>", ad.getName());
+    }
+}

segmentstore/server/host/src/main/java/io/pravega/segmentstore/server/host/admin/Parser.java

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.pravega.segmentstore.server.host.admin;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Scanner;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+/**
+ * Helps parse Strings into Commands.
+ */
+final class Parser {
+    private static final String SCANNER_PATTERN = "[^\"\\s]+|\"(\\\\.|[^\\\\\"])*\"";
+
+    /**
+     * Parses the given String into a Command, separating elements by spaces, and treating characters between double quotes(")
+     * as a single element. The first element is the Command Component, the second is the Command Name and the rest will
+     * be gathered as an ordered list of arguments.
+     *
+     * @param s The string to parse.
+     * @return A new instance of the Command class.
+     */
+    static Command parse(String s) {
+        Scanner scanner = new Scanner(s);
+        String component = scanner.findInLine(SCANNER_PATTERN);
+        String command = scanner.findInLine(SCANNER_PATTERN);
+        ArrayList<String> args = new ArrayList<>();
+        String arg;
+        while ((arg = scanner.findInLine(SCANNER_PATTERN)) != null) {
+            args.add(arg);
+        }
+
+        return new Command(component, command, Collections.unmodifiableList(args));
+    }
+
+    /**
+     * Represents a parsed Command.
+     */
+    @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+    @Getter
+    static class Command {
+        private final String component;
+        private final String name;
+        private final List<String> args;
+
+        @Override
+        public String toString() {
+            return String.format("%s %s (%s)", this.component, this.name, String.join(", ", this.args));
+        }
+    }
+}

segmentstore/server/host/src/main/java/io/pravega/segmentstore/server/host/admin/commands/AdminCommandState.java

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package io.pravega.segmentstore.server.host.admin.commands;
+
+import io.pravega.common.concurrent.ExecutorServiceHelpers;
+import io.pravega.segmentstore.server.store.ServiceBuilderConfig;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.ScheduledExecutorService;
+import lombok.Getter;
+
+/**
+ * Keeps state between commands.
+ */
+public class AdminCommandState implements AutoCloseable {
+    @Getter
+    private final ServiceBuilderConfig.Builder configBuilder;
+    @Getter
+    private final ScheduledExecutorService executor = ExecutorServiceHelpers.newScheduledThreadPool(3, "admin-tools");
+
+    /**
+     * Creates a new instance of the AdminCommandState class.
+     *
+     * @throws IOException If unable to read specified config properties file (assuming it exists).
+     */
+    public AdminCommandState() throws IOException {
+        this.configBuilder = ServiceBuilderConfig.builder();
+        try {
+            this.configBuilder.include(System.getProperty(ServiceBuilderConfig.CONFIG_FILE_PROPERTY_NAME, "config.properties"));
+        } catch (FileNotFoundException ex) {
+            // Nothing to do here.
+        }
+    }
+
+    @Override
+    public void close() {
+        this.executor.shutdown();
+    }
+}