Convert repository plugins to modules (elastic#81870)

Closes elastic#81652.

Convert the `repository-azure`, `repository-gcs` and `repository-s3`
plugins into modules, so that they are always included in the
Elasticsearch distribution. Also change plugin installation, removal
and syncing so that attempting to add or remove these plugins still
succeeds but is now a no-op.

Author: pugnascotia

.ci/scripts/packaging-test.sh

@@ -37,11 +37,6 @@ if [ -f "/etc/os-release" ] ; then
     if [[ "$ID" == "debian" || "$ID_LIKE" == "debian" ]] ; then
         # FIXME: The base image should not have rpm installed
         sudo rm -Rf /usr/bin/rpm
-        # Work around incorrect lintian version
-        #  https://github.com/elastic/elasticsearch/issues/48573
-        if [ $VERSION_ID == 10 ] ; then
-            sudo apt-get install -y --allow-downgrades lintian=2.15.0
-        fi
     fi
 else
     cat /etc/issue || true

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -88,7 +88,7 @@ if (providers.systemProperty('idea.active').forUseAtConfigurationTime().getOrNul
   tasks.register('buildDependencyArtifacts') {
     group = 'ide'
     description = 'Builds artifacts needed as dependency for IDE modules'
-    dependsOn ':client:rest-high-level:shadowJar', ':plugins:repository-hdfs:hadoop-client-api:shadowJar', ':plugins:repository-azure:azure-storage-blob:shadowJar'
+    dependsOn ':client:rest-high-level:shadowJar', ':plugins:repository-hdfs:hadoop-client-api:shadowJar', ':modules:repository-azure:azure-storage-blob:shadowJar'
   }
 
   idea {

distribution/docker/build.gradle

@@ -70,8 +70,7 @@ configurations {
   dockerSource
   log4jConfig
   tini
-  repositoryPlugins
-  nonRepositoryPlugins
+  allPlugins
   filebeat
   metricbeat
   cloudflareZlib
@@ -85,8 +84,7 @@ dependencies {
   dockerSource project(path: ":distribution:archives:linux-tar", configuration: 'default')
   log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini "krallin:tini:0.19.0:${tiniArch}"
-  repositoryPlugins project(path: ':plugins', configuration: 'repositoryPlugins')
-  nonRepositoryPlugins project(path: ':plugins', configuration: 'nonRepositoryPlugins')
+  allPlugins project(path: ':plugins', configuration: 'allPlugins')
   filebeat "beats:filebeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
   metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
   cloudflareZlib "cloudflare:zlib:${cloudflareZlibVersion}@tar.gz"
@@ -272,8 +270,6 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         String buildId = providers.systemProperty('build.id').forUseAtConfigurationTime().getOrNull()
         boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null
 
-        from configurations.repositoryPlugins
-
         if (includeBeats) {
           from configurations.filebeat
           from configurations.metricbeat
@@ -407,7 +403,7 @@ void addBuildEssDockerImageTask(Architecture architecture) {
       final Path projectDir = project.projectDir.toPath()
 
       into("plugins") {
-        from configurations.nonRepositoryPlugins
+        from configurations.allPlugins
       }
 
       from(projectDir.resolve("src/docker/Dockerfile.cloud-ess")) {

distribution/docker/src/docker/Dockerfile

@@ -137,18 +137,6 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
     find config -type f -exec chmod 0664 {} +
 
 <% if (docker_base == "cloud") { %>
-# Preinstall common plugins. Note that these are installed as root, meaning the `elasticsearch` user cannot delete them.
-COPY repository-s3-${version}.zip repository-gcs-${version}.zip repository-azure-${version}.zip /tmp/
-RUN bin/elasticsearch-plugin install --batch --verbose \\
-    file:/tmp/repository-s3-${version}.zip \\
-    file:/tmp/repository-gcs-${version}.zip \\
-    file:/tmp/repository-azure-${version}.zip
-# Generate a replacement example plugins config that reflects what is actually installed
-RUN echo "plugins:" > config/elasticsearch-plugins.example.yml && \\
-    echo "  - id: repository-azure" >> config/elasticsearch-plugins.example.yml && \\
-    echo "  - id: repository-gcs"   >> config/elasticsearch-plugins.example.yml && \\
-    echo "  - id: repository-s3"    >> config/elasticsearch-plugins.example.yml
-
 COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
 RUN set -eux ; \\
     for beat in filebeat metricbeat ; do \\

distribution/packages/build.gradle

@@ -291,8 +291,10 @@ Closure commonDebConfig(String architecture) {
   return {
     configure(commonPackageConfig('deb', architecture))
 
-    // jdeb does not provide a way to set the License control attribute, and ospackage
-    // silently ignores setting it. Instead, we set the license as "custom field"
+    // jdeb does not provide a way to set the License control attribute, and
+    // ospackage silently ignores setting it. This is probably because `License`
+    // is not actually a field in the Debian control file. So instead, we set
+    // the license as "custom field".
     customFields['License'] = 'Elastic-License'
 
     archiveVersion = project.version.replace('-', '~')

distribution/packages/src/common/scripts/postinst

@@ -8,6 +8,8 @@
 #       $1=0         : indicates a removal
 #       $1=1         : indicates an upgrade
 
+set -e
+
 # source the default env file
 if [ -f "@path.env@" ]; then
     . "@path.env@"

distribution/packages/src/common/scripts/postrm

@@ -9,6 +9,8 @@
 #       $1=0         : indicates a removal
 #       $1=1         : indicates an upgrade
 
+set -e
+
 # source the default env file
 if [ -f "@path.env@" ]; then
     . "@path.env@"

distribution/packages/src/common/scripts/posttrans

@@ -1,3 +1,5 @@
+set -e
+
 # source the default env file
 if [ -f "@path.env@" ]; then
     . "@path.env@"

distribution/packages/src/common/scripts/preinst

@@ -10,6 +10,8 @@
 #       $1=1       : indicates an new install
 #       $1=2       : indicates an upgrade
 
+set -e
+
 err_exit() {
     echo "$@" >&2
     exit 1

distribution/packages/src/common/scripts/prerm

@@ -9,6 +9,8 @@
 #       $1=0         : indicates a removal
 #       $1=1         : indicates an upgrade
 
+set -e
+
 # source the default env file
 if [ -f "@path.env@" ]; then
     . "@path.env@"

distribution/packages/src/deb/lintian/elasticsearch

@@ -1,10 +1,12 @@
 # we don't have a changelog, but we put our copyright file
 # under /usr/share/doc/elasticsearch, which triggers this warning
+# Note that this is renamed to `no-changelog` in newer versions of
+# lintian, but we still support Debian 8+, so we can't change this.
 changelog-file-missing-in-native-package
 
 # we intentionally copy our copyright file for all deb packages
 copyright-file-contains-full-apache-2-license
-copyright-should-refer-to-common-license-file-for-apache-2
+copyright-not-using-common-license-for-apache2
 copyright-without-copyright-notice
 
 # we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
@@ -21,7 +23,7 @@ non-standard-file-perm etc/elasticsearch/*
 non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
 
-# this lintian tag is simply wrong; contrary to the explanation, debian systemd
+# this lintian tag is simply wrong; contrary to the explanation, Debian systemd
 # does actually look at /usr/lib/systemd/system
 systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service
 
@@ -30,7 +32,6 @@ maintainer-script-calls-systemctl
 
 # bundled JDK
 embedded-library
-arch-dependent-file-in-usr-share usr/share/elasticsearch/jdk/*
 unstripped-binary-or-object usr/share/elasticsearch/jdk/*
 extra-license-file usr/share/elasticsearch/jdk/legal/*
 hardening-no-pie usr/share/elasticsearch/jdk/bin/*
@@ -41,3 +42,12 @@ unknown-java-class-version
 
 # elastic licensed modules contain elastic license
 extra-license-file usr/share/elasticsearch/modules/*
+
+# This dependency appears to have a packaging flaw, and includes a
+# generated source file alongside the compiled version
+jar-contains-source usr/share/elasticsearch/modules/repository-gcs/api-common*.jar *
+
+# There's no `License` field in Debian control files, but earlier versions
+# of `lintian` were more permissive. Override this warning so that we can
+# run `lintian` on different releases of Debian.
+unknown-field elasticsearch-*.deb License

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -153,6 +153,13 @@ public class InstallPluginAction implements Closeable {
         }
     }
 
+    /**
+     * IDs of plugins that have been migrated to modules and do not require installation. This data is
+     * maintained so that existing user workflows that install these plugins do not need to be updated
+     * immediately.
+     */
+    public static final Set<String> PLUGINS_CONVERTED_TO_MODULES = Set.of("repository-azure", "repository-gcs", "repository-s3");
+
     static final Set<PosixFilePermission> BIN_DIR_PERMS;
     static final Set<PosixFilePermission> BIN_FILES_PERMS;
     static final Set<PosixFilePermission> CONFIG_DIR_PERMS;
@@ -219,6 +226,15 @@ public void execute(List<PluginDescriptor> plugins) throws Exception {
                     handleInstallXPack(buildFlavor());
                 }
 
+                if (PLUGINS_CONVERTED_TO_MODULES.contains(pluginId)) {
+                    // This deliberately does not throw an exception in order to avoid failing automation that relies on installing this
+                    // plugin during deployment.
+                    terminal.errorPrintln(
+                        "[" + pluginId + "] is no longer a plugin but instead a module packaged with this distribution of Elasticsearch"
+                    );
+                    continue;
+                }
+
                 final List<Path> deleteOnFailure = new ArrayList<>();
                 deleteOnFailures.put(pluginId, deleteOnFailure);
 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginDescriptor.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.plugins.cli;
 
+import org.elasticsearch.common.Strings;
+
 import java.util.Objects;
 
 /**
@@ -27,7 +29,7 @@ public PluginDescriptor() {}
      *                 coordinates. Can be null for official plugins.
      */
     public PluginDescriptor(String id, String location) {
-        this.id = Objects.requireNonNull(id, "id cannot be null");
+        this.id = Strings.requireNonBlank(id, "plugin id cannot be null or blank");
         this.location = location;
     }
 
@@ -40,7 +42,7 @@ public String getId() {
     }
 
     public void setId(String id) {
-        this.id = id;
+        this.id = Strings.requireNonBlank(id, "plugin id cannot be null or blank");
     }
 
     public String getLocation() {

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugins.cli;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContent;
@@ -57,10 +58,12 @@ public void setProxy(String proxy) {
      * </ul>
      *
      * @param officialPlugins the plugins that can be installed by name only
+     * @param migratedPlugins plugins that were once official but have since become modules. These
+     *                        plugin IDs can still be specified, but do nothing.
      * @throws PluginSyncException if validation problems are found
      */
-    public void validate(Set<String> officialPlugins) throws PluginSyncException {
-        if (this.plugins.stream().anyMatch(each -> each == null || each.getId() == null || each.getId().isBlank())) {
+    public void validate(Set<String> officialPlugins, Set<String> migratedPlugins) throws PluginSyncException {
+        if (this.plugins.stream().anyMatch(each -> each == null || Strings.isNullOrBlank(each.getId()))) {
             throw new RuntimeException("Cannot have null or empty IDs in [elasticsearch-plugins.yml]");
         }
 
@@ -72,7 +75,9 @@ public void validate(Set<String> officialPlugins) throws PluginSyncException {
         }
 
         for (PluginDescriptor plugin : this.plugins) {
-            if (officialPlugins.contains(plugin.getId()) == false && plugin.getLocation() == null) {
+            if (officialPlugins.contains(plugin.getId()) == false
+                && migratedPlugins.contains(plugin.getId()) == false
+                && plugin.getLocation() == null) {
                 throw new PluginSyncException(
                     "Must specify location for non-official plugin [" + plugin.getId() + "] in [elasticsearch-plugins.yml]"
                 );

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java

@@ -26,10 +26,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.StringJoiner;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
+import static org.elasticsearch.plugins.cli.InstallPluginAction.PLUGINS_CONVERTED_TO_MODULES;
 
 /**
  * An action for the plugin CLI to remove plugins from Elasticsearch.
@@ -116,6 +116,7 @@ private void ensurePluginsNotUsedByOtherPlugins(List<PluginDescriptor> plugins)
 
     private void checkCanRemove(PluginDescriptor plugin) throws UserException {
         String pluginId = plugin.getId();
+
         final Path pluginDir = env.pluginsFile().resolve(pluginId);
         final Path pluginConfigDir = env.configFile().resolve(pluginId);
         final Path removing = env.pluginsFile().resolve(".removing-" + pluginId);
@@ -127,12 +128,19 @@ private void checkCanRemove(PluginDescriptor plugin) throws UserException {
          */
         if ((Files.exists(pluginDir) == false && Files.exists(pluginConfigDir) == false && Files.exists(removing) == false)
             || (Files.exists(pluginDir) == false && Files.exists(pluginConfigDir) && this.purge == false)) {
-            final String message = String.format(
-                Locale.ROOT,
-                "plugin [%s] not found; run 'elasticsearch-plugin list' to get list of installed plugins",
-                pluginId
-            );
-            throw new UserException(ExitCodes.CONFIG, message);
+
+            if (PLUGINS_CONVERTED_TO_MODULES.contains(pluginId)) {
+                terminal.errorPrintln(
+                    "plugin [" + pluginId + "] is no longer a plugin but instead a module packaged with this distribution of Elasticsearch"
+                );
+            } else {
+                final String message = String.format(
+                    Locale.ROOT,
+                    "plugin [%s] not found; run 'elasticsearch-plugin list' to get list of installed plugins",
+                    pluginId
+                );
+                throw new UserException(ExitCodes.CONFIG, message);
+            }
         }
 
         final Path pluginBinDir = env.binFile().resolve(pluginId);
@@ -159,15 +167,15 @@ private void removePlugin(PluginDescriptor plugin) throws IOException {
          */
         if (Files.exists(pluginDir)) {
             try (Stream<Path> paths = Files.list(pluginDir)) {
-                pluginPaths.addAll(paths.collect(Collectors.toList()));
+                pluginPaths.addAll(paths.toList());
             }
             terminal.println(VERBOSE, "removing [" + pluginDir + "]");
         }
 
         final Path pluginBinDir = env.binFile().resolve(pluginId);
         if (Files.exists(pluginBinDir)) {
             try (Stream<Path> paths = Files.list(pluginBinDir)) {
-                pluginPaths.addAll(paths.collect(Collectors.toList()));
+                pluginPaths.addAll(paths.toList());
             }
             pluginPaths.add(pluginBinDir);
             terminal.println(VERBOSE, "removing [" + pluginBinDir + "]");
@@ -176,7 +184,7 @@ private void removePlugin(PluginDescriptor plugin) throws IOException {
         if (Files.exists(pluginConfigDir)) {
             if (this.purge) {
                 try (Stream<Path> paths = Files.list(pluginConfigDir)) {
-                    pluginPaths.addAll(paths.collect(Collectors.toList()));
+                    pluginPaths.addAll(paths.toList());
                 }
                 pluginPaths.add(pluginConfigDir);
                 terminal.println(VERBOSE, "removing [" + pluginConfigDir + "]");