Merge branch 'main' into hello-world-job

.github/workflows/backport.yml

@@ -33,7 +33,7 @@ jobs:
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v2.1.0
+        uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
           head_template: backport/backport-<%= number %>-to-<%= base %>

.github/workflows/release-drafter.yml

@@ -0,0 +1,45 @@
+name: Release drafter
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  draft-a-release:
+    name: Draft a release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - id: get_data
+        run: |
+          echo "approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
+          echo "version=$(./gradlew getVersion  -Dbuild.snapshot=false -Dbuild -q | grep version= | cut -d'=' -f2)" >> $GITHUB_OUTPUT
+      - uses: trstringer/manual-approval@v1
+        with:
+          secret: ${{ github.TOKEN }}
+          approvers: ${{ steps.get_data.outputs.approvers }}
+          minimum-approvals: 2
+          issue-title: 'Release opensearch-sdk-java : ${{ steps.get_data.outputs.version }}'
+          issue-body: "Please approve or deny the release of opensearch-sdk-java. **VERSION**: ${{ steps.get_data.outputs.version }} **TAG**: ${{ github.ref_name }}  **COMMIT**: ${{ github.sha }}"
+          exclude-workflow-initiator-as-approver: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: 'gradle'
+      - name: Build with Gradle
+        run: |
+          ./gradlew --no-daemon -Dbuild.snapshot=false publishNebulaPublicationToLocalRepoRepository && tar -C build -cvf artifacts.tar.gz localRepo
+      - name: Draft a release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          generate_release_notes: true
+          files: |
+            artifacts.tar.gz

CREATE_YOUR_FIRST_EXTENSION.md

@@ -164,28 +164,28 @@ import org.opensearch.sdk.rest.BaseExtensionRestHandler;
 public class CrudAction extends BaseExtensionRestHandler {
 
     @Override
-    protected List<RouteHandler> routeHandlers() {
+    public List<NamedRoute> routes() {
         return List.of(
-            new RouteHandler(Method.PUT, "/sample", createHandler),
-            new RouteHandler(Method.GET, "/sample/{id}", readHandler),
-            new RouteHandler(Method.POST, "/sample/{id}", updateHandler),
-            new RouteHandler(Method.DELETE, "/sample/{id}", deleteHandler)
+            new NamedRoute.Builder().method(Method.PUT).path("/sample").uniqueName("extension1:sample/create").handler(createHandler).build(),
+            new NamedRoute.Builder().method(Method.GET).path("/sample/{id}").uniqueName("extension1:sample/get").handler(readHandler).build(),
+            new NamedRoute.Builder().method(Method.POST).path("/sample/{id}").uniqueName("extension1:sample/post").handler(updateHandler).build(),
+            new NamedRoute.Builder().method(Method.DELETE).path("/sample/{id}").uniqueName("extension1:sample/delete").handler(deleteHandler).build()
         );
     }
 
-    Function<RestRequest, ExtensionRestResponse> createHandler = (request) -> {
+    Function<RestRequest, RestResponse> createHandler = (request) -> {
         return new ExtensionRestResponse(request, RestStatus.OK, "To be implemented");
     };
 
-    Function<RestRequest, ExtensionRestResponse> readHandler = (request) -> {
+    Function<RestRequest, RestResponse> readHandler = (request) -> {
         return new ExtensionRestResponse(request, RestStatus.OK, "To be implemented");
     };
 
-    Function<RestRequest, ExtensionRestResponse> updateHandler = (request) -> {
+    Function<RestRequest, RestResponse> updateHandler = (request) -> {
         return new ExtensionRestResponse(request, RestStatus.OK, "To be implemented");
     };
 
-    Function<RestRequest, ExtensionRestResponse> deleteHandler = (request) -> {
+    Function<RestRequest, RestResponse> deleteHandler = (request) -> {
         return new ExtensionRestResponse(request, RestStatus.OK, "To be implemented");
     };
 }
@@ -248,7 +248,7 @@ return createJsonResponse(request, RestStatus.OK, "_id", response.id());
 Finally, you have the following code:
 
 ```java
-Function<RestRequest, ExtensionRestResponse> createHandler = (request) -> {
+Function<RestRequest, RestResponse> createHandler = (request) -> {
     IndexResponse response;
     try {
         BooleanResponse exists = client.indices().exists(new ExistsRequest.Builder().index("crudsample").build());

DESIGN.md

@@ -37,26 +37,26 @@ Extensions are designed to extend features through transport APIs, which are exp
 
 ### Discovery
 
-Extensions are discovered and configured in the `extensions.yml` file, which is similar to the `plugin-descriptor.properties` file that is read by OpenSearch during the node bootstrap. `ExtensionsManager` reads through the config file in the `~/extensions` directory and registers extensions within OpenSearch.
-
-The following is an example extension configuration in `extensions.yml`:
-
-```yaml
-extensions:
-  - name: sample-extension // extension name
-    uniqueId: opensearch-sdk-1 // identifier for the extension
-    hostAddress: '127.0.0.1' // host address
-    port: '4532' // port number
-    version: '1.0' // extension version
-    opensearchVersion: '3.0.0' // the OpenSearch version with which the extension is compiled
-    minimumCompatibleVersion: '3.0.0' // minimum version of OpenSearch with which the extension is wire compatible
+Extensions are registered through the below REST request within OpenSearch.
+
+```bash
+curl -XPOST "localhost:9200/_extensions/initialize" -H "Content-Type:application/json" --data '{ \
+"name":"hello-world", \                 // extension name
+"uniqueId":"hello-world", \             // identifier for the extension
+"hostAddress":"127.0.0.1", \            // host address
+"port":"4532", \                        // port number
+"version":"1.0", \                      // extension version
+"opensearchVersion":"3.0.0", \          // the OpenSearch version with which the extension is compiled
+"minimumCompatibleVersion":"3.0.0", \   // minimum version of OpenSearch with which the extension is wire compatible
+"dependencies":[{"uniqueId":"test1","version":"2.0.0"},{"uniqueId":"test2","version":"3.0.0"}] \  // required extensions for the host extension
+}'
 ```
 
 ### Communication
 
 Extensions use a `ServerSocket`, which binds them to listen on a host address and port defined in each extension's configuration file. Each type of incoming request invokes code from an associated handler.
 
-OpenSearch has its own `extensions.yml` configuration file that matches the extensions' addresses and ports. On startup, the `ExtensionsManager` uses the node's `TransportService` to send requests to each extension, with the first request initializing the extension and validating the host and port.
+`ExtensionsManager` uses the node's `TransportService` to send requests to each extension when the REST request to initialize an extension is invoked, with the first request initializing the extension and validating the host and port.
 
 Immediately following initialization, each extension establishes a connection to OpenSearch on its own transport service and sends its REST API to OpenSearch. The API contains a list of methods and URIs to which the extension will respond. These are registered with the `RestController`.
 
@@ -92,7 +92,7 @@ The `org.opensearch.sdk.sample.helloworld` package contains a sample `HelloWorld
 
 (8, 9, 10) During bootstrap, the OpenSearch `Node` instantiates a `RestController`, passing this to the `ExtensionsManager`, which subsequently passes it to a `RestActionsRequestHandler`.
 
-The `ExtensionsManager` reads a list of extensions present in `extensions.yml`. For each configured extension:
+The `ExtensionsManager` reads a list of extensions loaded through the REST request . For each configured extension:
 
 (11, 12) The `ExtensionsManager` initializes the extension using an `InitializeExtensionsRequest`/`InitializeExtensionsResponse`, establishing a two-way transport mechanism.
 

DEVELOPER_GUIDE.md

@@ -13,7 +13,6 @@
       * [Option 1](#option-1)
       * [Option 2](#option-2)
       * [Option 3](#option-3)
-    * [The `extensions.yml` file](#the-extensionsyml-file)
     * [Run OpenSearch](#run-opensearch)
   * [Send a REST request to the extension](#send-a-rest-request-to-the-extension)
 * [Developing your own extension](#developing-your-own-extension)
@@ -36,7 +35,6 @@ In general, running and using an extension can be broken down into the following
 1. Start OpenSearch:
     - [Clone the OpenSearch repository](#clone-the-opensearch-repository).
     - [Enable the extensions feature flag](#enable-the-extensions-feature-flag).
-    - Create the [`extensions.yml` file](#the-extensionsyml-file) and [run OpenSearch](#run-opensearch).
 1. Use the extension:
     - [Send a REST request to the extension](#send-a-rest-request-to-the-extension).
 
@@ -68,7 +66,7 @@ You can run the sample Hello World extension using the `helloWorld` task:
 
 Bound addresses will then be logged to the terminal:
 
-```
+```bash
 [main] INFO  transportservice.TransportService - publish_address {127.0.0.1:3333}, bound_addresses {[::1]:3333}, {127.0.0.1:3333}
 [main] INFO  transportservice.TransportService - profile [test]: publish_address {127.0.0.1:5555}, bound_addresses {[::1]:5555}, {127.0.0.1:5555}
 ```
@@ -92,7 +90,6 @@ path.home: <path/to/extension>
 Follow these steps to start OpenSearch:
 - [Clone the OpenSearch repository](#clone-the-opensearch-repository).
 - [Enable the extensions feature flag](#enable-the-extensions-feature-flag).
-- [Create the `extensions.yml` file](#the-extensionsyml-file).
 - [Run OpenSearch](#run-opensearch).
 
 #### Clone the OpenSearch repository
@@ -111,7 +108,7 @@ Extensions are experimental in OpenSearch 2.8, so you must enable them either be
 
 Add the experimental feature system property to `gradle/run.gradle`:
 
-```
+```bash
 testClusters {
   runTask {
     testDistribution = 'archive'
@@ -137,24 +134,6 @@ Enable the experimental feature flag by setting it to `true` in `opensearch.yml`
 - Search for `opensearch.experimental.feature.extensions.enabled`, uncomment it, and set it to `true`.
 - Run OpenSearch using `./bin/opensearch` when running from a local distribution.
 
-#### The `extensions.yml` file
-
-Every extension requires metadata stored in an extensions.yml file in order to be loaded successfully. To make the SDK look like an extension within OpenSearch, there must be an entry for the SDK within `extensions.yml`.
-
-The following is a sample `extensions.yml` file. The `uniqueId` will be used in REST paths. The name must match the `extensionName` field in the extension's `.yml` settings file (for the sample Hello World extension, the settings file is `helloworld-settings.yml`):
-
-```yaml
-extensions:
-  - name: hello-world
-    uniqueId: opensearch-sdk-java-1
-    hostAddress: '127.0.0.1'
-    port: '4532'
-    version: '1.0'
-    opensearchVersion: '3.0.0'
-    minimumCompatibleVersion: '3.0.0'
-```
-
-Before running OpenSearch with extensions, you need to create the `extensions.yml` file and place it in the appropriate directory, as explained in the next section.
 
 #### Run OpenSearch
 
@@ -164,21 +143,42 @@ To **run OpenSearch from a compiled binary**, follow these steps:
 
 - Start a separate terminal and navigate to the directory where OpenSearch has been cloned using `cd OpenSearch`.
 - Run `./gradlew assemble` to create a local distribution.
-- Navigate to the project root directory (for example, `cd distribution/archives/linux-tar/build/install/opensearch-3.0.0-SNAPSHOT/`). Note: On macOS, replace `linux-tar` with `darwin-tar`.
-- Check whether the `extensions` directory exists in OpenSearch using `ls`.
-- If the directory does not exist, create it using `mkdir extensions`.
-- Navigate to the extensions folder using `cd extensions`.
-- Manually create a file titled `extensions.yml` within the extensions directory using an IDE or an inline text editor.
-- Return to the OpenSearch directory using `cd ..`.
 - Start OpenSearch using `./bin/opensearch`.
+- Send the below sample REST API to initialize an extension
+```bash
+curl -XPOST "localhost:9200/_extensions/initialize" -H "Content-Type:application/json" --data '{
+"name":"hello-world",
+"uniqueId":"hello-world",
+"hostAddress":"127.0.0.1",
+"port":"4532",
+"version":"1.0",
+"opensearchVersion":"3.0.0",
+"minimumCompatibleVersion":"3.0.0",
+"dependencies":[{"uniqueId":"test1","version":"2.0.0"},{"uniqueId":"test2","version":"3.0.0"}] \
+}'
+```
 
 To **run OpenSearch from Gradle**, follow these steps:
-- Copy the `extensions.yml` file to the same directory indicated in the previous section.
-- Run `./gradlew run` to start OpenSearch. A log entry will indicate where OpenSearch is searching for `extensions.yml`.
+- Run `./gradlew run` to start OpenSearch.
+- Send the below sample REST API to initialize an extension
+```bash
+curl -XPOST "localhost:9200/_extensions/initialize" -H "Content-Type:application/json" --data '{
+"name":"hello-world",
+"uniqueId":"hello-world",
+"hostAddress":"127.0.0.1",
+"port":"4532",
+"version":"1.0",
+"opensearchVersion":"3.0.0",
+"minimumCompatibleVersion":"3.0.0",
+"dependencies":[{"uniqueId":"test1","version":"2.0.0"},{"uniqueId":"test2","version":"3.0.0"}]
+}'
+```
 
-During OpenSearch bootstrap, `ExtensionsManager` discovers the extension listening on a predefined port and executes the TCP handshake protocol to establish a data transfer connection. Then OpenSearch sends a request to the OpenSearch SDK for Java and, upon acknowledgment, the extension responds with its name. This name is logged in the terminal where OpenSearch is running:
+Note: If the Security plugin is initialized in OpenSearch, use admin credentials to send extension initialization request.
 
-```
+In response to the REST `/initialize` request, `ExtensionsManager` discovers the extension listening on a predefined port and executes the TCP handshake protocol to establish a data transfer connection. Then OpenSearch sends a request to the OpenSearch SDK for Java and, upon acknowledgment, the extension responds with its name. This name is logged in the terminal where OpenSearch is running:
+
+```bash
 [2022-06-16T21:30:18,857][INFO ][o.o.t.TransportService   ] [runTask-0] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
 [2022-06-16T21:30:18,978][INFO ][o.o.t.TransportService   ] [runTask-0] Action: internal:transport/handshake
 [2022-06-16T21:30:18,989][INFO ][o.o.t.TransportService   ] [runTask-0] TransportService:sendRequest action=internal:discovery/extensions
@@ -190,7 +190,7 @@ The OpenSearch SDK terminal also logs all requests and responses it receives fro
 
 - TCP handshake request:
 
-```
+```bash
 21:30:18.943 [opensearch[extension][transport_worker][T#7]] TRACE org.opensearch.latencytester.transportservice.netty4.OpenSearchLoggingHandler - [id: 0x37b22600, L:/127.0.0.1:4532 - R:/127.0.0.1:47766] READ: 55B
          +-------------------------------------------------+
          |  0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f |
@@ -205,7 +205,7 @@ MESSAGE RECEIVED:E«󀀀internal:tcp/handshake£·A
 
 - Extension name request/response:
 
-```
+```bash
 21:30:18.992 [opensearch[extension][transport_worker][T#6]] TRACE org.opensearch.latencytester.transportservice.netty4.OpenSearchLoggingHandler - [id: 0xb2be651b, L:/127.0.0.1:4532 - R:/127.0.0.1:47782] READ: 204B
          +-------------------------------------------------+
          |  0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f |
@@ -242,12 +242,12 @@ MESSAGE RECEIVED:ES-ǣ!internal:discovery/extensionsnode_extensionQSt9oKXFTSWqgX
 21:30:18.999 [opensearch[extension][transport_worker][T#6]] TRACE org.opensearch.transport.TransportService.tracer - [3][internal:discovery/extensions] sent response
 ```
 
-It is important to ensure that the OpenSearch SDK for Java is already running on a separate process prior to starting OpenSearch because extension discovery occurs only if the OpenSearch SDK for Java is already listening on a predefined port. Once the discovery is complete and the data transfer connection between both nodes has been established, OpenSearch and the OpenSearch SDK for Java can communicate with each other.
+It is important to ensure that the OpenSearch SDK for Java is already running on a separate process.
 
 ### Send a REST request to the extension
 
 The following request is configured to be handled by the sample `HelloWorldExtension` (note that its matching `uniqueId` is `opensearch-sdk-java-1`):
-```
+```bash
 curl -X GET localhost:9200/_extensions/_opensearch-sdk-java-1/hello
 ```
 
@@ -284,12 +284,12 @@ For information about launching and debugging from an IDE in OpenSearch, see [th
 ### Generating an artifact
 
 In `opensearch-sdk-java`, navigate to `build/distributions`. Look for the tarball in the form `opensearch-sdk-java-1.0.0-SNAPSHOT.tar`. If there is no such tarball, use the following command to create one:
-```
+```bash
 ./gradlew clean && ./gradlew build
 ```
 Once the tarball is generated, navigate to `/src/test/resources/sample` and look for `extension-settings.yml`. If the file is not present, create it.
 The tarball is generated in `/build/distributions`. To run the artifact (the tarball), use the following command:
-```
+```bash
 tar -xvf opensearch-sdk-java-1.0.0-SNAPSHOT.tar
 ```
 

PLUGIN_MIGRATION.md

@@ -67,14 +67,38 @@ XContentParser parser = XContentType.JSON
 Other potential initialization values are:
 ```java
 this.environmentSettings = extensionsRunner.getEnvironmentSettings();
-this.transportService = extensionsRunner.getExtensionTransportService();
+this.transportService = extensionsRunner.getSdkTransportService().getTransportService();
 this.restClient = anomalyDetectorExtension.getRestClient();
 this.sdkClusterService = new SDKClusterService(extensionsRunner);
 ```
 
 Many of these components are also available via Guice injection.
 
-Optionally, change the `routes()` to `routeHandlers()`. Change `prepareRequest()` to `handleRequest()`.
+### Replace `Route` with `NamedRoute`
+Change `routes()` to be NamedRoutes. Here is a sample of an existing route converted to a named route:
+Before:
+```
+public List<Route> routes() {
+    return ImmutableList.of(
+            new Route(GET, "/uri")
+        );
+}
+```
+With new scheme:
+```
+private Function<RestRequest, RestResponse> uriHandler = () -> {};
+public List<NamedRoute> routes() {
+    return ImmutableList.of(
+            new NamedRoute.Builder().method(GET).path("/uri").uniqueName("extension:uri").handler(uriHandler).build()
+        );
+}
+```
+
+You can optionally also add `actionNames()` to this route. These should correspond to any current actions defined as permissions in roles.
+`actionNames()` serve as a valuable tool for converting plugins into extensions while maintaining compatibility with pre-defined reserved roles.
+Ensure that these name-to-route mappings are easily accessible to the cluster admins to allow granting access to these APIs.
+
+Change `prepareRequest()` to `handleRequest()`.
 
 ### Replace `BytesRestResponse` with `ExtensionRestResponse`