Datafeed keys master

build.gradle

@@ -232,6 +232,7 @@ ext.libs = [
         dropwizard_jetty                    : "io.dropwizard:dropwizard-jetty", //was 1.0.6 now 1.1.0
         dropwizard_lifecycle                : "io.dropwizard:dropwizard-lifecycle", // version controlled by dropwizard-dependencies
         dropwizard_metrics_annotation       : "io.dropwizard.metrics:metrics-annotation", // version controlled by dropwizard-dependencies
+        dropwizard_metrics_caffeine3        : "io.dropwizard.metrics:metrics-caffeine3", // version controlled by dropwizard-dependencies
         dropwizard_metrics_core             : "io.dropwizard.metrics:metrics-core", // version controlled by dropwizard-dependencies
         dropwizard_metrics_healthchecks     : "io.dropwizard.metrics:metrics-healthchecks", // version controlled by dropwizard-dependencies
         dropwizard_servlets                 : "io.dropwizard:dropwizard-servlets", // version controlled by dropwizard-dependencies

setProxyLogLevels.sh

@@ -9,8 +9,8 @@ ADMIN_PORT_OVERRIDE=$ADMIN_PORT
 # shellcheck disable=SC1091
 source ./stroom-proxy/stroom-proxy-app/src/dist/config/scripts.env
 
-ADMIN_PORT=${ADMIN_PORT_OVERRIDE:-ADMIN_PORT}
+ADMIN_PORT=${ADMIN_PORT_OVERRIDE:-$ADMIN_PORT}
 
 # Override jar path for dev
-PATH_TO_JAR="./stroom-proxy/stroom-proxy-app//build/libs/stroom-proxy-app-all.jar" \
+PATH_TO_JAR="./stroom-proxy/stroom-proxy-app/build/libs/stroom-proxy-app-all.jar" \
   ./dist/set_log_levels.sh "$@"

settings.gradle

@@ -294,3 +294,4 @@ include 'stroom-view:stroom-view-impl'
 include 'stroom-ssl-test'
 include 'stroom-view:stroom-view-api'
 include 'stroom-index:stroom-index-api'
+include 'stroom-cache:stroom-cache-service-impl'

stroom-app/build.gradle

@@ -61,6 +61,7 @@ dependencies {
     implementation project(':stroom-bytebuffer')
     implementation project(':stroom-cache:stroom-cache-api')
     implementation project(':stroom-cache:stroom-cache-impl')
+    implementation project(':stroom-cache:stroom-cache-service-impl')
     implementation project(':stroom-cluster:stroom-cluster-api')
     implementation project(':stroom-cluster:stroom-cluster-impl')
     implementation project(':stroom-cluster:stroom-cluster-lock-api')

stroom-app/src/main/java/stroom/app/commands/CreateAccountCommand.java

@@ -3,9 +3,9 @@
 import stroom.config.app.Config;
 import stroom.event.logging.api.StroomEventLoggingService;
 import stroom.security.api.SecurityContext;
+import stroom.security.api.UserService;
 import stroom.security.identity.account.AccountService;
 import stroom.security.identity.shared.CreateAccountRequest;
-import stroom.security.impl.UserService;
 import stroom.util.logging.LogUtil;
 
 import com.google.inject.Injector;

stroom-app/src/main/java/stroom/app/commands/CreateApiKeyCommand.java

@@ -3,7 +3,7 @@
 import stroom.config.app.Config;
 import stroom.event.logging.api.StroomEventLoggingService;
 import stroom.security.api.SecurityContext;
-import stroom.security.impl.UserService;
+import stroom.security.api.UserService;
 import stroom.security.impl.apikey.ApiKeyService;
 import stroom.security.shared.CreateHashedApiKeyRequest;
 import stroom.security.shared.CreateHashedApiKeyResponse;

stroom-app/src/main/java/stroom/app/commands/ManageUsersCommand.java

@@ -3,9 +3,9 @@
 import stroom.config.app.Config;
 import stroom.event.logging.api.StroomEventLoggingService;
 import stroom.event.logging.api.StroomEventLoggingUtil;
+import stroom.security.api.AppPermissionService;
 import stroom.security.api.SecurityContext;
-import stroom.security.impl.AppPermissionService;
-import stroom.security.impl.UserService;
+import stroom.security.api.UserService;
 import stroom.security.shared.AppPermission;
 import stroom.util.NullSafe;
 import stroom.util.logging.LambdaLogger;

stroom-app/src/main/java/stroom/app/guice/BootStrapModule.java

@@ -8,6 +8,8 @@
 import stroom.config.global.impl.db.GlobalConfigDaoModule;
 import stroom.db.util.DbModule;
 import stroom.util.io.DirProvidersModule;
+import stroom.util.metrics.Metrics;
+import stroom.util.metrics.MetricsImpl;
 
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.health.HealthCheckRegistry;
@@ -70,6 +72,8 @@ protected void configure() {
 
         bind(Config.class).toInstance(configuration);
         bind(Environment.class).toInstance(environment);
+        bind(MetricRegistry.class).toInstance(environment.metrics());
+        bind(Metrics.class).to(MetricsImpl.class);
 
         final AppConfigModule appConfigModule = appConfigModuleFunc.apply(configHolder);
         install(appConfigModule);

stroom-app/src/main/java/stroom/app/guice/CoreModule.java

@@ -16,7 +16,8 @@ protected void configure() {
         install(new stroom.aws.s3.impl.S3ConfigHandlerModule());
         install(new stroom.aws.s3.impl.S3ConfigModule());
         install(new stroom.cache.impl.CacheModule());
-        install(new stroom.cache.impl.CacheResourceModule());
+        install(new stroom.cache.service.impl.CacheServiceModule());
+        install(new stroom.cache.service.impl.CacheResourceModule());
         install(new stroom.cluster.lock.impl.db.ClusterLockModule());
         install(new stroom.cluster.task.impl.ClusterTaskModule());
         install(new stroom.config.global.impl.ConfigProvidersModule());

stroom-app/src/main/resources/ui/noauth/swagger/stroom.json

@@ -4364,6 +4364,7 @@
     },
     "/feedStatus/v1/getFeedStatus" : {
       "post" : {
+        "deprecated" : true,
         "operationId" : "getFeedStatus",
         "requestBody" : {
           "content" : {
@@ -4392,6 +4393,36 @@
         "tags" : [ "Feed Status" ]
       }
     },
+    "/feedStatus/v2/getFeedStatus" : {
+      "post" : {
+        "operationId" : "getFeedStatus_1",
+        "requestBody" : {
+          "content" : {
+            "application/json" : {
+              "schema" : {
+                "$ref" : "#/components/schemas/GetFeedStatusRequestV2"
+              }
+            }
+          },
+          "description" : "GetFeedStatusRequest",
+          "required" : true
+        },
+        "responses" : {
+          "default" : {
+            "content" : {
+              "application/json" : {
+                "schema" : {
+                  "$ref" : "#/components/schemas/GetFeedStatusResponse"
+                }
+              }
+            },
+            "description" : "default response"
+          }
+        },
+        "summary" : "Submit a request to get the status of a feed",
+        "tags" : [ "Feed Status" ]
+      }
+    },
     "/fileTransfer/v1/fetchSnapshot" : {
       "post" : {
         "operationId" : "fetchSnapshot",
@@ -11113,7 +11144,7 @@
         "properties" : {
           "permission" : {
             "type" : "string",
-            "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION" ]
+            "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION", "CHECK_RECEIPT_STATUS" ]
           },
           "type" : {
             "type" : "string"
@@ -11788,15 +11819,15 @@
             "type" : "array",
             "items" : {
               "type" : "string",
-              "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION" ]
+              "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION", "CHECK_RECEIPT_STATUS" ]
             },
             "uniqueItems" : true
           },
           "permissions" : {
             "type" : "array",
             "items" : {
               "type" : "string",
-              "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION" ]
+              "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION", "CHECK_RECEIPT_STATUS" ]
             },
             "uniqueItems" : true
           },
@@ -11812,7 +11843,7 @@
             "type" : "array",
             "items" : {
               "type" : "string",
-              "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION" ]
+              "enum" : [ "ADMINISTRATOR", "ANNOTATIONS", "MANAGE_CACHE_PERMISSION", "VIEW_DATA_PERMISSION", "VIEW_DATA_WITH_PIPELINE_PERMISSION", "DELETE_DATA_PERMISSION", "IMPORT_DATA_PERMISSION", "EXPORT_DATA_PERMISSION", "MANAGE_USERS_PERMISSION", "CHANGE_OWNER_PERMISSION", "STEPPING_PERMISSION", "MANAGE_PROCESSORS_PERMISSION", "MANAGE_TASKS_PERMISSION", "DOWNLOAD_SEARCH_RESULTS_PERMISSION", "MANAGE_API_KEYS", "MANAGE_JOBS_PERMISSION", "MANAGE_PROPERTIES_PERMISSION", "MANAGE_POLICIES_PERMISSION", "MANAGE_NODES_PERMISSION", "MANAGE_INDEX_SHARDS_PERMISSION", "MANAGE_VOLUMES_PERMISSION", "MANAGE_DB_PERMISSION", "IMPORT_CONFIGURATION", "EXPORT_CONFIGURATION", "VIEW_SYSTEM_INFO_PERMISSION", "CHECK_RECEIPT_STATUS" ]
             },
             "uniqueItems" : true
           },
@@ -14698,13 +14729,19 @@
           "contextEncoding" : {
             "type" : "string"
           },
+          "contextFormat" : {
+            "type" : "string"
+          },
           "createTimeMs" : {
             "type" : "integer",
             "format" : "int64"
           },
           "createUser" : {
             "type" : "string"
           },
+          "dataFormat" : {
+            "type" : "string"
+          },
           "description" : {
             "type" : "string"
           },
@@ -14721,6 +14758,12 @@
             "type" : "integer",
             "format" : "int32"
           },
+          "schema" : {
+            "type" : "string"
+          },
+          "schemaVersion" : {
+            "type" : "string"
+          },
           "status" : {
             "type" : "string",
             "enum" : [ "RECEIVE", "REJECT", "DROP" ]
@@ -15799,6 +15842,23 @@
           }
         }
       },
+      "GetFeedStatusRequestV2" : {
+        "type" : "object",
+        "properties" : {
+          "attributeMap" : {
+            "type" : "object",
+            "additionalProperties" : {
+              "type" : "string"
+            }
+          },
+          "feedName" : {
+            "type" : "string"
+          },
+          "userDesc" : {
+            "$ref" : "#/components/schemas/UserDesc"
+          }
+        }
+      },
       "GetFeedStatusResponse" : {
         "type" : "object",
         "properties" : {
@@ -15811,7 +15871,7 @@
           },
           "stroomStatusCode" : {
             "type" : "string",
-            "enum" : [ "200 - 0 - OK", "406 - 100 - Feed must be specified", "406 - 101 - Feed is not defined", "406 - 102 - Data type is invalid", "406 - 110 - Feed is not set to receive data", "406 - 120 - Unexpected data type", "406 - 200 - Unknown compression", "401 - 300 - Client Certificate Required", "401 - 301 - Client Token Required", "401 - 302 - Client Token or Certificate Required", "401 - 310 - Client Certificate failed authentication", "401 - 311 - Client Token failed authentication", "401 - 312 - Client Token or Certificate failed authentication", "500 - 400 - Compressed stream invalid", "500 - 999 - Unknown error" ]
+            "enum" : [ "200 - 0 - OK", "406 - 100 - Feed must be specified", "406 - 101 - Feed is not defined", "406 - 102 - Data type is invalid", "406 - 103 - AccountId must be specified", "406 - 104 - Component must be specified", "406 - 105 - Data format is invalid", "406 - 110 - Feed is not set to receive data", "406 - 111 - Feed is not valid", "406 - 112 - AccountId is not valid", "406 - 120 - Unexpected data type", "406 - 200 - Unknown compression", "401 - 300 - Client Certificate Required", "401 - 301 - Client Token Required", "401 - 302 - Client token or certificate required", "401 - 303 - Client data feed key required", "401 - 304 - Authentication of the client is required", "401 - 310 - Client Certificate failed authentication", "401 - 311 - Client Token failed authentication", "401 - 312 - Client Token or Certificate failed authentication", "401 - 313 - Data feed key failed authentication", "500 - 400 - Compressed stream invalid", "500 - 999 - Unknown error" ]
           }
         }
       },

stroom-app/src/main/resources/ui/noauth/swagger/stroom.yaml

@@ -2987,6 +2987,7 @@ paths:
       - Feeds
   /feedStatus/v1/getFeedStatus:
     post:
+      deprecated: true
       operationId: getFeedStatus
       requestBody:
         content:
@@ -3005,6 +3006,26 @@ paths:
       summary: Submit a request to get the status of a feed
       tags:
       - Feed Status
+  /feedStatus/v2/getFeedStatus:
+    post:
+      operationId: getFeedStatus_1
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/GetFeedStatusRequestV2"
+        description: GetFeedStatusRequest
+        required: true
+      responses:
+        default:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/GetFeedStatusResponse"
+          description: default response
+      summary: Submit a request to get the status of a feed
+      tags:
+      - Feed Status
   /fileTransfer/v1/fetchSnapshot:
     post:
       operationId: fetchSnapshot
@@ -7662,6 +7683,7 @@ components:
           - IMPORT_CONFIGURATION
           - EXPORT_CONFIGURATION
           - VIEW_SYSTEM_INFO_PERMISSION
+          - CHECK_RECEIPT_STATUS
         type:
           type: string
         userRef:
@@ -8169,6 +8191,7 @@ components:
             - IMPORT_CONFIGURATION
             - EXPORT_CONFIGURATION
             - VIEW_SYSTEM_INFO_PERMISSION
+            - CHECK_RECEIPT_STATUS
           uniqueItems: true
         permissions:
           type: array
@@ -8200,6 +8223,7 @@ components:
             - IMPORT_CONFIGURATION
             - EXPORT_CONFIGURATION
             - VIEW_SYSTEM_INFO_PERMISSION
+            - CHECK_RECEIPT_STATUS
           uniqueItems: true
         userRef:
           $ref: "#/components/schemas/UserRef"
@@ -8236,6 +8260,7 @@ components:
             - IMPORT_CONFIGURATION
             - EXPORT_CONFIGURATION
             - VIEW_SYSTEM_INFO_PERMISSION
+            - CHECK_RECEIPT_STATUS
           uniqueItems: true
         inheritedPermissions:
           type: object
@@ -10615,11 +10640,15 @@ components:
           type: string
         contextEncoding:
           type: string
+        contextFormat:
+          type: string
         createTimeMs:
           type: integer
           format: int64
         createUser:
           type: string
+        dataFormat:
+          type: string
         description:
           type: string
         encoding:
@@ -10631,6 +10660,10 @@ components:
         retentionDayAge:
           type: integer
           format: int32
+        schema:
+          type: string
+        schemaVersion:
+          type: string
         status:
           type: string
           enum:
@@ -11387,6 +11420,17 @@ components:
           type: string
         senderDn:
           type: string
+    GetFeedStatusRequestV2:
+      type: object
+      properties:
+        attributeMap:
+          type: object
+          additionalProperties:
+            type: string
+        feedName:
+          type: string
+        userDesc:
+          $ref: "#/components/schemas/UserDesc"
     GetFeedStatusResponse:
       type: object
       properties:
@@ -11405,15 +11449,23 @@ components:
           - 406 - 100 - Feed must be specified
           - 406 - 101 - Feed is not defined
           - 406 - 102 - Data type is invalid
+          - 406 - 103 - AccountId must be specified
+          - 406 - 104 - Component must be specified
+          - 406 - 105 - Data format is invalid
           - 406 - 110 - Feed is not set to receive data
+          - 406 - 111 - Feed is not valid
+          - 406 - 112 - AccountId is not valid
           - 406 - 120 - Unexpected data type
           - 406 - 200 - Unknown compression
           - 401 - 300 - Client Certificate Required
           - 401 - 301 - Client Token Required
-          - 401 - 302 - Client Token or Certificate Required
+          - 401 - 302 - Client token or certificate required
+          - 401 - 303 - Client data feed key required
+          - 401 - 304 - Authentication of the client is required
           - 401 - 310 - Client Certificate failed authentication
           - 401 - 311 - Client Token failed authentication
           - 401 - 312 - Client Token or Certificate failed authentication
+          - 401 - 313 - Data feed key failed authentication
           - 500 - 400 - Compressed stream invalid
           - 500 - 999 - Unknown error
     GetPipelineForMetaRequest: