diff --git a/.gitignore b/.gitignore index 8e5a013..53f3078 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* /.README.md.html +/.project diff --git a/.project b/.project deleted file mode 100644 index 5660a58..0000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - streamsx.monitoring - - - - - - org.eclipse.xtext.ui.shared.xtextBuilder - - - - - - org.eclipse.xtext.ui.shared.xtextNature - - diff --git a/BUILD.md b/BUILD.md index 2671775..dc32a17 100644 --- a/BUILD.md +++ b/BUILD.md @@ -6,7 +6,7 @@ This toolkit uses Apache Ant 1.8 (or later) to build. This toolkit requires -* IBM Streams 4.1 or later +* IBM Streams 4.3 or later * Java 1.8 It is recommended to use the Java version that is part of the Streams installation. diff --git a/README.md b/README.md index 1157faa..c84d752 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ The toolkit contains operators that uses the **JMX API** to monitor applications Find the full documentation [here](https://ibmstreams.github.io/streamsx.monitoring/). -## Streaming Analytics service on IBM Cloud +## IBM Streams - IBM Cloud Private for Data -This toolkit is compatible with the Streaming Analytics service on IBM Cloud. +This toolkit is compatible with the IBM Streams running in IBM Cloud Private for Data. diff --git a/build.xml b/build.xml index 90e8085..b031042 100644 --- a/build.xml +++ b/build.xml @@ -27,7 +27,6 @@ - - @@ -131,7 +129,7 @@ destfile="${releasefilename}" basedir="${basedir}" includes="com.ibm.streamsx.monitoring/** samples/**" - excludes="**/.gitignore com.ibm.streamsx.monitoring/.settings/** com.ibm.streamsx.monitoring/.project com.ibm.streamsx.monitoring/.classpat*" + excludes="**/.gitignore com.ibm.streamsx.monitoring/.settings/** com.ibm.streamsx.monitoring/ext.libs com.ibm.streamsx.monitoring/.project com.ibm.streamsx.monitoring/.classpat*" /> diff --git a/com.ibm.streamsx.monitoring/.classpath_tpl b/com.ibm.streamsx.monitoring/.classpath_tpl index 606ace4..df464ce 100644 --- a/com.ibm.streamsx.monitoring/.classpath_tpl +++ b/com.ibm.streamsx.monitoring/.classpath_tpl @@ -8,9 +8,9 @@ - - - + + + diff --git a/com.ibm.streamsx.monitoring/build.xml b/com.ibm.streamsx.monitoring/build.xml index 0907067..a97a28d 100644 --- a/com.ibm.streamsx.monitoring/build.xml +++ b/com.ibm.streamsx.monitoring/build.xml @@ -14,6 +14,7 @@ + @@ -35,9 +36,9 @@ - - - + + + diff --git a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs.services/pe.spl b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs.services/pe.spl index a6a8354..95596d9 100644 --- a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs.services/pe.spl +++ b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs.services/pe.spl @@ -51,7 +51,7 @@ use com.ibm.streamsx.topology.topic::Publish; * PE failed notification. Suitable for submitting to an alert system * such as a Slack channel. * - * @param tag Tag placed at the beginning of the text alert. Can be set using the submission parameter `tag`. Defaults to an empty string meaning the tag for each alert will be the domain identifier followed by the instance identifier. + * @param tag Tag placed at the beginning of the text alert. Can be set using the submission parameter `tag`. Defaults to an empty string meaning the tag for each alert will be the instance identifier. * @param stoppedDuration Duration in seconds a PE must be stopped before a notification is issued. Can be set using the submission parameter `stoppedDuration`. Defaults to sixty seconds. */ public composite FailedPEService @@ -148,7 +148,7 @@ public composite FailedPEService mutable rstring failed_tag = ''; } onTuple Failed: - failed_tag = tag_ != '' ? tag_ : domainId + ':' + instanceId; + failed_tag = tag_ != '' ? tag_ : instanceId; output Alerts: string = failed_tag + ': ' + diff --git a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/JobStatusMonitor.spl b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/JobStatusMonitor.spl index 543d386..320a9b1 100644 --- a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/JobStatusMonitor.spl +++ b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/JobStatusMonitor.spl @@ -18,7 +18,7 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * The schema is defined by the ConnectionNotification type. * * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. + * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain instanceId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. * * @param user * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. @@ -27,20 +27,17 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param filterDocument - * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the domain, instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * If parameter is not set, then all jobs in current domain and instance are monitored. + * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * If parameter is not set, then all jobs in current instance are monitored. * * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. + * Specifies the connection URL. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the connection URL is used for the instance under which this operator is running. * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * @param instanceId + * Specifies the instance id that is monitored. If no instance id is specified, the instance id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - * @param credentials - * Specifies Streaming Analytics service credentials in JSON format. Relevant for IAM authentication case only. If parameter is set, then the parameters user and password are ignored. If the **applicationConfigurationName** parameter is specified, the application configuration with the property credentials can override this parameter value. + * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified the default value `TLSv1.2 is` used. * * @param timeOut * Specifies the number of seconds during which no duplicate of a tuple is emitted. @@ -53,11 +50,10 @@ public composite JobStatusMonitor (output stream OutStrea expression $applicationConfigurationName: ""; // optional, if user and password are set expression $user: ""; // optional, if set in application configuration expression $password: ""; // optional, if set in application configuration - expression $filterDocument; // optional, uses default filter to monitor all in current domain and instance - expression $connectionURL: ""; // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: ""; // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: ""; // optional, if not set, then domain settings are used, where the PE is running - expression $credentials: ""; + expression $filterDocument; // optional, uses default filter to monitor all in current instance + expression $connectionURL: ""; + expression $instanceId: ""; + expression $sslOption: ""; expression $timeOut: 5.0; graph @@ -74,15 +70,14 @@ public composite JobStatusMonitor (output stream OutStrea password: $password; filterDocument: $filterDocument; connectionURL: $connectionURL; - domainId: $domainId; + instanceId: $instanceId; sslOption: $sslOption; - credentials: $credentials; } stream OutStream = DeDuplicate(Notifications as I) { param timeOut: $timeOut; - key: notifyType, domainId, instanceId, jobId, jobName, resource, peId, peHealth, peStatus; + key: notifyType, instanceId, jobId, jobName, resource, peId, peHealth, peStatus; } } diff --git a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/Types.spl b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/Types.spl index 8a9a79b..90242d0 100644 --- a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/Types.spl +++ b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.jobs/Types.spl @@ -26,16 +26,11 @@ namespace com.ibm.streamsx.monitoring.jobs; * The notification type, like *com.ibm.streams.management.job.added*, * *com.ibm.streams.management.job.removed* and *com.ibm.streams.management.pe.changed*. * - * * **domainId** - * - * The notification belongs to a job that is running in a domain. The **domainId** - * attribute specifies the unique domain id. - * * * **instanceId** * - * The notification belongs to a job that is running in an instance that belongs - * to a domain. The **instanceId** attribute specifies the unique instance - * id for the domain. + * The notification belongs to a job that is running in an instance. + * The **instanceId** attribute specifies the unique instance + * id. * * * **jobId** * @@ -84,7 +79,6 @@ namespace com.ibm.streamsx.monitoring.jobs; */ type JobStatusNotification = tuple< rstring notifyType, - rstring domainId, rstring instanceId, int64 jobId, rstring jobName, diff --git a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics.services/MetricsIngestService.spl b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics.services/MetricsIngestService.spl index d2b9e4c..9216acc 100644 --- a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics.services/MetricsIngestService.spl +++ b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics.services/MetricsIngestService.spl @@ -20,7 +20,6 @@ use com.ibm.streamsx.topology.topic::Publish; * * Output JSON Schema: * \{ - * "domainId" : string, * "instanceId" : string, * "jobId" : string, * "jobName" : string, @@ -48,7 +47,7 @@ use com.ibm.streamsx.topology.topic::Publish; * * @param monitoringConfiguration * Application configuration name for the JMX connection details. Defaults to `monitoring` and can be set using the submission parameter `monitoringConfiguration`. - * The [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.3.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. + * The [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.3.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] can contain instanceId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. * * @param metricsTopic * The name of the topic this service is publishing to. If parameter is not set, then the topic name `streamsx/monitoring/metrics/values` is used. diff --git a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics/Types.spl b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics/Types.spl index 930ed70..74e0e47 100644 --- a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics/Types.spl +++ b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.metrics/Types.spl @@ -77,7 +77,7 @@ public composite Origin { /** * The notification type specifies all attributes that can be used to uniquely - * identify a changed metric value, starting with the domain and instance + * identify a changed metric value, starting with the instance * ids, job and other information, and finally the metric name and its * value. * @@ -86,16 +86,10 @@ public composite Origin { * The [com.ibm.streamsx.monitoring.metrics::MetricsSource|MetricsSource] operator * takes care to set existing attributes only. * - * * **domainId** - * - * The metric belongs to a job that is running in a domain. The **domainId** - * attribute specifies the unique domain id. - * * * **instanceId** * - * The metric belongs to a job that is running in an instance that belongs - * to a domain. The **instanceId** attribute specifies the unique instance - * id for the domain. + * The metric belongs to a job that is running in an instance. The **instanceId** + * attribute specifies the unique instance id. * * * **jobId** * @@ -179,7 +173,6 @@ public composite Origin { * */ type Notification = tuple< - rstring domainId, rstring instanceId, int64 jobId, rstring jobName, diff --git a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.system/Types.spl b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.system/Types.spl index e0f8aed..6df03d8 100644 --- a/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.system/Types.spl +++ b/com.ibm.streamsx.monitoring/com.ibm.streamsx.monitoring.system/Types.spl @@ -54,14 +54,10 @@ type SystemStatus = tuple< * The notification type com.ibm.streams.management.log.application.error or * com.ibm.streams.management.log.application.warning. * - * * **domainId** - * - * The domain ID. - * * * **instanceId** * * The **instanceId** attribute specifies the unique instance - * id for the domain. + * id * * * **jobId** * @@ -106,7 +102,6 @@ type SystemStatus = tuple< */ type LogNotification = tuple< rstring notifyType, - rstring domainId, rstring instanceId, int64 jobId, rstring resource, diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxOperator.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxOperator.java index c3a4565..5859e27 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxOperator.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxOperator.java @@ -46,7 +46,7 @@ public abstract class AbstractJmxOperator extends AbstractOperator { protected static final String DESC_PARAM_APPLICATION_CONFIGURATION_NAME = "Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] " - + "that can contain domainId, connectionURL, user, password, credentials and filterDocument " + + "that can contain instanceId, connectionURL, user, password and filterDocument " + "properties. The application configuration overrides values that " + "are specified with the corresponding parameters."; @@ -54,10 +54,7 @@ public abstract class AbstractJmxOperator extends AbstractOperator { "Specifies the connection URL as returned by the `streamtool " + "getjmxconnect` command. If the **applicationConfigurationName** " + "parameter is specified, the application configuration can " - + "override this parameter value." - + "If not specified and the domainId parameter value equals the domain " - + "id under which this operator is running, then the operator uses the " - + "`streamtool getjmxconnect` command to get the value."; + + "override this parameter value."; protected static final String DESC_PARAM_USER = "Specifies the user that is required for the JMX connection. If " @@ -72,22 +69,11 @@ public abstract class AbstractJmxOperator extends AbstractOperator { protected static final String DESC_PARAM_SSL_OPTION = "Specifies the sslOption that is required for the JMX connection. If " + "the **applicationConfigurationName** parameter is specified, " - + "the application configuration can override this parameter value." - + "If not specified and the domainId parameter value equals the domain " - + "id under which this operator is running, then the operator uses the " - + "`streamtool getdomainproperty` command to get the value."; + + "the application configuration can override this parameter value. Default is TLSv1.2"; - protected static final String DESC_PARAM_DOMAIN_ID = - "Specifies the domain id that is monitored. If no domain id is " - + "specified, the domain id under which this operator is running " - + "is used. If the operator is running in a standalone application " - + "the parameter is mandatory. If " - + "the **applicationConfigurationName** parameter is specified, " - + "the application configuration can override this parameter value."; - - protected static final String DESC_PARAM_INSTANCE_ID_FILTER = - "Specifies the instance id filter used to select the instance(s) that is monitored. If no value is " - + "specified, all instances are monitored per default. To monitor the instance id under which this operator is running, the value OWN can be specified when running in distributed mode." + protected static final String DESC_PARAM_INSTANCE_ID = + "Specifies the instance id used to select the instance that is monitored. If no value is " + + "specified, the instance id under which this operator is running is monitored." + " If the **applicationConfigurationName** parameter is specified, " + "the application configuration can override this parameter value."; @@ -97,22 +83,14 @@ public abstract class AbstractJmxOperator extends AbstractOperator { protected static final Object PARAMETER_PASSWORD = "password"; - protected static final Object PARAMETER_IAM_API_KEY = "iamApiKey"; - - protected static final Object PARAMETER_IAM_TOKEN_ENDPOINT = "iamTokenEndpoint"; - protected static final Object PARAMETER_SSL_OPTION = "sslOption"; protected static final Object PARAMETER_FILTER_DOCUMENT = "filterDocument"; - protected static final Object PARAMETER_DOMAIN_ID = "domainId"; - - protected static final Object PARAMETER_INSTANCE_ID_FILTER = "instanceIdFilter"; + protected static final Object PARAMETER_INSTANCE_ID = "instanceId"; protected static final String MISSING_VALUE = "The following value must be specified as parameter or in the application configuration: "; - protected static final String PARAMETER_CREDENTIALS = "credentials"; - // ------------------------------------------------------------------------ // Implementation. // ------------------------------------------------------------------------ @@ -124,8 +102,6 @@ public abstract class AbstractJmxOperator extends AbstractOperator { protected OperatorConfiguration _operatorConfiguration = new OperatorConfiguration(); - protected String _domainId = null; // domainId for this PE - /** * The base directory of the application */ @@ -193,21 +169,13 @@ public void setPassword(String password) { public void setSslOption(String sslOption) { _operatorConfiguration.set_sslOption(sslOption); } - - @Parameter( - optional=true, - description=AbstractJmxOperator.DESC_PARAM_DOMAIN_ID - ) - public void setDomainId(String domainId) { - _operatorConfiguration.set_domainId(domainId); - } @Parameter( optional=true, - description=AbstractJmxOperator.DESC_PARAM_INSTANCE_ID_FILTER + description=AbstractJmxOperator.DESC_PARAM_INSTANCE_ID ) - public void setInstanceIdFilter(String instanceIdFilter) { - _operatorConfiguration.set_instanceIdFilter(instanceIdFilter); + public void setInstanceId(String instanceId) { + _operatorConfiguration.set_instanceId(instanceId); } @Parameter( @@ -217,13 +185,6 @@ public void setInstanceIdFilter(String instanceIdFilter) { public void setApplicationConfigurationName(String applicationConfigurationName) { _operatorConfiguration.set_applicationConfigurationName(applicationConfigurationName); } - - @Parameter(optional=true, description = "Specifies Streaming Analytics service credentials. Credentials are provided in JSON format. Relevant for IAM authentication to Streaming Analytics service case only. If parameter is set, then the parameters user and password are ignored.") - public void setCredentials(String credentials) { - _operatorConfiguration.set_credentials(credentials); - } - - /** * Initialize this operator. Called once before any tuples are processed. @@ -243,14 +204,14 @@ public synchronized void initialize(OperatorContext context) // check required parameters if (null == _operatorConfiguration.get_applicationConfigurationName()) { - if ((null == _operatorConfiguration.get_user()) && (null == _operatorConfiguration.get_password()) && (null == _operatorConfiguration.get_credentials())) { - throw new com.ibm.streams.operator.DataException("The " + context.getName() + " operator requires parameters 'user' and 'password' or 'applicationConfigurationName' or 'credentials' be applied."); + if ((null == _operatorConfiguration.get_user()) && (null == _operatorConfiguration.get_password()) ) { + throw new com.ibm.streams.operator.DataException("The " + context.getName() + " operator requires parameters 'user' and 'password' or 'applicationConfigurationName' be applied."); } } else { if (context.getPE().isStandalone()) { if ((null == _operatorConfiguration.get_user()) && (null == _operatorConfiguration.get_password())) { - throw new com.ibm.streams.operator.DataException("The " + context.getName() + " operator requires parameters 'user' and 'password' and 'domainId' applied, when running in standalone mode. Application configuration is supported in distributed mode only."); + throw new com.ibm.streams.operator.DataException("The " + context.getName() + " operator requires parameters 'user' and 'password' and 'instanceId' applied, when running in standalone mode. Application configuration is supported in distributed mode only."); } } } @@ -258,59 +219,26 @@ public synchronized void initialize(OperatorContext context) this.baseDir = context.getPE().getApplicationDirectory(); /* - * The domainId parameter is optional. If the application developer does - * not set it, use the domain id under which the operator itself is + * The instanceId parameter is optional. If the application developer does + * not set it, use the instance id under which the operator itself is * running. */ - if (_operatorConfiguration.get_domainId() == null) { + if (_operatorConfiguration.get_instanceId() == null) { if (context.getPE().isStandalone()) { - throw new com.ibm.streams.operator.DataException("The " + context.getName() + " operator runs in standalone mode and can, therefore, not automatically determine a domain id. The following value must be specified as parameter: domainId"); + throw new com.ibm.streams.operator.DataException("The " + context.getName() + " operator runs in standalone mode and can, therefore, not automatically determine a instance id. The following value must be specified as parameter: instanceId"); } - String domainId = getApplicationConfigurationDomainId(); - if ("".equals(domainId)) { - _operatorConfiguration.set_domainId(context.getPE().getDomainId()); - _trace.info("The " + context.getName() + " operator automatically connects to the " + _operatorConfiguration.get_domainId() + " domain."); + String instanceId = getApplicationConfigurationInstanceId(); + if ("".equals(instanceId)) { + _operatorConfiguration.set_instanceId(context.getPE().getInstanceId()); + _trace.info("The " + context.getName() + " operator automatically connects to the " + _operatorConfiguration.get_instanceId() + " instance."); } else { - _operatorConfiguration.set_domainId(domainId); - _trace.info("The " + context.getName() + " operator connects to the " + _operatorConfiguration.get_domainId() + " domain specified by application configuration."); - } - } - // used to determine if configured domain is the domain where the PE runs - // if is running in standalone, then the domainId parameter/application configuration is used - _domainId = ((context.getPE().isStandalone()) ? _operatorConfiguration.get_domainId() : context.getPE().getDomainId()); - _operatorConfiguration.set_defaultFilterDomain(_domainId); - - // Set the instance filter, default is ".*" - if (!(context.getPE().isStandalone())) { - String instanceFilter = getApplicationConfigurationInstanceId(); - if (!"".equals(instanceFilter)) { - _operatorConfiguration.set_instanceIdFilter(instanceFilter); - } - if ("OWN".equals(_operatorConfiguration.get_instanceIdFilter())) { - _operatorConfiguration.set_instanceIdFilter(context.getPE().getInstanceId()); + _operatorConfiguration.set_instanceId(instanceId); + _trace.info("The " + context.getName() + " operator connects to the " + _operatorConfiguration.get_instanceId() + " instance specified by application configuration."); } } - _trace.info("The " + context.getName() + " operator uses the instance ID filter: " + _operatorConfiguration.get_instanceIdFilter()); - _operatorConfiguration.set_defaultFilterInstance(_operatorConfiguration.get_instanceIdFilter()); - + _operatorConfiguration.set_defaultFilterInstance(_operatorConfiguration.get_instanceId()); - - String credentials = getCredentials(); - if (!"".equals(credentials)) { - try { - // read the JSON service credentials applied in application configuration - String apikey = null; - JSONArtifact root = JSON.parse(credentials); - JSONObject json = (JSONObject)root; - Object apikeyObj = json.get("apikey"); - apikey = apikeyObj.toString(); - _trace.debug("apikey: " + apikey); - _operatorConfiguration.set_iamApiKey(apikey); - } catch (Exception e) { - _trace.error("Failed to parse credentials property from application configuration '" + _operatorConfiguration.get_applicationConfigurationName() + "'. ERROR: '" + e.getMessage() + "'"); - } - } } /** @@ -348,48 +276,18 @@ protected void setupClassPaths(OperatorContext context) throws Exception { } } - protected String getApplicationConfigurationDomainId() { - String result = ""; - String applicationConfigurationName = _operatorConfiguration.get_applicationConfigurationName(); - if (applicationConfigurationName != null) { - Map properties = getApplicationConfiguration(applicationConfigurationName); - if (properties.containsKey(PARAMETER_DOMAIN_ID)) { - result = properties.get(PARAMETER_DOMAIN_ID); - } - } - return result; - } protected String getApplicationConfigurationInstanceId() { String result = ""; String applicationConfigurationName = _operatorConfiguration.get_applicationConfigurationName(); if (applicationConfigurationName != null) { Map properties = getApplicationConfiguration(applicationConfigurationName); - if (properties.containsKey(PARAMETER_INSTANCE_ID_FILTER)) { - result = properties.get(PARAMETER_INSTANCE_ID_FILTER); - } - } - return result; - } - - protected String getCredentials() { - String result = ""; - // prio: operator parameter credentials first, check app config if credentials parameter is not set - if (null != _operatorConfiguration.get_credentials()) { - result = _operatorConfiguration.get_credentials(); - } - if ("".equals(result)) { - String applicationConfigurationName = _operatorConfiguration.get_applicationConfigurationName(); - if (applicationConfigurationName != null) { - Map properties = getApplicationConfiguration(applicationConfigurationName); - if (properties.containsKey(PARAMETER_CREDENTIALS)) { - result = properties.get(PARAMETER_CREDENTIALS); - } + if (properties.containsKey(PARAMETER_INSTANCE_ID)) { + result = properties.get(PARAMETER_INSTANCE_ID); } } return result; } - /** * Sets up a JMX connection. The connection URL, the user, and the password @@ -404,8 +302,6 @@ protected void setupJMXConnection() throws Exception { String user = _operatorConfiguration.get_user(); String password = _operatorConfiguration.get_password(); String sslOption = _operatorConfiguration.get_sslOption(); - String iamApiKey = _operatorConfiguration.get_iamApiKey(); - String iamTokenEndpoint = _operatorConfiguration.get_iamTokenEndpoint(); // Override defaults if the application configuration is specified String applicationConfigurationName = _operatorConfiguration.get_applicationConfigurationName(); if (applicationConfigurationName != null) { @@ -422,12 +318,6 @@ protected void setupJMXConnection() throws Exception { if (properties.containsKey(PARAMETER_SSL_OPTION)) { sslOption = properties.get(PARAMETER_SSL_OPTION); } - if (properties.containsKey(PARAMETER_IAM_API_KEY)) { - iamApiKey = properties.get(PARAMETER_IAM_API_KEY); - } - if (properties.containsKey(PARAMETER_IAM_TOKEN_ENDPOINT)) { - iamTokenEndpoint = properties.get(PARAMETER_IAM_TOKEN_ENDPOINT); - } } // Ensure a valid configuration. if (connectionURL == null) { @@ -437,20 +327,13 @@ protected void setupJMXConnection() throws Exception { throw new Exception(MISSING_VALUE + PARAMETER_CONNECTION_URL); } } - if (iamApiKey == null) { - if (user == null) { - throw new Exception(MISSING_VALUE + PARAMETER_USER + " or " + PARAMETER_IAM_API_KEY); - } - if (password == null) { - throw new Exception(MISSING_VALUE + PARAMETER_PASSWORD); - } + if (user == null) { + throw new Exception(MISSING_VALUE + PARAMETER_USER); } - else { - // use IAM authentication - String token = getAccessToken(iamApiKey, iamTokenEndpoint); - user = "streams-bearer"; - password = "bearertoken:"+token; + if (password == null) { + throw new Exception(MISSING_VALUE + PARAMETER_PASSWORD); } + /* * Prepare the JMX environment settings. */ @@ -458,21 +341,10 @@ protected void setupJMXConnection() throws Exception { String [] credentials = { user, password }; env.put("jmx.remote.credentials", credentials); env.put("jmx.remote.protocol.provider.pkgs", "com.ibm.streams.management"); - /* - * get the value from: streamtool getdomainproperty jmx.sslOption - * Code taken from: - * http://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.dev.doc/doc/jmxapi-lgop.html - */ + if (sslOption != null) { env.put("jmx.remote.tls.enabled.protocols", sslOption); } - else { - // not configured via parameter or application configuration - sslOption = autoDetectJmxSslOption(user, password); - if (sslOption != null) { - env.put("jmx.remote.tls.enabled.protocols", sslOption); - } - } /* * Setup the JMX connector and MBean connection. @@ -522,106 +394,12 @@ protected Map getApplicationConfiguration(String applicationConfi } private String autoDetectJmxConnect() throws Exception { - String result = null; - _trace.debug("_domainId=[" + _domainId + "]"); - _trace.debug("_operatorConfiguration.get_domainId()=[" + _operatorConfiguration.get_domainId() + "]"); - if (_operatorConfiguration.get_domainId().equals(_domainId)) { // running in same domain as configured - _trace.debug("get connectionURL with streamtool getjmxconnect"); - String cmd = "streamtool getjmxconnect -d "+ _domainId; - - StringBuffer output = new StringBuffer(); - Process p; - try { - p = Runtime.getRuntime().exec(cmd); - p.waitFor(); - BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line = ""; - while ((line = br.readLine())!= null) { - output.append(line).append(","); - } - } catch (Exception e) { - e.printStackTrace(); - } - result = output.toString(); - // Service URL must start with - if (result.startsWith("service:jmx:")) { - _trace.info("connectionURL=[" + result + "]"); - _operatorConfiguration.set_connectionURL(result); - } - else { - if (result.endsWith(",")) { - result = result.substring(0, result.length()-1); - } - _trace.error("Unable to determine "+PARAMETER_CONNECTION_URL+": " + result); - throw new Exception("Unable to determine "+PARAMETER_CONNECTION_URL); - } + // STREAMS_JMX_CONNECT + final String STREAMS_JMX_CONNECT = System.getenv("STREAMS_JMX_CONNECT"); + if (STREAMS_JMX_CONNECT == null || STREAMS_JMX_CONNECT.isEmpty()) { + _trace.error("STREAMS_JMX_CONNECT environment variable is not set."); } - return result; + return STREAMS_JMX_CONNECT; } - - private String getAccessToken(String apiKey, String url) throws Exception { - String token = null; - _trace.debug("get access token with apiKey:" + apiKey); - String[] cmd = { "/bin/sh", "-c", "curl -s -d grant_type=urn:ibm:params:oauth:grant-type:apikey -d apikey="+apiKey+" -H Content-type=application/x-www-form-urlencoded -H Accept=application/json "+url }; - - StringBuffer output = new StringBuffer(); - Process p; - try { - p = Runtime.getRuntime().exec(cmd); - p.waitFor(); - BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line = ""; - while ((line = br.readLine())!= null) { - output.append(line); - } - String cmdResult = output.toString(); - _trace.debug("cmdResult: " + cmdResult); - JSONArtifact root = JSON.parse(cmdResult); - JSONObject json = (JSONObject)root; - Object tokenObj = json.get("access_token"); - token = tokenObj.toString(); - - _trace.debug("token: " + token); - - } catch (Exception e) { - e.printStackTrace(); - } - if (null == token) { - _trace.error("Unable to determine IAM access token"); - throw new Exception("Unable to determine IAM access token"); - } - return token; - } - - private String autoDetectJmxSslOption(String user, String password) { - String sslOption = null; - if (_operatorConfiguration.get_domainId().equals(_domainId)) { // running in same domain as configured - _trace.debug("get jmx.sslOption with streamtool getdomainproperty"); - - String[] cmd = { "/bin/sh", "-c", "echo "+password+" | streamtool getdomainproperty -d "+ _domainId + " -U "+user+" jmx.sslOption" }; - - StringBuffer output = new StringBuffer(); - Process p; - try { - p = Runtime.getRuntime().exec(cmd); - p.waitFor(); - BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line = ""; - while ((line = br.readLine())!= null) { - output.append(line); - } - } catch (Exception e) { - e.printStackTrace(); - } - String cmdResult = output.toString(); - _trace.debug("cmdResult: " + cmdResult); - int idx = cmdResult.indexOf("="); - if (idx >=0) { - sslOption = cmdResult.substring(idx+1, cmdResult.length()); - _trace.info("sslOption=[" + sslOption + "]"); - } - } - return sslOption; - } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxSource.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxSource.java index 14a32ce..40481e3 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxSource.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/AbstractJmxSource.java @@ -20,7 +20,7 @@ import com.ibm.streams.operator.StreamingOutput; import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration.OpType; import com.ibm.streamsx.monitoring.jmx.internal.ConnectionNotificationTupleContainer; -import com.ibm.streamsx.monitoring.jmx.internal.DomainHandler; +import com.ibm.streamsx.monitoring.jmx.internal.InstanceHandler; import com.ibm.streamsx.monitoring.jmx.internal.JobStatusTupleContainer; import com.ibm.streamsx.monitoring.jmx.internal.LogTupleContainer; import com.ibm.streamsx.monitoring.jmx.internal.MetricsTupleContainer; @@ -55,43 +55,25 @@ public abstract class AbstractJmxSource extends AbstractJmxOperator { public static final String AUTHENTICATION_DESC = "\\n"+ "\\n+ Supported Authentication Schemes" + + "\\n"+ + "\\n# IBM Streams authentication\\n"+ + "\\nFor IBM Streams authentication the following authentication parameters should be used:\\n"+ + "\\n* user\\n"+ + "\\n* password\\n"+ "\\n"+ "\\nAuthentication can be configured with operator parameters or application configuration."+ "\\n"+ "\\n# Authentication with application configuration\\n"+ "\\n"+ - "**Create IBM Streaming Analytics Service Credentials**\\n" + - "\\nA service credential provides the necessary information to connect an application to Streaming Analytics service packaged in a JSON document. Service credentials are always associated with a Service ID, and new Service IDs can be created along with a new credential.\\n" + - "\\nUse the following steps to create a service credential:\\n" + - "\\n" + - " 1. Log in to the IBM Cloud console and navigate to your instance of Streaming Analytics service.\\n" + - " 2. In the side navigation, click Service Credentials.\\n" + - " 3. Click New credential and provide the necessary information.\\n" + - " 4. Click Add to generate service credential.\\n" + - " 5. Click View credentials and copy JSON into clipboard.\\n" + - "\\n"+ "**Save Credentials in Application Configuration Property**\\n" + - "\\n" + - "With this option, users can copy their IBM Streaming Analytics Credentials JSON from the IBM Streaming Analytics service and " - + "store it in an application configuration property called `credentials`. When the operator starts, " - + "it will look for that property and extract the information needed to connect. " + "\\n" + "The following steps outline how this can be done: \\n" + "\\n" + - " 1. Create an application configuration called `monitor`. You need to set the operator parameter `applicationConfigurationName`.\\n" + - " 2. Create a property in the `monitor` application configuration *named* `credentials`.\\n" + - " * The *value* of the property should be the raw IBM Streaming Analytics Service Credentials JSON\\n" + - " * The *value* of the property could be pasted from the clipboard if you have done the *Create IBM Streaming Analytics Service Credentials* steps above. \\n" + + " 1. Create an application configuration called `monitoring`. You need to set the operator parameter `applicationConfigurationName`.\\n" + + " 2. Create two properties in the `monitoring` application configuration *named* `user` and `password`.\\n" + " 3. The operator will look for an application configuration, if the parameter `applicationConfigurationName` is set and will extract " + "the information needed to connect.\\n" + - "\\n"+ - "\\n# Apply credentials as operator parameter\\n"+ - "\\nFor Streaming Analytics service (IAM authentication) the following parameter should be used:"+ - "\\n* credentials - JSON service credentials\\n"+ - "\\n"+ - "\\n# IBM Streams authentication\\n"+ - "\\nFor IBM Streams authentication the following authentication parameters should be used:\\n"+ - "\\n* user\\n"+ - "\\n* password\\n" + "\\n" ; // ------------------------------------------------------------------------ @@ -103,7 +85,7 @@ public abstract class AbstractJmxSource extends AbstractJmxOperator { */ private static Logger _trace = Logger.getLogger(AbstractJmxSource.class.getName()); - protected DomainHandler _domainHandler = null; + protected InstanceHandler _instanceHandler = null; /** * If the application configuration is used (applicationConfigurationName @@ -131,9 +113,9 @@ public synchronized void initialize(OperatorContext context) */ if (OpType.LOG_SOURCE != _operatorConfiguration.get_OperatorType()) { setupFilters(); - boolean isValidDomain = _operatorConfiguration.get_filters().matchesDomainId(_operatorConfiguration.get_domainId()); - if (!isValidDomain) { - throw new com.ibm.streams.operator.DataException("The " + _operatorConfiguration.get_domainId() + " domain does not match the specified filter criteria in " + _operatorConfiguration.get_filterDocument()); + boolean isValidInstance = _operatorConfiguration.get_filters().matchesInstanceId(_operatorConfiguration.get_instanceId()); + if (!isValidInstance) { + throw new com.ibm.streams.operator.DataException("The " + _operatorConfiguration.get_instanceId() + " instance does not match the specified filter criteria in " + _operatorConfiguration.get_filterDocument()); } } @@ -156,10 +138,10 @@ public synchronized void initialize(OperatorContext context) setupJMXConnection(); /* - * Further actions are handled in the domain handler that manages + * Further actions are handled in the instance handler that manages * instances that manages jobs, etc. */ - scanDomain(); + scanInstance(); } /** @@ -185,16 +167,16 @@ else if (!activeFilterDocumentFromApplicationConfiguration.equals(filterDocument } } if (isChanged) { - _domainHandler.close(); - _domainHandler = null; + _instanceHandler.close(); + _instanceHandler = null; setupFilters(); - scanDomain(); + scanInstance(); } } } - protected void scanDomain() { - _domainHandler = new DomainHandler(_operatorConfiguration, _operatorConfiguration.get_domainId()); + protected void scanInstance() { + _instanceHandler = new InstanceHandler(_operatorConfiguration, _operatorConfiguration.get_instanceId()); } /** @@ -263,13 +245,13 @@ protected void setupFilters() throws Exception { } } - protected void closeDomainHandler() { + protected void closeInstanceHandler() { try { - _domainHandler.close(); + _instanceHandler.close(); } catch (Exception ignore) { } - _domainHandler = null; + _instanceHandler = null; if (1 == get_isConnected().getValue()) { // update metric to indicate connection is broken get_nBrokenJMXConnections().increment(); diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/OperatorConfiguration.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/OperatorConfiguration.java index 5a5391f..0310d4b 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/OperatorConfiguration.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/OperatorConfiguration.java @@ -26,8 +26,8 @@ public class OperatorConfiguration { /** - * Specifies the connection URL as returned by the {@code streamtool - * getjmxconnect} command. + * Specifies the connection URL as returned by the {@code + * STREAMS_JMX_CONNECT} environment variable. */ private String _connectionURL = null; @@ -44,38 +44,12 @@ public class OperatorConfiguration { /** * Specifies the sslOption that is required for the JMX connection. */ - private String _sslOption = null; - - /** - * Specifies the domain that is monitored. - */ - private String _domainId = null; + private String _sslOption = "TLSv1.2"; /** * Specifies the instance that is monitored. */ - private String _instanceIdFilter = ".*"; - - /** - * Global Endpoint - */ - public static final String IAM_TOKEN_ENDPOINT = "https://iam.bluemix.net/identity/token"; - - /** - * Specifies the IAM token endpoint. - */ - - private String _iamTokenEndpoint = IAM_TOKEN_ENDPOINT; - - /** - * Specifies the IAM API key. - */ - private String _iamApiKey = null; - - /** - * Specifies the Streaming Analytics service credentials in JSON format. - */ - private String _credentials = null; + private String _instanceId = null; /** * Specifies the name of the application configuration object. @@ -84,7 +58,7 @@ public class OperatorConfiguration { /** * Specifies the path to a JSON-formatted document that specifies the - * domain, instance, job, operator, and metric name filters as regular + * instance, job, operator, and metric name filters as regular * expressions. Each regular expression must follow the rules that are * specified for Java Pattern. */ @@ -110,8 +84,6 @@ public class OperatorConfiguration { private String _defaultFilterInstance = null; - private String _defaultFilterDomain = null; - private MetricsTupleContainer _tupleContainerMetricsSource = null; private JobStatusTupleContainer _tupleContainerJobStatusSource = null; @@ -167,36 +139,6 @@ public void set_password(String password) { } } - public String get_iamTokenEndpoint() { - return _iamTokenEndpoint; - } - - public void set_iamTokenEndpoint(String iamTokenEndpoint) { - if (!("".equals(iamTokenEndpoint))) { - this._iamTokenEndpoint = iamTokenEndpoint; - } - } - - public String get_iamApiKey() { - return _iamApiKey; - } - - public void set_iamApiKey(String iamApiKey) { - if (!("".equals(iamApiKey))) { - this._iamApiKey = iamApiKey; - } - } - - public String get_credentials() { - return _credentials; - } - - public void set_credentials(String credentials) { - if (!("".equals(credentials))) { - this._credentials = credentials; - } - } - public String get_sslOption() { return _sslOption; } @@ -207,23 +149,13 @@ public void set_sslOption(String sslOption) { } } - public String get_domainId() { - return _domainId; + public String get_instanceId() { + return _instanceId; } - public void set_domainId(String domainId) { - if (!("".equals(domainId))) { - this._domainId = domainId; - } - } - - public String get_instanceIdFilter() { - return _instanceIdFilter; - } - - public void set_instanceIdFilter(String instanceIdFilter) { - if (!("".equals(instanceIdFilter))) { - this._instanceIdFilter = instanceIdFilter; + public void set_instanceId(String instanceId) { + if (!("".equals(instanceId))) { + this._instanceId = instanceId; } } @@ -338,25 +270,17 @@ public void set_defaultFilterInstance(String instance) { _defaultFilterInstance = instance; } - public void set_defaultFilterDomain(String domain) { - _defaultFilterDomain = domain; - } - public String get_defaultFilterDocument() { String result = null; if (_opType.equals(OpType.METRICS_SOURCE)) { - result = "[{\"domainIdPatterns\":\"" - + _defaultFilterDomain - + "\",\"instances\":[{\"instanceIdPatterns\":\"" + result = "[{\"instanceIdPatterns\":\"" + _defaultFilterInstance + "\",\"jobs\":[{\"jobNamePatterns\":\".*\"," + "\"pes\":[{\"metricNamePatterns\":\".*\",\"inputPorts\":[{\"portIndexes\":\"*\",\"metricNamePatterns\":\".*\"}],\"outputPorts\":[{\"portIndexes\":\"*\",\"metricNamePatterns\":\".*\"}],\"connections\":[{\"connectionIdPatterns\":\".*\",\"metricNamePatterns\":\".*\"}]}]," + "\"operators\":[{\"operatorNamePatterns\":\".*\",\"metricNamePatterns\":\".*\",\"inputPorts\":[{\"portIndexes\":\"*\",\"metricNamePatterns\":\".*\"}],\"outputPorts\":[{\"portIndexes\":\"*\",\"metricNamePatterns\":\".*\"}]}]}]}]}]"; } else if (_opType.equals(OpType.JOB_STATUS_SOURCE)) { - result = "[{\"domainIdPatterns\":\"" - + _defaultFilterDomain - + "\",\"instances\":[{\"instanceIdPatterns\":\"" + result = "[{\"instanceIdPatterns\":\"" + _defaultFilterInstance + "\",\"jobs\":[{\"jobNamePatterns\":\".*\",}]}]}]"; } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DeltaMetricEvaluator.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DeltaMetricEvaluator.java index 700d341..8b6d8f1 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DeltaMetricEvaluator.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DeltaMetricEvaluator.java @@ -32,12 +32,12 @@ public DeltaMetricEvaluator(long delta) { @Override public boolean isSubmittable(Metric metric) { - return (_lastValue == null) || (Math.abs(_lastValue - metric.getValue()) >= _delta); + return (_lastValue == null) || (Math.abs(_lastValue - metric.getValueAsLong()) >= _delta); } @Override public void updateStatus(Metric metric) { - _lastValue = metric.getValue(); + _lastValue = metric.getValueAsLong(); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DomainHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DomainHandler.java deleted file mode 100644 index 328721b..0000000 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/DomainHandler.java +++ /dev/null @@ -1,315 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2016, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -package com.ibm.streamsx.monitoring.jmx.internal; - -import javax.management.Notification; -import javax.management.NotificationFilterSupport; -import javax.management.NotificationListener; - -import org.apache.log4j.Logger; -import java.io.IOException; -import java.math.BigInteger; -import java.util.HashMap; -import java.util.Map; - -import javax.management.InstanceNotFoundException; -import javax.management.JMX; -import javax.management.ObjectName; - -import com.ibm.json.java.JSON; -import com.ibm.json.java.JSONArray; -import com.ibm.json.java.JSONObject; -import com.ibm.streams.management.Notifications; -import com.ibm.streams.management.ObjectNameBuilder; -import com.ibm.streams.management.domain.DomainMXBean; -import com.ibm.streams.operator.Tuple; -import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration; -import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration.OpType; - -/** - * Listen for the following domain notifications: - * - *
    - *
  • com.ibm.streams.management.instance.created - *

    - * The Source operator registers with the instance to receive - * instance-related notifications. - *

    • - *
  • com.ibm.streams.management.instance.deleted - *

    - * The Source operator removes all instance-related information, so - * jobs of this instance are not monitored anymore. - *

    • - *
- */ -public class DomainHandler implements NotificationListener, Closeable { - - /** - * Logger for tracing. - */ - private static Logger _trace = Logger.getLogger(DomainHandler.class.getName()); - - private OperatorConfiguration _operatorConfiguration = null; - - private String _domainId = null; - - private ObjectName _objName = null; - - private DomainMXBean _domain = null; - - private Map _instanceHandlers = new HashMap<>(); - - public DomainHandler(OperatorConfiguration operatorConfiguration, String domainId) { - - boolean isDebugEnabled = _trace.isDebugEnabled(); - if (isDebugEnabled) { - _trace.debug("DomainHandler(" + domainId + ")"); - } - - // Store parameters for later use. - _operatorConfiguration = operatorConfiguration; - _domainId = domainId; - - _objName = ObjectNameBuilder.domain(_domainId); - _domain = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), _objName, DomainMXBean.class, true); - - /* - * Register to get domain-related notifications. - */ - NotificationFilterSupport filter = new NotificationFilterSupport(); - if (OpType.LOG_SOURCE == _operatorConfiguration.get_OperatorType()) { - filter.enableType("com.ibm.streams.management.log.application.error"); - filter.enableType("com.ibm.streams.management.log.application.warning"); - } - else { - filter.enableType(Notifications.INSTANCE_CREATED); - filter.enableType(Notifications.INSTANCE_DELETED); - } - try { - if (_trace.isDebugEnabled()) { - _trace.debug("MBeanServerConnection.addNotificationListener()"); - } - _operatorConfiguration.get_mbeanServerConnection().addNotificationListener(_objName, this, filter, null); - } catch (InstanceNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - if (null != _operatorConfiguration.get_tupleContainerConnectionNotification()) { - try { - if (_trace.isDebugEnabled()) { - _trace.debug("JMXConnector.addConnectionNotificationListener()"); - } - // listen for potential lost notifications - operatorConfiguration.get_jmxConnector().addConnectionNotificationListener(this, null, null); - } catch (Exception e) { - e.printStackTrace(); - } - } - - if (OpType.LOG_SOURCE != _operatorConfiguration.get_OperatorType()) { - /* - * Register existing instances. - */ - for(String instanceId : _domain.getInstances()) { - addValidInstance(instanceId); - } - } - - } - - public void healthCheck() { - if (_trace.isDebugEnabled()) { - _trace.debug("healthCheck"); - } - com.ibm.streams.management.domain.DomainMXBean.Status status = _domain.getStatus(); - if (_trace.isDebugEnabled()) { - _trace.debug("DomainMXBean.Status="+status.toString()); - } - } - - /** - * {@inheritDoc} - * - * - */ - @Override - public void handleNotification(Notification notification, Object handback) { - if (_trace.isDebugEnabled()) { - _trace.debug("notification: " + notification + ", userData=" + notification.getUserData()); - } - - if (notification.getType().equals(Notifications.INSTANCE_CREATED)) { - String instanceId = getInstanceFromUserData(notification.getUserData()); - if (instanceId != "") { - /* - * Register instance. - */ - addValidInstance(instanceId); - } - else { - _trace.error("received INSTANCE_CREATED notification: user data does not contain instance name"); - } - } - else if (notification.getType().equals(Notifications.INSTANCE_DELETED)) { - String instanceId = getInstanceFromUserData(notification.getUserData()); - if (instanceId != "") { - /* - * Unregister existing instance. - */ - if(_instanceHandlers.containsKey(instanceId)) { - _instanceHandlers.remove(instanceId); - if (_trace.isInfoEnabled()) { - _trace.info("The following instance is deleted: domain=" + _domainId + ", instance=" + instanceId); - } - } - } - else { - _trace.error("received INSTANCE_DELETED notification: user data does not contain instance name"); - } - } - else { - if ((OpType.LOG_SOURCE == _operatorConfiguration.get_OperatorType()) && - (notification.getType().contains("com.ibm.streams.management.log.application"))) { - // emit tuple - try { - String instance = null; - String resource = null; - BigInteger pe = null; - BigInteger job = null; - String operator = null; - String text = null; - boolean messagesSkipped = false; - if (null != notification.getUserData()) { - if ((notification.getMessage() != null) && (notification.getMessage().length() > 0)) { - // javax.management.Notification[source=com.ibm.streams.management:type=domain,name="xxx"] - // [type=com.ibm.streams.management.log.application.error][message=xxx], - // userData={"instance":"xxx","resource":"xxx","pe":"1","domain":"xxx","job":"1","operator":"xxx"} - if (_trace.isDebugEnabled()) { - _trace.debug("parse userData"); - } - JSONObject obj = (JSONObject)JSON.parse(notification.getUserData().toString()); - instance = obj.get("instance").toString(); - resource = obj.get("resource").toString(); - pe = new BigInteger(obj.get("pe").toString()); - job = new BigInteger(obj.get("job").toString()); - operator = obj.get("operator").toString(); - } - else { - // if message is not set then the message is part of userData - // javax.management.Notification[source=com.ibm.streams.management:type=domain,name="xxx"] - // [type=com.ibm.streams.management.log.application.error][message=], - // userData={"messages":[{"instance":"xxx","resource":"xxx","pe":"xx","domain":"xxx","text":"xxx","job":"xx","operator":"xxx","timestamp":1515687171914}],"messagesSkipped":false} - if (_trace.isDebugEnabled()) { - _trace.debug("parse userData for messages and messagesSkipped"); - } - JSONObject obj1 = (JSONObject)JSON.parse(notification.getUserData().toString()); - messagesSkipped = (obj1.get("messagesSkipped").toString().equalsIgnoreCase("true")) ? true : false; - Object json = obj1.get("messages"); - if (json instanceof JSONArray) { - for (Object obj : (JSONArray)json) { - if (obj instanceof JSONObject) { - JSONObject jsonobj = (JSONObject) obj; - instance = jsonobj.get("instance").toString(); - resource = jsonobj.get("resource").toString(); - pe = new BigInteger(jsonobj.get("pe").toString()); - job = new BigInteger(jsonobj.get("job").toString()); - operator = jsonobj.get("operator").toString(); - text = jsonobj.get("text").toString(); - final Tuple tuple = _operatorConfiguration.get_tupleContainerLogSource().getTuple(notification, _domainId, instance, resource, pe, job, operator, text, messagesSkipped); - _operatorConfiguration.get_tupleContainerLogSource().submit(tuple); - } - } - } - } - } - if (text == null) { - final Tuple tuple = _operatorConfiguration.get_tupleContainerLogSource().getTuple(notification, _domainId, instance, resource, pe, job, operator, text, messagesSkipped); - _operatorConfiguration.get_tupleContainerLogSource().submit(tuple); - } - } - catch (Exception e) { - _trace.error("Error in parsing userData: " + e); - } - } - else { - if ((null != _operatorConfiguration.get_tupleContainerConnectionNotification()) && - (notification.getType().contains("jmx.remote.connection"))) { - // Notification emitted when a client connection has failed unexpectedly or when client connection has potentially lost notifications. - final Tuple tuple = _operatorConfiguration.get_tupleContainerConnectionNotification().getTuple(notification); - _operatorConfiguration.get_tupleContainerConnectionNotification().submit(tuple); - } - else { - _trace.error("notification: " + notification + ", userData=" + notification.getUserData()); - } - } - } - } - - private String getInstanceFromUserData(Object userDataObj) { - String result = ""; - if (null != userDataObj) { - String userData = userDataObj.toString(); - // com.ibm.streams.management:type=domain.instance,domain="xxx",name="yyy" - int idx = userData.lastIndexOf("name="); - if ((idx > -1) && (idx+6 < userData.length())) { - result = userData.substring(idx+6, userData.length()-1); - } - } - return result; - } - - protected void addValidInstance(String instanceId) { - boolean matches = _operatorConfiguration.get_filters().matchesInstanceId(_domainId, instanceId); - if (_trace.isInfoEnabled()) { - if (matches) { - _trace.info("The following instance meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + instanceId); - } - else { - _trace.info("The following instance does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + instanceId); - } - } - if(matches) { - _instanceHandlers.put(instanceId, new InstanceHandler(_operatorConfiguration, _domainId, instanceId)); - } - } - - /** - * Iterate all instances to capture the job metrics. - * @throws Exception - */ - public void captureMetrics() throws Exception { - boolean isDebugEnabled = _trace.isDebugEnabled(); - if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ")"); - } - _operatorConfiguration.get_tupleContainerMetricsSource().setDomainId(_domainId); - for(String instanceId : _instanceHandlers.keySet()) { - _instanceHandlers.get(instanceId).captureMetrics(); - } - if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ")"); - } - } - - /** - * Remove notification listeners from this and child objects. - */ - @Override - public void close() throws Exception { - // Remove the notification listener. - _operatorConfiguration.get_mbeanServerConnection().removeNotificationListener(_objName, this); - // Close all resources of all child objects. - for(InstanceHandler handler : _instanceHandlers.values()) { - handler.close(); - } - _instanceHandlers.clear(); - } - -} diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/InstanceHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/InstanceHandler.java index 4568184..1afb1e8 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/InstanceHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/InstanceHandler.java @@ -18,6 +18,10 @@ import java.util.HashMap; import java.util.Map; +import com.ibm.json.java.JSON; +import com.ibm.json.java.JSONArray; +import com.ibm.json.java.JSONObject; + import javax.management.InstanceNotFoundException; import javax.management.JMX; import javax.management.ObjectName; @@ -27,9 +31,10 @@ import com.ibm.streams.management.job.JobMXBean; import com.ibm.streams.operator.Tuple; import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration; +import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration.OpType; /** - * Listen for the following domain notifications: + * Listen for the following notifications: * *
    *
  • com.ibm.streams.management.job.added @@ -53,36 +58,39 @@ public class InstanceHandler implements NotificationListener, Closeable { private OperatorConfiguration _operatorConfiguration = null; - private String _domainId = null; - private String _instanceId = null; private ObjectName _objName = null; private InstanceMXBean _instance = null; - private Map _jobHandlers = new HashMap<>(); + private Map _jobHandlers = new HashMap<>(); - public InstanceHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId) { + public InstanceHandler(OperatorConfiguration operatorConfiguration, String instanceId) { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("InstanceHandler(" + domainId + "," + instanceId + ")"); + _trace.debug("InstanceHandler(" + instanceId + ")"); } // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; - _objName = ObjectNameBuilder.instance(_domainId, _instanceId); + _objName = ObjectNameBuilder.instance(_instanceId); _instance = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), _objName, InstanceMXBean.class, true); /* * Register to get instance-related notifications. */ NotificationFilterSupport filter = new NotificationFilterSupport(); - filter.enableType(Notifications.JOB_ADDED); - filter.enableType(Notifications.JOB_REMOVED); + if (OpType.LOG_SOURCE == _operatorConfiguration.get_OperatorType()) { + filter.enableType(Notifications.LOG_APPLICATION_ERROR); + filter.enableType(Notifications.LOG_APPLICATION_WARNING); + } + else { + filter.enableType(Notifications.JOB_ADDED); + filter.enableType(Notifications.JOB_REMOVED); + } try { _operatorConfiguration.get_mbeanServerConnection().addNotificationListener(_objName, this, filter, null); } catch (InstanceNotFoundException e) { @@ -90,12 +98,23 @@ public InstanceHandler(OperatorConfiguration operatorConfiguration, String domai } catch (IOException e) { e.printStackTrace(); } -// TODO jmxc.addConnectionNotificationListener(this, null, null); // listen for potential lost notifications + if (null != _operatorConfiguration.get_tupleContainerConnectionNotification()) { + try { + if (_trace.isInfoEnabled()) { + _trace.info("JMXConnector.addConnectionNotificationListener()"); + } + // listen for potential lost notifications + operatorConfiguration.get_jmxConnector().addConnectionNotificationListener(this, null, null); + } catch (Exception e) { + e.printStackTrace(); + } + } + /* * Register existing jobs. */ - for(BigInteger jobId : _instance.getJobs()) { + for(String jobId : _instance.getJobs()) { addValidJob(jobId); } @@ -115,10 +134,11 @@ public void handleNotification(Notification notification, Object handback) { /* * Register existing jobs. */ - BigInteger jobId = (BigInteger)notification.getUserData(); + BigInteger jobIdLong = (BigInteger)notification.getUserData(); + String jobId = jobIdLong.toString(); if (null != _operatorConfiguration.get_tupleContainerJobStatusSource()) { - final Tuple tuple = _operatorConfiguration.get_tupleContainerJobStatusSource().getTuple(notification, handback, _domainId, _instanceId, jobId, null, null, null, null, null); + final Tuple tuple = _operatorConfiguration.get_tupleContainerJobStatusSource().getTuple(notification, handback, _instanceId, jobId, null, null, null, null, null); _operatorConfiguration.get_tupleContainerJobStatusSource().submit(tuple); } @@ -136,11 +156,12 @@ else if (notification.getType().equals(Notifications.JOB_REMOVED)) { /* * Unregister existing jobs. */ - BigInteger jobId = (BigInteger)notification.getUserData(); + BigInteger jobIdLong = (BigInteger)notification.getUserData(); + String jobId = jobIdLong.toString(); if (_jobHandlers.containsKey(jobId)) { if (null != _operatorConfiguration.get_tupleContainerJobStatusSource()) { - final Tuple tuple = _operatorConfiguration.get_tupleContainerJobStatusSource().getTuple(notification, handback, _domainId, _instanceId, jobId, null, null, null, null, null); + final Tuple tuple = _operatorConfiguration.get_tupleContainerJobStatusSource().getTuple(notification, handback, _instanceId, jobId, null, null, null, null, null); _operatorConfiguration.get_tupleContainerJobStatusSource().submit(tuple); } @@ -158,33 +179,106 @@ else if (isInfoEnabled) { } } else { - _trace.error("notification: " + notification + ", userData=" + notification.getUserData()); + if ((OpType.LOG_SOURCE == _operatorConfiguration.get_OperatorType()) && + (notification.getType().contains("com.ibm.streams.management.log.application"))) { + // emit tuple + try { + String instance = null; + String resource = null; + BigInteger pe = null; + BigInteger job = null; + String operator = null; + String text = null; + boolean messagesSkipped = false; + if (null != notification.getUserData()) { + if ((notification.getMessage() != null) && (notification.getMessage().length() > 0)) { + // javax.management.Notification[source=com.ibm.streams.management:type=domain,name="xxx"] + // [type=com.ibm.streams.management.log.application.error][message=xxx], + // userData={"instance":"xxx","resource":"xxx","pe":"1","domain":"xxx","job":"1","operator":"xxx"} + if (_trace.isDebugEnabled()) { + _trace.debug("parse userData"); + } + JSONObject obj = (JSONObject)JSON.parse(notification.getUserData().toString()); + instance = obj.get("instance").toString(); + resource = obj.get("resource").toString(); + pe = new BigInteger(obj.get("pe").toString()); + job = new BigInteger(obj.get("job").toString()); + operator = obj.get("operator").toString(); + } + else { + // if message is not set then the message is part of userData + // javax.management.Notification[source=com.ibm.streams.management:type=domain,name="xxx"] + // [type=com.ibm.streams.management.log.application.error][message=], + // userData={"messages":[{"instance":"xxx","resource":"xxx","pe":"xx","domain":"xxx","text":"xxx","job":"xx","operator":"xxx","timestamp":1515687171914}],"messagesSkipped":false} + if (_trace.isDebugEnabled()) { + _trace.debug("parse userData for messages and messagesSkipped"); + } + JSONObject obj1 = (JSONObject)JSON.parse(notification.getUserData().toString()); + messagesSkipped = (obj1.get("messagesSkipped").toString().equalsIgnoreCase("true")) ? true : false; + Object json = obj1.get("messages"); + if (json instanceof JSONArray) { + for (Object obj : (JSONArray)json) { + if (obj instanceof JSONObject) { + JSONObject jsonobj = (JSONObject) obj; + instance = jsonobj.get("instance").toString(); + resource = jsonobj.get("resource").toString(); + pe = new BigInteger(jsonobj.get("pe").toString()); + job = new BigInteger(jsonobj.get("job").toString()); + operator = jsonobj.get("operator").toString(); + text = jsonobj.get("text").toString(); + final Tuple tuple = _operatorConfiguration.get_tupleContainerLogSource().getTuple(notification, instance, resource, pe, job, operator, text, messagesSkipped); + _operatorConfiguration.get_tupleContainerLogSource().submit(tuple); + } + } + } + } + } + if (text == null) { + final Tuple tuple = _operatorConfiguration.get_tupleContainerLogSource().getTuple(notification, instance, resource, pe, job, operator, text, messagesSkipped); + _operatorConfiguration.get_tupleContainerLogSource().submit(tuple); + } + } + catch (Exception e) { + _trace.error("Error in parsing userData: " + e); + } + } + else { + if ((null != _operatorConfiguration.get_tupleContainerConnectionNotification()) && + (notification.getType().contains("jmx.remote.connection"))) { + // Notification emitted when a client connection has failed unexpectedly or when client connection has potentially lost notifications. + final Tuple tuple = _operatorConfiguration.get_tupleContainerConnectionNotification().getTuple(notification); + _operatorConfiguration.get_tupleContainerConnectionNotification().submit(tuple); + } + else { + _trace.error("notification: " + notification + ", userData=" + notification.getUserData()); + } + } } } - protected void addValidJob(BigInteger jobId) { + protected void addValidJob(String jobId) { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { _trace.debug("--> addValidJob(" + jobId + ")"); } // Registering the job must be done before attempting to access any of // the job-related beans. - _instance.registerJob(jobId); + _instance.registerJobById(jobId); // Special handling required because we do not have the job name easily accessible. - ObjectName jobObjName = ObjectNameBuilder.job(_domainId, _instanceId, jobId); + ObjectName jobObjName = ObjectNameBuilder.job(_instanceId, jobId); JobMXBean job = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), jobObjName, JobMXBean.class, true); String jobName = job.getName(); - boolean matches = _operatorConfiguration.get_filters().matchesJobName(_domainId, _instanceId, jobName); + boolean matches = _operatorConfiguration.get_filters().matchesJobName(_instanceId, jobName); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following job meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=" + jobName + ", jobId=" + jobId); + _trace.info("The following job meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=" + jobName + ", jobId=" + jobId); } else { - _trace.info("The following job does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=" + jobName + ", jobId=" + jobId); + _trace.info("The following job does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=" + jobName + ", jobId=" + jobId); } } if (matches) { - _jobHandlers.put(jobId, new JobHandler(_operatorConfiguration, _domainId, _instanceId, jobId)); + _jobHandlers.put(jobId, new JobHandler(_operatorConfiguration, _instanceId, jobId)); } if (isDebugEnabled) { _trace.debug("<-- addValidJob(" + jobId + ")"); @@ -198,14 +292,14 @@ protected void addValidJob(BigInteger jobId) { public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ",instance=" + _instanceId + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ")"); } _operatorConfiguration.get_tupleContainerMetricsSource().setInstanceId(_instanceId); - for(BigInteger jobId : _jobHandlers.keySet()) { + for(String jobId : _jobHandlers.keySet()) { _jobHandlers.get(jobId).captureMetrics(); } if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ",instance=" + _instanceId + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ")"); } } @@ -223,4 +317,14 @@ public void close() throws Exception { _jobHandlers.clear(); } + public void healthCheck() { + if (_trace.isDebugEnabled()) { + _trace.debug("healthCheck"); + } + com.ibm.streams.management.instance.InstanceMXBean.Status status = _instance.getStatus(); + if (_trace.isDebugEnabled()) { + _trace.debug("InstanceMXBean.Status="+status.toString()); + } + } + } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobHandler.java index 1d51c53..2d64d17 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobHandler.java @@ -38,7 +38,7 @@ import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration.OpType; /** - * Listen for the following domain notifications: + * Listen for the following notifications: * *
      *
    • com.ibm.streams.management.job.added @@ -62,13 +62,11 @@ public class JobHandler implements NotificationListener, Closeable { private OperatorConfiguration _operatorConfiguration = null; - private String _domainId = null; - private String _instanceId = null; private ObjectName _objName = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; @@ -76,21 +74,20 @@ public class JobHandler implements NotificationListener, Closeable { private Map _operatorHandlers = new HashMap<>(); - private Map _peHandlers = new HashMap<>(); + private Map _peHandlers = new HashMap<>(); - public JobHandler(OperatorConfiguration applicationConfiguration, String domainId, String instanceId, BigInteger jobId) { + public JobHandler(OperatorConfiguration applicationConfiguration, String instanceId, String jobId) { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("JobHandler(" + domainId + "," + instanceId + "," + jobId + ")"); + _trace.debug("JobHandler(" + instanceId + "," + jobId + ")"); } // Store parameters for later use. _operatorConfiguration = applicationConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; - _objName = ObjectNameBuilder.job(_domainId, _instanceId, _jobId); + _objName = ObjectNameBuilder.job(_instanceId, _jobId); _job = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), _objName, JobMXBean.class, true); _jobName = _job.getName(); @@ -120,7 +117,7 @@ public JobHandler(OperatorConfiguration applicationConfiguration, String domainI /* * Create handlers for operators that match the filter criteria. */ - for(BigInteger peId : _job.getPes()) { + for(String peId : _job.getPes()) { addPE(peId); } @@ -143,32 +140,32 @@ public void handleNotification(Notification notification, Object handback) { } protected void addValidOperator(String operatorName) { - boolean matches = _operatorConfiguration.get_filters().matchesOperatorName(_domainId, _instanceId, _jobName, operatorName); + boolean matches = _operatorConfiguration.get_filters().matchesOperatorName(_instanceId, _jobName, operatorName); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following operator meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + operatorName); + _trace.info("The following operator meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + operatorName); } else { - _trace.info("The following operator does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + operatorName); + _trace.info("The following operator does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + operatorName); } } if (matches) { - _operatorHandlers.put(operatorName, new OperatorHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, operatorName)); + _operatorHandlers.put(operatorName, new OperatorHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, operatorName)); } } - protected void addPE(BigInteger peId) { - boolean matches = _operatorConfiguration.get_filters().matchesPeId(_domainId, _instanceId, _jobName, peId); + protected void addPE(String peId) { + boolean matches = _operatorConfiguration.get_filters().matchesPeId(_instanceId, _jobName, peId); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following PE meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + peId); + _trace.info("The following PE meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + peId); } else { - _trace.info("The following PE does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + peId); + _trace.info("The following PE does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + peId); } } if (matches) { - _peHandlers.put(peId, new PeHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, peId)); + _peHandlers.put(peId, new PeHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, peId)); } } @@ -182,7 +179,7 @@ protected void addPE(BigInteger peId) { public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ",instance=" + _instanceId + ",jobId=" + _jobId + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ",jobId=" + _jobId + ")"); } MetricsTupleContainer schema = _operatorConfiguration.get_tupleContainerMetricsSource(); schema.setJobId(_jobId); @@ -190,66 +187,12 @@ public void captureMetrics() throws Exception { for(String operatorName : _operatorHandlers.keySet()) { _operatorHandlers.get(operatorName).captureMetrics(); } - for(BigInteger peId : _peHandlers.keySet()) { + for(String peId : _peHandlers.keySet()) { _peHandlers.get(peId).captureMetrics(); } if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ",instance=" + _instanceId + ",jobId=" + _jobId + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ",jobId=" + _jobId + ")"); } -// ----------------------------------------------------------------------------- -// SNAPSHOT METRICS CODE (BEGIN) -// ----------------------------------------------------------------------------- -// String uri = _job.snapshotMetrics(); -// try { -// _trace.error("[" + _domainId + "," + _instanceId + "," + _jobName + "] capture metrics from URI: " + uri); -// -// // TODO streamtool getdomainproperty jmx.sslOption -// String sslOption = "TLSv1"; -// // set up trust manager to validate certificate issuer -// TrustManager [] tm = new TrustManager[] {new JmxTrustManager()}; -// SSLContext ctxt = SSLContext.getInstance(sslOption); -// ctxt.init(null, tm, null); -// -// // set up host name verifier to trust the server from which the certificate was sent -// HostnameVerifier hv = new HostnameVerifier() { -// public boolean verify(String urlHostName, SSLSession session) { -// // return false to reject -// return true; -// } -// }; -// -// URL url = new URL(uri); -// HttpsURLConnection conn = (HttpsURLConnection)url.openConnection(); -// conn.setSSLSocketFactory(ctxt.getSocketFactory()); -// conn.setHostnameVerifier(hv); -// conn.setRequestMethod("GET"); -// conn.connect(); -// InputStream response = conn.getInputStream(); -// BufferedReader reader = new BufferedReader(new InputStreamReader(response)); -// String line; -// while ((line = reader.readLine()) != null) { -// System.out.println(line); -// } -// reader.close(); -// response.close(); -// conn.disconnect(); -// -// if (_jobIdAttributeIndex != -1) { -// tuple.setLong(_jobIdAttributeIndex, _jobId.longValue()); -// } -// if (_jobNameAttributeIndex != -1) { -// tuple.setString(_jobNameAttributeIndex, _jobName); -// } -// // Submit tuple to output stream -// port.submit(tuple); -// -// } -// catch(Exception e) { -// e.printStackTrace(); -// } - // ----------------------------------------------------------------------------- - // SNAPSHOT METRICS CODE (END) - // ----------------------------------------------------------------------------- } /** diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobStatusTupleContainer.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobStatusTupleContainer.java index 9f97175..65a8f7a 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobStatusTupleContainer.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/JobStatusTupleContainer.java @@ -33,11 +33,6 @@ public class JobStatusTupleContainer { */ private StreamingOutput _port = null; - /** - * Index of the domainId attribute. - */ - private Integer _domainIdAttributeIndex = null; - /** * Index of the instanceId attribute. */ @@ -120,11 +115,6 @@ public JobStatusTupleContainer(OperatorContext context, StreamingOutput getAttributes( final Notification notification, final Object handback, - final String domainId, final String instanceId, - final BigInteger jobId, + final String jobId, final String jobName, final String resource, - final BigInteger peId, + final String peId, final Object peHealth, final Object peStatus) { final Map attributes = new HashMap(); - if (_domainIdAttributeIndex != -1) { - if (domainId != null) { - attributes.put("domainId", new RString(domainId)); - } - } if (_instanceIdAttributeIndex != -1) { if (instanceId != null) { attributes.put("instanceId", new RString(instanceId)); @@ -254,7 +236,7 @@ protected Map getAttributes( } if (_jobIdAttributeIndex != -1) { if (jobId != null) { - attributes.put("jobId", jobId.longValue()); + attributes.put("jobId", Long.parseLong(jobId)); } } if (_jobNameAttributeIndex != -1) { @@ -269,7 +251,7 @@ protected Map getAttributes( } if (_peIdAttributeIndex != -1) { if (peId != null) { - attributes.put("peId", peId.longValue()); + attributes.put("peId", Long.parseLong(peId)); } } if (_peHealthAttributeIndex != -1) { diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/LogTupleContainer.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/LogTupleContainer.java index 5459b3f..a049f4d 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/LogTupleContainer.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/LogTupleContainer.java @@ -32,11 +32,7 @@ public class LogTupleContainer { */ private StreamingOutput _port = null; - /** - * Index of the domainId attribute. - */ - private Integer _domainIdAttributeIndex = null; - + /** * Index of the instanceId attribute. */ @@ -99,11 +95,6 @@ public LogTupleContainer(OperatorContext context, StreamingOutput p _port = port; StreamSchema schema = port.getStreamSchema(); - // Domain-related attributes. - if (_domainIdAttributeIndex == null) { - Attribute attribute = schema.getAttribute("domainId"); - _domainIdAttributeIndex = Integer.valueOf(attribute != null && attribute.getType().getMetaType() == Type.MetaType.RSTRING ? attribute.getIndex() : -1); - } // Instance-related attributes. if (_instanceIdAttributeIndex == null) { Attribute attribute = schema.getAttribute("instanceId"); @@ -163,7 +154,6 @@ public void punctuate(Punctuation mark) throws Exception { */ public Tuple getTuple( final Notification notification, - final String domainId, final String instance, final String resource, final BigInteger pe, @@ -173,7 +163,6 @@ public Tuple getTuple( final boolean messagesSkipped) { return _port.getStreamSchema().getTuple(getAttributes( notification, - domainId, instance, resource, pe, @@ -192,7 +181,6 @@ public Tuple getTuple( */ protected Map getAttributes( final Notification notification, - final String domainId, final String instanceId, final String resource, final BigInteger peId, @@ -204,11 +192,6 @@ protected Map getAttributes( final Map attributes = new HashMap(); - if (_domainIdAttributeIndex != -1) { - if (domainId != null) { - attributes.put("domainId", new RString(domainId)); - } - } if (_instanceIdAttributeIndex != -1) { if (instanceId != null) { attributes.put("instanceId", new RString(instanceId)); diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricOwningHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricOwningHandler.java index f309651..b948dc9 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricOwningHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricOwningHandler.java @@ -86,7 +86,7 @@ protected MetricOwningHandler(MetricsRegistrationMode metricsRegistrationMode) { /** * Set the metric-relevant tuple attributes and submit the tuple. - * All other domain-, instance-, job-, operator-, port-relevant + * All other instance-, job-, operator-, port-relevant * tuple attributes are already set. * * @param metric @@ -100,7 +100,7 @@ protected void submitMetric(Metric metric) throws Exception { tc.setMetricType(metric.getMetricType()); tc.setMetricKind(metric.getMetricKind()); tc.setMetricName(metric.getName()); - tc.setMetricValue(metric.getValue()); + tc.setMetricValue(metric.getValueAsLong()); tc.setLastTimeRetrieved(metric.getLastTimeRetrieved()); tc.submit(); } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricsTupleContainer.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricsTupleContainer.java index 77f6e97..54ffd3a 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricsTupleContainer.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/MetricsTupleContainer.java @@ -19,10 +19,6 @@ public class MetricsTupleContainer { - /** - * Index of the domainId attribute. - */ - private Integer _domainIdAttributeIndex = null; /** * Index of the instanceId attribute. @@ -119,11 +115,6 @@ public MetricsTupleContainer(StreamingOutput port) { _port = port; _tuple = port.newTuple(); StreamSchema schema = port.getStreamSchema(); - // Domain-related attributes. - if (_domainIdAttributeIndex == null) { - Attribute attribute = schema.getAttribute("domainId"); - _domainIdAttributeIndex = Integer.valueOf(attribute != null && attribute.getType().getMetaType() == Type.MetaType.RSTRING ? attribute.getIndex() : -1) ; - } // Instance-related attributes. if (_instanceIdAttributeIndex == null) { Attribute attribute = schema.getAttribute("instanceId"); @@ -194,16 +185,6 @@ public MetricsTupleContainer(StreamingOutput port) { } } - /** - * Optionally set the domain id in the output tuple. - * - * @param domainId - */ - public void setDomainId(String domainId) { - if (_domainIdAttributeIndex != -1) { - _tuple.setString(_domainIdAttributeIndex, domainId); - } - } /** * Optionally set the instance id in the output tuple. @@ -226,6 +207,11 @@ public void setJobId(BigInteger jobId) { _tuple.setLong(_jobIdAttributeIndex, jobId.longValue()); } } + public void setJobId(String jobId) { + if (_jobIdAttributeIndex != -1) { + _tuple.setLong(_jobIdAttributeIndex, Long.parseLong(jobId)); + } + } /** * Optionally set the job name in the output tuple. @@ -315,6 +301,11 @@ public void setPeId(BigInteger peId) { _tuple.setLong(_peIdAttributeIndex, peId.longValue()); } } + public void setPeId(String peId) { + if (_peIdAttributeIndex != -1) { + _tuple.setLong(_peIdAttributeIndex, Long.parseLong(peId)); + } + } /** * Optionally set the metric type in the output tuple. diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorHandler.java index b0029e3..6b1269b 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorHandler.java @@ -38,11 +38,9 @@ public class OperatorHandler extends MetricOwningHandler implements Notification */ private static Logger _trace = Logger.getLogger(OperatorHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; @@ -56,22 +54,21 @@ public class OperatorHandler extends MetricOwningHandler implements Notification private Map _outputPortHandlers = new HashMap<>(); - public OperatorHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, String operatorName) { + public OperatorHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String operatorName) { super(MetricsRegistrationMode.DynamicMetricsRegistration); if (_trace.isDebugEnabled()) { - _trace.debug("OperatorHandler(" + domainId + "," + instanceId + ")"); + _trace.debug("OperatorHandler(" + instanceId + ")"); } // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; _operatorName = operatorName; - _objName = ObjectNameBuilder.operator(_domainId, _instanceId, _jobId, _operatorName); + _objName = ObjectNameBuilder.operator(_instanceId, _jobId, _operatorName); _operator = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), _objName, OperatorMXBean.class, true); /* @@ -141,13 +138,13 @@ else if (notification.getType().equals(Notifications.OPERATOR_CONNECTION_REMOVED @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesOperatorMetricName(_domainId, _instanceId, _jobName, _operatorName, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesOperatorMetricName(_instanceId, _jobName, _operatorName, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following operator custom metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", metric=" + metricName); + _trace.info("The following operator custom metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", metric=" + metricName); } else { - _trace.info("The following operator custom metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", metric=" + metricName); + _trace.info("The following operator custom metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", metric=" + metricName); } } return isRelevant; @@ -160,32 +157,32 @@ protected Set retrieveMetrics() { } protected void addValidInputPort(Integer portIndex) { - boolean matches = _operatorConfiguration.get_filters().matchesOperatorInputPortIndex(_domainId, _instanceId, _jobName, _operatorName, portIndex); + boolean matches = _operatorConfiguration.get_filters().matchesOperatorInputPortIndex(_instanceId, _jobName, _operatorName, portIndex); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following input port meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); + _trace.info("The following input port meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); } else { - _trace.info("The following input port does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); + _trace.info("The following input port does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); } } if (matches) { - _inputPortHandlers.put(portIndex, new OperatorInputPortHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, _operatorName, portIndex)); + _inputPortHandlers.put(portIndex, new OperatorInputPortHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, _operatorName, portIndex)); } } protected void addValidOutputPort(Integer portIndex) { - boolean matches = _operatorConfiguration.get_filters().matchesOperatorOutputPortIndex(_domainId, _instanceId, _jobName, _operatorName, portIndex); + boolean matches = _operatorConfiguration.get_filters().matchesOperatorOutputPortIndex(_instanceId, _jobName, _operatorName, portIndex); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following output port meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); + _trace.info("The following output port meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); } else { - _trace.info("The following output port does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); + _trace.info("The following output port does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + portIndex); } } if (matches) { - _outputPortHandlers.put(portIndex, new OperatorOutputPortHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, _operatorName, portIndex)); + _outputPortHandlers.put(portIndex, new OperatorOutputPortHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, _operatorName, portIndex)); } } @@ -201,7 +198,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); @@ -225,7 +222,7 @@ public void captureMetrics() throws Exception { } if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorInputPortHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorInputPortHandler.java index 654c02d..99d7d6c 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorInputPortHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorInputPortHandler.java @@ -30,11 +30,9 @@ public class OperatorInputPortHandler extends MetricOwningHandler implements Clo */ private static Logger _trace = Logger.getLogger(OperatorInputPortHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; @@ -44,7 +42,7 @@ public class OperatorInputPortHandler extends MetricOwningHandler implements Clo private OperatorInputPortMXBean _port = null; - public OperatorInputPortHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, String operatorName, Integer portIndex) { + public OperatorInputPortHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String operatorName, Integer portIndex) { super(MetricsRegistrationMode.DynamicMetricsRegistration); @@ -53,7 +51,6 @@ public OperatorInputPortHandler(OperatorConfiguration operatorConfiguration, Str // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; @@ -61,10 +58,10 @@ public OperatorInputPortHandler(OperatorConfiguration operatorConfiguration, Str _portIndex = portIndex; if (isDebugEnabled) { - _trace.debug("--> InputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("--> InputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } - ObjectName operatorObjName = ObjectNameBuilder.operatorInputPort(_domainId, _instanceId, _jobId, _operatorName, _portIndex); + ObjectName operatorObjName = ObjectNameBuilder.operatorInputPort(_instanceId, _jobId, _operatorName, _portIndex); _port = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), operatorObjName, OperatorInputPortMXBean.class, true); /* @@ -73,19 +70,19 @@ public OperatorInputPortHandler(OperatorConfiguration operatorConfiguration, Str registerMetrics(); if (isDebugEnabled) { - _trace.debug("<-- InputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("<-- InputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } } @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesOperatorInputPortMetricName(_domainId, _instanceId, _jobName, _operatorName, _portIndex, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesOperatorInputPortMetricName(_instanceId, _jobName, _operatorName, _portIndex, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following input port metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following input port metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); } else { - _trace.info("The following input port metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following input port metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); } } return isRelevant; @@ -107,7 +104,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); tc.setOrigin("OperatorInputPort"); @@ -115,7 +112,7 @@ public void captureMetrics() throws Exception { captureAndSubmitChangedMetrics(); if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorOutputPortHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorOutputPortHandler.java index e29bb3b..9858470 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorOutputPortHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/OperatorOutputPortHandler.java @@ -30,11 +30,9 @@ public class OperatorOutputPortHandler extends MetricOwningHandler implements Cl */ private static Logger _trace = Logger.getLogger(OperatorOutputPortHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; @@ -44,7 +42,7 @@ public class OperatorOutputPortHandler extends MetricOwningHandler implements Cl private OperatorOutputPortMXBean _port = null; - public OperatorOutputPortHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, String operatorName, Integer portIndex) { + public OperatorOutputPortHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String operatorName, Integer portIndex) { super(MetricsRegistrationMode.DynamicMetricsRegistration); @@ -53,7 +51,6 @@ public OperatorOutputPortHandler(OperatorConfiguration operatorConfiguration, St // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; @@ -61,10 +58,10 @@ public OperatorOutputPortHandler(OperatorConfiguration operatorConfiguration, St _portIndex = portIndex; if (isDebugEnabled) { - _trace.debug("--> OutputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("--> OutputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } - ObjectName operatorObjName = ObjectNameBuilder.operatorOutputPort(_domainId, _instanceId, _jobId, _operatorName, _portIndex); + ObjectName operatorObjName = ObjectNameBuilder.operatorOutputPort(_instanceId, _jobId, _operatorName, _portIndex); _port = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), operatorObjName, OperatorOutputPortMXBean.class, true); /* @@ -73,19 +70,19 @@ public OperatorOutputPortHandler(OperatorConfiguration operatorConfiguration, St registerMetrics(); if (isDebugEnabled) { - _trace.debug("<-- OutputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("<-- OutputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } } @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesOperatorOutputPortMetricName(_domainId, _instanceId, _jobName, _operatorName, _portIndex, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesOperatorOutputPortMetricName(_instanceId, _jobName, _operatorName, _portIndex, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following output port metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following output port metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); } else { - _trace.info("The following output port metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following output port metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], operator=" + _operatorName + ", port=" + _portIndex + ", metric=" + metricName); } } return isRelevant; @@ -107,7 +104,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); tc.setOrigin("OperatorOutputPort"); @@ -115,7 +112,7 @@ public void captureMetrics() throws Exception { captureAndSubmitChangedMetrics(); if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", operator=" + _operatorName + ", port=" + _portIndex + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeConnectionHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeConnectionHandler.java index a6be01a..7ae0944 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeConnectionHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeConnectionHandler.java @@ -30,21 +30,19 @@ public class PeConnectionHandler extends MetricOwningHandler implements Closeabl */ private static Logger _trace = Logger.getLogger(PeConnectionHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; - private BigInteger _peId = null; + private String _peId = null; private String _connectionId = null; private PeConnectionMXBean _connection = null; - public PeConnectionHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, BigInteger peId, String connectionId) { + public PeConnectionHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String peId, String connectionId) { super(MetricsRegistrationMode.DynamicMetricsRegistration); @@ -53,7 +51,6 @@ public PeConnectionHandler(OperatorConfiguration operatorConfiguration, String d // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; @@ -61,10 +58,10 @@ public PeConnectionHandler(OperatorConfiguration operatorConfiguration, String d _connectionId = connectionId; if (isDebugEnabled) { - _trace.debug("--> ConnectionHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); + _trace.debug("--> ConnectionHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); } - ObjectName objName = ObjectNameBuilder.peConnection(_domainId, _instanceId, _connectionId); + ObjectName objName = ObjectNameBuilder.peConnection(_instanceId, _connectionId); _connection = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), objName, PeConnectionMXBean.class, true); /* @@ -73,19 +70,19 @@ public PeConnectionHandler(OperatorConfiguration operatorConfiguration, String d registerMetrics(); if (isDebugEnabled) { - _trace.debug("<-- PeConnectionHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); + _trace.debug("<-- PeConnectionHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); } } @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesPeConnectionMetricName(_domainId, _instanceId, _jobName, _peId, _connectionId, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesPeConnectionMetricName(_instanceId, _jobName, _peId, _connectionId, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following input port metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connectionId=" + _connectionId + ", metric=" + metricName); + _trace.info("The following input port metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connectionId=" + _connectionId + ", metric=" + metricName); } else { - _trace.info("The following input port metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connetionId=" + _connectionId + ", metric=" + metricName); + _trace.info("The following input port metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connetionId=" + _connectionId + ", metric=" + metricName); } } return isRelevant; @@ -107,7 +104,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); tc.setOrigin("PeConnection"); @@ -115,7 +112,7 @@ public void captureMetrics() throws Exception { captureAndSubmitChangedMetrics(); if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", connectionId=" + _connectionId + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeHandler.java index 4165f15..5647693 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeHandler.java @@ -40,17 +40,15 @@ public class PeHandler extends MetricOwningHandler implements NotificationListen */ private static Logger _trace = Logger.getLogger(PeHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; private ObjectName _objName = null; - private BigInteger _peId = null; + private String _peId = null; private PeMXBean _pe = null; @@ -60,21 +58,20 @@ public class PeHandler extends MetricOwningHandler implements NotificationListen private Map _connectionHandlers = new HashMap<>(); - public PeHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, BigInteger peId) { + public PeHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String peId) { super(MetricsRegistrationMode.DynamicMetricsRegistration); if (_trace.isDebugEnabled()) { - _trace.debug("PeHandler(" + domainId + "," + instanceId + ")"); + _trace.debug("PeHandler(" + instanceId + ")"); } // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; _peId = peId; - _objName = ObjectNameBuilder.pe(_domainId, _instanceId, _peId); + _objName = ObjectNameBuilder.pe(_instanceId, _peId); _pe = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), _objName, PeMXBean.class, true); /* @@ -131,20 +128,20 @@ public void handleNotification(Notification notification, Object handback) { _trace.debug("notification: " + notification + ", userData=" + notification.getUserData()); } if (OpType.JOB_STATUS_SOURCE == _operatorConfiguration.get_OperatorType()) { - final Tuple tuple = _operatorConfiguration.get_tupleContainerJobStatusSource().getTuple(notification, handback, _domainId, _instanceId, _jobId, _jobName, _pe.getResource(), _peId, _pe.getHealth(), _pe.getStatus()); + final Tuple tuple = _operatorConfiguration.get_tupleContainerJobStatusSource().getTuple(notification, handback, _instanceId, _jobId, _jobName, _pe.getResource(), _peId, _pe.getHealth(), _pe.getStatus()); _operatorConfiguration.get_tupleContainerJobStatusSource().submit(tuple); } } @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesPeMetricName(_domainId, _instanceId, _jobName, _peId, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesPeMetricName(_instanceId, _jobName, _peId, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following pe metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", metric=" + metricName); + _trace.info("The following pe metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", metric=" + metricName); } else { - _trace.info("The following pe metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", metric=" + metricName); + _trace.info("The following pe metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", metric=" + metricName); } } return isRelevant; @@ -157,47 +154,47 @@ protected Set retrieveMetrics() { } protected void addValidInputPort(Integer portIndex) { - boolean matches = _operatorConfiguration.get_filters().matchesPeInputPortIndex(_domainId, _instanceId, _jobName, _peId, portIndex); + boolean matches = _operatorConfiguration.get_filters().matchesPeInputPortIndex(_instanceId, _jobName, _peId, portIndex); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following input port meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); + _trace.info("The following input port meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); } else { - _trace.info("The following input port does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); + _trace.info("The following input port does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); } } if (matches) { - _inputPortHandlers.put(portIndex, new PeInputPortHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, _peId, portIndex)); + _inputPortHandlers.put(portIndex, new PeInputPortHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, _peId, portIndex)); } } protected void addValidOutputPort(Integer portIndex) { - boolean matches = _operatorConfiguration.get_filters().matchesPeOutputPortIndex(_domainId, _instanceId, _jobName, _peId, portIndex); + boolean matches = _operatorConfiguration.get_filters().matchesPeOutputPortIndex(_instanceId, _jobName, _peId, portIndex); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following output port meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); + _trace.info("The following output port meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); } else { - _trace.info("The following output port does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); + _trace.info("The following output port does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + portIndex); } } if (matches) { - _outputPortHandlers.put(portIndex, new PeOutputPortHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, _peId, portIndex)); + _outputPortHandlers.put(portIndex, new PeOutputPortHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, _peId, portIndex)); } } protected void addValidConnection(String connectionId) { - boolean matches = _operatorConfiguration.get_filters().matchesPeConnectionId(_domainId, _instanceId, _jobName, _peId, connectionId); + boolean matches = _operatorConfiguration.get_filters().matchesPeConnectionId(_instanceId, _jobName, _peId, connectionId); if (_trace.isInfoEnabled()) { if (matches) { - _trace.info("The following output port meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connection=" + connectionId); + _trace.info("The following output port meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connection=" + connectionId); } else { - _trace.info("The following output port does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connection=" + connectionId); + _trace.info("The following output port does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", connection=" + connectionId); } } if (matches) { - _connectionHandlers.put(connectionId, new PeConnectionHandler(_operatorConfiguration, _domainId, _instanceId, _jobId, _jobName, _peId, connectionId)); + _connectionHandlers.put(connectionId, new PeConnectionHandler(_operatorConfiguration, _instanceId, _jobId, _jobName, _peId, connectionId)); } } @@ -212,7 +209,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); tc.setOperatorName(""); @@ -238,7 +235,7 @@ public void captureMetrics() throws Exception { } if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeInputPortHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeInputPortHandler.java index 597290b..2b0133f 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeInputPortHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeInputPortHandler.java @@ -30,21 +30,19 @@ public class PeInputPortHandler extends MetricOwningHandler implements Closeable */ private static Logger _trace = Logger.getLogger(PeInputPortHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; - private BigInteger _peId = null; + private String _peId = null; private Integer _portIndex = null; private PeInputPortMXBean _port = null; - public PeInputPortHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, BigInteger peId, Integer portIndex) { + public PeInputPortHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String peId, Integer portIndex) { super(MetricsRegistrationMode.DynamicMetricsRegistration); @@ -53,7 +51,6 @@ public PeInputPortHandler(OperatorConfiguration operatorConfiguration, String do // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; @@ -61,10 +58,10 @@ public PeInputPortHandler(OperatorConfiguration operatorConfiguration, String do _portIndex = portIndex; if (isDebugEnabled) { - _trace.debug("--> InputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("--> InputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } - ObjectName objName = ObjectNameBuilder.peInputPort(_domainId, _instanceId, _peId, _portIndex); + ObjectName objName = ObjectNameBuilder.peInputPort(_instanceId, _peId, _portIndex); _port = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), objName, PeInputPortMXBean.class, true); /* @@ -73,19 +70,19 @@ public PeInputPortHandler(OperatorConfiguration operatorConfiguration, String do registerMetrics(); if (isDebugEnabled) { - _trace.debug("<-- InputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("<-- InputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } } @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesPeInputPortMetricName(_domainId, _instanceId, _jobName, _peId, _portIndex, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesPeInputPortMetricName(_instanceId, _jobName, _peId, _portIndex, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following input port metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following input port metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); } else { - _trace.info("The following input port metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following input port metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); } } return isRelevant; @@ -107,7 +104,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); tc.setOrigin("PeInputPort"); @@ -115,7 +112,7 @@ public void captureMetrics() throws Exception { captureAndSubmitChangedMetrics(); if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeOutputPortHandler.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeOutputPortHandler.java index 4876d63..48a4f8a 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeOutputPortHandler.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/PeOutputPortHandler.java @@ -30,21 +30,19 @@ public class PeOutputPortHandler extends MetricOwningHandler implements Closeabl */ private static Logger _trace = Logger.getLogger(PeOutputPortHandler.class.getName()); - private String _domainId = null; - private String _instanceId = null; - private BigInteger _jobId = null; + private String _jobId = null; private String _jobName = null; - private BigInteger _peId = null; + private String _peId = null; private Integer _portIndex = null; private PeOutputPortMXBean _port = null; - public PeOutputPortHandler(OperatorConfiguration operatorConfiguration, String domainId, String instanceId, BigInteger jobId, String jobName, BigInteger peId, Integer portIndex) { + public PeOutputPortHandler(OperatorConfiguration operatorConfiguration, String instanceId, String jobId, String jobName, String peId, Integer portIndex) { super(MetricsRegistrationMode.DynamicMetricsRegistration); @@ -53,7 +51,6 @@ public PeOutputPortHandler(OperatorConfiguration operatorConfiguration, String d // Store parameters for later use. _operatorConfiguration = operatorConfiguration; - _domainId = domainId; _instanceId = instanceId; _jobId = jobId; _jobName = jobName; @@ -61,10 +58,10 @@ public PeOutputPortHandler(OperatorConfiguration operatorConfiguration, String d _portIndex = portIndex; if (isDebugEnabled) { - _trace.debug("--> OutputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("--> OutputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } - ObjectName objName = ObjectNameBuilder.peOutputPort(_domainId, _instanceId, _peId, _portIndex); + ObjectName objName = ObjectNameBuilder.peOutputPort(_instanceId, _peId, _portIndex); _port = JMX.newMXBeanProxy(_operatorConfiguration.get_mbeanServerConnection(), objName, PeOutputPortMXBean.class, true); /* @@ -73,19 +70,19 @@ public PeOutputPortHandler(OperatorConfiguration operatorConfiguration, String d registerMetrics(); if (isDebugEnabled) { - _trace.debug("<-- OutputPortHandler(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("<-- OutputPortHandler(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } } @Override protected boolean isRelevantMetric(String metricName) { - boolean isRelevant = _operatorConfiguration.get_filters().matchesPeOutputPortMetricName(_domainId, _instanceId, _jobName, _peId, _portIndex, metricName); + boolean isRelevant = _operatorConfiguration.get_filters().matchesPeOutputPortMetricName(_instanceId, _jobName, _peId, _portIndex, metricName); if (_trace.isInfoEnabled()) { if (isRelevant) { - _trace.info("The following output port metric meets the filter criteria and is therefore, monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following output port metric meets the filter criteria and is therefore, monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); } else { - _trace.info("The following output port metric does not meet the filter criteria and is therefore, not monitored: domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); + _trace.info("The following output port metric does not meet the filter criteria and is therefore, not monitored: instance=" + _instanceId + ", job=[" + _jobId + "][" + _jobName + "], peId=" + _peId + ", port=" + _portIndex + ", metric=" + metricName); } } return isRelevant; @@ -107,7 +104,7 @@ public void captureMetrics() throws Exception { boolean isDebugEnabled = _trace.isDebugEnabled(); if (isDebugEnabled) { - _trace.debug("--> captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("--> captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } MetricsTupleContainer tc = _operatorConfiguration.get_tupleContainerMetricsSource(); tc.setOrigin("PeOutputPort"); @@ -115,7 +112,7 @@ public void captureMetrics() throws Exception { captureAndSubmitChangedMetrics(); if (isDebugEnabled) { - _trace.debug("<-- captureMetrics(domain=" + _domainId + ", instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); + _trace.debug("<-- captureMetrics(instance=" + _instanceId + ", job=[" + _jobId + "]:" + _jobName + ", peId=" + _peId + ", port=" + _portIndex + ")"); } } diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/DomainFilter.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/DomainFilter.java deleted file mode 100644 index 5cdb4a0..0000000 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/DomainFilter.java +++ /dev/null @@ -1,250 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2016, 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -package com.ibm.streamsx.monitoring.jmx.internal.filters; - -import java.math.BigInteger; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.regex.PatternSyntaxException; - -import org.apache.log4j.Logger; - -final class DomainFilter extends PatternMatcher { - - /** - * Logger for tracing. - */ - private static Logger _trace = Logger.getLogger(DomainFilter.class.getName()); - - /** - * A domain has many instances. - */ - protected Map _instanceFilters = new HashMap<>(); - - public DomainFilter(String regularExpression, Set filters) throws PatternSyntaxException { - super(regularExpression); - for(InstanceFilter instanceFilter : filters) { - _instanceFilters.put(instanceFilter.getRegularExpression(), instanceFilter); - } - } - - public boolean matchesDomainId(String domainId) { - boolean matches = matches(domainId); - return matches; - } - - public boolean matchesInstanceId(String domainId, String instanceId) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesInstanceId(instanceId); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesJobName(String domainId, String instanceId, String jobName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesJobName(instanceId, jobName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesOperatorName(String domainId, String instanceId, String jobName, String operatorName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesOperatorName(instanceId, jobName, operatorName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesOperatorMetricName(String domainId, String instanceId, String jobName, String operatorName, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesOperatorMetricName(instanceId, jobName, operatorName, metricName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesOperatorInputPortIndex(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesOperatorInputPortIndex(instanceId, jobName, operatorName, portIndex); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesOperatorInputPortMetricName(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesOperatorInputPortMetricName(instanceId, jobName, operatorName, portIndex, metricName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesOperatorOutputPortIndex(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesOperatorOutputPortIndex(instanceId, jobName, operatorName, portIndex); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesOperatorOutputPortMetricName(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesOperatorOutputPortMetricName(instanceId, jobName, operatorName, portIndex, metricName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeId(String domainId, String instanceId, String jobName, BigInteger peId) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeId(instanceId, jobName, peId); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeMetricName(String domainId, String instanceId, String jobName, BigInteger peId, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeMetricName(instanceId, jobName, peId, metricName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeInputPortIndex(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeInputPortIndex(instanceId, jobName, peId, portIndex); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeInputPortMetricName(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeInputPortMetricName(instanceId, jobName, peId, portIndex, metricName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeOutputPortIndex(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeOutputPortIndex(instanceId, jobName, peId, portIndex); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeOutputPortMetricName(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeOutputPortMetricName(instanceId, jobName, peId, portIndex, metricName); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeConnectionId(String domainId, String instanceId, String jobName, BigInteger peId, String connectionId) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeConnectionId(instanceId, jobName, peId, connectionId); - if (matches) { - break; - } - } - } - return matches; - } - - public boolean matchesPeConnectionMetricName(String domainId, String instanceId, String jobName, BigInteger peId, String connectionId, String metricName) { - boolean matches = matchesDomainId(domainId) && (_instanceFilters.size() > 0); - if (matches) { - for(InstanceFilter filter : _instanceFilters.values()) { - matches = filter.matchesPeConnectionMetricName(instanceId, jobName, peId, connectionId, metricName); - if (matches) { - break; - } - } - } - return matches; - } - -} diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/DomainParser.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/DomainParser.java deleted file mode 100644 index 8d08a4f..0000000 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/DomainParser.java +++ /dev/null @@ -1,81 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2016, 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -package com.ibm.streamsx.monitoring.jmx.internal.filters; - -import java.util.HashSet; -import java.util.Set; - -import org.apache.log4j.Logger; - -import com.ibm.json.java.JSONArtifact; -import com.ibm.json.java.JSONObject; -import com.ibm.streamsx.monitoring.jmx.OperatorConfiguration.OpType; - -public class DomainParser extends AbstractParser { - - private static Logger _logger = Logger.getLogger(DomainParser.class.getName()); - - private static final String DOMAIN_ID_PATTERNS = "domainIdPatterns"; - - private static final String INSTANCES = "instances"; - - private InstanceParser _instanceParser; - - protected DomainParser(OpType aType) { - _instanceParser = new InstanceParser(aType); - - setMandatoryItem(DOMAIN_ID_PATTERNS); - setMandatoryItem(INSTANCES); - - setValidationRule(DOMAIN_ID_PATTERNS, new IValidator() { - - @Override - public boolean validate(String key, Object object) { - return verifyPatterns(key, object); - } - - }); - - setValidationRule(INSTANCES, new IValidator() { - - @Override - public boolean validate(String key, Object object) { - boolean result = true; - if (object instanceof JSONArtifact) { - result = _instanceParser.validate((JSONArtifact)object); - } - else { - result = false; - logger().error("filterDocument: The parsed object must be a JSONArtifact. Details: key=" + key + ", object=" + object); - } - return result; - } - - }); - } - - @Override - protected Logger logger() { - return _logger; - } - - @SuppressWarnings("unchecked") - @Override - protected Set buildFilters(JSONObject json) { -// logger().error("Domain.JSON=" + json); - Set patterns = buildPatternList(json.get(DOMAIN_ID_PATTERNS)); - Set filters = _instanceParser.buildFilters((JSONArtifact)json.get(INSTANCES)); - Set result = new HashSet<>(); - for (String pattern : patterns) { -// logger().error("create domain filter, pattern=" + pattern); - result.add(new DomainFilter(pattern, filters)); - } - return result; - } - -} diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/Filters.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/Filters.java index d4cbe15..9e2ea75 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/Filters.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/Filters.java @@ -10,7 +10,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; @@ -29,15 +28,15 @@ public class Filters { /** * */ - protected Map _domainFilters = new HashMap<>(); + protected Map _instanceFilters = new HashMap<>(); public Filters() { } - public boolean matchesDomainId(String domainId) { + public boolean matchesInstanceId(String instanceId) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesDomainId(domainId); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesInstanceId(instanceId); if (matches) { break; } @@ -45,10 +44,10 @@ public boolean matchesDomainId(String domainId) { return matches; } - public boolean matchesInstanceId(String domainId, String instanceId) { + public boolean matchesJobName(String instanceId, String jobName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesInstanceId(domainId, instanceId); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesJobName(instanceId, jobName); if (matches) { break; } @@ -56,10 +55,10 @@ public boolean matchesInstanceId(String domainId, String instanceId) { return matches; } - public boolean matchesJobName(String domainId, String instanceId, String jobName) { + public boolean matchesOperatorName(String instanceId, String jobName, String operatorName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesJobName(domainId, instanceId, jobName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesOperatorName(instanceId, jobName, operatorName); if (matches) { break; } @@ -67,10 +66,10 @@ public boolean matchesJobName(String domainId, String instanceId, String jobName return matches; } - public boolean matchesOperatorName(String domainId, String instanceId, String jobName, String operatorName) { + public boolean matchesOperatorMetricName(String instanceId, String jobName, String operatorName, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesOperatorName(domainId, instanceId, jobName, operatorName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesOperatorMetricName(instanceId, jobName, operatorName, metricName); if (matches) { break; } @@ -78,10 +77,10 @@ public boolean matchesOperatorName(String domainId, String instanceId, String jo return matches; } - public boolean matchesOperatorMetricName(String domainId, String instanceId, String jobName, String operatorName, String metricName) { + public boolean matchesOperatorInputPortIndex(String instanceId, String jobName, String operatorName, Integer portIndex) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesOperatorMetricName(domainId, instanceId, jobName, operatorName, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesOperatorInputPortIndex(instanceId, jobName, operatorName, portIndex); if (matches) { break; } @@ -89,10 +88,10 @@ public boolean matchesOperatorMetricName(String domainId, String instanceId, Str return matches; } - public boolean matchesOperatorInputPortIndex(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex) { + public boolean matchesOperatorInputPortMetricName(String instanceId, String jobName, String operatorName, Integer portIndex, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesOperatorInputPortIndex(domainId, instanceId, jobName, operatorName, portIndex); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesOperatorInputPortMetricName(instanceId, jobName, operatorName, portIndex, metricName); if (matches) { break; } @@ -100,10 +99,10 @@ public boolean matchesOperatorInputPortIndex(String domainId, String instanceId, return matches; } - public boolean matchesOperatorInputPortMetricName(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex, String metricName) { + public boolean matchesOperatorOutputPortIndex(String instanceId, String jobName, String operatorName, Integer portIndex) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesOperatorInputPortMetricName(domainId, instanceId, jobName, operatorName, portIndex, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesOperatorOutputPortIndex(instanceId, jobName, operatorName, portIndex); if (matches) { break; } @@ -111,10 +110,10 @@ public boolean matchesOperatorInputPortMetricName(String domainId, String instan return matches; } - public boolean matchesOperatorOutputPortIndex(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex) { + public boolean matchesOperatorOutputPortMetricName(String instanceId, String jobName, String operatorName, Integer portIndex, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesOperatorOutputPortIndex(domainId, instanceId, jobName, operatorName, portIndex); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesOperatorOutputPortMetricName(instanceId, jobName, operatorName, portIndex, metricName); if (matches) { break; } @@ -122,10 +121,10 @@ public boolean matchesOperatorOutputPortIndex(String domainId, String instanceId return matches; } - public boolean matchesOperatorOutputPortMetricName(String domainId, String instanceId, String jobName, String operatorName, Integer portIndex, String metricName) { + public boolean matchesPeId(String instanceId, String jobName, String peId) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesOperatorOutputPortMetricName(domainId, instanceId, jobName, operatorName, portIndex, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeId(instanceId, jobName, peId); if (matches) { break; } @@ -133,10 +132,10 @@ public boolean matchesOperatorOutputPortMetricName(String domainId, String insta return matches; } - public boolean matchesPeId(String domainId, String instanceId, String jobName, BigInteger peId) { + public boolean matchesPeMetricName(String instanceId, String jobName, String peId, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeId(domainId, instanceId, jobName, peId); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeMetricName(instanceId, jobName, peId, metricName); if (matches) { break; } @@ -144,10 +143,10 @@ public boolean matchesPeId(String domainId, String instanceId, String jobName, B return matches; } - public boolean matchesPeMetricName(String domainId, String instanceId, String jobName, BigInteger peId, String metricName) { + public boolean matchesPeInputPortIndex(String instanceId, String jobName, String peId, Integer portIndex) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeMetricName(domainId, instanceId, jobName, peId, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeInputPortIndex(instanceId, jobName, peId, portIndex); if (matches) { break; } @@ -155,10 +154,10 @@ public boolean matchesPeMetricName(String domainId, String instanceId, String jo return matches; } - public boolean matchesPeInputPortIndex(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex) { + public boolean matchesPeInputPortMetricName(String instanceId, String jobName, String peId, Integer portIndex, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeInputPortIndex(domainId, instanceId, jobName, peId, portIndex); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeInputPortMetricName(instanceId, jobName, peId, portIndex, metricName); if (matches) { break; } @@ -166,10 +165,10 @@ public boolean matchesPeInputPortIndex(String domainId, String instanceId, Strin return matches; } - public boolean matchesPeInputPortMetricName(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeOutputPortIndex(String instanceId, String jobName, String peId, Integer portIndex) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeInputPortMetricName(domainId, instanceId, jobName, peId, portIndex, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeOutputPortIndex(instanceId, jobName, peId, portIndex); if (matches) { break; } @@ -177,21 +176,10 @@ public boolean matchesPeInputPortMetricName(String domainId, String instanceId, return matches; } - public boolean matchesPeOutputPortIndex(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex) { + public boolean matchesPeOutputPortMetricName(String instanceId, String jobName, String peId, Integer portIndex, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeOutputPortIndex(domainId, instanceId, jobName, peId, portIndex); - if (matches) { - break; - } - } - return matches; - } - - public boolean matchesPeOutputPortMetricName(String domainId, String instanceId, String jobName, BigInteger peId, Integer portIndex, String metricName) { - boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeOutputPortMetricName(domainId, instanceId, jobName, peId, portIndex, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeOutputPortMetricName(instanceId, jobName, peId, portIndex, metricName); if (matches) { break; } @@ -199,10 +187,10 @@ public boolean matchesPeOutputPortMetricName(String domainId, String instanceId, return matches; } - public boolean matchesPeConnectionId(String domainId, String instanceId, String jobName, BigInteger peId, String connectionId) { + public boolean matchesPeConnectionId(String instanceId, String jobName, String peId, String connectionId) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeConnectionId(domainId, instanceId, jobName, peId, connectionId); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeConnectionId(instanceId, jobName, peId, connectionId); if (matches) { break; } @@ -210,10 +198,10 @@ public boolean matchesPeConnectionId(String domainId, String instanceId, String return matches; } - public boolean matchesPeConnectionMetricName(String domainId, String instanceId, String jobName, BigInteger peId, String connectionId, String metricName) { + public boolean matchesPeConnectionMetricName(String instanceId, String jobName, String peId, String connectionId, String metricName) { boolean matches = false; - for(DomainFilter filter : _domainFilters.values()) { - matches = filter.matchesPeConnectionMetricName(domainId, instanceId, jobName, peId, connectionId, metricName); + for(InstanceFilter filter : _instanceFilters.values()) { + matches = filter.matchesPeConnectionMetricName(instanceId, jobName, peId, connectionId, metricName); if (matches) { break; } @@ -227,12 +215,12 @@ public boolean matchesPeConnectionMetricName(String domainId, String instanceId, * * @param filterDocument * Specifies the absolute path of a JSON-formatted file, which contains - * the specification of the filter criteria for domain, instance, job, + * the specification of the filter criteria for instance, job, * operator, and metric names. * * @return * A tree of filter objects that is used to evaluate whether a given - * domain, instance, job, operator, or metric name matches the specified + * instance, job, operator, or metric name matches the specified * filter criteria. * * @throws IOException @@ -256,12 +244,12 @@ static public Filters setupFilters(String filterDocument, OpType aType) throws I * * @param inputStream * Specifies the input stream that provides a JSON-formatted text, which - * contains the specification of the filter criteria for domain, instance, + * contains the specification of the filter criteria for instance, * job, operator, and metric names. * * @return * A tree of filter objects that is used to evaluate whether a given - * domain, instance, job, operator, or metric name matches the specified + * instance, job, operator, or metric name matches the specified * filter criteria. * * @throws IOException @@ -275,11 +263,11 @@ static public Filters setupFilters(String filterDocument, OpType aType) throws I static public Filters setupFilters(InputStream inputStream, OpType aType) throws IOException { Filters filters = new Filters(); JSONArtifact root = JSON.parse(inputStream); - DomainParser domainParser = new DomainParser(aType); - if (domainParser.validate(root)) { - Set domainFilters = domainParser.buildFilters(root); - for (DomainFilter domainFilter : domainFilters) { - filters._domainFilters.put(domainFilter.getRegularExpression(), domainFilter); + InstanceParser instanceParser = new InstanceParser(aType); + if (instanceParser.validate(root)) { + Set instanceFilters = instanceParser.buildFilters(root); + for (InstanceFilter instanceFilter : instanceFilters) { + filters._instanceFilters.put(instanceFilter.getRegularExpression(), instanceFilter); } } return filters; diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/InstanceFilter.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/InstanceFilter.java index a96f0d8..7aafe71 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/InstanceFilter.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/InstanceFilter.java @@ -130,7 +130,7 @@ public boolean matchesOperatorOutputPortMetricName(String instanceId, String job return matches; } - public boolean matchesPeId(String instanceId, String jobName, BigInteger peId) { + public boolean matchesPeId(String instanceId, String jobName, String peId) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -143,7 +143,7 @@ public boolean matchesPeId(String instanceId, String jobName, BigInteger peId) { return matches; } - public boolean matchesPeMetricName(String instanceId, String jobName, BigInteger peId, String metricName) { + public boolean matchesPeMetricName(String instanceId, String jobName, String peId, String metricName) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -156,7 +156,7 @@ public boolean matchesPeMetricName(String instanceId, String jobName, BigInteger return matches; } - public boolean matchesPeInputPortIndex(String instanceId, String jobName, BigInteger peId, Integer portIndex) { + public boolean matchesPeInputPortIndex(String instanceId, String jobName, String peId, Integer portIndex) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -169,7 +169,7 @@ public boolean matchesPeInputPortIndex(String instanceId, String jobName, BigInt return matches; } - public boolean matchesPeInputPortMetricName(String instanceId, String jobName, BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeInputPortMetricName(String instanceId, String jobName, String peId, Integer portIndex, String metricName) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -182,7 +182,7 @@ public boolean matchesPeInputPortMetricName(String instanceId, String jobName, B return matches; } - public boolean matchesPeOutputPortIndex(String instanceId, String jobName, BigInteger peId, Integer portIndex) { + public boolean matchesPeOutputPortIndex(String instanceId, String jobName, String peId, Integer portIndex) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -195,7 +195,7 @@ public boolean matchesPeOutputPortIndex(String instanceId, String jobName, BigIn return matches; } - public boolean matchesPeOutputPortMetricName(String instanceId, String jobName, BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeOutputPortMetricName(String instanceId, String jobName, String peId, Integer portIndex, String metricName) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -208,7 +208,7 @@ public boolean matchesPeOutputPortMetricName(String instanceId, String jobName, return matches; } - public boolean matchesPeConnectionId(String instanceId, String jobName, BigInteger peId, String connectionId) { + public boolean matchesPeConnectionId(String instanceId, String jobName, String peId, String connectionId) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { @@ -221,7 +221,7 @@ public boolean matchesPeConnectionId(String instanceId, String jobName, BigInteg return matches; } - public boolean matchesPeConnectionMetricName(String instanceId, String jobName, BigInteger peId, String peConnection, String metricName) { + public boolean matchesPeConnectionMetricName(String instanceId, String jobName, String peId, String peConnection, String metricName) { boolean matches = matchesInstanceId(instanceId) && (_jobFilters.size() > 0); if (matches) { for(JobFilter filter : _jobFilters.values()) { diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/JobFilter.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/JobFilter.java index 831285e..477ffae 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/JobFilter.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/JobFilter.java @@ -7,7 +7,6 @@ package com.ibm.streamsx.monitoring.jmx.internal.filters; -import java.math.BigInteger; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -133,7 +132,7 @@ public boolean matchesOperatorOutputPortMetricName(String jobName, String operat return matches; } - public boolean matchesPeId(String jobName, BigInteger peId) { + public boolean matchesPeId(String jobName, String peId) { boolean matches = false; if (_type == OpType.METRICS_SOURCE) { matches = matchesJobName(jobName) && (_peFilters.size() > 0); @@ -144,7 +143,7 @@ else if (_type == OpType.JOB_STATUS_SOURCE) { return matches; } - public boolean matchesPeMetricName(String jobName, BigInteger peId, String metricName) { + public boolean matchesPeMetricName(String jobName, String peId, String metricName) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { @@ -157,7 +156,7 @@ public boolean matchesPeMetricName(String jobName, BigInteger peId, String metri return matches; } - public boolean matchesPeInputPortIndex(String jobName, BigInteger peId, Integer portIndex) { + public boolean matchesPeInputPortIndex(String jobName, String peId, Integer portIndex) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { @@ -170,7 +169,7 @@ public boolean matchesPeInputPortIndex(String jobName, BigInteger peId, Integer return matches; } - public boolean matchesPeInputPortMetricName(String jobName, BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeInputPortMetricName(String jobName, String peId, Integer portIndex, String metricName) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { @@ -183,7 +182,7 @@ public boolean matchesPeInputPortMetricName(String jobName, BigInteger peId, Int return matches; } - public boolean matchesPeOutputPortIndex(String jobName, BigInteger peId, Integer portIndex) { + public boolean matchesPeOutputPortIndex(String jobName, String peId, Integer portIndex) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { @@ -196,7 +195,7 @@ public boolean matchesPeOutputPortIndex(String jobName, BigInteger peId, Integer return matches; } - public boolean matchesPeOutputPortMetricName(String jobName, BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeOutputPortMetricName(String jobName, String peId, Integer portIndex, String metricName) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { @@ -209,7 +208,7 @@ public boolean matchesPeOutputPortMetricName(String jobName, BigInteger peId, In return matches; } - public boolean matchesPeConnectionId(String jobName, BigInteger peId, String connectionId) { + public boolean matchesPeConnectionId(String jobName, String peId, String connectionId) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { @@ -222,7 +221,7 @@ public boolean matchesPeConnectionId(String jobName, BigInteger peId, String con return matches; } - public boolean matchesPeConnectionMetricName(String jobName, BigInteger peId, String connectionId, String metricName) { + public boolean matchesPeConnectionMetricName(String jobName, String peId, String connectionId, String metricName) { boolean matches = matchesPeId(jobName, peId); if (matches) { for(PeFilter filter : _peFilters) { diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/PeFilter.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/PeFilter.java index ef74734..1708934 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/PeFilter.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jmx/internal/filters/PeFilter.java @@ -60,7 +60,7 @@ public PeFilter(Set metricFilters, Set inputPortFilter } } - public boolean matchesPeMetricName(BigInteger peId, String metricName) { + public boolean matchesPeMetricName(String peId, String metricName) { boolean matches = false; for(MetricFilter filter : _metricFilters.values()) { matches = filter.matchesMetricName(metricName); @@ -71,7 +71,7 @@ public boolean matchesPeMetricName(BigInteger peId, String metricName) { return matches; } - public boolean matchesPeInputPortIndex(BigInteger peId, Integer portIndex) { + public boolean matchesPeInputPortIndex(String peId, Integer portIndex) { boolean matches = false; for(PortFilter filter : _inputPortFilters.values()) { matches = filter.matchesPortIndex(portIndex); @@ -82,7 +82,7 @@ public boolean matchesPeInputPortIndex(BigInteger peId, Integer portIndex) { return matches; } - public boolean matchesPeInputPortMetricName(BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeInputPortMetricName(String peId, Integer portIndex, String metricName) { boolean matches = false; for(PortFilter filter : _inputPortFilters.values()) { matches = filter.matchesPortMetricName(portIndex, metricName); @@ -93,7 +93,7 @@ public boolean matchesPeInputPortMetricName(BigInteger peId, Integer portIndex, return matches; } - public boolean matchesPeOutputPortIndex(BigInteger peId, Integer portIndex) { + public boolean matchesPeOutputPortIndex(String peId, Integer portIndex) { boolean matches = false; for(PortFilter filter : _outputPortFilters.values()) { matches = filter.matchesPortIndex(portIndex); @@ -104,7 +104,7 @@ public boolean matchesPeOutputPortIndex(BigInteger peId, Integer portIndex) { return matches; } - public boolean matchesPeOutputPortMetricName(BigInteger peId, Integer portIndex, String metricName) { + public boolean matchesPeOutputPortMetricName(String peId, Integer portIndex, String metricName) { boolean matches = false; for(PortFilter filter : _outputPortFilters.values()) { matches = filter.matchesPortMetricName(portIndex, metricName); @@ -115,7 +115,7 @@ public boolean matchesPeOutputPortMetricName(BigInteger peId, Integer portIndex, return matches; } - public boolean matchesPeConnectionId(BigInteger peId, String connectionId) { + public boolean matchesPeConnectionId(String peId, String connectionId) { boolean matches = false; for(ConnectionFilter filter : _connectionFilters.values()) { matches = filter.matchesConnectionId(connectionId); @@ -126,7 +126,7 @@ public boolean matchesPeConnectionId(BigInteger peId, String connectionId) { return matches; } - public boolean matchesPeConnectionMetricName(BigInteger peId, String peConnection, String metricName) { + public boolean matchesPeConnectionMetricName(String peId, String peConnection, String metricName) { boolean matches = false; for(ConnectionFilter filter : _connectionFilters.values()) { matches = filter.matchesConnectionMetricName(peConnection, metricName); diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jobs/JobStatusSource.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jobs/JobStatusSource.java index ec92508..8a72120 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jobs/JobStatusSource.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/jobs/JobStatusSource.java @@ -86,9 +86,9 @@ public class JobStatusSource extends AbstractJmxSource { + "* com.ibm.streams.management.pe.changed\\n" + "\\n" + "Per default, the JobStatusSource operator monitors all jobs " - + "in the current domain and instance, if filter document is not specified.\\n" + + "in the current instance, if filter document is not specified.\\n" + "\\n" - + "The JobStatusSource operator monitors filter-matching domains, " + + "The JobStatusSource operator monitors filter-matching " + "instances, and jobs that are running while the application that " + "uses the JobStatusSource operator, starts. Furthermore, the " + "operator gets notifications for created and deleted instances, " @@ -98,15 +98,9 @@ public class JobStatusSource extends AbstractJmxSource { + "\\n" + "+ Filter document\\n" + "\\n" - + "The filter document specifies patterns for domain, instance and " + + "The filter document specifies patterns for instance and " + "job and their relations.\\n" + "\\n" - + "It also specifies " - + "which name patterns are related, for example: For a domain X " - + "monitor all instances, whereas in each instance only jobs with " - + "a Y in their names shall be evaluated. For another domain Z, " - + "only jobs with a name ending with XYZ, are monitored, etc.\\n" - + "\\n" + "The filter document is a JSON-encoded text file or JSON-encoded String that is " + "configured with the **filterDocument** parameter.\\n" + "\\n" @@ -122,37 +116,25 @@ public class JobStatusSource extends AbstractJmxSource { + "\\n" + " [\\n" + " {\\n" - + " \\\"domainIdPatterns\\\":\\\".*\\\",\\n" - + " \\\"instances\\\":\\n" + + " \\\"instanceIdPatterns\\\":\\\".*\\\",\\n" + + " \\\"jobs\\\":\\n" + " [\\n" + " {\\n" - + " \\\"instanceIdPatterns\\\":\\\".*\\\",\\n" - + " \\\"jobs\\\":\\n" - + " [\\n" - + " {\\n" - + " \\\"jobNamePatterns\\\":\\\".*\\\",\\n" - + " }\\n" - + " ]\\n" + + " \\\"jobNamePatterns\\\":\\\".*\\\",\\n" + " }\\n" + " ]\\n" + " }\\n" + " ]\\n" + "\\n" - + "The following filter document example selects events from jobs in the *StreamsInstance* instance and *StreamsDomain* domain only:\\n" + + "The following filter document example selects events from jobs in the *StreamsInstance* instance only:\\n" + "\\n" + " [\\n" + " {\\n" - + " \\\"domainIdPatterns\\\":\\\"StreamsDomain\\\",\\n" - + " \\\"instances\\\":\\n" + + " \\\"instanceIdPatterns\\\":\\\"StreamsInstance\\\",\\n" + + " \\\"jobs\\\":\\n" + " [\\n" + " {\\n" - + " \\\"instanceIdPatterns\\\":\\\"StreamsInstance\\\",\\n" - + " \\\"jobs\\\":\\n" - + " [\\n" - + " {\\n" - + " \\\"jobNamePatterns\\\":\\\".*\\\",\\n" - + " }\\n" - + " ]\\n" + + " \\\"jobNamePatterns\\\":\\\".*\\\",\\n" + " }\\n" + " ]\\n" + " }\\n" @@ -170,7 +152,7 @@ public class JobStatusSource extends AbstractJmxSource { private static final String DESC_PARAM_FILTER_DOCUMENT = "Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies " - + "the domain, instance and job as " + + "the instance and job as " + "regular expressions. Each regular expression must follow the " + "rules that are specified for Java " + "[https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. " @@ -246,11 +228,11 @@ public void run() { _trace.warn("Reconnect"); setupJMXConnection(); _connected = true; - scanDomain(); // create new DomainHandler + scanInstance(); // create new InstanceHandler } if (_connected) { - _domainHandler.healthCheck(); + _instanceHandler.healthCheck(); if (_operatorConfiguration.get_applicationConfigurationName() != null) { detectAndProcessChangedFilterDocumentInApplicationConfiguration(); @@ -260,7 +242,7 @@ public void run() { catch (Exception e) { _trace.error("JMX connection error ", e); _connected = false; - closeDomainHandler(); + closeInstanceHandler(); } } }, 5000l, Double.valueOf(_operatorConfiguration.get_checkPeriod() * 1000.0).longValue(), TimeUnit.MILLISECONDS); diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/MetricsSource.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/MetricsSource.java index 57153d6..7598fee 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/MetricsSource.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/MetricsSource.java @@ -89,24 +89,19 @@ public class MetricsSource extends AbstractJmxSource { + "metric changes as tuple stream.\\n" + "\\n" + "As an application developer, you provide the so-called filter " - + "document. The filter document specifies patterns for domain, " - + "instance, job, operator, and metric names. It also specifies " - + "which name patterns are related, for example: For a domain X " - + "monitor all instances, whereas in each instance only jobs with " - + "a Y in their names shall be evaluated. For another domain Z, " - + "only jobs with a name ending with XYZ, are monitored, etc.\\n" - + "\\n" + + "document. The filter document specifies patterns for " + + "instance, job, operator, and metric names.\\n" + "If the MetricsSource evaluates whether, for example, a custom " + "metric of an operator shall be retrieved periodically, the name " + "patterns are applied. A custom metric is uniquely identified " - + "with the domain, instance, job, operator, and metric name. All " + + "with the instance, job, operator, and metric name. All " + "these parts must match to the corresponding set of related name " + "patterns.\\n" + "\\n" + "Per default, the MetricsSource operator monitors all metrics " - + "in the current domain and instance, if filter document is not specified." + + "in the current instance, if filter document is not specified." + "\\n" - + "The MetricsSource operator monitors filter-matching domains, " + + "The MetricsSource operator monitors filter-matching " + "instances, and jobs that are running while the application that " + "uses the MetricsSource operator, starts. Furthermore, the " + "operator gets notifications for created and deleted instances, " @@ -116,18 +111,12 @@ public class MetricsSource extends AbstractJmxSource { + "\\n" + "+ Filter document\\n" + "\\n" - + "The filter document specifies patterns for domain, instance, " + + "The filter document specifies patterns for instance, " + "job, operator, and metric names, and their relations.\\n" + "\\n" + "Only those objects (and their parents) that " + "match the specified filters, are monitored.\\n" + "\\n" - + "It also specifies " - + "which name patterns are related, for example: For a domain X " - + "monitor all instances, whereas in each instance only jobs with " - + "a Y in their names shall be evaluated. For another domain Z, " - + "only jobs with a name ending with XYZ, are monitored, etc.\\n" - + "\\n" + "The filter document is a JSON-encoded text file or JSON-encoded String that is " + "configured with the **filterDocument** parameter.\\n" + "\\n" @@ -143,10 +132,6 @@ public class MetricsSource extends AbstractJmxSource { + "\\n" + " [\\n" + " {\\n" - + " \\\"domainIdPatterns\\\":\\\".*\\\",\\n" - + " \\\"instances\\\":\\n" - + " [\\n" - + " {\\n" + " \\\"instanceIdPatterns\\\":\\\".*\\\",\\n" + " \\\"jobs\\\":\\n" + " [\\n" @@ -207,13 +192,9 @@ public class MetricsSource extends AbstractJmxSource { + " }\\n" + " ]\\n" + "\\n" - + "The following filter document example selects two custom metrics names *nBytesWritten* and *nObjects* of a custom *Storage* operator in the *StreamsInstance* instance and *StreamsDomain* domain only:\\n" + + "The following filter document example selects two custom metrics names *nBytesWritten* and *nObjects* of a custom *Storage* operator in the *StreamsInstance* instance only:\\n" + "\\n" + " [\\n" - + " {\\n" - + " \\\"domainIdPatterns\\\":\\\"StreamsDomain\\\",\\n" - + " \\\"instances\\\":\\n" - + " [\\n" + " {\\n" + " \\\"instanceIdPatterns\\\":\\\"StreamsInstance\\\",\\n" + " \\\"jobs\\\":\\n" @@ -252,7 +233,7 @@ public class MetricsSource extends AbstractJmxSource { private static final String DESC_PARAM_FILTER_DOCUMENT = "Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies " - + "the domain, instance, job, operator, and metric name filters as " + + "the instance, job, operator, and metric name filters as " + "regular expressions. Each regular expression must follow the " + "rules that are specified for Java " + "[https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. " @@ -427,18 +408,18 @@ private void produceTuples() throws Exception { _trace.warn("Reconnect"); setupJMXConnection(); connected = true; - scanDomain(); // create new DomainHandler + scanInstance(); // create new InstanceHandler } if (connected) { - _domainHandler.healthCheck(); - _domainHandler.captureMetrics(); + _instanceHandler.healthCheck(); + _instanceHandler.captureMetrics(); } } catch (Exception e) { _trace.error("JMX connection error ", e); connected = false; - closeDomainHandler(); + closeInstanceHandler(); setupFilters(); } /* diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Filters.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Filters.java index 6c02a94..4911d37 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Filters.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Filters.java @@ -18,7 +18,7 @@ /** * Filters is a container for the filters inside a ThresholdObject. - * There can be many filters defined inside a ThresholdObject (eg. domainId, + * There can be many filters defined inside a ThresholdObject (e.g. * instanceId, jobName, etc.). */ public class Filters { diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Metrics.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Metrics.java index 1cfa1d6..3b23f16 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Metrics.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/metrics/internal/monitor/Metrics.java @@ -25,7 +25,7 @@ public class Metrics { */ public static final String METRIC_NAME = "metricName", METRIC_VALUE = "metricValue", LAST_TIME_RETRIEVED = "lastTimeRetrieved"; - public static final String[] METRIC_TYPES = new String[] { "domainId", "instanceId", "jobId", "jobName", "resource", "peId", "operatorName", + public static final String[] METRIC_TYPES = new String[] { "instanceId", "jobId", "jobName", "resource", "peId", "operatorName", "channel", "portIndex", "connectionId", "metricType", "metricKind", "metricName", "metricValue", "lastTimeRetrieved"}; diff --git a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/system/LogSource.java b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/system/LogSource.java index 8be9521..92c0bed 100644 --- a/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/system/LogSource.java +++ b/com.ibm.streamsx.monitoring/impl/java/src/com/ibm/streamsx/monitoring/system/LogSource.java @@ -141,16 +141,16 @@ public void run() { _trace.warn("Reconnect"); setupJMXConnection(); _connected = true; - scanDomain(); // create new DomainHandler + scanInstance(); // create new InstanceHandler } if (_connected) { - _domainHandler.healthCheck(); + _instanceHandler.healthCheck(); } } catch (Exception e) { _trace.error("JMX connection error ", e); _connected = false; - closeDomainHandler(); + closeInstanceHandler(); } } }, 3000l, Double.valueOf(5 * 1000.0).longValue(), TimeUnit.MILLISECONDS); diff --git a/com.ibm.streamsx.monitoring/info.xml b/com.ibm.streamsx.monitoring/info.xml index 929f57f..6c312d6 100644 --- a/com.ibm.streamsx.monitoring/info.xml +++ b/com.ibm.streamsx.monitoring/info.xml @@ -6,134 +6,14 @@ /4.1.0.0/bin/streamsprofile.sh - -# About this task - -After the location of the toolkit is communicated to the compiler, the SPL -artifacts that are specified in the toolkit can be used by an application. -The application can include a `use` directive to bring the necessary namespaces -into scope. Alternatively, you can fully qualify the operators that are provided -by toolkit with their namespaces as prefixes. - -# Procedure - -1. Configure the SPL compiler to find the toolkit root directory. Use one of the - following methods: - - * Set the **STREAMS_SPLPATH** environment variable to the root directory of a - toolkit or multiple toolkits (with : as a separator). For example: - - export STREAMS_SPLPATH=$HOME/toolkits/com.ibm.streamsx.monitoring - - * Specify the **-t** or **--spl-path** command parameter when you run the - **sc** command. For example: - - sc -t $HOME/toolkits/com.ibm.streamsx.monitoring -M MyMain - - where MyMain is the name of the SPL main composite. - - **Note**: These command parameters override the **STREAMS_SPLPATH** - environment variable. - - * Add the toolkit location in IBM Streams Studio. - -2. Develop your application. To avoid the need to fully qualify the operators, - add a use directive in your application. - - * For example, you can add the following clause in your SPL source file: - - use com.ibm.streamsx.monitoring.metrics::*; - - You can also specify a use clause for individual operators by replacing the - asterisk (\*) with the operator name. For example: - - use com.ibm.streamsx.monitoring.metrics::MetricsSource; - -3. Build your application. You can use the **sc** command or Streams Studio. - -4. Start the IBM Streams instance. - -5. Run the application. You can submit the application as a job by using the - **streamtool submitjob** command or by using Streams Studio. + Using the sample applications @@ -146,26 +26,9 @@ The samples directory contains the following sample applications demonstrating t Find a description of each sample in the README.md file of the sample directory. -Either build the samples after importing the project into Streams Studio or from command line using the Makefile. -The sample applications can be launched in standalone or distributed mode. - -+ Using the microservices - -The toolkit provides applications that can be launched as microservice. Find the microservices description in the following namespaces: - -* [tk$com.ibm.streamsx.monitoring/ns$com.ibm.streamsx.monitoring.jobs.services.html|com.ibm.streamsx.monitoring.jobs.services] -* [tk$com.ibm.streamsx.monitoring/ns$com.ibm.streamsx.monitoring.metrics.services.html|com.ibm.streamsx.monitoring.metrics.services] - -The recommended way to build and launch the microservices is using **streamsx-runner**. -From command line you could launch the application with [http://ibmstreams.github.io/streamsx.topology/doc/pythondoc/scripts/runner.html|streamsx-runner] to the Streaming Analytics service. - -For example, launch the MetricsIngestService from command line using relative path to the `com.ibm.streamsx.monitoring` toolkit location: - - streamsx-runner --service-name $STREAMING_ANALYTICS_SERVICE_NAME --main-composite com.ibm.streamsx.monitoring.metrics.services::MetricsIngestService --toolkits com.ibm.streamsx.monitoring --trace info - ]]> - 2.0.0 - 4.1.0.0 + 3.0.0 + 4.3.0.0 diff --git a/samples/Makefile b/samples/Makefile new file mode 100644 index 0000000..70aea1a --- /dev/null +++ b/samples/Makefile @@ -0,0 +1,20 @@ +# Licensed Materials - Property of IBM +# Monitoring Toolkit for IBM Streams +# (c) Copyright IBM Corp. 2019 +# All rights reserved. + +DIRS := $(shell for x in *; do if [ -d $$x ]; then echo -n "$$x "; fi; done) +$(info Samples to build: $(DIRS)) +$(info ) + +.PHONY: all clean + +all: + for x in $(DIRS); do echo -e "\n######## make sample $$x ########"; cd $$x; make all; cd -; done + +clean: + for x in $(DIRS); do cd $$x; make clean; cd -; done + +tkidx: + for x in $(DIRS); do cd $$x; make tkidx; cd -; done + diff --git a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Makefile b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Makefile index 81e3d7e..0c824ff 100644 --- a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Makefile +++ b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Makefile @@ -43,3 +43,5 @@ configure: -streamtool rmappconfig --noprompt monitoring streamtool mkappconfig --property user=$(STREAMS_USER) --property password=$(STREAMS_PASSWORD) monitoring +tkidx: + $(STREAMS_INSTALL)/bin/spl-make-toolkit -i . diff --git a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/README.md b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/README.md index aea4a14..cc288e2 100644 --- a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/README.md +++ b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/README.md @@ -4,55 +4,44 @@ This sample SPL application demonstrates the use of JobStatusMonitor operator to ### Use -Build **Monitor** and **SampleJob** applications: - -`make` - -### Run Monitor application - -Launch the **Monitor** application first. - -#### Standalone mode +#### Launch with python script -The monitor application in standalone mode can not determine the `domainId`. Therefore you need to specify the `domainId` parameter. +```` +cd .. +python3 launch_job_status_monitor_sample.py +```` -Update `domainId`, `user` and `password` parameters for your Streams environment. +#### Launch with Streams Console -`./output/monitor/bin/standalone domainId=StreamsDomain user=streamsadmin password=password` +Build **Monitor** and **SampleJob** applications: -#### Distributed mode +`make` -For example with `user` and `password` submission parameters: +##### Run Monitor application -`streamtool submitjob output/monitor/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor.Monitor.sab -P user=streamsadmin -P password=password` +Launch the **Monitor** application first. Update `user` and `password` submission parameters for your Streams environment. -Alternative you can specify the `domainId`, `connectionURL`, `user`, `password`, and `filterDocument` properties in an [application configuration](https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html). +Alternative you can specify the `instanceId`, `connectionURL`, `user`, `password`, and `filterDocument` properties in an [application configuration](https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html). Advantage of the use of an application configuration is, that you can update the `filterDocument` at runtime. -***Application configuration is supported in distributed mode only.*** - -`streamtool submitjob output/monitor/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor.Monitor.sab -P applicationConfigurationName=monitoring` - -### Run sample application to be monitored +##### Run sample application to be monitored Afterwards launch the **SampleJob** application to be monitored in distributed mode. -`streamtool submitjob output/sample/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor.SampleJob.sab` - ### Result Verify the notification events in the console output. -*In the Streaming Analytics service, go to the Log Viewer and Click on the PE's Console Log to view output.* +*In the Streams Console, go to the Log Viewer and Click on the PE's Console Log to view output.* One PE of the sample application has been forced to restart and the output contains "com.ibm.streams.management.pe.changed" notifications: ... - {notifyType="com.ibm.streams.management.pe.changed",domainId="StreamsDomain",instanceId="StreamsInstance",jobId=0,jobName="com.ibm.streamsx.monitoring.jobs.sample::SampleJob_0",resource="streamshost.ibm.com",peId=2,peHealth="partiallyUnhealthy",peStatus="restarting",eventTimestamp=(1505980604,904000000,0)} + {notifyType="com.ibm.streams.management.pe.changed",instanceId="StreamsInstance",jobId=0,jobName="com.ibm.streamsx.monitoring.jobs.sample::SampleJob_0",resource="streamshost.ibm.com",peId=2,peHealth="partiallyUnhealthy",peStatus="restarting",eventTimestamp=(1505980604,904000000,0)} ... - {notifyType="com.ibm.streams.management.pe.changed",domainId="StreamsDomain",instanceId="StreamsInstance",jobId=0,jobName="com.ibm.streamsx.monitoring.jobs.sample::SampleJob_0",resource="streamshost.ibm.com",peId=2,peHealth="healthy",peStatus="running",eventTimestamp=(1505980606,474000000,0)} + {notifyType="com.ibm.streams.management.pe.changed",instanceId="StreamsInstance",jobId=0,jobName="com.ibm.streamsx.monitoring.jobs.sample::SampleJob_0",resource="streamshost.ibm.com",peId=2,peHealth="healthy",peStatus="running",eventTimestamp=(1505980606,474000000,0)} ### Clean: diff --git a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Monitor.spl b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Monitor.spl index 1e87bb3..3b23998 100644 --- a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Monitor.spl +++ b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/Monitor.spl @@ -14,7 +14,7 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * This sample application demonstrates the usage of the JobStatusMonitor operator and monitors all jobs and reports the Job/PE status events * * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. + * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain instanceId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. * * @param user * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. @@ -23,32 +23,25 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param filterDocument - * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the domain, instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * If parameter is not set, then all jobs in current domain and instance are monitored. + * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * If parameter is not set, then all jobs in current instance are monitored. * * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - * @param credentials - * Specifies Streaming Analytics service credentials in JSON format. Relevant for IAM authentication case only. If parameter is set, then the parameters user and password are ignored. If the **applicationConfigurationName** parameter is specified, the application configuration with the property credentials can override this parameter value. + * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * */ -composite Monitor { +public composite Monitor { param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set + expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", "monitoring"); // optional, if user and password are set expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration expression $filterDocument: getSubmissionTimeValue("filterDocument", ""); // uses default, if not set - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $credentials: getSubmissionTimeValue("credentials", ""); + expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then instance settings are used, where the PE is running + expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then instance settings are used, where the PE is running graph @@ -64,9 +57,7 @@ composite Monitor { password: $password; filterDocument: $filterDocument; connectionURL: $connectionURL; - domainId: $domainId; sslOption: $sslOption; - credentials: $credentials; } /* diff --git a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/etc/JobStatusSource_MonitorEverything.json b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/etc/JobStatusSource_MonitorEverything.json index 9e7b198..ac55a1e 100644 --- a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/etc/JobStatusSource_MonitorEverything.json +++ b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/etc/JobStatusSource_MonitorEverything.json @@ -1,16 +1,10 @@ [ { - "domainIdPatterns":".*", - "instances": + "instanceIdPatterns":".*", + "jobs": [ { - "instanceIdPatterns":".*", - "jobs": - [ - { - "jobNamePatterns":".*", - } - ] + "jobNamePatterns":".*", } ] } diff --git a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/info.xml b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/info.xml index 1693f1a..1cd73ee 100644 --- a/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/info.xml +++ b/samples/com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor/info.xml @@ -4,13 +4,13 @@ com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor This sample SPL application demonstrates the use of JobStatusMonitor operator to get the notification of PE status changes and added/removed jobs. - 1.0.0 - 4.1.0.0 + 1.2.0 + 4.3.0.0 com.ibm.streamsx.monitoring - 1.6.0 + 3.0.0 diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Makefile b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Makefile index 26de933..7dac256 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Makefile +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Makefile @@ -40,6 +40,9 @@ clean: $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) rm -rf $(OUTPUT_DIR) +tkidx: + $(STREAMS_INSTALL)/bin/spl-make-toolkit -i . + configure: -streamtool rmappconfig --noprompt monitoring streamtool mkappconfig --property connectionURL=`streamtool getjmxconnect` --property user=$(STREAMS_USER) --property password=$(STREAMS_PASSWORD) monitoring diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/README.md b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/README.md index 1bf179f..444e412 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/README.md +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/README.md @@ -4,53 +4,43 @@ This sample SPL application demonstrates the use of MetricsSource operator. ### Use -Build **Monitor** and **SampleJob** applications: - -`make` - -### Run Monitor application - -Launch the **Monitor** application first. +#### Launch with python script -#### Standalone mode +```` +cd .. +python3 launch_metrics_monitor_sample.py +```` -The monitor application in standalone mode can not determine the `domainId`. Therefore you need to specify the `domainId` parameter. +#### Launch with Streams Console -Update `domainId`, `user` and `password` parameters for your Streams environment. - -`./output/monitor/bin/standalone domainId=StreamsDomain user=streamsadmin password=password` +Build **Monitor** and **SampleJob** applications: -#### Distributed mode +`make` -For example with `user` and `password` submission parameters: +###### Run Monitor application -`streamtool submitjob output/monitor/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource.Monitor.sab -P user=streamsadmin -P password=password` +Launch the **Monitor** application first. Update `user` and `password` submission parameters for your Streams environment. -Alternative you can specify the `domainId`, `connectionURL`, `user`, `password`, and `filterDocument` properties in an [application configuration](https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html). +Alternative you can specify the `instanceId`, `connectionURL`, `user`, `password`, and `filterDocument` properties in an [application configuration](https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html). Advantage of the use of an application configuration is, that you can update the `filterDocument` at runtime. -***Application configuration is supported in distributed mode only.*** - -`streamtool submitjob output/monitor/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource.Monitor.sab -P applicationConfigurationName=monitoring` - -### Run sample application to be monitored +##### Run sample application to be monitored Afterwards launch the **SampleJob** application to be monitored in distributed mode. -`streamtool submitjob output/sample/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource.SampleJob.sab` ### Result Verify the metrics notification tuples in the console output. -*In the Streaming Analytics service, go to the Log Viewer and Click on the PE's Console Log to view output.* +*In the Streams Console, go to the Log Viewer and Click on the PE's Console Log to view output.* The MetricsSource operator emits a metric tuple for each metric, for which the operator identifies a changed value. After each scan cycle, the operator emits a WindowMarker to this port. - MetricNotification: {domainId="StreamsDomain",instanceId="StreamsInstance",jobId=6,jobName="com.ibm.streamsx.monitoring.metrics.sample.MetricsSource::SampleJob_6",resource="streamshost.ibm.com",peId=10,origin=OperatorOutputPort,operatorName="Numbers",channel=-1,portIndex=0,connectionId="",metricType="system",metricKind="counter",metricName="nTuplesSubmitted",metricValue=3,lastTimeRetrieved=1505995655000} + MetricNotification: {instanceId="StreamsInstance",jobId=6,jobName="com.ibm.streamsx.monitoring.metrics.sample.MetricsSource::SampleJob_6",resource="streamshost.ibm.com",peId=10,origin=OperatorOutputPort,operatorName="Numbers",channel=-1,portIndex=0,connectionId="",metricType="system",metricKind="counter",metricName="nTuplesSubmitted",metricValue=3,lastTimeRetrieved=1505995655000} ... MetricNotifications: WindowMarker diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Monitor.spl b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Monitor.spl index d4801d6..06aad7a 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Monitor.spl +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/Monitor.spl @@ -15,7 +15,7 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * This sample application demonstrate how to use the MetricsSource operator * * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. + * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain instanceId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. * * @param user * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. @@ -24,34 +24,26 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param filterDocument - * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the domain, instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * If parameter is not set, then all metrics from all jobs in current domain and instance are monitored. + * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * If parameter is not set, then all metrics from all jobs in current instance are monitored. * * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - * @param credentials - * Specifies Streaming Analytics service credentials in JSON format. Relevant for IAM authentication case only. If parameter is set, then the parameters user and password are ignored. If the **applicationConfigurationName** parameter is specified, the application configuration with the property credentials can override this parameter value. + * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param emitMetricTuple * Specifies when to emit a tuple for a metric. Supported modes are the following string values: "onChangedValue" (default) and "periodic". */ -composite Monitor { +public composite Monitor { param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set + expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", "monitoring"); // optional, if user and password are set expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration expression $filterDocument: getSubmissionTimeValue("filterDocument", ""); // uses default, if not set - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $credentials: getSubmissionTimeValue("credentials", ""); + expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then instance settings are used, where the PE is running + expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then instance settings are used, where the PE is running expression $emitMetricTuple: getSubmissionTimeValue("emitMetricTuple", "onChangedValue"); graph @@ -67,10 +59,8 @@ composite Monitor { password: $password; filterDocument: $filterDocument; connectionURL: $connectionURL; - domainId: $domainId; sslOption: $sslOption; emitMetricTuple: $emitMetricTuple; - credentials: $credentials; } /* diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorEverything.json b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorEverything.json index 62dd8b9..d14bc77 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorEverything.json +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorEverything.json @@ -1,60 +1,54 @@ [ { - "domainIdPatterns":".*", - "instances": + "instanceIdPatterns":".*", + "jobs": [ { - "instanceIdPatterns":".*", - "jobs": + "jobNamePatterns":".*", + "pes": [ { - "jobNamePatterns":".*", - "pes": + "metricNamePatterns":".*", + "inputPorts": [ { - "metricNamePatterns":".*", - "inputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ], - "outputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ], - "connections": - [ - { - "connectionIdPatterns":".*", - "metricNamePatterns":".*" - } - ] + "portIndexes":"*", + "metricNamePatterns":".*" } ], - "operators": + "outputPorts": [ { - "operatorNamePatterns":".*", - "metricNamePatterns":".*", - "inputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ], - "outputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ] + "portIndexes":"*", + "metricNamePatterns":".*" + } + ], + "connections": + [ + { + "connectionIdPatterns":".*", + "metricNamePatterns":".*" + } + ] + } + ], + "operators": + [ + { + "operatorNamePatterns":".*", + "metricNamePatterns":".*", + "inputPorts": + [ + { + "portIndexes":"*", + "metricNamePatterns":".*" + } + ], + "outputPorts": + [ + { + "portIndexes":"*", + "metricNamePatterns":".*" } ] } diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorMemoryMetrics.json b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorMemoryMetrics.json index 3f63ad7..a4a689a 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorMemoryMetrics.json +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorMemoryMetrics.json @@ -1,22 +1,16 @@ [ { - "domainIdPatterns":".*", - "instances": + "instanceIdPatterns":".*", + "jobs": [ { - "instanceIdPatterns":".*", - "jobs": + "jobNamePatterns":".*", + "pes": [ { - "jobNamePatterns":".*", - "pes": + "metricNamePatterns": [ - { - "metricNamePatterns": - [ - "n(Resident)?MemoryConsumption" - ] - } + "n(Resident)?MemoryConsumption" ] } ] diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorOperatorMetrics.json b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorOperatorMetrics.json index 8a2fbd8..4f3f056 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorOperatorMetrics.json +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorOperatorMetrics.json @@ -1,25 +1,19 @@ [ { - "domainIdPatterns":".*", - "instances": + "instanceIdPatterns":".*", + "jobs": [ { - "instanceIdPatterns":".*", - "jobs": + "jobNamePatterns":".*", + "operators": [ { - "jobNamePatterns":".*", - "operators": + "operatorNamePatterns":".*", + "outputPorts": [ { - "operatorNamePatterns":".*", - "outputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ] + "portIndexes":"*", + "metricNamePatterns":".*" } ] } diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorSomeCustomMetrics.json b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorSomeCustomMetrics.json index 119dda5..98a9ec0 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorSomeCustomMetrics.json +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/etc/MetricsSource_MonitorSomeCustomMetrics.json @@ -1,25 +1,19 @@ [ { - "domainIdPatterns":".*", - "instances": + "instanceIdPatterns":".*", + "jobs": [ { - "instanceIdPatterns":".*", - "jobs": + "jobNamePatterns":".*", + "operators": [ { - "jobNamePatterns":".*", - "operators": + "operatorNamePatterns":"Storage", + "metricNamePatterns": [ - { - "operatorNamePatterns":"Storage", - "metricNamePatterns": - [ - "rnd", - "inc", - "const" - ] - } + "rnd", + "inc", + "const" ] } ] diff --git a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/info.xml b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/info.xml index ec6a6f3..94b682f 100644 --- a/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/info.xml +++ b/samples/com.ibm.streamsx.monitoring.metrics.sample.MetricsSource/info.xml @@ -8,13 +8,13 @@ This sample application demonstrate how to use the MetricsSource operator. Application Configuration requires Streams 4.2 or later. ]]> - 1.0.0 - 4.2.0.0 + 1.1.0 + 4.3.0.0 com.ibm.streamsx.monitoring - 1.6.0 + 3.0.0 diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/Makefile b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/Makefile index 1b2ca0b..041806e 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/Makefile +++ b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/Makefile @@ -43,3 +43,5 @@ configure: -streamtool rmappconfig --noprompt monitoring streamtool mkappconfig --property user=$(STREAMS_USER) --property password=$(STREAMS_PASSWORD) monitoring +tkidx: + $(STREAMS_INSTALL)/bin/spl-make-toolkit -i . diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/README.md b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/README.md index 7673365..433aab0 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/README.md +++ b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/README.md @@ -4,51 +4,40 @@ This sample SPL application demonstrates the use of LogSource operator to get th ### Use -Build **Monitor** and **SampleJob** applications: - -`make` - -### Run Monitor application - -Launch the **Monitor** application first. - -#### Standalone mode +#### Launch with python script -The monitor application in standalone mode can not determine the `domainId`. Therefore you need to specify the `domainId` parameter. +```` +cd .. +python3 launch_logs_monitor_sample.py +```` -Update `domainId`, `user` and `password` parameters for your Streams environment. +#### Launch with Streams Console -`./output/monitor/bin/standalone domainId=StreamsDomain user=streamsadmin password=password` +Build **Monitor** and **SampleJob** applications: -#### Distributed mode +`make` -For example with `user` and `password` submission parameters: +###### Run Monitor application -`streamtool submitjob output/monitor/com.ibm.streamsx.monitoring.system.sample.LogSource.Monitor.sab -P user=streamsadmin -P password=password` +Launch the **Monitor** application first. Update `user` and `password` submission parameters for your Streams environment. -Alternative you can specify the `domainId`, `connectionURL`, `user` and `password` properties in an [application configuration](https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html). +Alternative you can specify the `instanceId`, `connectionURL`, `user` and `password` properties in an [application configuration](https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html). -***Application configuration is supported in distributed mode only.*** - -`streamtool submitjob output/monitor/com.ibm.streamsx.monitoring.system.sample.LogSource.Monitor.sab -P applicationConfigurationName=monitoring` - -### Run sample application to be monitored +###### Run sample application to be monitored Afterwards launch the **SampleJob** application to be monitored in distributed mode. -`streamtool submitjob output/sample/com.ibm.streamsx.monitoring.system.sample.LogSource.SampleJob.sab` - ### Result Verify the application log notification tuples in the console output. -*In the Streaming Analytics service, go to the Log Viewer and Click on the PE's Console Log to view output.* +*In the Streams Console, go to the Log Viewer and Click on the PE's Console Log to view output.* ... - {notifyType="com.ibm.streams.management.log.application.error",domainId="StreamsDomain",instanceId="StreamsInstance",jobId=0,resource="streamshost.ibm.com",peId=0,operatorName="Logger",sequence=109,eventTimestamp=(1506002769,408000000,0),message="This is error log #.43"} - {notifyType="com.ibm.streams.management.log.application.warning",domainId="StreamsDomain",instanceId="StreamsInstance",jobId=0,resource="streamshost.ibm.com",peId=0,operatorName="Logger",sequence=110,eventTimestamp=(1506002770,471000000,0),message="This is warning log #.43"} + {notifyType="com.ibm.streams.management.log.application.error",instanceId="StreamsInstance",jobId=0,resource="streamshost.ibm.com",peId=0,operatorName="Logger",sequence=109,eventTimestamp=(1506002769,408000000,0),message="This is error log #.43"} + {notifyType="com.ibm.streams.management.log.application.warning",instanceId="StreamsInstance",jobId=0,resource="streamshost.ibm.com",peId=0,operatorName="Logger",sequence=110,eventTimestamp=(1506002770,471000000,0),message="This is warning log #.43"} ... diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/com.ibm.streamsx.monitoring.system.sample.LogSource/Monitor.spl b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/com.ibm.streamsx.monitoring.system.sample.LogSource/Monitor.spl index 61d0629..51ba910 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/com.ibm.streamsx.monitoring.system.sample.LogSource/Monitor.spl +++ b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/com.ibm.streamsx.monitoring.system.sample.LogSource/Monitor.spl @@ -15,7 +15,7 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * This sample application demonstrates the usage of the LogSource operator and monitors log events * * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. + * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain instanceId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. * * @param user * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. @@ -24,26 +24,19 @@ use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. + * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value * * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. + * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. * - * @param credentials - * Specifies Streaming Analytics service credentials in JSON format. Relevant for IAM authentication case only. If parameter is set, then the parameters user and password are ignored. If the **applicationConfigurationName** parameter is specified, the application configuration with the property credentials can override this parameter value. */ -composite Monitor { +public composite Monitor { param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set + expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", "monitoring"); // optional, if user and password are set expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $credentials: getSubmissionTimeValue("credentials", ""); + expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then instance settings are used, where the PE is running + expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then instance settings are used, where the PE is running graph @@ -58,9 +51,7 @@ composite Monitor { user: $user; password: $password; connectionURL: $connectionURL; - domainId: $domainId; sslOption: $sslOption; - credentials: $credentials; } /* diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/info.xml b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/info.xml index 165a944..fb88f09 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/info.xml +++ b/samples/com.ibm.streamsx.monitoring.system.sample.LogSource/info.xml @@ -4,13 +4,13 @@ com.ibm.streamsx.monitoring.system.sample.LogSource This sample SPL application demonstrates the use of LogSource operator to get the notification of application logs. - 1.0.0 - 4.2.0.0 + 1.1.0 + 4.3.0.0 com.ibm.streamsx.monitoring - 1.6.0 + 3.0.0 diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Makefile b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Makefile index 3340b44..a7cbc8b 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Makefile +++ b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Makefile @@ -37,3 +37,5 @@ clean: $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) rm -rf $(OUTPUT_DIR) +tkidx: + $(STREAMS_INSTALL)/bin/spl-make-toolkit -i . diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/README.md b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/README.md index 24719a5..0baa0dc 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/README.md +++ b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/README.md @@ -22,7 +22,7 @@ Build the **Monitor** application: Verify the **SystemMonitorSource** tuples in the console output. -*In the Streaming Analytics service, go to the Log Viewer and Click on the PE's Console Log to view output.* +*In the Streams Console, go to the Log Viewer and Click on the PE's Console Log to view output.* Date and time , CPU usage % , Memory usage (%) , Memory total (KB) , Memory used (KB) ... diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Monitor.spl b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Monitor.spl index 7d8bde7..3c78289 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Monitor.spl +++ b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/Monitor.spl @@ -13,7 +13,7 @@ use com.ibm.streamsx.monitoring.system::* ; * The status attribute contains addtional information retrieved from top command if the detail parameter is set to true. * Otherwise the status attribute contains the date and time only. */ -composite Monitor +public composite Monitor { param expression $period :(int32) getSubmissionTimeValue("period", "3") ; @@ -38,7 +38,7 @@ composite Monitor stream printLogs = Custom(SysMon) { logic state : { - mutable int32 i = 0; + mutable int64 i = 0l; mutable rstring headerLine = "Date and time , CPU usage % , Memory usage (%) , Memory total (KB) , Memory used (KB)" ; mutable rstring result = ""; @@ -47,7 +47,7 @@ composite Monitor onTuple SysMon: { result = status + " , " +(rstring) cpuUsage + " , " +(rstring) memUsage + " , " +(rstring)memTotal + " , " +(rstring) memUsed; - if (i == 0) { + if (i == 0l) { result = headerLine + "\n" + result; // add header to result } diff --git a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/info.xml b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/info.xml index 692adcf..e2c3777 100644 --- a/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/info.xml +++ b/samples/com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource/info.xml @@ -4,13 +4,13 @@ com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource This sample SPL application demonstrates the use of SystemMonitorSource operator to get the CPU and Memory usage from the current running system. - 1.0.0 - 4.1.0.0 + 1.1.0 + 4.3.0.0 com.ibm.streamsx.monitoring - 1.2.0 + 3.0.0 diff --git a/samples/launch_job_status_monitor_sample.py b/samples/launch_job_status_monitor_sample.py new file mode 100644 index 0000000..d662415 --- /dev/null +++ b/samples/launch_job_status_monitor_sample.py @@ -0,0 +1,37 @@ + +#Imports +from streamsx.topology.topology import * +from streamsx.topology.context import * +from streamsx.topology.schema import CommonSchema, StreamSchema +from streamsx.topology import context +import streamsx.spl.op as op + + +monitoring_toolkit = '../com.ibm.streamsx.monitoring' +sample_toolkit = 'com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor' + +def _launch(main): + cfg = {} + cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + rc = streamsx.topology.context.submit('DISTRIBUTED', main, cfg) + +def monitor_app(): + topo = Topology('JobStatusMonitorSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor::Monitor', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + +def sample_app(): + topo = Topology('CrashSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor::SampleJob', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + + +monitor_app() +sample_app() + + + diff --git a/samples/launch_logs_monitor_sample.py b/samples/launch_logs_monitor_sample.py new file mode 100644 index 0000000..c45c312 --- /dev/null +++ b/samples/launch_logs_monitor_sample.py @@ -0,0 +1,37 @@ + +#Imports +from streamsx.topology.topology import * +from streamsx.topology.context import * +from streamsx.topology.schema import CommonSchema, StreamSchema +from streamsx.topology import context +import streamsx.spl.op as op + + +monitoring_toolkit = '../com.ibm.streamsx.monitoring' +sample_toolkit = 'com.ibm.streamsx.monitoring.system.sample.LogSource' + +def _launch(main): + cfg = {} + cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + rc = streamsx.topology.context.submit('DISTRIBUTED', main, cfg) + +def monitor_app(): + topo = Topology('LogsMonitorSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.system.sample.LogSource::Monitor', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + +def sample_app(): + topo = Topology('LogsSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.system.sample.LogSource::SampleJob', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + + +monitor_app() +sample_app() + + + diff --git a/samples/launch_metrics_monitor_sample.py b/samples/launch_metrics_monitor_sample.py new file mode 100644 index 0000000..f391aff --- /dev/null +++ b/samples/launch_metrics_monitor_sample.py @@ -0,0 +1,37 @@ + +#Imports +from streamsx.topology.topology import * +from streamsx.topology.context import * +from streamsx.topology.schema import CommonSchema, StreamSchema +from streamsx.topology import context +import streamsx.spl.op as op + + +monitoring_toolkit = '../com.ibm.streamsx.monitoring' +sample_toolkit = 'com.ibm.streamsx.monitoring.metrics.sample.MetricsSource' + +def _launch(main): + cfg = {} + cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + rc = streamsx.topology.context.submit('DISTRIBUTED', main, cfg) + +def monitor_app(): + topo = Topology('MetricsMonitorSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.metrics.sample.MetricsSource::Monitor', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + +def sample_app(): + topo = Topology('MetricsSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.metrics.sample.MetricsSource::SampleJob', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + + +monitor_app() +sample_app() + + + diff --git a/samples/launch_system_monitor_sample.py b/samples/launch_system_monitor_sample.py new file mode 100644 index 0000000..9b2da1d --- /dev/null +++ b/samples/launch_system_monitor_sample.py @@ -0,0 +1,29 @@ + +#Imports +from streamsx.topology.topology import * +from streamsx.topology.context import * +from streamsx.topology.schema import CommonSchema, StreamSchema +from streamsx.topology import context +import streamsx.spl.op as op + + +monitoring_toolkit = '../com.ibm.streamsx.monitoring' +sample_toolkit = 'com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource' + +def _launch(main): + cfg = {} + cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + rc = streamsx.topology.context.submit('DISTRIBUTED', main, cfg) + +def sample_app(): + topo = Topology('SystemMonitorSample') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + streamsx.spl.toolkit.add_toolkit(topo, sample_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource::Monitor', toolkits=[monitoring_toolkit,sample_toolkit]) + _launch(r[0]) + + +sample_app() + + + diff --git a/tests/build.xml b/tests/build.xml deleted file mode 100644 index de99bec..0000000 --- a/tests/build.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/cloud-test/README.md b/tests/cloud-test/README.md index 5892c2a..2f6cbd4 100644 --- a/tests/cloud-test/README.md +++ b/tests/cloud-test/README.md @@ -1,44 +1,25 @@ -# Setup for testing with Streaming Analytics service +# Setup for testing with ICP4D Streams instance ## Before launching the test -Ensure that you have Python 3.5 installed. For example, you can get Python 3.5 from the [Anaconda archive page](https://repo.continuum.io/archive/index.html). +Setup your environment: -Ensure that the bin directory is added to the PATH environment variable. If necessary, add the bin directory by entering the following command on the command line: +* Python 3.6 installed +* streamsx package 1.12.9 or later installed - export PATH="~/anaconda3/bin:$PATH" - -Ensure that you have set the following environment variables: - -* `STREAMING_ANALYTICS_SERVICE_NAME` - name of your Streaming Analytics service -* `VCAP_SERVICES` - [VCAP](https://console.bluemix.net/docs/services/StreamingAnalytics/r_vcap_services.html#r_vcap_services) information in JSON format or a JSON file - -Install the latest streamsx package with pip, a package manager for Python, by entering the following command on the command line: - - pip install --user --upgrade streamsx - - -### Starting a Streaming Analytics service - -Make sure that your Streaming Analytics service is running. - -* If you have a Streaming Analytics service in IBM Cloud, make sure that it is started and running. -* To create a new Streaming Analytics service: - * Go to the IBM Cloud web portal and sign in (or sign up for a free IBM Cloud account). - * Click Catalog, browse for the Streaming Analytics service, and then click it. - * Enter the service name and then click Create to set up your service. The service dashboard opens and your service starts automatically. The service name appears as the title of the service dashboard. +Set `STREAMS_USERNAME`, `STREAMS_PASSWORD` and `STREAMS_REST_URL` environment variables. ### Create Application Configuration Test applications require an application configuration with the name "monitoring". -The "monitoring" application configuration needs to have a single property with the name "credentials" containing the JSON from the Streaming Analytics service credentials. - # Run the tests ``` ant test ``` + python3 -u -m unittest test_monitoring_icp.TestDistributed.test_metrics_monitor + # Clean-up Delete generated files of test suites. diff --git a/tests/cloud-test/build.xml b/tests/cloud-test/build.xml index ff2761f..9305d37 100644 --- a/tests/cloud-test/build.xml +++ b/tests/cloud-test/build.xml @@ -18,7 +18,7 @@ - + @@ -28,17 +28,12 @@ - - - - - - + - + diff --git a/tests/cloud-test/test_monitoring/test.system/TestLogsSource.spl b/tests/cloud-test/test_monitoring/test.system/TestLogsSource.spl index fde1fde..e102bc3 100644 --- a/tests/cloud-test/test_monitoring/test.system/TestLogsSource.spl +++ b/tests/cloud-test/test_monitoring/test.system/TestLogsSource.spl @@ -34,7 +34,7 @@ public composite TestLogsSource(output ResultStream) { if (I.notifyType == "com.ibm.streams.management.log.application.warning") { warnLogReceived = true; } - if ((!done1Sent) && (warnLogReceived) && (errLogReceived)) { + if ((!done1Sent) && (errLogReceived)) { submit({result="TEST_RESULT_PASS"}, ResultStream); done1Sent = true; } diff --git a/tests/cloud-test/test_streaming_analytics.py b/tests/cloud-test/test_monitoring_icp.py similarity index 70% rename from tests/cloud-test/test_streaming_analytics.py rename to tests/cloud-test/test_monitoring_icp.py index 0af6205..55e5a4d 100644 --- a/tests/cloud-test/test_streaming_analytics.py +++ b/tests/cloud-test/test_monitoring_icp.py @@ -8,18 +8,11 @@ import streamsx.rest as sr from streamsx.topology import context -class TestCloud(unittest.TestCase): - """ Test invocations of composite operators in Streaming Analytics Service """ - - @classmethod - def setUpClass(self): - # start streams service - connection = sr.StreamingAnalyticsConnection() - service = connection.get_streaming_analytics() - result = service.start_instance() +class TestDistributed(unittest.TestCase): + """ Test invocations of composite operators in IBM Streams """ def setUp(self): - Tester.setup_streaming_analytics(self, force_remote_build=False) + Tester.setup_distributed(self) def _add_toolkits(self, topo): tk.add_toolkit(topo, './test_monitoring') @@ -39,7 +32,10 @@ def _build_launch_validate(self, name, composite_name): tester.tuple_count(test_op.stream, 1, exact=True) tester.contents(test_op.stream, [{'result':'TEST_RESULT_PASS'}] ) - tester.test(self.test_ctxtype, self.test_config) + self.test_config[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + job_config = streamsx.topology.context.JobConfig(tracing='info') + job_config.add(self.test_config) + tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True) def _launch_sample_job(self): # this job is monitored by test.jobs::TestJobStatusSource application @@ -48,11 +44,9 @@ def _launch_sample_job(self): self._add_toolkits(topo) # Call the crash composite test_op = op.Source(topo, "test.jobs::SampleCrashSource", 'tuple') - # prepare config and submit the job to Streaming Analytics service config={} - sc = sr.StreamingAnalyticsConnection() - config[context.ConfigParams.STREAMS_CONNECTION] = sc - return context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=config) + config[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + return context.submit(context.ContextTypes.DISTRIBUTED, topo, config=config) def test_metrics_monitor(self): diff --git a/tests/monitoring_microservices.py b/tests/monitoring_microservices.py new file mode 100644 index 0000000..a9b423b --- /dev/null +++ b/tests/monitoring_microservices.py @@ -0,0 +1,53 @@ + +#Imports +from streamsx.topology.topology import * +from streamsx.topology.context import * +from streamsx.topology.schema import CommonSchema, StreamSchema +from streamsx.topology import context +import streamsx.spl.op as op + + +monitoring_toolkit = '../com.ibm.streamsx.monitoring' + +def _launch(main): + cfg = {} + cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False + rc = streamsx.topology.context.submit('DISTRIBUTED', main, cfg) + +def metrics_ingest_service(): + topo = Topology('MetricsIngestService') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.metrics.services::MetricsIngestService', toolkits=[monitoring_toolkit]) + _launch(r[0]) + +def job_status_service(): + topo = Topology('JobStatusService') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.jobs.services::JobStatusService', toolkits=[monitoring_toolkit]) + _launch(r[0]) + +def failed_pe_service(): + topo = Topology('FailedPEService') + streamsx.spl.toolkit.add_toolkit(topo, monitoring_toolkit) + r = op.main_composite(kind='com.ibm.streamsx.monitoring.jobs.services::FailedPEService', toolkits=[monitoring_toolkit]) + _launch(r[0]) + + +metrics_ingest_service() +job_status_service() +failed_pe_service() + + +topo = Topology("SubscribeSample") +jobs_status_schema = StreamSchema('tuple') +ts = topo.subscribe('streamsx/monitoring/jobs/status', schema=jobs_status_schema) +ts.print() +ts.isolate() + +s = topo.subscribe('streamsx/monitoring/metrics/values', schema=CommonSchema.Json) +s.print() +s.isolate() + +_launch(topo) + + diff --git a/tests/spl-test/JobStatusMonitor/__init__.py b/tests/spl-test/JobStatusMonitor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/JobStatusMonitor/test_app_config/Makefile b/tests/spl-test/JobStatusMonitor/test_app_config/Makefile deleted file mode 100644 index 42b779f..0000000 --- a/tests/spl-test/JobStatusMonitor/test_app_config/Makefile +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -configure: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - streamtool mkappconfig --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - -configure-none: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname sampleAppConfigJobStatus - -start-monitor: - streamtool submitjob output/monitor/Monitor.sab --jobname monitorAppConfigJobStatus -P applicationConfigurationName=com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - -stop-sample: - streamtool canceljob --jobnames sampleAppConfigJobStatus --force - -stop-monitor: - streamtool canceljob --jobnames monitorAppConfigJobStatus --collectlogs - diff --git a/tests/spl-test/JobStatusMonitor/test_app_config/Monitor.spl b/tests/spl-test/JobStatusMonitor/test_app_config/Monitor.spl deleted file mode 100644 index 795ec6b..0000000 --- a/tests/spl-test/JobStatusMonitor/test_app_config/Monitor.spl +++ /dev/null @@ -1,96 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -use com.ibm.streamsx.monitoring.jobs::*; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This sample application demonstrates the usage of the JobStatusMonitor operator and monitors all jobs and reports the Job/PE status events - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param filterDocument - * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the domain, instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * If parameter is not set, then all jobs in current domain and instance are monitored. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - */ -composite Monitor { - - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $filterDocument: getSubmissionTimeValue("filterDocument", ""); // uses default, if not set - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - - /* - * The JobStatusMonitor generates a tuple for each notified PE status change. - */ - (stream ChangeNotifications; - stream ConnectionNotifications - ) = JobStatusMonitor() { - param - applicationConfigurationName: $applicationConfigurationName; - user: $user; - password: $password; - filterDocument: $filterDocument; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - } - - /* - * Verify the received notifications. - */ - (stream SaveDone1) as ChangeNotificationTracer = Custom(ChangeNotifications as I; ConnectionNotifications as C) { - logic - state: { - mutable boolean done1Sent = false; - } - onTuple I: { - printStringLn((rstring)I); - if (I.notifyType == "com.ibm.streams.management.pe.changed") { - if (!done1Sent) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - } - } - onTuple C: { - printStringLn((rstring)C); - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - config - placement : partitionColocation("MONITOR"); - -} diff --git a/tests/spl-test/JobStatusMonitor/test_app_config/SampleJob.spl b/tests/spl-test/JobStatusMonitor/test_app_config/SampleJob.spl deleted file mode 100644 index 14080f0..0000000 --- a/tests/spl-test/JobStatusMonitor/test_app_config/SampleJob.spl +++ /dev/null @@ -1,38 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * This application is monitored by the Monitor application and - * forces an operator crash and PE restart. - */ -public composite SampleJob -{ - - graph - - stream Triggers as O = Beacon() { - param - period: 1.0; - iterations: 9u; - initDelay: 10.0; - } - - () as CrashSink = Custom(Triggers as I) { - logic - onTuple I: { - appLog(Log.error, "Operator is requested to fail."); - abort(); - } - config - restartable : true; - } - - - config - placement : partitionExlocation("TEST"); -} diff --git a/tests/spl-test/JobStatusMonitor/test_app_config/__init__.py b/tests/spl-test/JobStatusMonitor/test_app_config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/JobStatusMonitor/test_app_config/data/.gitignore b/tests/spl-test/JobStatusMonitor/test_app_config/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/JobStatusMonitor/test_app_config/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/JobStatusMonitor/test_app_config/test_app_config.py b/tests/spl-test/JobStatusMonitor/test_app_config/test_app_config.py deleted file mode 100644 index 785f809..0000000 --- a/tests/spl-test/JobStatusMonitor/test_app_config/test_app_config.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest -import os -import testharness as th - -class JobStatusMonitorAppConfigTest(unittest.TestCase): - - result_file_1 = "done_1" - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_files("StreamsLogsJob*.tgz") - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.stop_monitor() - th.stop_sample() - th.rm_app_config() - - def test_distributed(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.create_app_config() - th.make_applications() - th.start_monitor() - th.start_sample() - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/Makefile b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/Makefile deleted file mode 100644 index aebb21d..0000000 --- a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -configure: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - streamtool mkappconfig --property domainId=$(TEST_DOMAIN) --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - -start-test-domain: - streamtool startdomain -d $(TEST_DOMAIN) - streamtool startinst -d $(TEST_DOMAIN) -i $(TEST_INSTANCE) - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname sampleJmxReconnectJobStatus -i $(TEST_INSTANCE) -d $(TEST_DOMAIN) - -start-monitor: - streamtool submitjob output/monitor/Monitor.sab --jobname monitorJmxReconnectJobStatus -P domainId=$(TEST_DOMAIN) -P user=$(STREAMS_USERNAME) -P password=$(STREAMS_PASSWORD) -P connectionURL=`streamtool getjmxconnect -d $(TEST_DOMAIN)` - -stop-test-domain: - streamtool stopinst -d $(TEST_DOMAIN) -i $(TEST_INSTANCE) - streamtool stopdomain -d $(TEST_DOMAIN) - -stop-sample: - streamtool canceljob --jobnames sampleJmxReconnectJobStatus --force -i $(TEST_INSTANCE) -d $(TEST_DOMAIN) - -stop-monitor: - streamtool canceljob --jobnames monitorJmxReconnectJobStatus --collectlogs - diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/Monitor.spl b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/Monitor.spl deleted file mode 100644 index 3c59e02..0000000 --- a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/Monitor.spl +++ /dev/null @@ -1,104 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -use com.ibm.streamsx.monitoring.jobs::*; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This application tests the JMX reconnect in JobStatusMonitor operator - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - */ -composite Monitor { - - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - - /* - * The JobStatusMonitor generates a tuple for each notified PE status change. - */ - (stream ChangeNotifications; - stream ConnectionNotifications - ) = JobStatusMonitor() { - param - applicationConfigurationName: $applicationConfigurationName; - user: $user; - password: $password; - filterDocument: getThisToolkitDir()+"/etc" + "/JobStatusSource_MonitorEverything.json"; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - } - - /* - * Verify the received notifications. - */ - (stream SaveDone1;stream SaveDone2) as ChangeNotificationTracer = Custom(ChangeNotifications as I; ConnectionNotifications as C) { - logic - state: { - mutable boolean done1Sent = false; - mutable boolean done2Sent = false; - mutable boolean receivedConnectionNotification = false; - } - onTuple I: { - printStringLn((rstring)I); - if (I.notifyType == "com.ibm.streams.management.pe.changed") { - if (!done1Sent) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - if ((!done2Sent) && (receivedConnectionNotification)) { - submit({result="TEST_RESULT_PASS"}, SaveDone2); - submit(Sys.WindowMarker, SaveDone2); - done2Sent = true; - } - } - } - onTuple C: { - printStringLn((rstring)C); - // expect ConnectionNotification when domain is stopped and JMX connections is closed - receivedConnectionNotification = true; - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - () as Done2 = FileSink(SaveDone2 as I) { - param file: "done_2"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - config - placement : partitionColocation("MONITOR"); - -} diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/SampleJob.spl b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/SampleJob.spl deleted file mode 100644 index 14080f0..0000000 --- a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/SampleJob.spl +++ /dev/null @@ -1,38 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * This application is monitored by the Monitor application and - * forces an operator crash and PE restart. - */ -public composite SampleJob -{ - - graph - - stream Triggers as O = Beacon() { - param - period: 1.0; - iterations: 9u; - initDelay: 10.0; - } - - () as CrashSink = Custom(Triggers as I) { - logic - onTuple I: { - appLog(Log.error, "Operator is requested to fail."); - abort(); - } - config - restartable : true; - } - - - config - placement : partitionExlocation("TEST"); -} diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/__init__.py b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/data/.gitignore b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/etc/JobStatusSource_MonitorEverything.json b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/etc/JobStatusSource_MonitorEverything.json deleted file mode 100644 index 9e7b198..0000000 --- a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/etc/JobStatusSource_MonitorEverything.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "domainIdPatterns":".*", - "instances": - [ - { - "instanceIdPatterns":".*", - "jobs": - [ - { - "jobNamePatterns":".*", - } - ] - } - ] - } -] diff --git a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/test_jmx_reconnect.py b/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/test_jmx_reconnect.py deleted file mode 100644 index 1102993..0000000 --- a/tests/spl-test/JobStatusMonitor/test_jmx_reconnect/test_jmx_reconnect.py +++ /dev/null @@ -1,56 +0,0 @@ -import unittest -import os -import testharness as th - - -@unittest.skipIf(th.checkDomain() > 0, "Missing configuration for second Streams domain and instance. Environment variables TEST_DOMAIN and TEST_INSTANCE are not set.") -class JobStatusMonitorJmxReconnectTest(unittest.TestCase): - - result_file_1 = "done_1" - result_file_2 = "done_2" - - @classmethod - def setUpClass(cls): - th.checkEnvJMX() - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_f(self.result_file_2) - th.remove_files("StreamsLogsJob*.tgz") - th.start_test_domain() - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_f(self.result_file_2) - th.stop_monitor() - th.stop_sample() - th.stop_test_domain() - - def test_with_two_domains(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.make_applications() - th.start_monitor() - th.start_sample() - - # test if monitor app receives notifications - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - - # force JMX connection failure and monitor app needs to reconnect - th.stop_test_domain() - th.start_test_domain() - - th.start_sample() - - # test if monitor app receives notifications - th.wait_for_file(self.result_file_2) - err = th.test_result_file(self.result_file_2) - self.assertEqual(err, 0) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/JobStatusMonitor/test_restart_notification/Makefile b/tests/spl-test/JobStatusMonitor/test_restart_notification/Makefile deleted file mode 100644 index 4d0afa8..0000000 --- a/tests/spl-test/JobStatusMonitor/test_restart_notification/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -configure: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - streamtool mkappconfig --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) com.ibm.streamsx.monitoring.jobs.JobStatusMonitor.ApplicationConfiguration - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname restartNotificationJobStatus - -stop-sample: - streamtool canceljob --jobnames restartNotificationJobStatus --force diff --git a/tests/spl-test/JobStatusMonitor/test_restart_notification/Monitor.spl b/tests/spl-test/JobStatusMonitor/test_restart_notification/Monitor.spl deleted file mode 100644 index ab72a51..0000000 --- a/tests/spl-test/JobStatusMonitor/test_restart_notification/Monitor.spl +++ /dev/null @@ -1,108 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -use com.ibm.streamsx.monitoring.jobs::*; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This application tests JobStatusMonitor operator in standalone mode - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param filterDocument - * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the domain, instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * If parameter is not set, then all jobs in current domain and instance are monitored. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - */ -composite Monitor { - - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $filterDocument: getSubmissionTimeValue("filterDocument", ""); // uses default, if not set - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - - /* - * The JobStatusMonitor generates a tuple for each notified PE status change. - */ - (stream ChangeNotifications; - stream ConnectionNotifications - ) = JobStatusMonitor() { - param - applicationConfigurationName: $applicationConfigurationName; - user: $user; - password: $password; - filterDocument: $filterDocument; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - } - - /* - * Verify the received notifications. - */ - () as ChangeNotificationTracer = Custom(ChangeNotifications as I; ConnectionNotifications as C; TimeoutStream as T) { - logic - state: { - mutable boolean passed = false; - } - onTuple I: { - printStringLn((rstring)I); - if ((I.notifyType == "com.ibm.streams.management.pe.changed") && (I.peStatus == "restarting")) { - if ((length(I.jobName) > 0) && (length(I.domainId) > 0) && (length(I.instanceId) > 0) && (length(I.resource) > 0)) { - printStringLn("TEST_RESULT_PASS"); - passed = true; - shutdownPE(); - } - else { - printStringLn("ERROR: MISSING ATTRIBUTE VALUES"); - printStringLn("TEST_RESULT_FAIL"); - abort(); - } - } - } - onTuple T: { - if ((false == passed) && (false == isShutdown())) { - printStringLn("ERROR: TIMEOUT"); - printStringLn("TEST_RESULT_FAIL"); - abort(); - } - } - } - - stream TimeoutStream = Beacon() { - param - iterations: 1u; - initDelay: 60.0; - } - - - config - placement : partitionColocation("MONITOR"); - -} diff --git a/tests/spl-test/JobStatusMonitor/test_restart_notification/SampleJob.spl b/tests/spl-test/JobStatusMonitor/test_restart_notification/SampleJob.spl deleted file mode 100644 index 14080f0..0000000 --- a/tests/spl-test/JobStatusMonitor/test_restart_notification/SampleJob.spl +++ /dev/null @@ -1,38 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * This application is monitored by the Monitor application and - * forces an operator crash and PE restart. - */ -public composite SampleJob -{ - - graph - - stream Triggers as O = Beacon() { - param - period: 1.0; - iterations: 9u; - initDelay: 10.0; - } - - () as CrashSink = Custom(Triggers as I) { - logic - onTuple I: { - appLog(Log.error, "Operator is requested to fail."); - abort(); - } - config - restartable : true; - } - - - config - placement : partitionExlocation("TEST"); -} diff --git a/tests/spl-test/JobStatusMonitor/test_restart_notification/__init__.py b/tests/spl-test/JobStatusMonitor/test_restart_notification/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/JobStatusMonitor/test_restart_notification/test_restart_notification.py b/tests/spl-test/JobStatusMonitor/test_restart_notification/test_restart_notification.py deleted file mode 100644 index 5fe91b6..0000000 --- a/tests/spl-test/JobStatusMonitor/test_restart_notification/test_restart_notification.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -import os -import testharness as th - -class JobStatusMonitorRestartNotificationTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.stop_sample() - - def test_standalone(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.make_applications() - th.start_sample() - stdout, stderr, err = th.run_monitor_standalone(args=['user='+os.environ["STREAMS_USERNAME"], 'password='+os.environ["STREAMS_PASSWORD"], 'domainId='+os.environ["STREAMS_DOMAIN_ID"]]) - th.assert_pass(err == 0 and (str(stdout).find('TEST_RESULT_PASS') != -1), stdout, stderr) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/LogSource/__init__.py b/tests/spl-test/LogSource/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/LogSource/test_app_config/Main.spl b/tests/spl-test/LogSource/test_app_config/Main.spl deleted file mode 100644 index 2bd6ee1..0000000 --- a/tests/spl-test/LogSource/test_app_config/Main.spl +++ /dev/null @@ -1,81 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2016, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.monitoring.system::LogSource; -use com.ibm.streamsx.monitoring.system::*; - -/** - * This application tests the LogSource operator with applicationConfigurationName parameter - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - */ -composite Main { - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", "com.ibm.streamsx.monitoring.LogSource.ApplicationConfiguration"); - - graph - - /* - * The LogSource generates a tuple for each notified log event. - */ - (stream Logs - ) = LogSource() { - param - applicationConfigurationName: $applicationConfigurationName; - } - - /* - * Verify the received notifications. - */ - (stream SaveDone1) as NotificationTracer = Custom(Logs as I) { - logic - state: { - mutable boolean done1Sent = false; - mutable boolean errLogReceived = false; - mutable boolean warnLogReceived = false; - } - onTuple I: { - printStringLn((rstring)I); - if (I.notifyType == "com.ibm.streams.management.log.application.error") { - errLogReceived = true; - } - if (I.notifyType == "com.ibm.streams.management.log.application.warning") { - warnLogReceived = true; - } - if ((!done1Sent) && (warnLogReceived) && (errLogReceived)) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - stream Triggers as O = Beacon() { - param - period: 2.5; - } - - () as Logger = Custom(Triggers as I) { - logic - state: { - mutable int64 count = 0l; - } - onTuple I: { - count++; - appLog(Log.error, "This is error log #."+(rstring)count); - block((float64)1ul); - appLog(Log.warn, "This is warning log #."+(rstring)count); - } - } - -} diff --git a/tests/spl-test/LogSource/test_app_config/Makefile b/tests/spl-test/LogSource/test_app_config/Makefile deleted file mode 100644 index b8534d8..0000000 --- a/tests/spl-test/LogSource/test_app_config/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Main - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR) - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -configure: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.LogSource.ApplicationConfiguration - streamtool mkappconfig --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) com.ibm.streamsx.monitoring.LogSource.ApplicationConfiguration - -configure-none: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.LogSource.ApplicationConfiguration - -start-monitor: - streamtool submitjob output/Main.sab --jobname logSourceAppConfig - -stop-monitor: - streamtool canceljob --jobnames logSourceAppConfig --collectlogs - diff --git a/tests/spl-test/LogSource/test_app_config/__init__.py b/tests/spl-test/LogSource/test_app_config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/LogSource/test_app_config/data/.gitignore b/tests/spl-test/LogSource/test_app_config/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/LogSource/test_app_config/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/LogSource/test_app_config/test_app_config.py b/tests/spl-test/LogSource/test_app_config/test_app_config.py deleted file mode 100644 index a0487dd..0000000 --- a/tests/spl-test/LogSource/test_app_config/test_app_config.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest -import os -import testharness as th - -class LogSourceAppConfigTest(unittest.TestCase): - - result_file_1 = "done_1" - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_files("StreamsLogsJob*.tgz") - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.stop_monitor() - th.rm_app_config() - - def test_distributed(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.create_app_config() - th.make_applications() - th.start_monitor() - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/LogSource/test_jmx_reconnect/Makefile b/tests/spl-test/LogSource/test_jmx_reconnect/Makefile deleted file mode 100644 index 7efcbfc..0000000 --- a/tests/spl-test/LogSource/test_jmx_reconnect/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -start-test-domain: - streamtool startdomain -d $(TEST_DOMAIN) - streamtool startinst -d $(TEST_DOMAIN) -i $(TEST_INSTANCE) - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname sampleJmxReconnectLogSource -i $(TEST_INSTANCE) -d $(TEST_DOMAIN) - -start-monitor: - streamtool submitjob output/monitor/Monitor.sab --jobname monitorJmxReconnectLogSource -P domainId=$(TEST_DOMAIN) -P user=$(STREAMS_USERNAME) -P password=$(STREAMS_PASSWORD) -P connectionURL=`streamtool getjmxconnect -d $(TEST_DOMAIN)` - -stop-test-domain: - streamtool stopinst -d $(TEST_DOMAIN) -i $(TEST_INSTANCE) - streamtool stopdomain -d $(TEST_DOMAIN) - -stop-sample: - streamtool canceljob --jobnames sampleJmxReconnectLogSource --force -i $(TEST_INSTANCE) -d $(TEST_DOMAIN) - -stop-monitor: - streamtool canceljob --jobnames monitorJmxReconnectLogSource --collectlogs - diff --git a/tests/spl-test/LogSource/test_jmx_reconnect/Monitor.spl b/tests/spl-test/LogSource/test_jmx_reconnect/Monitor.spl deleted file mode 100644 index 17e4a41..0000000 --- a/tests/spl-test/LogSource/test_jmx_reconnect/Monitor.spl +++ /dev/null @@ -1,101 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.monitoring.system::LogSource; -use com.ibm.streamsx.monitoring.system::*; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This application tests the JMX reconnect in MetricsSource operator - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - */ -composite Monitor { - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - - /* - * The LogSource generates a tuple for each notified log event. - */ - (stream Logs; - stream ConnectionNotifications - ) = LogSource() { - param - applicationConfigurationName: $applicationConfigurationName; - user: $user; - password: $password; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - } - - /* - * Verify the received notifications. - */ - (stream SaveDone1; stream SaveDone2) as NotificationTracer = Custom(Logs as I; ConnectionNotifications as C) { - logic - state: { - mutable boolean done1Sent = false; - mutable boolean done2Sent = false; - mutable boolean receivedConnectionNotification = false; - } - onTuple I: { - printStringLn((rstring)I); - if (I.notifyType == "com.ibm.streams.management.log.application.error") { - if (!done1Sent) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - if ((!done2Sent) && (receivedConnectionNotification)) { - submit({result="TEST_RESULT_PASS"}, SaveDone2); - submit(Sys.WindowMarker, SaveDone2); - done2Sent = true; - } - } - } - onTuple C: { - printStringLn((rstring)C); - // expect ConnectionNotification when domain is stopped and JMX connections is closed - receivedConnectionNotification = true; - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - () as Done2 = FileSink(SaveDone2 as I) { - param file: "done_2"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - -} diff --git a/tests/spl-test/LogSource/test_jmx_reconnect/SampleJob.spl b/tests/spl-test/LogSource/test_jmx_reconnect/SampleJob.spl deleted file mode 100644 index be36e34..0000000 --- a/tests/spl-test/LogSource/test_jmx_reconnect/SampleJob.spl +++ /dev/null @@ -1,35 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * This application is monitored by the Monitor application creates application logs - */ -public composite SampleJob -{ - - graph - - stream Triggers as O = Beacon() { - param - period: 2.5; - } - - () as Logger = Custom(Triggers as I) { - logic - state: { - mutable int64 count = 0l; - } - onTuple I: { - count++; - appLog(Log.error, "This is error log #."+(rstring)count); - block((float64)1ul); - appLog(Log.warn, "This is warning log #."+(rstring)count); - } - } - -} diff --git a/tests/spl-test/LogSource/test_jmx_reconnect/__init__.py b/tests/spl-test/LogSource/test_jmx_reconnect/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/LogSource/test_jmx_reconnect/data/.gitignore b/tests/spl-test/LogSource/test_jmx_reconnect/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/LogSource/test_jmx_reconnect/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/LogSource/test_jmx_reconnect/test_jmx_reconnect.py b/tests/spl-test/LogSource/test_jmx_reconnect/test_jmx_reconnect.py deleted file mode 100644 index e3b9d9b..0000000 --- a/tests/spl-test/LogSource/test_jmx_reconnect/test_jmx_reconnect.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -import os -import testharness as th - -@unittest.skipIf(th.checkDomain() > 0, "Missing configuration for second Streams domain and instance. Environment variables TEST_DOMAIN and TEST_INSTANCE are not set.") -class LogSourceJmxReconnectTest(unittest.TestCase): - - result_file_1 = "done_1" - result_file_2 = "done_2" - - @classmethod - def setUpClass(cls): - th.checkEnvJMX() - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_f(self.result_file_2) - th.remove_files("StreamsLogsJob*.tgz") - th.start_test_domain() - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_f(self.result_file_2) - th.stop_monitor() - th.stop_sample() - th.stop_test_domain() - - def test_with_two_domains(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.make_applications() - th.start_monitor() - th.start_sample() - - # test if monitor app receives notifications - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - - # force JMX connection failure and monitor app needs to reconnect - th.stop_test_domain() - th.start_test_domain() - - th.start_sample() - - # test if monitor app receives notifications - th.wait_for_file(self.result_file_2) - err = th.test_result_file(self.result_file_2) - self.assertEqual(err, 0) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/LogSource/test_log_notification/Makefile b/tests/spl-test/LogSource/test_log_notification/Makefile deleted file mode 100644 index d38c6ca..0000000 --- a/tests/spl-test/LogSource/test_log_notification/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname appLogNotifications - -stop-sample: - streamtool canceljob --jobnames appLogNotifications --force diff --git a/tests/spl-test/LogSource/test_log_notification/Monitor.spl b/tests/spl-test/LogSource/test_log_notification/Monitor.spl deleted file mode 100644 index 4292a59..0000000 --- a/tests/spl-test/LogSource/test_log_notification/Monitor.spl +++ /dev/null @@ -1,107 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.monitoring.system::LogSource; -use com.ibm.streamsx.monitoring.system::*; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This application tests LogSource operator in standalone mode - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - */ -composite Monitor { - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - - /* - * The LogSource generates a tuple for each notified log event. - */ - (stream Logs - ) = LogSource() { - param - applicationConfigurationName: $applicationConfigurationName; - user: $user; - password: $password; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - } - - /* - * Verify the received notifications. - */ - () as NotificationTracer = Custom(Logs as I; TimeoutStream as T) { - logic - state: { - mutable boolean passed = false; - mutable boolean errLogReceived = false; - mutable boolean warnLogReceived = false; - } - onTuple I: { - printStringLn((rstring)I); - if (I.notifyType == "com.ibm.streams.management.log.application.error") { - errLogReceived = true; - } - if (I.notifyType == "com.ibm.streams.management.log.application.warning") { - warnLogReceived = true; - } - - if ((warnLogReceived) && (errLogReceived)) { - if ((length(I.domainId) > 0) && (length(I.instanceId) > 0) && (length(I.resource) > 0) && (length(I.message) > 0) && (length(I.operatorName) > 0)) { - printStringLn("TEST_RESULT_PASS"); - passed = true; - shutdownPE(); - } - else { - printStringLn("ERROR: MISSING ATTRIBUTE VALUES"); - printStringLn("RECEIVED TUPLE:["+(rstring)I+"]"); - printStringLn("TEST_RESULT_FAIL"); - abort(); - } - } - } - onTuple T: { - if ((false == passed) && (false == isShutdown())) { - printStringLn("ERROR: TIMEOUT"); - printStringLn("TEST_RESULT_FAIL"); - abort(); - } - } - } - - stream TimeoutStream = Beacon() { - param - iterations: 1u; - initDelay: 60.0; - } - -} diff --git a/tests/spl-test/LogSource/test_log_notification/SampleJob.spl b/tests/spl-test/LogSource/test_log_notification/SampleJob.spl deleted file mode 100644 index be36e34..0000000 --- a/tests/spl-test/LogSource/test_log_notification/SampleJob.spl +++ /dev/null @@ -1,35 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * This application is monitored by the Monitor application creates application logs - */ -public composite SampleJob -{ - - graph - - stream Triggers as O = Beacon() { - param - period: 2.5; - } - - () as Logger = Custom(Triggers as I) { - logic - state: { - mutable int64 count = 0l; - } - onTuple I: { - count++; - appLog(Log.error, "This is error log #."+(rstring)count); - block((float64)1ul); - appLog(Log.warn, "This is warning log #."+(rstring)count); - } - } - -} diff --git a/tests/spl-test/LogSource/test_log_notification/__init__.py b/tests/spl-test/LogSource/test_log_notification/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/LogSource/test_log_notification/test_log_notification.py b/tests/spl-test/LogSource/test_log_notification/test_log_notification.py deleted file mode 100644 index 08d56de..0000000 --- a/tests/spl-test/LogSource/test_log_notification/test_log_notification.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -import os -import testharness as th - -class LogSourceNotificationTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.stop_sample() - - def test_standalone(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.make_applications() - th.start_sample() - stdout, stderr, err = th.run_monitor_standalone(args=['user='+os.environ["STREAMS_USERNAME"], 'password='+os.environ["STREAMS_PASSWORD"], 'domainId='+os.environ["STREAMS_DOMAIN_ID"]]) - th.assert_pass(err == 0 and (str(stdout).find('TEST_RESULT_PASS') != -1), stdout, stderr) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/MetricsMonitor/__init__.py b/tests/spl-test/MetricsMonitor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/MetricsMonitor/test_microservices/Main.spl b/tests/spl-test/MetricsMonitor/test_microservices/Main.spl deleted file mode 100644 index 58ac3a0..0000000 --- a/tests/spl-test/MetricsMonitor/test_microservices/Main.spl +++ /dev/null @@ -1,91 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.topology.topic::Subscribe; - -/** - * Create the custom application metrics. The function is usable in the state - * logic of an operator. - */ -stateful boolean createMetrics() { - createCustomMetric("inc", "The incrementing number.", Sys.Counter, 0l); - return true; -} - -/** - * This test application subscribes the topic of the MetricsMonitor microservice - * - * @param metricsMonitorTopic - * Specifies the topic of the MetricsMonitor microservice - * - */ -composite Main { - param - expression $metricsMonitorTopic : getSubmissionTimeValue("metricsMonitorTopic", "streamsx/monitoring/metrics/alerts"); - - graph - - stream AlertStrings = Subscribe() { - param - topic: $metricsMonitorTopic; - streamType: com.ibm.streamsx.topology::String; - } - - stream AlertMessages = Subscribe() { - param - topic: $metricsMonitorTopic; - streamType: com.ibm.streamsx.monitoring.metrics::Alert; - } - - (stream SaveDone1) as Validator = Custom(AlertStrings as I; AlertMessages as M) { - logic - state: { - mutable boolean done1Sent = false; - } - onTuple I: { - printStringLn((rstring)I); - if (!done1Sent) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - } - onTuple M: { - printStringLn((rstring)M); - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - /* - * The Beacon generates incrementing. - */ - stream Numbers as O = Beacon() { - param period: 0.5; - output O: - incrementingNumber = (int64)IterationCount(); - } - - /* - * The Custom stores the received numbers in metrics. - */ - () as Storage = Custom(Numbers as I) { - logic - state: { - boolean created = createMetrics(); - } - onTuple I: { - setCustomMetricValue("inc", I.incrementingNumber); - } - } -} - - - - diff --git a/tests/spl-test/MetricsMonitor/test_microservices/Makefile b/tests/spl-test/MetricsMonitor/test_microservices/Makefile deleted file mode 100644 index b51c69a..0000000 --- a/tests/spl-test/MetricsMonitor/test_microservices/Makefile +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -STREAMSX_JSON_TOOLKIT ?=$(STREAMS_INSTALL)/toolkits/com.ibm.streamsx.json -STREAMSX_TOPOLOGY_TOOLKIT ?=$(STREAMS_INSTALL)/toolkits/com.ibm.streamsx.topology -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT):$(STREAMSX_JSON_TOOLKIT):$(STREAMSX_TOPOLOGY_TOOLKIT) - -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Main - -build: distributed - -all: clean build - $(SPLC) $(SPLC_FLAGS) -M com.ibm.streamsx.monitoring.metrics.services::MetricsIngestService $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/MetricsIngestService - $(SPLC) $(SPLC_FLAGS) -M com.ibm.streamsx.monitoring.metrics.services::MetricsMonitorService $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/MetricsMonitorService - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR) - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -configure: - -streamtool rmappconfig --noprompt thresholds - streamtool mkappconfig --property thresholdDocument=`cat etc/microservices.json | perl -e 'my @content = ; my $$str = join("", @content); $$str =~ s/\s//g; print $$str;'` thresholds - -streamtool rmappconfig --noprompt monitoring - streamtool mkappconfig --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) monitoring - -configure-none: - -streamtool rmappconfig --noprompt monitoring - -streamtool rmappconfig --noprompt thresholds - -start-monitor: - streamtool submitjob output/MetricsIngestService/com.ibm.streamsx.monitoring.metrics.services.MetricsIngestService.sab --jobname microserviceMetrics - streamtool submitjob output/MetricsMonitorService/com.ibm.streamsx.monitoring.metrics.services.MetricsMonitorService.sab --jobname microserviceMonitor - streamtool submitjob output/Main.sab --jobname testMicroservices - -stop-monitor: - streamtool canceljob --jobnames microserviceMetrics --collectlogs - streamtool canceljob --jobnames microserviceMonitor --collectlogs - streamtool canceljob --jobnames testMicroservices --collectlogs - diff --git a/tests/spl-test/MetricsMonitor/test_microservices/__init__.py b/tests/spl-test/MetricsMonitor/test_microservices/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/MetricsMonitor/test_microservices/data/.gitignore b/tests/spl-test/MetricsMonitor/test_microservices/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/MetricsMonitor/test_microservices/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/MetricsMonitor/test_microservices/etc/microservices.json b/tests/spl-test/MetricsMonitor/test_microservices/etc/microservices.json deleted file mode 100644 index c0f282b..0000000 --- a/tests/spl-test/MetricsMonitor/test_microservices/etc/microservices.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "metricName":"inc", - "thresholds": - { - "value":">100" - } - } -] diff --git a/tests/spl-test/MetricsMonitor/test_microservices/test_microservices.py b/tests/spl-test/MetricsMonitor/test_microservices/test_microservices.py deleted file mode 100644 index b6202fc..0000000 --- a/tests/spl-test/MetricsMonitor/test_microservices/test_microservices.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -import os -import testharness as th - -class MetricsMonitorMicroservicesTest(unittest.TestCase): - - result_file_1 = "done_1" - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_files("StreamsLogsJob*.tgz") - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.stop_monitor() - th.rm_app_config() - - def test_distributed(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.create_app_config() - th.make_applications() - th.start_monitor() - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/MetricsSource/__init__.py b/tests/spl-test/MetricsSource/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/MetricsSource/test_app_config/Main.spl b/tests/spl-test/MetricsSource/test_app_config/Main.spl deleted file mode 100644 index 8671ccd..0000000 --- a/tests/spl-test/MetricsSource/test_app_config/Main.spl +++ /dev/null @@ -1,73 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2016, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.monitoring.metrics::MetricsSource; -use com.ibm.streamsx.monitoring.metrics::Notification; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This sample application demonstrate how to use the MetricsSource operator - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - */ -composite Main { - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", "com.ibm.streamsx.monitoring.MetricsSource.ApplicationConfiguration"); - expression $filterDocument: getSubmissionTimeValue("filterDocument", ""); // uses default, if not set - - graph - /* - * The MetricsSource generates tuple for the specified metrics. - */ - (stream MetricNotifications - ) = MetricsSource() { - param - applicationConfigurationName: $applicationConfigurationName; - filterDocument: $filterDocument; // test needs to apply etc/MetricsSource_MonitorOperatorMetrics.json as submission param - emitMetricTuple: "onChangedValue"; - } - - /* - * Verify the received metric value notifications. - */ - (stream SaveDone1) as NotificationTracer = Custom(MetricNotifications as I) { - logic - state: { - mutable boolean done1Sent = false; - mutable boolean metricNotificationReceived = false; - } - onTuple I: { - printStringLn((rstring)I); - if ((I.metricName=="nTuplesSubmitted") && (-1 != findFirst(I.operatorName, "AnnotatedSource", 0))) { - metricNotificationReceived = true; - } - } - onPunct I: { - if ((!done1Sent) && (metricNotificationReceived)) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - - /* - * A @parallel-annotated source. - */ - @parallel(width=3) - (stream Data) as AnnotatedSource = Beacon() { - param period: 3.0; - } - -} diff --git a/tests/spl-test/MetricsSource/test_app_config/Makefile b/tests/spl-test/MetricsSource/test_app_config/Makefile deleted file mode 100644 index 1c11c59..0000000 --- a/tests/spl-test/MetricsSource/test_app_config/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Main -START_MON_ARGS ?= - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR) - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -configure: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.MetricsSource.ApplicationConfiguration - streamtool mkappconfig --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) com.ibm.streamsx.monitoring.MetricsSource.ApplicationConfiguration - -configure-json: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.MetricsSource.ApplicationConfiguration - streamtool mkappconfig --property user=$(STREAMS_USERNAME) --property password=$(STREAMS_PASSWORD) --property filterDocument=`cat etc/MetricsSource_MonitorOperatorMetrics.json | perl -e 'my @content = ; my $$str = join("", @content); $$str =~ s/\s//g; print $$str;'` com.ibm.streamsx.monitoring.MetricsSource.ApplicationConfiguration - -configure-none: - -streamtool rmappconfig --noprompt com.ibm.streamsx.monitoring.MetricsSource.ApplicationConfiguration - -start-monitor: - streamtool submitjob output/Main.sab --jobname metricsAppConfig $(START_MON_ARGS) - -stop-monitor: - streamtool canceljob --jobnames metricsAppConfig --collectlogs - diff --git a/tests/spl-test/MetricsSource/test_app_config/__init__.py b/tests/spl-test/MetricsSource/test_app_config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/MetricsSource/test_app_config/data/.gitignore b/tests/spl-test/MetricsSource/test_app_config/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/MetricsSource/test_app_config/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/MetricsSource/test_app_config/etc/MetricsSource_MonitorOperatorMetrics.json b/tests/spl-test/MetricsSource/test_app_config/etc/MetricsSource_MonitorOperatorMetrics.json deleted file mode 100644 index 8a2fbd8..0000000 --- a/tests/spl-test/MetricsSource/test_app_config/etc/MetricsSource_MonitorOperatorMetrics.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "domainIdPatterns":".*", - "instances": - [ - { - "instanceIdPatterns":".*", - "jobs": - [ - { - "jobNamePatterns":".*", - "operators": - [ - { - "operatorNamePatterns":".*", - "outputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ] - } - ] - } - ] - } - ] - } -] diff --git a/tests/spl-test/MetricsSource/test_app_config/test_app_config.py b/tests/spl-test/MetricsSource/test_app_config/test_app_config.py deleted file mode 100644 index 0f780a2..0000000 --- a/tests/spl-test/MetricsSource/test_app_config/test_app_config.py +++ /dev/null @@ -1,61 +0,0 @@ -import unittest -import os -import testharness as th - -class MetricsSourceAppConfigTest(unittest.TestCase): - - result_file_1 = "done_1" - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_files("StreamsLogsJob*.tgz") - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.stop_monitor() - th.rm_app_config() - - def test_distributed(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.create_app_config() - th.make_applications() - th.start_monitor() - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - - def test_distributed_filter_doc_in_app_config(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.create_app_config_json() - th.make_applications() - th.start_monitor() - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - - def test_distributed_filter_doc_param_file(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.create_app_config() - th.make_applications() - th.start_monitor(['START_MON_ARGS=-P filterDocument=etc/MetricsSource_MonitorOperatorMetrics.json']) - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/Makefile b/tests/spl-test/MetricsSource/test_jmx_reconnect/Makefile deleted file mode 100644 index 42181c7..0000000 --- a/tests/spl-test/MetricsSource/test_jmx_reconnect/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) -SPL_DATA_DIR = ./data - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --data-directory $(SPL_DATA_DIR) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -start-test-domain: - streamtool startdomain -d $(TEST_DOMAIN) - streamtool startinst -d $(TEST_DOMAIN) -i $(TEST_INSTANCE) - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname sampleJmxReconnectMetricsSource -i $(TEST_INSTANCE) -d $(TEST_DOMAIN) - -start-monitor: - streamtool submitjob output/monitor/Monitor.sab --jobname monitorJmxReconnectMetricsSource -P domainId=$(TEST_DOMAIN) -P user=$(STREAMS_USERNAME) -P password=$(STREAMS_PASSWORD) -P connectionURL=`streamtool getjmxconnect -d $(TEST_DOMAIN)` - -stop-test-domain: - streamtool stopinst -d $(TEST_DOMAIN) -i $(TEST_INSTANCE) - streamtool stopdomain -d $(TEST_DOMAIN) - -stop-sample: - streamtool canceljob --jobnames sampleJmxReconnectMetricsSource --force -i $(TEST_INSTANCE) -d $(TEST_DOMAIN) - -stop-monitor: - streamtool canceljob --jobnames monitorJmxReconnectMetricsSource --collectlogs - diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/Monitor.spl b/tests/spl-test/MetricsSource/test_jmx_reconnect/Monitor.spl deleted file mode 100644 index e804ab2..0000000 --- a/tests/spl-test/MetricsSource/test_jmx_reconnect/Monitor.spl +++ /dev/null @@ -1,93 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.monitoring.metrics::*; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This application tests the JMX reconnect in MetricsSource operator - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - */ -composite Monitor { - param - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - - (stream MetricNotifications; stream ConnectionNotifications - ) = MetricsSource() { - param - user: $user; - password: $password; - filterDocument: getThisToolkitDir()+"/etc" + "/MetricsSource_MonitorEverything.json"; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - emitMetricTuple: "onChangedValue"; - } - - /* - * Verify the received notifications. - */ - (stream SaveDone1; stream SaveDone2) as NotificationTracer = Custom(MetricNotifications as I; ConnectionNotifications as C) { - logic - state: { - mutable boolean done1Sent = false; - mutable boolean done2Sent = false; - mutable boolean receivedConnectionNotification = false; - } - onTuple I: { - printStringLn((rstring)I); - if ((I.metricName=="nTuplesSubmitted") && (-1 != findFirst(I.operatorName, "AnnotatedSource", 0))) { - if (!done1Sent) { - submit({result="TEST_RESULT_PASS"}, SaveDone1); - submit(Sys.WindowMarker, SaveDone1); - done1Sent = true; - } - if ((!done2Sent) && (receivedConnectionNotification)) { - submit({result="TEST_RESULT_PASS"}, SaveDone2); - submit(Sys.WindowMarker, SaveDone2); - done2Sent = true; - } - } - } - onTuple C: { - printStringLn((rstring)C); - // expect ConnectionNotification when domain is stopped and JMX connections is closed - receivedConnectionNotification = true; - } - } - - () as Done1 = FileSink(SaveDone1 as I) { - param file: "done_1"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - () as Done2 = FileSink(SaveDone2 as I) { - param file: "done_2"; format: csv; flush: 1u; quoteStrings : false; writePunctuations: false; closeMode: punct; moveFileToDirectory: dataDirectory()+"/.."; - } - - -} diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/SampleJob.spl b/tests/spl-test/MetricsSource/test_jmx_reconnect/SampleJob.spl deleted file mode 100644 index 8eb176f..0000000 --- a/tests/spl-test/MetricsSource/test_jmx_reconnect/SampleJob.spl +++ /dev/null @@ -1,25 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * This application is monitored by the Monitor application - */ -public composite SampleJob -{ - - graph - - /* - * A @parallel-annotated source. - */ - @parallel(width=3) - (stream Data) as AnnotatedSource = Beacon() { - param period: 3.0; - } - -} diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/__init__.py b/tests/spl-test/MetricsSource/test_jmx_reconnect/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/data/.gitignore b/tests/spl-test/MetricsSource/test_jmx_reconnect/data/.gitignore deleted file mode 100644 index 57c3d00..0000000 --- a/tests/spl-test/MetricsSource/test_jmx_reconnect/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -done* diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/etc/MetricsSource_MonitorEverything.json b/tests/spl-test/MetricsSource/test_jmx_reconnect/etc/MetricsSource_MonitorEverything.json deleted file mode 100644 index 62dd8b9..0000000 --- a/tests/spl-test/MetricsSource/test_jmx_reconnect/etc/MetricsSource_MonitorEverything.json +++ /dev/null @@ -1,65 +0,0 @@ -[ - { - "domainIdPatterns":".*", - "instances": - [ - { - "instanceIdPatterns":".*", - "jobs": - [ - { - "jobNamePatterns":".*", - "pes": - [ - { - "metricNamePatterns":".*", - "inputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ], - "outputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ], - "connections": - [ - { - "connectionIdPatterns":".*", - "metricNamePatterns":".*" - } - ] - } - ], - "operators": - [ - { - "operatorNamePatterns":".*", - "metricNamePatterns":".*", - "inputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ], - "outputPorts": - [ - { - "portIndexes":"*", - "metricNamePatterns":".*" - } - ] - } - ] - } - ] - } - ] - } -] diff --git a/tests/spl-test/MetricsSource/test_jmx_reconnect/test_jmx_reconnect.py b/tests/spl-test/MetricsSource/test_jmx_reconnect/test_jmx_reconnect.py deleted file mode 100644 index fc0dd0b..0000000 --- a/tests/spl-test/MetricsSource/test_jmx_reconnect/test_jmx_reconnect.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -import os -import testharness as th - -@unittest.skipIf(th.checkDomain() > 0, "Missing configuration for second Streams domain and instance. Environment variables TEST_DOMAIN and TEST_INSTANCE are not set.") -class MetricsSourceJmxReconnectTest(unittest.TestCase): - - result_file_1 = "done_1" - result_file_2 = "done_2" - - @classmethod - def setUpClass(cls): - th.checkEnvJMX() - - def setUp(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_f(self.result_file_2) - th.remove_files("StreamsLogsJob*.tgz") - th.start_test_domain() - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.remove_f(self.result_file_1) - th.remove_f(self.result_file_2) - th.stop_monitor() - th.stop_sample() - th.stop_test_domain() - - def test_with_two_domains(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - th.make_applications() - th.start_monitor() - th.start_sample() - - # test if monitor app receives notifications - th.wait_for_file(self.result_file_1) - err = th.test_result_file(self.result_file_1) - self.assertEqual(err, 0) - - # force JMX connection failure and monitor app needs to reconnect - th.stop_test_domain() - th.start_test_domain() - - th.start_sample() - - # test if monitor app receives notifications - th.wait_for_file(self.result_file_2) - err = th.test_result_file(self.result_file_2) - self.assertEqual(err, 0) - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/MetricsSource/test_metric_notification/Makefile b/tests/spl-test/MetricsSource/test_metric_notification/Makefile deleted file mode 100644 index 5b7d5d9..0000000 --- a/tests/spl-test/MetricsSource/test_metric_notification/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2017, International Business Machines Corporation. -# All Rights Reserved. - -.PHONY: build all clean distributed - -SPLC_FLAGS = -a -OUTPUT_DIR = output -ifeq ($(STREAMS_INSTALL),) -$(error error: environment variable STREAMS_INSTALL has to be set) -endif -STREAMSX_MONITORING_TOOLKIT ?=../../../../com.ibm.streamsx.monitoring -SPLC = $(STREAMS_INSTALL)/bin/sc -SPL_PATH = $(STREAMSX_MONITORING_TOOLKIT) -SPLC_FLAGS += -t $(SPL_PATH) - -SPL_CMD_ARGS ?= -SPL_MAIN_COMPOSITE = Monitor -SPL_MAIN_COMPOSITE1 = SampleJob - -build: distributed - -all: clean build - -distributed: - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE) $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/monitor - $(SPLC) $(SPLC_FLAGS) -M $(SPL_MAIN_COMPOSITE1) $(SPL_CMD_ARGS) --output-directory=$(OUTPUT_DIR)/sample - -clean: - $(SPLC) $(SPLC_FLAGS) -C -M $(SPL_MAIN_COMPOSITE) - rm -rf $(OUTPUT_DIR) - -start-sample: - streamtool submitjob output/sample/SampleJob.sab --jobname customMetrics - -stop-sample: - streamtool canceljob --jobnames customMetrics --force diff --git a/tests/spl-test/MetricsSource/test_metric_notification/Monitor.spl b/tests/spl-test/MetricsSource/test_metric_notification/Monitor.spl deleted file mode 100644 index fca80e2..0000000 --- a/tests/spl-test/MetricsSource/test_metric_notification/Monitor.spl +++ /dev/null @@ -1,107 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2016, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - -use com.ibm.streamsx.monitoring.metrics::MetricsSource; -use com.ibm.streamsx.monitoring.metrics::Notification; -use com.ibm.streamsx.monitoring.jmx::ConnectionNotification; - -/** - * This application tests MetricsSource operator in standalone mode - * - * @param applicationConfigurationName - * Specifies the name of [https://www.ibm.com/support/knowledgecenter/en/SSCRJU_4.2.0/com.ibm.streams.admin.doc/doc/creating-secure-app-configs.html|application configuration object] that can contain domainId, connectionURL, user, password, and filterDocument properties. The application configuration overrides values that are specified with the corresponding parameters. - * - * @param user - * Specifies the user that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param password - * Specifies the password that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param filterDocument - * Specifies the either a path to a JSON-formatted document or a JSON-formatted String that specifies the domain, instance and job filters as regular expressions. Each regular expression must follow the rules that are specified for Java [https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html|Pattern]. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * If parameter is not set, then all metrics from all jobs in current domain and instance are monitored. - * - * @param connectionURL - * Specifies the connection URL as returned by the `streamtool getjmxconnect` command. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value.If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getjmxconnect` command to get the value. - * - * @param domainId - * Specifies the domain id that is monitored. If no domain id is specified, the domain id under which this operator is running is used. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. - * - * @param sslOption - * Specifies the sslOption that is required for the JMX connection. If the **applicationConfigurationName** parameter is specified, the application configuration can override this parameter value. If not specified and the domainId parameter value equals the domain id under which this operator is running, then the operator uses the `streamtool getdomainproperty` command to get the value. - * - */ -composite Monitor { - param - expression $applicationConfigurationName: getSubmissionTimeValue("applicationConfigurationName", ""); // optional, if user and password are set - expression $user: getSubmissionTimeValue("user", ""); // optional, if set in application configuration - expression $password: getSubmissionTimeValue("password", ""); // optional, if set in application configuration - expression $filterDocument: getSubmissionTimeValue("filterDocument", ""); // uses default, if not set - expression $connectionURL: getSubmissionTimeValue("connectionURL", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $domainId: getSubmissionTimeValue("domainId", ""); // optional, if not set, then domain settings are used, where the PE is running - expression $sslOption: getSubmissionTimeValue("sslOption", ""); // optional, if not set, then domain settings are used, where the PE is running - - graph - /* - * The MetricsSource generates tuple for the specified metrics. - */ - (stream MetricNotifications - ) = MetricsSource() { - param - applicationConfigurationName: $applicationConfigurationName; - user: $user; - password: $password; - filterDocument: $filterDocument; - connectionURL: $connectionURL; - domainId: $domainId; - sslOption: $sslOption; - emitMetricTuple: "onChangedValue"; - } - - /* - * Verify the received metric value notifications. - */ - () as NotificationTracer = Custom(MetricNotifications as I; TimeoutStream as T) { - logic - state: { - mutable boolean passed = false; - } - onTuple I: { - printStringLn("MetricNotification: " + (rstring)I); - if ((I.metricName == "inc") && (I.metricType == "custom")) { - //TODO check metricKind != "unknown" - if ((length(I.jobName) > 0) && (length(I.domainId) > 0) && (length(I.instanceId) > 0) && (length(I.resource) > 0)) { - printStringLn("TEST_RESULT_PASS"); - passed = true; - shutdownPE(); - } - else { - printStringLn("ERROR: MISSING ATTRIBUTE VALUES"); - printStringLn("TEST_RESULT_FAIL"); - abort(); - } - } - } - onTuple T: { - if ((false == passed) && (false == isShutdown())) { - printStringLn("ERROR: TIMEOUT"); - printStringLn("TEST_RESULT_FAIL"); - abort(); - } - } - } - - stream TimeoutStream = Beacon() { - param - iterations: 1u; - initDelay: 60.0; - } - - - config - placement : partitionColocation("MONITOR"); -} diff --git a/tests/spl-test/MetricsSource/test_metric_notification/SampleJob.spl b/tests/spl-test/MetricsSource/test_metric_notification/SampleJob.spl deleted file mode 100644 index 10dbcca..0000000 --- a/tests/spl-test/MetricsSource/test_metric_notification/SampleJob.spl +++ /dev/null @@ -1,70 +0,0 @@ -// -// **************************************************************************** -// * Copyright (C) 2017, International Business Machines Corporation * -// * All rights reserved. * -// **************************************************************************** -// - - -/** - * Several numbers that are created by the Beacon and stored in metrics. - */ -type MetricNumbers = tuple< - int64 incrementingNumber, - int64 randomNumber ->; - -/** - * Create the custom application metrics. The function is usable in the state - * logic of an operator. - */ -stateful boolean createMetrics() { - createCustomMetric("inc", "The incrementing number.", Sys.Counter, 0l); - createCustomMetric("rnd", "The random number.", Sys.Counter, 0l); - createCustomMetric("const", "The constant number.", Sys.Counter, 5l); - return true; -} - -/** - * Set the custom application metrics. - * - * @param numbers - * Specifies the numbers that are created by the Beacon operator. - */ -stateful void setMetrics(MetricNumbers numbers) { - setCustomMetricValue("inc", numbers.incrementingNumber); - setCustomMetricValue("rnd", numbers.randomNumber); -} - -/** - * This application is monitored by the Monitor application. - * Custom metrics are created and incremented. - */ -public composite SampleJob -{ - - graph - /* - * The Beacon generates incrementing and random numbers. - */ - stream Numbers as O = Beacon() { - param period: 5.0; - output O: - incrementingNumber = (int64)IterationCount(), - randomNumber = (int64)(random() * 1000.0); - } - - /* - * The Custom stores the received numbers in metrics. - */ - () as Storage = Custom(Numbers as I) { - logic - state: { - boolean created = createMetrics(); - } - onTuple I: { - setMetrics(I); - } - } - -} diff --git a/tests/spl-test/MetricsSource/test_metric_notification/__init__.py b/tests/spl-test/MetricsSource/test_metric_notification/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/MetricsSource/test_metric_notification/etc/MetricsSource_MonitorSomeCustomMetrics.json b/tests/spl-test/MetricsSource/test_metric_notification/etc/MetricsSource_MonitorSomeCustomMetrics.json deleted file mode 100644 index 119dda5..0000000 --- a/tests/spl-test/MetricsSource/test_metric_notification/etc/MetricsSource_MonitorSomeCustomMetrics.json +++ /dev/null @@ -1,29 +0,0 @@ -[ - { - "domainIdPatterns":".*", - "instances": - [ - { - "instanceIdPatterns":".*", - "jobs": - [ - { - "jobNamePatterns":".*", - "operators": - [ - { - "operatorNamePatterns":"Storage", - "metricNamePatterns": - [ - "rnd", - "inc", - "const" - ] - } - ] - } - ] - } - ] - } -] diff --git a/tests/spl-test/MetricsSource/test_metric_notification/test_metric_notification.py b/tests/spl-test/MetricsSource/test_metric_notification/test_metric_notification.py deleted file mode 100644 index d044e88..0000000 --- a/tests/spl-test/MetricsSource/test_metric_notification/test_metric_notification.py +++ /dev/null @@ -1,37 +0,0 @@ -import unittest -import os -import testharness as th - -class MetricsSourceNotificationTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - - def tearDown(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.stop_sample() - - def test_standalone(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.make_applications() - th.start_sample() - stdout, stderr, err = th.run_monitor_standalone(args=['user='+os.environ["STREAMS_USERNAME"], 'password='+os.environ["STREAMS_PASSWORD"], 'domainId='+os.environ["STREAMS_DOMAIN_ID"]]) - th.assert_pass(err == 0 and (str(stdout).find('TEST_RESULT_PASS') != -1), stdout, stderr) - - def test_standalone_filter_doc(self): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - th.make_applications() - th.start_sample() - stdout, stderr, err = th.run_monitor_standalone(args=['user='+os.environ["STREAMS_USERNAME"], 'password='+os.environ["STREAMS_PASSWORD"], 'domainId='+os.environ["STREAMS_DOMAIN_ID"], 'filterDocument=etc/MetricsSource_MonitorSomeCustomMetrics.json']) - th.assert_pass(err == 0 and (str(stdout).find('TEST_RESULT_PASS') != -1), stdout, stderr) - - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/spl-test/README.md b/tests/spl-test/README.md deleted file mode 100644 index 7b92e30..0000000 --- a/tests/spl-test/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Setup for tests - -The test scripts require **python 3.x** for python unittest. Ensure that `python3` command is in your `PATH`. - -You need to set the following environment variables: - -* `STREAMS_USERNAME` - user name for JMX connection -* `STREAMS_PASSWORD` - password for JMX connection - -## Optional - -The test suites `test_jmx_reconnect` require the following environment variables: - -* `TEST_DOMAIN` - domain ID for a second *Streams Domain*. It is required that the property `jmx.port` is **not** set to value `0`. -* `TEST_INSTANCE` - instance ID of a *Streams Instance* in the `TEST_DOMAIN` - -If not set, then the test suites are skipped. - -## Before launching the test - -**The test scripts does not create, modify or delete a Streams Domain or Instance.** - -* It is required that the domain (`STREAMS_DOMAIN_ID`) and instance (`STREAMS_INSTANCE_ID`) are started. -* *If present, the domain (`TEST_DOMAIN`) and instance (`TEST_INSTANCE`) should be stopped.* - - -# Run the tests -``` -ant test -``` - -To run a single test, select one of the test suites, for example: -``` -python3 -m unittest JobStatusMonitor/test_jmx_reconnect/test_jmx_reconnect.py -v -``` - -# Clean-up - -Delete generated files of test suites. -``` -ant clean -``` diff --git a/tests/spl-test/__init__.py b/tests/spl-test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/spl-test/build.xml b/tests/spl-test/build.xml deleted file mode 100644 index ad1e968..0000000 --- a/tests/spl-test/build.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/spl-test/testharness.py b/tests/spl-test/testharness.py deleted file mode 100755 index 2f7ca48..0000000 --- a/tests/spl-test/testharness.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (C) 2017 International Business Machines Corporation. -# All Rights Reserved. - -import sys, os, time -from subprocess import call, Popen, PIPE - -class TestFailure(Exception): - def __init__(self, out, err): - self.stdout = out - self.stderr = err - - def say(self, test_name): - print (test_name + ' fail:\n' + '\tstdout: ' + self.stdout + '\n' + '\tstderr: ' + self.stderr + '\n') - -def exec_noexit(seq): - p = Popen(seq, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - return stdout, stderr, p.returncode - -def assert_pass(condition, stdout, stderr): - if not condition: - raise TestFailure(stdout, stderr) - -def start_sample(args=list()): - stdout, stderr, err = exec_noexit(['make', 'start-sample'] + args) - assert_pass(err == 0, stdout, stderr) - -def stop_sample(args=list()): - exec_noexit(['make', 'stop-sample'] + args) - -def start_monitor(args=list()): - stdout, stderr, err = exec_noexit(['make', 'start-monitor'] + args) - assert_pass(err == 0, stdout, stderr) - -def start_test_domain(args=list()): - exec_noexit(['make', 'start-test-domain'] + args) - -def rm_app_config(): - stdout, stderr, err = exec_noexit(['make', 'configure-none']) - assert_pass(err == 0, stdout, stderr) - -def create_app_config(): - stdout, stderr, err = exec_noexit(['make', 'configure']) - assert_pass(err == 0, stdout, stderr) - -def create_app_config_json(): - stdout, stderr, err = exec_noexit(['make', 'configure-json']) - assert_pass(err == 0, stdout, stderr) - -def stop_test_domain(args=list()): - exec_noexit(['make', 'stop-test-domain'] + args) - -def stop_monitor(args=list()): - exec_noexit(['make', 'stop-monitor'] + args) - -def make_applications(): - stdout, stderr, err = exec_noexit(['make', 'all']) - assert_pass(err == 0, stdout, stderr) - -def make_clean(): - stdout, stderr, err = exec_noexit(['make', 'clean']) - assert_pass(err == 0, stdout, stderr) - -def run_standalone(args=list()): - return exec_noexit(['./output/bin/standalone'] + args) - -def run_monitor_standalone(args=list()): - return exec_noexit(['./output/monitor/bin/standalone'] + args) - -def remove_f(name): - '''Removes a file but ignores not existing files errors''' - try: - os.remove(name) - except OSError as e: - if e.errno != 2: - raise - -def remove_files(name): - Popen(["rm -f " + name], shell=True, stdout=PIPE).communicate() - -def wait_for_file(name): - timeout = 180 # [seconds] - timeout_start = time.time() - while time.time() < timeout_start + timeout: - if os.path.exists (name): - break - time.sleep(1) - -def test_result_file(name): - err = 0 - try: - fh = open(name, 'r') - result = fh.read() - fh.close() - except IOError: - err = 1 - return err - -def checkEnvJMX(): - try: - os.environ["STREAMS_USERNAME"] - os.environ["STREAMS_PASSWORD"] - except KeyError: - print ("ERROR: Please set the environment variables STREAMS_USERNAME and STREAMS_PASSWORD") - raise - -def checkDomain(): - result = 0 - try: - os.environ["TEST_DOMAIN"] - os.environ["TEST_INSTANCE"] - except KeyError: - result = 1 - return result - diff --git a/updateWeb.pl b/updateWeb.pl deleted file mode 100755 index 928dd48..0000000 --- a/updateWeb.pl +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/perl - -use strict; -use Getopt::Long; - - -my $debug = 0; -my $dryrun = 0; -my $doPush; -my $doCommit=1; -my $message = "Update documents"; -my $help; -my $useTemp; -my $deleteTemp = 1; -my $makeDoc = 1; -my %samples; - -GetOptions('push!'=>\$doPush, - 'commit!'=>\$doCommit, - 'dryrun' => \$dryrun, - 'message|m=s' => \$message, - 'makedoc!' => \$makeDoc, - 'help|h|?'=>\$help, - 'deleteTemp!' => \$deleteTemp, - 'createTempRepository' => \$useTemp); -if ($dryrun) { - $doPush = 0; - $doCommit = 0; -} - -if ($doPush && !$doCommit) { - print STDERR "If --push is specified, then commit must be enabled\n"; -} - -if ($help) { - print STDOUT "updateWeb.pl runs spl-make-doc in the current repository for all toolkits and samples,\n"; - print STDOUT "and then commits the doc updates to a second document repository (one set to gh-pages).\n"; - print STDOUT "Usage: updateWeb.pl [--commit|--nocommit] [--push|--nopush] [--message ] \n"; - exit 0; -} -if (scalar @ARGV != 1 && !$useTemp) { - print STDERR "Usage: updateWeb.pl \n"; - exit 1; -} -if ($useTemp && $deleteTemp) { - $doCommit or die "Must do a commit if making a temporary repository"; - $doPush or die "Must do a push if using a temporary repository"; -} -my $pagesLocation = $ARGV[0]; -die "$pagesLocation not a valid directory" unless $useTemp || (-e $pagesLocation && -d $pagesLocation); - -sub updateIndexHtml() { - - my $infile = "$pagesLocation/index.html"; - my $outfile = "$pagesLocation/index.html.tmp"; - open(IN,"<$pagesLocation/index.html") or die "Could not open $infile"; - open(OUT,">$pagesLocation/index.html.tmp") or die "Could not open $outfile"; - while() { - if (//) { - print OUT $_; - my $foundEnd = 0; - print OUT "
        \n"; - for my $app (keys %samples) { - print OUT "
      • $app\n"; - } - print OUT "
      \n"; - while() { - if (//) { - $foundEnd = 1; - print OUT $_; - last; - } - } - die "Did not find END SAMPLE LIST" if !$foundEnd; - } - else { - print OUT $_; - } - } - - close OUT; - close IN; - system("cp $outfile $infile"); - system("cd $pagesLocation; git add index.html"); -} - -sub lookForApp($$) { - my ($dir,$exclude) = @_; - $debug && print "lookForApp($dir,$exclude)\n"; - my @files = `ls $dir`; - my $changes = 0; - for my $f (@files) { - chomp $f; - my $fullName = "$dir/$f"; - $debug && print "Trying file $fullName\n"; - next if (defined $exclude && $dir =~ /$exclude/); - next unless (-d "$fullName"); - if (-e "$fullName/doc/spldoc") { - if ($fullName =~ /samples/) { - $debug && print "$fullName is a sample\n"; - $samples{$f}="$fullName/doc/spldoc/html/index.html"; - } - $debug && print "directory $fullName appears to have a documentation\n"; - if ($dryrun) { - print "Would copy $fullName/doc to $pagesLocation\n"; - print "cp -r $fullName/doc $pagesLocation/$fullName\n"; - print "cd $pagesLocation; git add -A $fullName/doc\n"; - } - else { - system("mkdir -p $pagesLocation/$fullName"); - system("cp -r $fullName/doc $pagesLocation/$fullName"); - system("cd $pagesLocation; git add -A $fullName/doc"); - } - $? >> 8 == 0 or die "Problem adding."; - } - else { - $debug && print "no docs found in $fullName\n"; - lookForApp("$fullName"); - } - } -} - -sub main() { - if ($makeDoc) { - system("ant spldoc"); - } - $? >> 8 == 0 or die "Could not build spl doc"; - # Make sure the branch is checked out in location given on the command - - if ($useTemp) { - my $line = `git remote show origin | grep Fetch`; - $line =~ /Fetch URL:\s+(.*)$/; - my $url = $1; - $url =~ /github.com[:\/]\w+\/(.+)\.git$/; - my $repoName = $1; - - $pagesLocation = "/tmp/$repoName"; - die "Cannot create repository since $pagesLocation already exists" if (-e $pagesLocation); - system("cd /tmp; git clone $url"); - } - - system("cd $pagesLocation; git checkout gh-pages"); - $? >> 8 == 0 or die "Cannot set branch to gh-pages in $pagesLocation"; - # handle the toolkit; need a better way of specifying which toolkit. - lookForApp(".","test"); - updateIndexHtml(); - my $changes = `cd $pagesLocation; git status -s | grep -v "^?" | wc -l`; - chomp $changes; - print "$changes files changed\n"; - if ($changes > 0) { - if ($dryrun) { - print "DryRun: $changes files changed\n"; - } - elsif ($changes >0 && $doCommit) { - system("cd $pagesLocation; git commit -a -m \"$message\""); - if ($doPush) { - system("cd $pagesLocation; git push origin gh-pages"); - } - } - } - if ($useTemp && $deleteTemp) { - if ($pagesLocation =~ /tmp/) { - system("rm -rf $pagesLocation"); - } - else { - die "Unexpected temporary repository structure; not deleting $pagesLocation"; - } - } -} - -main();