From 9826911e0e7c5f47f4891309d11d48b00f0fe60f Mon Sep 17 00:00:00 2001 From: Alan Storm Date: Thu, 26 Aug 2021 08:59:47 -0700 Subject: [PATCH] feat: sns instrumentation (#2157) * Implements implementation of Amazon SNS for the v2 APIs https://github.com/elastic/apm-agent-nodejs/issues/1955 Co-authored-by: Trent Mick --- CHANGELOG.asciidoc | 3 + docs/supported-technologies.asciidoc | 2 +- lib/instrumentation/modules/aws-sdk.js | 4 +- lib/instrumentation/modules/aws-sdk/sns.js | 151 +++++++ .../modules/aws-sdk/fixtures/sns.js | 46 ++ .../modules/aws-sdk/sns.test.js | 409 ++++++++++++++++++ 6 files changed, 613 insertions(+), 2 deletions(-) create mode 100644 lib/instrumentation/modules/aws-sdk/sns.js create mode 100644 test/instrumentation/modules/aws-sdk/fixtures/sns.js create mode 100644 test/instrumentation/modules/aws-sdk/sns.test.js diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index bd078c2d29b..1d8bec88c0f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -88,6 +88,9 @@ Notes: option. Before this change only incoming requests supporting HTTP/2 would be traced. ({pull}2143[#2143]) +* Add instrumentation of the AWS SNS publish method when using the + https://www.npmjs.com/package/aws-sdk[JavaScript AWS SDK v2] (`aws-sdk`). ({pull}2157[#2157]) + [float] ===== Bug fixes diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index efd54a88d78..838587d793d 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -72,7 +72,7 @@ The Node.js agent will automatically instrument the following modules to give yo [options="header"] |======================================================================= |Module |Version |Note -|https://www.npmjs.com/package/aws-sdk[aws-sdk] |>1 <3 |Will instrument SQS send/receive/delete messages, all S3 methods, and all DynamoDB methods +|https://www.npmjs.com/package/aws-sdk[aws-sdk] |>1 <3 |Will instrument SQS send/receive/delete messages, all S3 methods, all DynamoDB methods, and the SNS publish method |https://www.npmjs.com/package/cassandra-driver[cassandra-driver] |>=3.0.0 |Will instrument all queries |https://www.npmjs.com/package/elasticsearch[elasticsearch] |>=8.0.0 |Will instrument all queries |https://www.npmjs.com/package/@elastic/elasticsearch[@elastic/elasticsearch] |>=7.0.0 <8.0.0 |Will instrument all queries diff --git a/lib/instrumentation/modules/aws-sdk.js b/lib/instrumentation/modules/aws-sdk.js index ae7dd7f3444..2478ba2eb16 100644 --- a/lib/instrumentation/modules/aws-sdk.js +++ b/lib/instrumentation/modules/aws-sdk.js @@ -4,11 +4,13 @@ const shimmer = require('../shimmer') const { instrumentationS3 } = require('./aws-sdk/s3') const { instrumentationSqs } = require('./aws-sdk/sqs') const { instrumentationDynamoDb } = require('./aws-sdk/dynamodb.js') +const { instrumentationSns } = require('./aws-sdk/sns.js') const instrumentorFromSvcId = { s3: instrumentationS3, sqs: instrumentationSqs, - dynamodb: instrumentationDynamoDb + dynamodb: instrumentationDynamoDb, + sns: instrumentationSns } // Called in place of AWS.Request.send and AWS.Request.promise diff --git a/lib/instrumentation/modules/aws-sdk/sns.js b/lib/instrumentation/modules/aws-sdk/sns.js new file mode 100644 index 00000000000..7389d16ffa2 --- /dev/null +++ b/lib/instrumentation/modules/aws-sdk/sns.js @@ -0,0 +1,151 @@ +const constants = require('../../../constants') + +const TYPE = 'messaging' +const SUBTYPE = 'sns' +const ACTION = 'publish' +const PHONE_NUMBER = '' + +function getArnOrPhoneNumberFromRequest (request) { + let arn = request && request.params && request.params.TopicArn + if (!arn) { + arn = request && request.params && request.params.TargetArn + } + if (!arn) { + arn = request && request.params && request.params.PhoneNumber + } + return arn +} + +function getRegionFromRequest (request) { + return request && request.service && + request.service.config && request.service.config.region +} + +function getSpanNameFromRequest (request) { + const topicName = getDestinationNameFromRequest(request) + return `SNS PUBLISH to ${topicName}` +} + +function getMessageContextFromRequest (request) { + return { + queue: { + name: getDestinationNameFromRequest(request) + } + } +} + +function getAddressFromRequest (request) { + return request && request.service && request.service.endpoint && + request.service.endpoint.hostname +} + +function getPortFromRequest (request) { + return request && request.service && request.service.endpoint && + request.service.endpoint.port +} + +function getMessageDestinationContextFromRequest (request) { + return { + address: getAddressFromRequest(request), + port: getPortFromRequest(request), + service: { + resource: `${SUBTYPE}/${getDestinationNameFromRequest(request)}`, + type: TYPE, + name: SUBTYPE + }, + cloud: { + region: getRegionFromRequest(request) + } + } +} + +function getDestinationNameFromRequest (request) { + const phoneNumber = request && request.params && request.params.PhoneNumber + if (phoneNumber) { + return PHONE_NUMBER + } + + const topicArn = request && request.params && request.params.TopicArn + const targetArn = request && request.params && request.params.TargetArn + + if (topicArn) { + const parts = topicArn.split(':') + const topicName = parts.pop() + return topicName + } + + if (targetArn) { + const fullName = targetArn.split(':').pop() + if (fullName.lastIndexOf('/') !== -1) { + return fullName.substring(0, fullName.lastIndexOf('/')) + } else { + return fullName + } + } +} + +function shouldIgnoreRequest (request, agent) { + if (request.operation !== 'publish') { + return true + } + + // is the named topic on our ignore list? + if (agent._conf && agent._conf.ignoreMessageQueuesRegExp) { + const queueName = getArnOrPhoneNumberFromRequest(request) + if (queueName) { + for (const rule of agent._conf.ignoreMessageQueuesRegExp) { + if (rule.test(queueName)) { + return true + } + } + } + } + + return false +} + +function instrumentationSns (orig, origArguments, request, AWS, agent, { version, enabled }) { + const type = TYPE + const subtype = SUBTYPE + const action = ACTION + + if (shouldIgnoreRequest(request, agent)) { + return orig.apply(request, origArguments) + } + + const name = getSpanNameFromRequest(request) + const span = agent.startSpan(name, type, subtype, action) + if (!span) { + return orig.apply(request, origArguments) + } + + span.setDestinationContext(getMessageDestinationContextFromRequest(request)) + span.setMessageContext(getMessageContextFromRequest(request)) + + request.on('complete', function (response) { + if (response && response.error) { + const errOpts = { + skipOutcome: true + } + agent.captureError(response.error, errOpts) + span._setOutcomeFromErrorCapture(constants.OUTCOME_FAILURE) + } + + // we'll need to manually mark this span as async. The actual async hop + // is captured by the agent's async hooks instrumentation + span.sync = false + span.end() + }) + + return orig.apply(request, origArguments) +} + +module.exports = { + instrumentationSns, + + // exported for testing + getSpanNameFromRequest, + getDestinationNameFromRequest, + getMessageDestinationContextFromRequest, + getArnOrPhoneNumberFromRequest +} diff --git a/test/instrumentation/modules/aws-sdk/fixtures/sns.js b/test/instrumentation/modules/aws-sdk/fixtures/sns.js new file mode 100644 index 00000000000..ada25ea8ce6 --- /dev/null +++ b/test/instrumentation/modules/aws-sdk/fixtures/sns.js @@ -0,0 +1,46 @@ +'use strict' +module.exports = { + publish: { + response: ` + + + 32a3b682-ce0c-5b2a-b97f-efe5112e9f06 + + + 8e87dc3a-07f7-54f0-ae0d-855dd8d5f0dc + + `, + httpStatusCode: 200 + }, + publishNoTopic: { + response: ` + + + Sender + NotFound + Topic does not exist + + 02672fe4-577a-5c2a-9a11-7683bd8777e1 + `, + httpStatusCode: 404 + }, + listTopics: { + response: ` + + + + + arn:aws:sns:us-west-2:111111111111:topic-name + + + arn:aws:sns:us-west-2:111111111111:dynamodb + + + + + 1cb1f1f2-48fa-523b-a01a-a895b59d244b + + `, + httpStatusCode: 200 + } +} diff --git a/test/instrumentation/modules/aws-sdk/sns.test.js b/test/instrumentation/modules/aws-sdk/sns.test.js new file mode 100644 index 00000000000..c42fd802cc6 --- /dev/null +++ b/test/instrumentation/modules/aws-sdk/sns.test.js @@ -0,0 +1,409 @@ +const agent = require('../../../..').start({ + serviceName: 'test', + secretToken: 'test', + captureExceptions: false, + metricsInterval: 0, + centralConfig: 'none', + logLevel: 'off', + cloudProvider: 'none', + ignoreMessageQueues: [ + 'arn:aws:sns:us-west-2:111111111111:ignore-name' + ] +}) + +const tape = require('tape') +const express = require('express') +const bodyParser = require('body-parser') +const AWS = require('aws-sdk') + +const { + getSpanNameFromRequest, getDestinationNameFromRequest, + getArnOrPhoneNumberFromRequest, getMessageDestinationContextFromRequest +} = require('../../../../lib/instrumentation/modules/aws-sdk/sns') +const fixtures = require('./fixtures/sns') +const mockClient = require('../../../_mock_http_client') + +initializeAwsSdk() + +function initializeAwsSdk () { + // SDk requires a region to be set + AWS.config.update({ region: 'us-west-2' }) + + // without fake credentials the aws-sdk will attempt to fetch + // credentials as though it was on an EC2 instance + process.env.AWS_ACCESS_KEY_ID = 'fake-1' + process.env.AWS_SECRET_ACCESS_KEY = 'fake-2' +} + +function createMockServer (fixture) { + const app = express() + app.use(bodyParser.urlencoded({ extended: false })) + app.post('/', (req, res) => { + res.status(fixture.httpStatusCode) + res.setHeader('Content-Type', 'text/xml') + res.send(fixture.response) + }) + return app +} + +function resetAgent (cb) { + agent._instrumentation.currentTransaction = null + agent._transport = mockClient(cb) +} + +tape.test('AWS SNS: Unit Test Functions', function (test) { + test.test('getArnOrPhoneNumberFromRequest tests', function (t) { + t.equals(getArnOrPhoneNumberFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many like it but this one is mine', + TopicArn: 'foo' + } + }), 'foo') + + t.equals(getArnOrPhoneNumberFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many like it but this one is mine', + TargetArn: 'bar' + } + }), 'bar') + + t.equals(getArnOrPhoneNumberFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many like it but this one is mine', + PhoneNumber: '1-555-555-5555' + } + }), '1-555-555-5555') + + t.end() + }) + + test.test('getDestinationNameFromRequest tests', function (t) { + t.equals(getDestinationNameFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many like it but this one is mine', + TopicArn: 'arn:aws:sns:us-west-2:111111111111:topic-name' + } + }), 'topic-name') + + t.equals(getDestinationNameFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many lot like it but this one is mine', + TargetArn: 'arn:aws:sns:us-west-2:123456789012:endpoint/GCM/gcmpushapp/5e3e9847-3183-3f18-a7e8-671c3a57d4b3' + } + }), 'endpoint/GCM/gcmpushapp') + + // unlikely we'll receive a targetArn without /, but we should + // do something reasonable, just in case + t.equals(getDestinationNameFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many lot like it but this one is mine', + TargetArn: 'arn:aws:sns:us-west-2:123456789012:endpoint:GCM' + } + }), 'GCM') + + t.equals(getDestinationNameFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many lot like it but this one is mine', + TopicArn: 'arn:aws:sns:us-west-2:111111111111:foo/withslashes' + } + }), 'foo/withslashes') + + t.equals(getDestinationNameFromRequest({ + operation: 'publish', + params: { + Message: 'work test', + Subject: 'Admin', + PhoneNumber: '15037299028' + } + }), '') + + t.equals(getDestinationNameFromRequest(null), undefined) + t.equals(getDestinationNameFromRequest({}), undefined) + t.equals(getDestinationNameFromRequest({ params: {} }), undefined) + t.end() + }) + + test.test('getSpanNameFromRequest tests', function (t) { + t.equals(getSpanNameFromRequest({ + operation: 'publish', + params: { + Message: 'work test', + Subject: 'Admin', + PhoneNumber: '15555555555' + } + }), 'SNS PUBLISH to ') + + t.equals(getSpanNameFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many lot like it but this one is mine', + TargetArn: 'arn:aws:sns:us-west-2:123456789012:endpoint/GCM/gcmpushapp/5e3e9847-3183-3f18-a7e8-671c3a57d4b3' + } + }), 'SNS PUBLISH to endpoint/GCM/gcmpushapp') + + t.equals(getSpanNameFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many lot like it but this one is mine', + TopicArn: 'arn:aws:sns:us-west-2:111111111111:foo:topic-name' + } + }), 'SNS PUBLISH to topic-name') + + t.equals(getSpanNameFromRequest(null), 'SNS PUBLISH to undefined') + t.equals(getSpanNameFromRequest({}), 'SNS PUBLISH to undefined') + t.equals(getSpanNameFromRequest({ params: {} }), 'SNS PUBLISH to undefined') + t.end() + }) + + test.test('getMessageDestinationContextFromRequest tests', function (t) { + t.deepEquals( + getMessageDestinationContextFromRequest({ + operation: 'publish', + params: { + Message: 'this is my test, there are many lot like it but this one is mine', + TopicArn: 'arn:aws:sns:us-west-2:111111111111:foo:topic-name' + }, + service: { + config: { + region: 'us-west-2' + }, + endpoint: { + hostname: 'example.com', + port: 1234 + } + } + }), + { + address: 'example.com', + port: 1234, + service: { + resource: 'sns/topic-name', + type: 'messaging', + name: 'sns' + }, + cloud: { region: 'us-west-2' } + } + ) + + t.deepEquals( + getMessageDestinationContextFromRequest(null), + { + address: null, + port: null, + service: { + resource: 'sns/undefined', + type: 'messaging', + name: 'sns' + }, + cloud: { region: null } + } + ) + + t.deepEquals( + getMessageDestinationContextFromRequest({}), + { + address: undefined, + port: undefined, + service: { + resource: 'sns/undefined', + type: 'messaging', + name: 'sns' + }, + cloud: { region: undefined } + } + ) + t.end() + }) + + test.end() +}) + +tape.test('AWS SNS: End to End Test', function (test) { + test.test('API: publish', function (t) { + const params = { + Message: 'this is my test, there are many like it but this one is mine', /* required */ + TopicArn: 'arn:aws:sns:us-west-2:111111111111:topic-name' + } + + const app = createMockServer( + fixtures.publish + ) + const listener = app.listen(0, function () { + const port = listener.address().port + resetAgent(function (data) { + const span = data.spans.filter((span) => span.type === 'messaging').pop() + t.equals(span.name, 'SNS PUBLISH to topic-name', 'span named correctly') + t.equals(span.type, 'messaging', 'span type correctly set') + t.equals(span.subtype, 'sns', 'span subtype set correctly') + t.equals(span.action, 'publish', 'span action set correctly') + t.equals(span.context.message.queue.name, 'topic-name') + t.equals(span.context.destination.service.resource, 'sns/topic-name') + t.equals(span.context.destination.service.type, 'messaging') + t.equals(span.context.destination.service.name, 'sns') + t.equals(span.context.destination.address, 'localhost') + t.equals(span.context.destination.port, port) + t.equals(span.context.destination.cloud.region, 'us-west-2') + t.end() + }) + + AWS.config.update({ + endpoint: `http://localhost:${port}` + }) + agent.startTransaction('myTransaction') + const publishTextPromise = new AWS.SNS({ apiVersion: '2010-03-31' }) + .publish(params).promise() + + // Handle promise's fulfilled/rejected states + publishTextPromise.then(function (data) { + agent.endTransaction() + listener.close() + }).catch(function (err) { + t.error(err) + agent.endTransaction() + listener.close() + }) + }) + }) + + test.test('API: no transaction', function (t) { + const params = { + Message: 'this is my test, there are many like it but this one is mine', /* required */ + TopicArn: 'arn:aws:sns:us-west-2:111111111111:topic-name' + } + + const app = createMockServer( + fixtures.publish + ) + const listener = app.listen(0, function () { + resetAgent(function (data) { + const span = data.spans.filter((span) => span.type === 'messaging').pop() + t.ok(!span, 'no messaging span without a transaction') + t.end() + }) + const port = listener.address().port + AWS.config.update({ + endpoint: `http://localhost:${port}` + }) + + const publishTextPromise = new AWS.SNS({ apiVersion: '2010-03-31' }) + .publish(params).promise() + + // Handle promise's fulfilled/rejected states + publishTextPromise.then(function (data) { + listener.close() + }).catch(function (err) { + t.error(err) + listener.close() + }) + }) + }) + + test.test('API: error', function (t) { + const params = { + Message: 'this is my test, there are many like it but this one is mine', /* required */ + TopicArn: 'arn:aws:sns:us-west-2:111111111111:topic-name-not-exists' + } + + const app = createMockServer( + fixtures.publishNoTopic + ) + const listener = app.listen(0, function () { + resetAgent(function (data) { + const span = data.spans.filter((span) => span.type === 'messaging').pop() + t.equals(span.outcome, 'failure', 'error produces outcome=failure span') + t.end() + }) + const port = listener.address().port + AWS.config.update({ + endpoint: `http://localhost:${port}` + }) + agent.startTransaction('myTransaction') + const publishTextPromise = new AWS.SNS({ apiVersion: '2010-03-31' }) + .publish(params).promise() + + // Handle promise's fulfilled/rejected states + publishTextPromise.then(function (data) { + agent.endTransaction() + listener.close() + }).catch(function (err) { + t.ok(err, 'error expected') + agent.endTransaction() + listener.close() + }) + }) + }) + + test.test('API: listTopics', function (t) { + const app = createMockServer( + fixtures.listTopics + ) + const listener = app.listen(0, function () { + resetAgent(function (data) { + const span = data.spans.filter((span) => span.type === 'messaging').pop() + t.ok(!span, 'only publish operation creates spans') + t.end() + }) + const port = listener.address().port + AWS.config.update({ + endpoint: `http://localhost:${port}` + }) + agent.startTransaction('myTransaction') + const publishTextPromise = new AWS.SNS({ apiVersion: '2010-03-31' }) + .listTopics().promise() + + // Handle promise's fulfilled/rejected states + publishTextPromise.then(function (data) { + agent.endTransaction() + listener.close() + }).catch(function (err) { + t.error(err) + agent.endTransaction() + listener.close() + }) + }) + }) + + test.test('API: ignored queue', function (t) { + const params = { + Message: 'this is my test, there are many like it but this one is mine', /* required */ + TopicArn: 'arn:aws:sns:us-west-2:111111111111:ignore-name' + } + + const app = createMockServer( + fixtures.publish + ) + const listener = app.listen(0, function () { + resetAgent(function (data) { + const span = data.spans.filter((span) => span.type === 'messaging').pop() + t.ok(!span, 'ignores configured topic name') + t.end() + }) + const port = listener.address().port + AWS.config.update({ + endpoint: `http://localhost:${port}` + }) + agent.startTransaction('myTransaction') + const publishTextPromise = new AWS.SNS({ apiVersion: '2010-03-31' }) + .publish(params).promise() + + // Handle promise's fulfilled/rejected states + publishTextPromise.then(function (data) { + agent.endTransaction() + listener.close() + }).catch(function (err) { + t.error(err) + agent.endTransaction() + listener.close() + }) + }) + }) + + test.end() +})