From 42c15cbbb5a35a14a01f727476ca4a3b72d634e5 Mon Sep 17 00:00:00 2001 From: Georgy Berezhnoy Date: Thu, 26 Aug 2021 17:18:18 +0300 Subject: [PATCH 1/6] Add event timestamp to repetitions --- lib/utils.js | 12 ++++-- lib/utils.test.js | 8 ++++ ...0826165556-add-timestamp-to-repetitions.js | 43 +++++++++++++++++++ workers/grouper/src/index.ts | 4 +- 4 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 migrations/20210826165556-add-timestamp-to-repetitions.js diff --git a/lib/utils.js b/lib/utils.js index c44488b3..e792dc4c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -26,13 +26,14 @@ module.exports.deepMerge = deepMerge; * * @param {object|Array} source - source object * @param {object|Array} target - target object + * @param {Array} [requiredFields] - fields to leave in diff object * @returns {object} */ -function deepDiff(source, target) { +function deepDiff(source, target, requiredFields= []) { if (typeOf(target) === 'array') { return arrayDiff(source, target); } else if (typeOf(target) === 'object') { - return objectDiff(source, target); + return objectDiff(source, target, requiredFields); } else if (source !== target) { return target; } else { @@ -62,10 +63,11 @@ function arrayDiff(source, target) { * * @param {object} objectA - first object for comparing * @param {object} objectB - second object for comparing + * @param {Array} requiredFields - fields to leave * * @returns {object} */ -function objectDiff(objectA, objectB) { +function objectDiff(objectA, objectB, requiredFields = []) { const diffObject = {}; /** @@ -93,6 +95,10 @@ function objectDiff(objectA, objectB) { } if (objectAItem === objectBItem) { + if (requiredFields.includes(prop)) { + diffObject[prop] = objectAItem; + } + return; } diff --git a/lib/utils.test.js b/lib/utils.test.js index a0b49099..037c0f5a 100644 --- a/lib/utils.test.js +++ b/lib/utils.test.js @@ -135,4 +135,12 @@ describe('Utils', () => { expect(merge).toEqual(testCase.expectedMerge); }); }); + + test('should leave required fields', () => { + const data = dataProvider[1]; + + const diff = utils.deepDiff(data.sourceObject, data.targetObject, ['a']); + + expect(diff.a).toEqual(data.sourceObject.a); + }) }); diff --git a/migrations/20210826165556-add-timestamp-to-repetitions.js b/migrations/20210826165556-add-timestamp-to-repetitions.js new file mode 100644 index 00000000..d4d9cf80 --- /dev/null +++ b/migrations/20210826165556-add-timestamp-to-repetitions.js @@ -0,0 +1,43 @@ +/** + * This migration updates date from format `dd-mm-YYYY` to midnight unixtime + * so that each client with different timezone could convert it to local time + */ +module.exports = { + async up(db) { + const collections = await db.listCollections({}, { + authorizedCollections: true, + nameOnly: true, + }).toArray(); + + const projectIds = []; + const REPETITIONS = 'repetitions'; + const EVENTS = 'events'; + + collections.forEach((collection) => { + if (/repetitions/.test(collection.name)) { + projectIds.push(collection.name.split(':')[1]); + } + }); + + for (const projectId of projectIds) { + const originalEvents = await db.collection(`${EVENTS}:${projectId}`).find({}).toArray(); + const repetitions = await db.collection(`${REPETITIONS}:${projectId}`).find({ + 'payload.timestamp': { $eq: null } + }).toArray(); + + for (const event of originalEvents) { + const eventRepetitions = repetitions.filter(rep => rep.groupHash === event.groupHash); + + for (const repetition of eventRepetitions) { + await db.collection(`${REPETITIONS}:${projectId}`).updateOne({ + _id: repetition._id, + }, { + $set: { + 'payload.timestamp': event.payload.timestamp, + }, + }); + } + } + } + }, +}; diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index abf3f418..632ddf5b 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -146,8 +146,10 @@ export default class GrouperWorker extends Worker { /** * Save event's repetitions + * + * Leave timestamp in diff for database queries */ - const diff = utils.deepDiff(existedEvent.payload, task.event); + const diff = utils.deepDiff(existedEvent.payload, task.event, [ 'timestamp' ]); const newRepetition = { groupHash: uniqueEventHash, From e34acb47b95ae2e3ab6a2f23bf1d43fcd2f8254b Mon Sep 17 00:00:00 2001 From: Georgy Berezhnoy Date: Thu, 26 Aug 2021 18:05:20 +0300 Subject: [PATCH 2/6] Use batching --- ...0826165556-add-timestamp-to-repetitions.js | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/migrations/20210826165556-add-timestamp-to-repetitions.js b/migrations/20210826165556-add-timestamp-to-repetitions.js index d4d9cf80..674773d2 100644 --- a/migrations/20210826165556-add-timestamp-to-repetitions.js +++ b/migrations/20210826165556-add-timestamp-to-repetitions.js @@ -1,9 +1,8 @@ /** - * This migration updates date from format `dd-mm-YYYY` to midnight unixtime - * so that each client with different timezone could convert it to local time + * This migration sets timestamp for event repetitions if it was omitted because it's the same as original event one */ module.exports = { - async up(db) { + async up(db, client) { const collections = await db.listCollections({}, { authorizedCollections: true, nameOnly: true, @@ -19,25 +18,33 @@ module.exports = { } }); - for (const projectId of projectIds) { - const originalEvents = await db.collection(`${EVENTS}:${projectId}`).find({}).toArray(); - const repetitions = await db.collection(`${REPETITIONS}:${projectId}`).find({ - 'payload.timestamp': { $eq: null } - }).toArray(); + const session = client.startSession(); - for (const event of originalEvents) { - const eventRepetitions = repetitions.filter(rep => rep.groupHash === event.groupHash); + try { + session.withTransaction(async () => { + for (const projectId of projectIds) { + const originalEvents = await db.collection(`${EVENTS}:${projectId}`).find({}).toArray(); + const repetitions = await db.collection(`${REPETITIONS}:${projectId}`).find({ + 'payload.timestamp': {$eq: null} + }).toArray(); - for (const repetition of eventRepetitions) { - await db.collection(`${REPETITIONS}:${projectId}`).updateOne({ - _id: repetition._id, - }, { - $set: { - 'payload.timestamp': event.payload.timestamp, - }, - }); + for (const event of originalEvents) { + const eventRepetitions = repetitions.filter(rep => rep.groupHash === event.groupHash); + + for (const repetition of eventRepetitions) { + await db.collection(`${REPETITIONS}:${projectId}`).updateOne({ + _id: repetition._id, + }, { + $set: { + 'payload.timestamp': event.payload.timestamp, + }, + }); + } + } } - } + }) + } finally { + session.endSession(); } }, }; From 64bd905584a8154212e68486c11a3f7d2c686e3d Mon Sep 17 00:00:00 2001 From: Georgy Berezhnoy Date: Thu, 26 Aug 2021 18:11:54 +0300 Subject: [PATCH 3/6] Update node catcher --- yarn.lock | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/yarn.lock b/yarn.lock index 5dd99ad9..13fa6a96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -279,13 +279,22 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" -"@hawk.so/nodejs@^2.1.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@hawk.so/nodejs/-/nodejs-2.2.0.tgz#d72f028b2b39eaf6ac1a8b5f711ab9faa7649cb3" +"@hawk.so/nodejs@^3.0.10": + version "3.0.10" + resolved "https://registry.yarnpkg.com/@hawk.so/nodejs/-/nodejs-3.0.10.tgz#11e34f2572f53088913b1d45e3abfaa391386904" + integrity sha512-KpVk34n6ttX21KEJlGmKi7Pg4bh+x3x1Ot8O1Xy4ZRb+ob208PU/ThDEOeJgf/QH3QBTTNnNWK/HvjSW81TkHg== dependencies: - axios "^0.19.2" + "@hawk.so/types" "^0.1.15" + axios "^0.21.1" stack-trace "^0.0.10" +"@hawk.so/types@^0.1.15": + version "0.1.15" + resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.15.tgz#9a561dda47fab91f3093e32cbdfaf83b1eb000c1" + integrity sha512-LHZdjmyIbAQ+0Og9Px34BTYF5YM0fto9bSluONmgVVr6Ctu2nPqPibNF1s9PEEisrtRxzuUQX8ea/OAmOJhK3w== + dependencies: + "@types/mongodb" "^3.5.34" + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -1147,12 +1156,6 @@ aws4@^1.8.0: version "1.11.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" -axios@^0.19.2: - version "0.19.2" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27" - dependencies: - follow-redirects "1.5.10" - axios@^0.20.0: version "0.20.0" resolved "https://registry.yarnpkg.com/axios/-/axios-0.20.0.tgz#057ba30f04884694993a8cd07fa394cff11c50bd" @@ -1960,12 +1963,6 @@ debug@4.1.1: dependencies: ms "^2.1.1" -debug@=3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - dependencies: - ms "2.0.0" - debug@^2.2.0, debug@^2.3.3, debug@^2.6.9, debug@~2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -2714,12 +2711,6 @@ fn.name@1.x.x: version "1.1.0" resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" -follow-redirects@1.5.10: - version "1.5.10" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" - dependencies: - debug "=3.1.0" - follow-redirects@^1.10.0: version "1.13.0" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.0.tgz#b42e8d93a2a7eea5ed88633676d6597bc8e384db" From 209f85acba2eebf0ab7f06fa7de2c472409d95f7 Mon Sep 17 00:00:00 2001 From: Georgy Berezhnoy Date: Thu, 26 Aug 2021 18:27:29 +0300 Subject: [PATCH 4/6] Fix catcher problems --- package.json | 2 +- workers/javascript/src/index.ts | 2 +- workers/limiter/src/index.ts | 2 +- workers/paymaster/src/index.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index bdc03ad0..fabb865e 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "run-limiter": "yarn worker hawk-worker-limiter" }, "dependencies": { - "@hawk.so/nodejs": "^2.1.0", + "@hawk.so/nodejs": "^3.0.10", "@types/amqplib": "^0.5.13", "@types/jest": "^25.2.1", "@types/mongodb": "^3.5.15", diff --git a/workers/javascript/src/index.ts b/workers/javascript/src/index.ts index 1f792707..b33c3400 100644 --- a/workers/javascript/src/index.ts +++ b/workers/javascript/src/index.ts @@ -119,7 +119,7 @@ export default class JavascriptEventWorker extends EventWorker { */ HawkCatcher.send(error, { payload: event.payload, - }); + } as any); return event.payload.backtrace[index]; }); diff --git a/workers/limiter/src/index.ts b/workers/limiter/src/index.ts index d2492fc2..2eb95a4d 100644 --- a/workers/limiter/src/index.ts +++ b/workers/limiter/src/index.ts @@ -327,7 +327,7 @@ export default class LimiterWorker extends Worker { HawkCatcher.send(error, { workspaceId: workspace._id, - }); + } as any); throw error; } diff --git a/workers/paymaster/src/index.ts b/workers/paymaster/src/index.ts index 910df472..f6e05bf3 100644 --- a/workers/paymaster/src/index.ts +++ b/workers/paymaster/src/index.ts @@ -167,7 +167,7 @@ export default class PaymasterWorker extends Worker { HawkCatcher.send(error, { workspaceId: workspace._id, - }); + } as any); return false; } From 74e9ad70aa0a85f29ad15ea069efd6cae3e64a4f Mon Sep 17 00:00:00 2001 From: Georgy Berezhnoy Date: Thu, 26 Aug 2021 18:27:42 +0300 Subject: [PATCH 5/6] Remove migrations from dockerignore --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 963fdd60..ec0945f6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,6 @@ node_modules coverage /**/tests -migrations tools bin From c837204ddbc4bc2ce8628036c3e5e0fceb27caaa Mon Sep 17 00:00:00 2001 From: Georgy Berezhnoy Date: Thu, 26 Aug 2021 18:32:33 +0300 Subject: [PATCH 6/6] Remove mongo migration config from dockerignore --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index ec0945f6..9edfda96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,7 +6,6 @@ tools bin # config files -migrate-mongo-config.js jest-mongodb-config.js jest.config.js ecosystem.config.js