Skip to content

Commit 9192f93

Browse files
huntharohuntharo
committed
Initial commit
0 parents  commit 9192f93

26 files changed

+13762
-0
lines changed

‎.eslintignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
node_modules
2+
dist

‎.eslintrc

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"root": true,
3+
"parser": "@typescript-eslint/parser",
4+
"parserOptions": {
5+
"project": "./tsconfig.json"
6+
},
7+
"plugins": ["@typescript-eslint", "prettier"],
8+
"extends": [
9+
"eslint:recommended",
10+
"plugin:@typescript-eslint/eslint-recommended",
11+
"plugin:@typescript-eslint/recommended",
12+
"prettier"
13+
],
14+
"env": {
15+
"jest": true
16+
},
17+
"rules": {
18+
"no-console": 1, // Means warning
19+
"prettier/prettier": 2, // Means error
20+
"@typescript-eslint/ban-ts-comment": "off",
21+
"@typescript-eslint/no-floating-promises": ["error"],
22+
"@typescript-eslint/no-misused-promises": ["error"],
23+
"@typescript-eslint/promise-function-async": ["error"],
24+
"@typescript-eslint/require-await": ["error"],
25+
// note you must disable the base rule as it can report incorrect errors
26+
"no-return-await": "off",
27+
"@typescript-eslint/return-await": ["error"],
28+
// Don't allow awaiting non-Promises
29+
"@typescript-eslint/await-thenable": "error"
30+
},
31+
"ignorePatterns": ["dist", "cdk.out"]
32+
}

‎.github/workflows/ci.yml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Build - CI
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
11+
jobs:
12+
build:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout code
16+
uses: actions/checkout@v3
17+
18+
- name: Use Node.js 16
19+
uses: actions/setup-node@v3
20+
with:
21+
node-version: 16
22+
23+
- name: Install Modules
24+
run: npm ci
25+
26+
- name: Build
27+
run: npm run build
28+
29+
- name: Lint
30+
run: npm run lint
31+
32+
- name: Test
33+
run: npm run test

‎.github/workflows/docs.yml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Publish Docs
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
workflow_dispatch:
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- name: Checkout code
14+
uses: actions/checkout@v3
15+
16+
- name: Use Node.js 16
17+
uses: actions/setup-node@v3
18+
with:
19+
node-version: 16
20+
21+
- name: Install Modules
22+
run: npm ci
23+
24+
- name: Build Docs
25+
run: npm run build:docs
26+
27+
- name: Deploy to GitHub Pages
28+
uses: peaceiris/actions-gh-pages@v3
29+
with:
30+
github_token: ${{ secrets.GITHUB_TOKEN }}
31+
publish_dir: ./docs

‎.github/workflows/publish.yml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: Package and Publish
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
jobs:
8+
build:
9+
runs-on: ubuntu-latest
10+
permissions:
11+
contents: read
12+
packages: write
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v3
16+
17+
- name: Use Node.js 16
18+
uses: actions/setup-node@v3
19+
with:
20+
node-version: 16
21+
22+
- name: Use the Release Tag Version
23+
run: |
24+
npm version from-git --allow-same-version --no-git-tag-version
25+
26+
- name: Install Modules
27+
run: npm ci
28+
29+
- name: Build
30+
run: npm run build
31+
32+
- name: Lint
33+
run: npm run lint
34+
35+
- name: Test
36+
run: npm run test
37+
38+
- name: Publish with CLI to Code Artifact
39+
run: |
40+
npm publish --access public --ignore-scripts --dry-run

‎.gitignore

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.idea/
2+
node_modules
3+
dist/
4+
*.tsbuildinfo
5+
*.log
6+
.nyc_output/
7+
coverage/
8+
.DS_Store
9+
docs/

‎.nvmrc

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
v16.17.1

‎.prettierrc

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"semi": true,
3+
"trailingComma": "all",
4+
"singleQuote": true,
5+
"printWidth": 100
6+
}

‎.vscode/settings.json

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"files.exclude": {
3+
// "**/node_modules/*": true,
4+
// "cdk.out": true
5+
// "node_modules": true,
6+
// "**/node_modules": true
7+
},
8+
"editor.defaultFormatter": "esbenp.prettier-vscode",
9+
"editor.formatOnSave": true,
10+
"editor.insertSpaces": true,
11+
"editor.tabSize": 2,
12+
"files.associations": {
13+
"Dockerfile*": "dockerfile"
14+
},
15+
"[typescript]": {
16+
"editor.defaultFormatter": "esbenp.prettier-vscode"
17+
},
18+
"yaml.customTags": [
19+
"!And",
20+
"!And sequence",
21+
"!If",
22+
"!If sequence",
23+
"!Not",
24+
"!Not sequence",
25+
"!Equals",
26+
"!Equals sequence",
27+
"!Or",
28+
"!Or sequence",
29+
"!FindInMap",
30+
"!FindInMap sequence",
31+
"!Base64",
32+
"!Join",
33+
"!Join sequence",
34+
"!Cidr",
35+
"!Ref",
36+
"!Sub",
37+
"!Sub sequence",
38+
"!GetAtt",
39+
"!GetAZs",
40+
"!ImportValue",
41+
"!ImportValue sequence",
42+
"!Select",
43+
"!Select sequence",
44+
"!Split",
45+
"!Split sequence"
46+
],
47+
"[xml]": {
48+
"editor.defaultFormatter": "DotJoshJohnson.xml"
49+
}
50+
}

‎CODE_OF_CONDUCT.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Contributor Code of Conduct
2+
3+
The Shutterstock team is committed to fostering a welcoming community.
4+
5+
This project is governed by Shutterstock’s [Code of Conduct](https://github.com/shutterstock/code-of-conduct). All contributors and participants agree to abide by its terms.

‎ISSUE_TEMPLATE.md

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
### Expected behavior
2+
3+
### Actual behavior
4+
5+
### Steps to reproduce the behavior
6+
7+
### Additional specs (e.g. browser, version, etc.)
8+

‎LICENSE.md

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
The MIT License (MIT)
2+
=====================
3+
Copyright (c) `2022` `Shutterstock, Inc.`
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy of
6+
this software and associated documentation files (the "Software"), to deal in
7+
the Software without restriction, including without limitation the rights to
8+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
9+
of the Software, and to permit persons to whom the Software is furnished to do
10+
so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎README.md

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) [![API Docs](https://img.shields.io/badge/API%20Docs-View%20Here-blue)](https://tech.shutterstock.com/kinesis-helpers/) [![Build - CI](https://github.com/shutterstock/kinesis-helpers/actions/workflows/ci.yml/badge.svg)](https://github.com/shutterstock/kinesis-helpers/actions/workflows/ci.yml) [![Package and Publish](https://github.com/shutterstock/kinesis-helpers/actions/workflows/publish.yml/badge.svg)](https://github.com/shutterstock/kinesis-helpers/actions/workflows/publish.yml) [![Publish Docs](https://github.com/shutterstock/kinesis-helpers/actions/workflows/docs.yml/badge.svg)](https://github.com/shutterstock/kinesis-helpers/actions/workflows/docs.yml)
2+
3+
# Overview
4+
5+
Helper classes for the AWS v3 SDK for JS Kinesis Client (`@aws-sdk/client-kinesis`).
6+
7+
`KinesisRetrier` will retry failed records with a "successful" `PutRecordsCommand` that has items marked as failed within the response, which then need to be retried.
8+
9+
`KinesisBackgroundWriter` will accept writes immediately if there is a concurrent writer slot available, and will block the caller until a slot becomes available if all slots are in use.
10+
11+
# Getting Started
12+
13+
## Installation
14+
15+
The package is available on npm as [@shutterstock/kinesis-helpers](https://www.npmjs.com/package/@shutterstock/kinesis-helpers)
16+
17+
`npm i @shutterstock/kinesis-helpers`
18+
19+
## Importing
20+
21+
```typescript
22+
import { KinesisBackgroundWriter, KinesisRetrier } from '@shutterstock/kinesis-helpers';
23+
```
24+
25+
## API Documentation
26+
27+
After installing the package, you might want to look at our [API Documentation](https://tech.shutterstock.com/kinesis-helpers/) to learn about all the features available.
28+
29+
# Features
30+
31+
- [KinesisRetrier](https://tech.shutterstock.com/kinesis-helpers/classes/KinesisRetrier.html)
32+
- Kinesis batch put retrier
33+
- Batch puts return a list of items that were throttled
34+
- The batch is marked as succeeded (200 status code) even though items failed
35+
- As a result, the built-in AWS SDK retry logic will not retry the items in the batch that were throttle
36+
- [KinesisBackgroundWriter](https://tech.shutterstock.com/kinesis-helpers/classes/KinesisBackgroundWriter.html)
37+
- Accepts writes immediately if there is a concurrent writer slot available
38+
- Blocks the caller until a slot becomes available if all slots are in use
39+
- Collects and exposes errors so the caller can detect if errors have been happening
40+
41+
# Contributing
42+
43+
## Setting up Build Environment
44+
45+
- `nvm use`
46+
- `npm i`
47+
- `npm run build`
48+
- `npm run lint`
49+
- `npm run test`
50+
51+
## Running Examples
52+
53+
### kinesis-retrier
54+
55+
1. Create Kinesis Data Stream using AWS Console or any other method
56+
1. Default name is `kinesis-helpers-test-stream`
57+
2. 1 shard is sufficient
58+
3. 1 day retention is sufficient
59+
4. No encryption is sufficient
60+
5. On-demand throughput is sufficient
61+
2. `npm run example:kinesis-retrier`
62+
1. If the stream name was changed: `KINESIS_STREAM_NAME=my-stream-name npm run example:kinesis-retrier`
63+
64+
### kinesis-background-writer
65+
66+
1. Create Kinesis Data Stream using AWS Console or any other method
67+
1. Default name is `kinesis-helpers-test-stream`
68+
2. 1 shard is sufficient
69+
3. 1 day retention is sufficient
70+
4. No encryption is sufficient
71+
5. On-demand throughput is sufficient
72+
2. `npm run example:kinesis-background-writer`
73+
1. If the stream name was changed: `KINESIS_STREAM_NAME=my-stream-name npm run example:kinesis-background-writer`

‎examples/kinesis-background-writer.ts

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import * as kinesis from '@aws-sdk/client-kinesis';
2+
import { KinesisBackgroundWriter, KinesisRetrier } from '@shutterstock/kinesis-helpers';
3+
4+
const kinesisClient = new kinesis.KinesisClient({});
5+
const { KINESIS_STREAM_NAME = 'kinesis-helpers-test-stream', RECORDS_TO_WRITE = '1000000' } =
6+
process.env;
7+
const RECORDS_TO_WRITE_NUM = parseInt(RECORDS_TO_WRITE, 10);
8+
const RECORDS_PER_BATCH = 500;
9+
10+
async function main() {
11+
// Use a KinesisRetrier so that we do not get throttling exceptions
12+
// within the background batch writes
13+
const kinesisRetrier = new KinesisRetrier({
14+
kinesisClient,
15+
});
16+
17+
const backgroundWriter = new KinesisBackgroundWriter({
18+
kinesisClient: kinesisRetrier,
19+
concurrency: 4,
20+
});
21+
22+
const records: kinesis.PutRecordsCommandInput = {
23+
StreamName: KINESIS_STREAM_NAME,
24+
Records: [],
25+
};
26+
27+
// Thanks TypeScript? I guess? The value is assigned above but
28+
// we are getting "possibly undefined" in the loop below
29+
records.Records = [];
30+
31+
for (let i = 0; i < RECORDS_PER_BATCH; i++) {
32+
records.Records.push({
33+
Data: Buffer.from('123', 'utf-8'),
34+
PartitionKey: '123',
35+
});
36+
}
37+
38+
// Send a whole lot of records so we start getting throttled within the batches
39+
for (let i = 0; i < RECORDS_TO_WRITE_NUM; i += RECORDS_PER_BATCH) {
40+
await backgroundWriter.send(new kinesis.PutRecordsCommand(records));
41+
}
42+
43+
// Need to wait until the backgroundWriter is idle (has finished any pending requests)
44+
await backgroundWriter.onIdle();
45+
46+
// TODO: If there were any errors, log them
47+
}
48+
49+
void main();

0 commit comments

Comments
 (0)