From bc819a2fdecf759300772c354c497b0458547153 Mon Sep 17 00:00:00 2001 From: Klaus Trainer Date: Mon, 26 Sep 2016 23:15:52 +0200 Subject: [PATCH] feat: add `hexEncodeTableName` option If the `hexEncodeTableName` DynamoDB option is set to `true`, DynamoDBDOWN encodes table names in hexadecimal. This can be useful if one would like to pass `location` parameter values to `levelup` that aren't compatible with DynamoDB's restrictions on table names. --- README.md | 64 +++++++++++++++++++++++++++++++++++++---------------- index.js | 34 ++++++++++++++++++++-------- iterator.js | 2 +- 3 files changed, 71 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index f3adea6..4a07339 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,44 @@ When running the above example, you should get the following console output: read stream closed ``` -### Table Creation ### +## Hash Keys ## + +In DynamoDB, keys consist of two parts: a *hash key* and a *range key*. To achieve LevelDB-like behaviour, all keys in a database instance are given the same hash key. That means that you can't do range queries over keys with different hash keys. + +The default hash key is `!`. You can specify it by putting a `$` in the `location` argument. The `$` separates the table name from the hash key. + +### Example ### + +```js +const levelup = require('levelup') +const DynamoDBDOWN = require('dynamodbdown') + +const dynamoDBOptions = { + region: 'eu-west-1', + secretAccessKey: 'abc', + accessKeyId: '123' +} + +const options = { + db: DynamoDBDOWN, + dynamodb: dynamoDBOptions // required AWS configuration +} + +const db = levelup('tableName$hashKey', options) + +db.put('some key', 'some value', => err { + // the DynamoDB object would now look like this: + // { + // hkey: 'hashKey', + // rkey: 'some key', + // value: 'some value' + // } +}) +``` + +If you are fine with sharing capacity units across multiple database instances or applications, you can reuse a table by specifying the same table name, but different hash keys. + +## Table Creation ## If the table doesn't exist, DynamoDBDOWN will try to create a table. You can specify the read/write throughput. If not specified, it will default to `1/1`. If the table already exists, the specified throughput will have no effect. Throughput can be changed for tables that already exist by using the DynamoDB API or the AWS Console. @@ -69,18 +106,16 @@ const dynamoDBOptions = { } const options = { - db: location => DynamoDBDOWN, + db: DynamoDBDOWN, dynamodb: dynamoDBOptions // required AWS configuration } const db = levelup('tableName', options) ``` -### Hash Keys ### +## Table Name Encoding ## -In DynamoDB, keys consist of two parts: a *hash key* and a *range key*. To achieve LevelDB-like behaviour, all keys in a database instance are given the same hash key. That means that you can't do range queries over keys with different hash keys. - -The default hash key is `!`. You can specify it by putting a `$` in the `location` argument. The `$` separates the table name from the hash key. +DynamoDBDOWN encodes table names in hexadecimal if you set the `dynamodb.hexEncodeTableName` option to `true`. This can be useful if you'd like pass `location` parameter values to `levelup` that aren't compatible with DynamoDB's restrictions on table names (see [here](docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html)). ### Example ### @@ -91,7 +126,8 @@ const DynamoDBDOWN = require('dynamodbdown') const dynamoDBOptions = { region: 'eu-west-1', secretAccessKey: 'abc', - accessKeyId: '123' + accessKeyId: '123', + hexEncodeTableName: true } const options = { @@ -99,20 +135,10 @@ const options = { dynamodb: dynamoDBOptions // required AWS configuration } -const db = levelup('tableName/hashKey', options) - -db.put('some key', 'some value', => err { - // the DynamoDB object would now look like this: - // { - // hkey: 'hashKey', - // rkey: 'some key', - // value: 'some value' - // } -}) +const db = levelup('tableName', options) // the DynaoDB table name will + // be '7461626c654e616d65' ``` -If you are fine with sharing capacity units across multiple database instances or applications, you can reuse a table by specifying the same table name, but different hash keys. - ## Changelog ## See [here](https://github.com/KlausTrainer/dynamodbdown/releases). diff --git a/index.js b/index.js index 4134065..8b9504b 100644 --- a/index.js +++ b/index.js @@ -28,6 +28,16 @@ function DynamoDBDOWN (location) { inherits(DynamoDBDOWN, AbstractLevelDOWN) +function hexEncodeTableName (str) { + var hex = '' + + for (var pos = 0; pos < str.length; pos++) { + hex += String(str.charCodeAt(pos).toString(16)) + } + + return hex +} + DynamoDBDOWN.prototype._open = function (options, cb) { if (!options.dynamodb) { return cb(new Error('`open` requires `options` argument with "dynamodb" key')) @@ -37,7 +47,13 @@ DynamoDBDOWN.prototype._open = function (options, cb) { this.tableName = this.tableName.replace(options.prefix, '') } - const dynamodbOptions = Object.assign({tableName: this.tableName}, options.dynamodb) + if (options.dynamodb.hexEncodeTableName === true) { + this.encodedTableName = hexEncodeTableName(this.tableName) + } else { + this.encodedTableName = this.tableName + } + + const dynamodbOptions = Object.assign({tableName: this.encodedTableName}, options.dynamodb) this.dynamoDb = new AWS.DynamoDB(dynamodbOptions) @@ -64,7 +80,7 @@ DynamoDBDOWN.prototype._close = function (cb) { DynamoDBDOWN.prototype._put = function (key, value, options, cb) { const params = { - TableName: this.tableName, + TableName: this.encodedTableName, Item: { hkey: {S: this.hashKey.toString()}, rkey: {S: key.toString()}, @@ -77,7 +93,7 @@ DynamoDBDOWN.prototype._put = function (key, value, options, cb) { DynamoDBDOWN.prototype._get = function (key, options, cb) { const params = { - TableName: this.tableName, + TableName: this.encodedTableName, Key: { hkey: {S: this.hashKey.toString()}, rkey: {S: key.toString()} @@ -99,7 +115,7 @@ DynamoDBDOWN.prototype._get = function (key, options, cb) { DynamoDBDOWN.prototype._del = function (key, options, cb) { const params = { - TableName: this.tableName, + TableName: this.encodedTableName, Key: { hkey: {S: this.hashKey.toString()}, rkey: {S: key.toString()} @@ -172,8 +188,8 @@ DynamoDBDOWN.prototype._batch = function (array, options, cb) { const reqs = [] - if (data && data.UnprocessedItems && data.UnprocessedItems[this.tableName]) { - reqs.push.apply(reqs, data.UnprocessedItems[this.tableName]) + if (data && data.UnprocessedItems && data.UnprocessedItems[this.encodedTableName]) { + reqs.push.apply(reqs, data.UnprocessedItems[this.encodedTableName]) } reqs.push.apply(reqs, ops.splice(0, 25 - reqs.length)) @@ -182,7 +198,7 @@ DynamoDBDOWN.prototype._batch = function (array, options, cb) { return cb() } - params.RequestItems[this.tableName] = reqs + params.RequestItems[this.encodedTableName] = reqs this.dynamoDb.batchWriteItem(params, loop) } @@ -195,7 +211,7 @@ DynamoDBDOWN.prototype._iterator = function (options) { DynamoDBDOWN.prototype.createTable = function (opts, cb) { const params = { - TableName: this.tableName, + TableName: this.encodedTableName, AttributeDefinitions: [ {AttributeName: 'hkey', AttributeType: 'S'}, {AttributeName: 'rkey', AttributeType: 'S'} @@ -218,7 +234,7 @@ DynamoDBDOWN.destroy = function (name, callback) { const store = globalStore[name] if (store) { - store.dynamoDb.deleteTable({TableName: store.tableName}, (err, data) => { + store.dynamoDb.deleteTable({TableName: store.encodedTableName}, (err, data) => { if (err) { callback() } else { diff --git a/iterator.js b/iterator.js index fefa0ee..af5bddf 100644 --- a/iterator.js +++ b/iterator.js @@ -139,7 +139,7 @@ DynamoDBIterator.prototype.getRange = function (opts, cb) { const rkey = createRKey(opts) const params = { - TableName: this.db.tableName, + TableName: this.db.encodedTableName, KeyConditions: { hkey: { ComparisonOperator: 'EQ',