Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 75fb587

Browse files
bigbitbusbigbitbus
committedMay 1, 2024
Various changes
1 parent b5a2a9a commit 75fb587

8 files changed

+8269
-10732
lines changed
 

‎package-lock.json

+8,228-10,693
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+8-9
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,18 @@
11
{
22
"name": "@roboflow/tfrecords",
3-
"version": "1.0.4",
3+
"version": "1.0.5",
44
"description": "TensorFlow record (.tfrecord) API for Node.JS and Browsers",
55
"main": "src/tensorFlowBuilder.js",
66
"dependencies": {
77
"buffer-reverse": "^1.0.1",
8-
"crypto-js": "^3.1.9-1",
9-
"google-protobuf": "^3.6.1",
8+
"google-protobuf": "3.21.2",
109
"node-int64": "^0.4.0",
1110
"shortid": "^2.2.14"
1211
},
1312
"types": "src/index.d.ts",
1413
"scripts": {
1514
"test": "jest --config jestconfig.json",
16-
"build": "tsc",
15+
"build": "tsc --project tsconfig.build.json",
1716
"format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"",
1817
"lint": "tslint -p tsconfig.json",
1918
"prepare": "npm run build",
@@ -37,14 +36,14 @@
3736
},
3837
"homepage": "https://github.com/JacopoMangiavacchi/TFRecords#readme",
3938
"devDependencies": {
40-
"@types/node": "^10.12.7",
41-
"@types/jest": "^24.0.23",
42-
"jest": "^24.9.0",
39+
"@types/jest": "^29.5.12",
40+
"@types/node": "^20.12.7",
41+
"jest": "^29.7.0",
4342
"prettier": "^1.16.4",
44-
"ts-jest": "^24.2.0",
43+
"ts-jest": "29.1.2",
4544
"tslint": "^5.12.1",
4645
"tslint-config-prettier": "^1.18.0",
47-
"typescript": "^3.3.3"
46+
"typescript": "^5.4.5"
4847
},
4948
"publishConfig": {
5049
"registry": "https://registry.npmjs.org"

‎src/guard.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ class Guard {
77
* @param paramName - The name of the parameter to validate
88
* @param message - The error message to return on invalid value
99
*/
10-
static emtpy(value, paramName, message) {
11-
if ((!!value === false || value.trim().length === 0)) {
10+
static empty(value, paramName, message) {
11+
if (value === null || !value || value.trim().length === 0) {
1212
message = message || (`'${paramName || "value"}' cannot be null or empty`);
1313
throw new Error(message);
1414
}
@@ -20,7 +20,7 @@ class Guard {
2020
* @param message - The error message to return on invalid value
2121
*/
2222
static null(value, paramName, message) {
23-
if ((!!value === false)) {
23+
if (!value) {
2424
message = message || (`'${paramName || "value"}' cannot be null or undefined`);
2525
throw new Error(message);
2626
}
@@ -33,7 +33,7 @@ class Guard {
3333
* @param message - The error message to return on invalid value
3434
*/
3535
static expression(value, predicate, paramName, message) {
36-
if (!!value === false || !predicate(value)) {
36+
if (!value || !predicate(value)) {
3737
message = message || (`'${paramName || "value"}' is not a valid value`);
3838
throw new Error(message);
3939
}

‎src/guard.test.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
66
const guard_1 = __importDefault(require("./guard"));
77
describe("Guard", () => {
88
function methodWithRequiredName(name) {
9-
guard_1.default.emtpy(name);
9+
guard_1.default.empty(name);
1010
}
1111
function methodWithRequiredNameWithParam(name) {
12-
guard_1.default.emtpy(name, "name", "Name is required");
12+
guard_1.default.empty(name, "name", "Name is required");
1313
}
1414
function methodWithRequiredObject(options) {
1515
guard_1.default.null(options);

‎src/tensorFlowBuilder.js

+12-11
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"use strict";
22
Object.defineProperty(exports, "__esModule", { value: true });
3+
exports.TFRecordsBuilder = exports.FeatureType = void 0;
34
const tensorFlowRecordsProtoBuf_pb_1 = require("./tensorFlowRecordsProtoBuf_pb");
45
const tensorFlowHelpers_1 = require("./tensorFlowHelpers");
56
const stream_1 = require("stream");
@@ -17,15 +18,12 @@ var FeatureType;
1718
FeatureType[FeatureType["Binary"] = 1] = "Binary";
1819
FeatureType[FeatureType["Int64"] = 2] = "Int64";
1920
FeatureType[FeatureType["Float"] = 3] = "Float";
20-
})(FeatureType = exports.FeatureType || (exports.FeatureType = {}));
21+
})(FeatureType || (exports.FeatureType = FeatureType = {}));
2122
/**
2223
* @name - TFRecords Builder Class
2324
* @description - Create a TFRecords object
2425
*/
2526
class TFRecordsBuilder {
26-
constructor() {
27-
this.features = new tensorFlowRecordsProtoBuf_pb_1.Features();
28-
}
2927
/**
3028
* @records - An Array of TFRecord Buffer created with releaseTFRecord()
3129
* @description - Return a Buffer representation of a TFRecords object
@@ -34,9 +32,9 @@ class TFRecordsBuilder {
3432
return Buffer.concat(records.map((record) => {
3533
const length = record.length;
3634
// Get TFRecords CRCs for TFRecords Header and Footer
37-
const bufferLength = tensorFlowHelpers_1.getInt64Buffer(length);
38-
const bufferLengthMaskedCRC = tensorFlowHelpers_1.getInt32Buffer(tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(bufferLength)));
39-
const bufferDataMaskedCRC = tensorFlowHelpers_1.getInt32Buffer(tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(record)));
35+
const bufferLength = (0, tensorFlowHelpers_1.getInt64Buffer)(length);
36+
const bufferLengthMaskedCRC = (0, tensorFlowHelpers_1.getInt32Buffer)((0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)(bufferLength)));
37+
const bufferDataMaskedCRC = (0, tensorFlowHelpers_1.getInt32Buffer)((0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)(record)));
4038
// Concatenate all TFRecords Header, Data and Footer buffer
4139
return Buffer.concat([bufferLength,
4240
bufferLengthMaskedCRC,
@@ -58,14 +56,17 @@ class TFRecordsBuilder {
5856
transform: (record, encoding, callback) => {
5957
const length = record.length;
6058
// Get TFRecords CRCs for TFRecords Header and Footer
61-
const bufferLength = tensorFlowHelpers_1.getInt64Buffer(length);
62-
const bufferLengthMaskedCRC = tensorFlowHelpers_1.getInt32Buffer(tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(bufferLength)));
63-
const bufferDataMaskedCRC = tensorFlowHelpers_1.getInt32Buffer(tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(record)));
59+
const bufferLength = (0, tensorFlowHelpers_1.getInt64Buffer)(length);
60+
const bufferLengthMaskedCRC = (0, tensorFlowHelpers_1.getInt32Buffer)((0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)(bufferLength)));
61+
const bufferDataMaskedCRC = (0, tensorFlowHelpers_1.getInt32Buffer)((0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)(record)));
6462
callback(undefined, Buffer.concat([bufferLength, bufferLengthMaskedCRC, record, bufferDataMaskedCRC]));
6563
},
6664
highWaterMark,
6765
});
6866
}
67+
constructor() {
68+
this.features = new tensorFlowRecordsProtoBuf_pb_1.Features();
69+
}
6970
/**
7071
* @key - Feature Key
7172
* @type - Feature Type
@@ -87,7 +88,7 @@ class TFRecordsBuilder {
8788
case FeatureType.String:
8889
const stringList = new tensorFlowRecordsProtoBuf_pb_1.BytesList();
8990
values.forEach((value) => {
90-
stringList.addValue(tensorFlowHelpers_1.textEncode(value));
91+
stringList.addValue((0, tensorFlowHelpers_1.textEncode)(value));
9192
});
9293
feature.setBytesList(stringList);
9394
break;

‎src/tensorFlowHelpers.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
1818
return (mod && mod.__esModule) ? mod : { "default": mod };
1919
};
2020
Object.defineProperty(exports, "__esModule", { value: true });
21+
exports.readInt64 = exports.textDecode = exports.textEncode = exports.getInt32Buffer = exports.getInt64Buffer = exports.maskCrc = exports.crc32c = void 0;
2122
// maskDelta is a magic number taken from
2223
// https://github.com/tensorflow/tensorflow/blob/754048a0453a04a761e112ae5d99c149eb9910dd/
2324
// tensorflow/core/lib/hash/crc32c.h#L33.
@@ -139,7 +140,7 @@ exports.textDecode = textDecode;
139140
function readInt64(buffer) {
140141
guard_1.default.null(buffer);
141142
guard_1.default.expression(buffer.length, (num) => num >= 8);
142-
buffer = buffer_reverse_1.default(buffer.slice(0, 8));
143+
buffer = (0, buffer_reverse_1.default)(buffer.slice(0, 8));
143144
const int64 = new node_int64_1.default(buffer, 0);
144145
return int64.toNumber(true);
145146
}

‎src/tensorFlowHelpers.test.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -13,38 +13,38 @@ const tensorFlowHelpers_1 = require("./tensorFlowHelpers");
1313
describe("TFRecords Helper Functions", () => {
1414
describe("Run getInt64Buffer method test", () => {
1515
it("Check getInt64Buffer for number 164865", () => __awaiter(void 0, void 0, void 0, function* () {
16-
expect(tensorFlowHelpers_1.getInt64Buffer(164865)).toEqual(new Buffer([1, 132, 2, 0, 0, 0, 0, 0]));
16+
expect((0, tensorFlowHelpers_1.getInt64Buffer)(164865)).toEqual(new Buffer([1, 132, 2, 0, 0, 0, 0, 0]));
1717
}));
1818
});
1919
describe("Run getInt32Buffer method test", () => {
2020
it("Check getInt32Buffer for number 164865", () => __awaiter(void 0, void 0, void 0, function* () {
21-
expect(tensorFlowHelpers_1.getInt32Buffer(164865)).toEqual(new Buffer([1, 132, 2, 0]));
21+
expect((0, tensorFlowHelpers_1.getInt32Buffer)(164865)).toEqual(new Buffer([1, 132, 2, 0]));
2222
}));
2323
});
2424
describe("Run crc32c method test", () => {
2525
it("Check crc32c for number 164865", () => __awaiter(void 0, void 0, void 0, function* () {
26-
expect(tensorFlowHelpers_1.crc32c(new Buffer([1, 132, 2, 0, 0, 0, 0, 0]))).toEqual(1310106699);
26+
expect((0, tensorFlowHelpers_1.crc32c)(new Buffer([1, 132, 2, 0, 0, 0, 0, 0]))).toEqual(1310106699);
2727
}));
2828
});
2929
describe("Run maskCrc method test", () => {
3030
it("Check maskCrc for crc 1310106699", () => __awaiter(void 0, void 0, void 0, function* () {
31-
expect(tensorFlowHelpers_1.maskCrc(1310106699)).toEqual(3944318725);
31+
expect((0, tensorFlowHelpers_1.maskCrc)(1310106699)).toEqual(3944318725);
3232
}));
3333
});
3434
describe("Run integration of getInt32Buffer(maskCrc(crc32c(getInt64Buffer())) methods test", () => {
3535
it("Check maskCrc for for number 164865", () => __awaiter(void 0, void 0, void 0, function* () {
36-
expect(tensorFlowHelpers_1.getInt32Buffer(tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(tensorFlowHelpers_1.getInt64Buffer(164865)))))
36+
expect((0, tensorFlowHelpers_1.getInt32Buffer)((0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)((0, tensorFlowHelpers_1.getInt64Buffer)(164865)))))
3737
.toEqual(new Buffer([5, 135, 25, 235]));
3838
}));
3939
});
4040
describe("Run textEncode method test", () => {
4141
it("Check textEncode for string 'ABC123'", () => __awaiter(void 0, void 0, void 0, function* () {
42-
expect(tensorFlowHelpers_1.textEncode("ABC123")).toEqual(new Uint8Array([65, 66, 67, 49, 50, 51]));
42+
expect((0, tensorFlowHelpers_1.textEncode)("ABC123")).toEqual(new Uint8Array([65, 66, 67, 49, 50, 51]));
4343
}));
4444
});
4545
describe("Run textDecode method test", () => {
4646
it("Check textDecode for array [65, 66, 67, 49, 50, 51]", () => __awaiter(void 0, void 0, void 0, function* () {
47-
expect(tensorFlowHelpers_1.textDecode(new Uint8Array([65, 66, 67, 49, 50, 51]))).toEqual("ABC123");
47+
expect((0, tensorFlowHelpers_1.textDecode)(new Uint8Array([65, 66, 67, 49, 50, 51]))).toEqual("ABC123");
4848
}));
4949
});
5050
});

‎src/tensorFlowReader.js

+6-5
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
33
return (mod && mod.__esModule) ? mod : { "default": mod };
44
};
55
Object.defineProperty(exports, "__esModule", { value: true });
6+
exports.TFRecordsReader = void 0;
67
const guard_1 = __importDefault(require("./guard"));
78
const tensorFlowBuilder_1 = require("./tensorFlowBuilder");
89
const tensorFlowHelpers_1 = require("./tensorFlowHelpers");
@@ -18,8 +19,8 @@ class TFRecordsReader {
1819
let position = 0;
1920
while (position < tfrecords.length) {
2021
const lengthBuffer = tfrecords.slice(position, position + 8);
21-
const dataLength = tensorFlowHelpers_1.readInt64(lengthBuffer);
22-
const lengthCrc = tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(lengthBuffer));
22+
const dataLength = (0, tensorFlowHelpers_1.readInt64)(lengthBuffer);
23+
const lengthCrc = (0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)(lengthBuffer));
2324
position += 8;
2425
const expectedLengthCrc = tfrecords.readUInt32LE(position);
2526
position += 4;
@@ -28,7 +29,7 @@ class TFRecordsReader {
2829
break;
2930
}
3031
const dataBuffer = tfrecords.slice(position, position + dataLength);
31-
const dataCrc = tensorFlowHelpers_1.maskCrc(tensorFlowHelpers_1.crc32c(dataBuffer));
32+
const dataCrc = (0, tensorFlowHelpers_1.maskCrc)((0, tensorFlowHelpers_1.crc32c)(dataBuffer));
3233
position += dataLength;
3334
const expectedDataCrc = tfrecords.readUInt32LE(position);
3435
position += 4;
@@ -64,7 +65,7 @@ class TFRecordsReader {
6465
const feature = message.getContext().getFeatureMap().get(key);
6566
switch (type) {
6667
case tensorFlowBuilder_1.FeatureType.String:
67-
return tensorFlowHelpers_1.textDecode(feature.getBytesList().array[0][0]);
68+
return (0, tensorFlowHelpers_1.textDecode)(feature.getBytesList().array[0][0]);
6869
case tensorFlowBuilder_1.FeatureType.Binary:
6970
return feature.getBytesList().array[0][0];
7071
case tensorFlowBuilder_1.FeatureType.Int64:
@@ -84,7 +85,7 @@ class TFRecordsReader {
8485
const feature = message.getContext().getFeatureMap().get(key);
8586
switch (type) {
8687
case tensorFlowBuilder_1.FeatureType.String:
87-
return feature.getBytesList().array[0].map((buffer) => tensorFlowHelpers_1.textDecode(buffer));
88+
return feature.getBytesList().array[0].map((buffer) => (0, tensorFlowHelpers_1.textDecode)(buffer));
8889
case tensorFlowBuilder_1.FeatureType.Binary:
8990
return feature.getBytesList().array[0];
9091
case tensorFlowBuilder_1.FeatureType.Int64:

0 commit comments

Comments
 (0)
Please sign in to comment.