Skip to content

[HCK-10854]Fix/ RE/FE and gremlin with node 22 #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 24, 2025
Merged

[HCK-10854]Fix/ RE/FE and gremlin with node 22 #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8e6a183
we don't use ssh in cosmosdb (at least not here)
bigorn0 Apr 16, 2025
7cabd8a
Update deps and fix gremlin with node 22
bigorn0 Apr 16, 2025
a4a8fc0
Cleanup
bigorn0 Apr 16, 2025
580b40b
Fix plugin RE/FE apply
bigorn0 Apr 17, 2025
4941766
Add backward compatibility
bigorn0 Apr 18, 2025
aa2c04c
fix: skip throughput option on specific capacity modes
chulanovskyi-bs Apr 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions forward_engineering/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ module.exports = {
}
const progress = createLogger(logger, containerProps.dbId, graphName);

const cosmosClient = applyToInstanceHelper(_).setUpDocumentClient(data);
const cosmosClient = applyToInstanceHelper().setUpDocumentClient(data);

progress('Create database if not exists ...');

Expand All @@ -41,9 +41,9 @@ module.exports = {

const containerResponse = await cosmosClient.database(containerProps.dbId).containers.createIfNotExists({
id: graphName,
partitionKey: getPartitionKey(_)(data.containerData),
...applyToInstanceHelper(_).getContainerThroughputProps(containerProps),
defaultTtl: applyToInstanceHelper(_).getTTL(containerProps),
partitionKey: getPartitionKey(data.containerData),
...applyToInstanceHelper().getContainerThroughputProps(containerProps),
defaultTtl: applyToInstanceHelper().getTTL(containerProps),
});

progress('Applying Cosmos DB script ...');
Expand All @@ -66,19 +66,19 @@ module.exports = {
const storedProcs = _.get(cosmosDBScript, 'Stored Procedures', []);
if (storedProcs.length) {
progress('Upload stored procs ...');
await applyToInstanceHelper(_).createStoredProcs(storedProcs, containerResponse.container);
await applyToInstanceHelper().createStoredProcs(storedProcs, containerResponse.container);
}

const udfs = _.get(cosmosDBScript, 'User Defined Functions', []);
if (udfs.length) {
progress('Upload user defined functions ...');
await applyToInstanceHelper(_).createUDFs(udfs, containerResponse.container);
await applyToInstanceHelper().createUDFs(udfs, containerResponse.container);
}

const triggers = _.get(cosmosDBScript, 'Triggers', []);
if (triggers.length) {
progress('Upload triggers ...');
await applyToInstanceHelper(_).createTriggers(triggers, containerResponse.container);
await applyToInstanceHelper().createTriggers(triggers, containerResponse.container);
}

if (!gremlinScript) {
Expand All @@ -87,16 +87,16 @@ module.exports = {

progress('Applying Gremlin script ...');

const { labels, edges } = applyToInstanceHelper(_).parseScriptStatements(gremlinScript);
const gremlinClient = await applyToInstanceHelper(_).getGremlinClient(data, containerProps.dbId, graphName);
const { labels, edges } = applyToInstanceHelper().parseScriptStatements(gremlinScript);
const gremlinClient = await applyToInstanceHelper().getGremlinClient(data, containerProps.dbId, graphName);

progress('Uploading labels ...');

await applyToInstanceHelper(_).runGremlinQueries(gremlinClient, labels);
await applyToInstanceHelper().runGremlinQueries(gremlinClient, labels);

progress('Uploading edges ...');

await applyToInstanceHelper(_).runGremlinQueries(gremlinClient, edges);
await applyToInstanceHelper().runGremlinQueries(gremlinClient, edges);

cb();
} catch (err) {
Expand All @@ -109,8 +109,8 @@ module.exports = {
logger.clear();
logger.log('info', connectionInfo, 'Test connection', connectionInfo.hiddenKeys);
try {
const client = applyToInstanceHelper(_).setUpDocumentClient(connectionInfo);
await applyToInstanceHelper(_).testConnection(client);
const client = applyToInstanceHelper().setUpDocumentClient(connectionInfo);
await applyToInstanceHelper().testConnection(client);
return cb();
} catch (err) {
logger.log('error', mapError(err), 'Connection failed');
Expand Down
17 changes: 8 additions & 9 deletions forward_engineering/applyToInstanceHelper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,26 @@
const { CosmosClient, StoredProcedure, UserDefinedFunction, Trigger } = require('@azure/cosmos');
const gremlin = require('gremlin');
const _ = require('lodash');

const applyToInstanceHelper = _ => ({
const applyToInstanceHelper = () => ({
setUpDocumentClient(connectionInfo) {
const dbNameRegExp = /wss:\/\/(\S*).gremlin\.cosmos\./i;
const dbName = dbNameRegExp.exec(connectionInfo.gremlinEndpoint);
if (!dbName?.[1]) {
throw new Error('Incorrect endpoint provided. Expected format: wss://<account name>.gremlin.cosmos.');
}
const endpoint = `https://${dbName[1]}.documents.azure.com:443/`;
const dbName = connectionInfo.azureCosmosdbAccount;
const endpoint = `https://${dbName}.documents.azure.com:443/`;
const key = connectionInfo.accountKey;

return new CosmosClient({ endpoint, key });
},

async getGremlinClient(connectionInfo, databaseId, collection) {
const gremlinEndpoint = `wss://${connectionInfo.azureCosmosdbAccount}.gremlin.cosmos.azure.com`;
const traversalSource = 'g';

const authenticator = new gremlin.driver.auth.PlainTextSaslAuthenticator(
`/dbs/${databaseId}/colls/${collection}`,
connectionInfo.accountKey,
);

const client = new gremlin.driver.Client(connectionInfo.gremlinEndpoint, {
const client = new gremlin.driver.Client(gremlinEndpoint, {
authenticator,
traversalSource,
rejectUnauthorized: true,
Expand Down Expand Up @@ -154,7 +152,8 @@ const applyToInstanceHelper = _ => ({
},

getContainerThroughputProps(containerProps) {
if (containerProps.capacityMode === 'Serverless') {
const capacityModesToSkip = ['Serverless', 'Provisioned throughput'];
if (capacityModesToSkip.includes(containerProps.capacityMode)) {
return {};
}
if (containerProps.autopilot) {
Expand Down
37 changes: 22 additions & 15 deletions forward_engineering/generateContainerScript.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ const generateEdges = (collections, relationships, jsonData) => {
return edges.join(';\n\n') + ';';
};

const getGremlinScript = (_, data) => {
const getGremlinScript = data => {
let { collections, relationships, jsonData, containerData, options } = data;
let resultScript = '';
const traversalSource = _.get(containerData, [0, 'traversalSource'], 'g');
Expand Down Expand Up @@ -147,16 +147,23 @@ const getGremlinScript = (_, data) => {
return resultScript;
};

const getCosmosDbScript = (_, containerData) => {
return JSON.stringify(
{
partitionKey: getPartitionKey(_)(containerData),
indexingPolicy: getIndexPolicyScript(_)(containerData),
...scriptHelper.addItems(_)(containerData),
},
null,
2,
);
const getCosmosDbScript = containerData => {
const partitionKey = getPartitionKey(containerData);

const getContainerConfig = () => {
const baseConfig = {
indexingPolicy: getIndexPolicyScript(containerData),
...scriptHelper.addItems(containerData),
};
if (partitionKey) {
return {
partitionKey,
...baseConfig,
};
}
return baseConfig;
};
return JSON.stringify(getContainerConfig(), null, 2);
};

const generateContainerScript = (data, logger, cb, app) => {
Expand All @@ -167,16 +174,16 @@ const generateContainerScript = (data, logger, cb, app) => {
if (data.options.origin === 'ui') {
cb(null, [
{
script: getGremlinScript(_, data),
script: getGremlinScript(data),
},
{
script: getCosmosDbScript(_, data.containerData),
script: getCosmosDbScript(data.containerData),
},
]);
} else if (scriptId === 'cosmosdb') {
cb(null, getCosmosDbScript(_, data.containerData));
cb(null, getCosmosDbScript(data.containerData));
} else {
cb(null, getGremlinScript(_, data));
cb(null, getGremlinScript(data));
}
} catch (e) {
logger.log('error', { message: e.message, stack: e.stack }, 'Forward-Engineering Error');
Expand Down
110 changes: 52 additions & 58 deletions forward_engineering/getIndexPolicyScript.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
const _ = require('lodash');

const add = (key, value) => obj => {
if (value === undefined || value === '' || (Array.isArray(value) && value.length === 0)) {
return obj;
Expand Down Expand Up @@ -61,73 +63,65 @@ const filterDeactivated = items => {
});
};

const getIncludedPath =
_ =>
(includedPaths = []) => {
return filterDeactivated(includedPaths)
.map(item => {
return _.flow(add('path', getPath(item.indexIncludedPath)))({});
})
.filter(item => !_.isEmpty(item));
};
const getIncludedPath = (includedPaths = []) => {
return filterDeactivated(includedPaths)
.map(item => {
return _.flow(add('path', getPath(item.indexIncludedPath)))({});
})
.filter(item => !_.isEmpty(item));
};

const getExcludedPath =
_ =>
(excludedPaths = []) => {
return filterDeactivated(excludedPaths)
.map(item => {
return _.flow(add('path', getPath(item.indexExcludedPath)))({});
})
.filter(item => !_.isEmpty(item));
};
const getExcludedPath = (excludedPaths = []) => {
return filterDeactivated(excludedPaths)
.map(item => {
return _.flow(add('path', getPath(item.indexExcludedPath)))({});
})
.filter(item => !_.isEmpty(item));
};

const getCompositeIndexes =
_ =>
(compositeIndexes = []) => {
return filterDeactivated(compositeIndexes)
.map(item => {
if (!Array.isArray(item.compositeFieldPath)) {
return;
}

return _.uniqWith(
item.compositeFieldPath.map(item => {
const path = item.name.split('/');

return {
path: ['', ...path.slice(1).map(prepareName)].join('/'),
order: item.type || 'ascending',
};
}),
(a, b) => a.path === b.path,
).filter(item => !_.isEmpty(item));
})
.filter(item => !_.isEmpty(item));
};
const getCompositeIndexes = (compositeIndexes = []) => {
return filterDeactivated(compositeIndexes)
.map(item => {
if (!Array.isArray(item.compositeFieldPath)) {
return;
}

const getSpatialIndexes =
_ =>
(spatialIndexes = []) => {
return filterDeactivated(spatialIndexes)
.map(item => {
return _.flow(
add('path', getPath(item.indexIncludedPath)),
add('types', (item.dataTypes || []).map(dataType => dataType.spatialType).filter(Boolean)),
)({});
})
.filter(item => !_.isEmpty(item) && item.path);
};
return _.uniqWith(
item.compositeFieldPath.map(item => {
const path = item.name.split('/');

return {
path: ['', ...path.slice(1).map(prepareName)].join('/'),
order: item.type || 'ascending',
};
}),
(a, b) => a.path === b.path,
).filter(item => !_.isEmpty(item));
})
.filter(item => !_.isEmpty(item));
};

const getSpatialIndexes = (spatialIndexes = []) => {
return filterDeactivated(spatialIndexes)
.map(item => {
return _.flow(
add('path', getPath(item.indexIncludedPath)),
add('types', (item.dataTypes || []).map(dataType => dataType.spatialType).filter(Boolean)),
)({});
})
.filter(item => !_.isEmpty(item) && item.path);
};

const getIndexPolicyScript = _ => containerData => {
const getIndexPolicyScript = containerData => {
const indexTab = containerData[1] || {};

return _.flow(
add('automatic', indexTab.indexingAutomatic === 'true'),
add('indexingMode', indexTab.indexingMode),
add('includedPaths', getIncludedPath(_)(indexTab.includedPaths)),
add('excludedPaths', getExcludedPath(_)(indexTab.excludedPaths)),
add('spatialIndexes', getSpatialIndexes(_)(indexTab.spatialIndexes)),
add('compositeIndexes', getCompositeIndexes(_)(indexTab.compositeIndexes)),
add('includedPaths', getIncludedPath(indexTab.includedPaths)),
add('excludedPaths', getExcludedPath(indexTab.excludedPaths)),
add('spatialIndexes', getSpatialIndexes(indexTab.spatialIndexes)),
add('compositeIndexes', getCompositeIndexes(indexTab.compositeIndexes)),
)({});
};

Expand Down
6 changes: 4 additions & 2 deletions forward_engineering/helpers/getPartitionKey.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
const getPartitionKey = _ => containerData => {
return _.get(containerData, '[0].partitionKey[0].name', '').trim().replace(/\/$/, '');
const _ = require('lodash');

const getPartitionKey = containerData => {
return _.get(containerData, '[0].partitionKey[0].name', '').trim().replace(/\/$/, null);
};

module.exports = {
Expand Down
Loading