From b74961497530f4e0ce4d579ad595565dad21e147 Mon Sep 17 00:00:00 2001 From: Gerald Baulig Date: Wed, 24 Jan 2024 13:31:18 +0100 Subject: [PATCH] fix(transformer): fix demo-shop catalog transform.js --- .github/workflows/test.yaml | 17 +- datasets/demo-shop/README.md | 24 +- .../demo-shop/data/seed-data/addresses.yaml | 8 +- .../demo-shop/data/seed-data/customers.yaml | 14 +- .../data/seed-data/organizations.yaml | 4 +- .../demo-shop/generator/catalog/package.json | 1 - .../demo-shop/generator/catalog/transform.js | 217 +++++----------- datasets/demo-shop/jobs/job_catalog.json | 11 +- datasets/demo-shop/jobs/job_identity.json | 3 +- datasets/demo-shop/jobs/job_master.json | 12 +- datasets/system/.gitignore | 4 - datasets/system/README.md | 5 + .../data/seed-data/contact_point_types.yaml | 4 +- datasets/system/data/seed-data/taxes.yaml | 234 ++++++++++-------- datasets/system/jobs/job_extra.json | 5 +- datasets/system/jobs/job_identity.json | 7 +- datasets/system/jobs/job_master.json | 15 +- import.js | 25 +- package.json | 7 +- 19 files changed, 258 insertions(+), 359 deletions(-) delete mode 100644 datasets/system/.gitignore diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 227d660..d991f4a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,17 +22,8 @@ jobs: - name: Install Dependencies run: npm ci - - name: Import system master data - run: | - API_KEY=$(docker logs restorecommerce_facade_srv 2>/dev/null | grep API | awk '{print $7}') - node import.js import -t $API_KEY -d system -j master + - name: Extract Bootstrap API-KEY + run: npm run env:token - - name: Import system identity data - run: | - API_KEY=$(docker logs restorecommerce_facade_srv 2>/dev/null | grep API | awk '{print $7}') - node import.js import -t $API_KEY -d system -j identity - - - name: Import extra data - run: | - API_KEY=$(docker logs restorecommerce_facade_srv 2>/dev/null | grep API | awk '{print $7}') - node import.js import -t $API_KEY -d system -j extra + - name: Import all + run: npm run import diff --git a/datasets/demo-shop/README.md b/datasets/demo-shop/README.md index 1a1d4bc..03ef92c 100644 --- a/datasets/demo-shop/README.md +++ b/datasets/demo-shop/README.md @@ -24,11 +24,11 @@ to import the data locally or into the production environment. ### Current supported jobs are -- master (imports resources `organizations`, `addresses`, `contact_points`, `shops`, `customers`) +- master (imports `organizations`, `addresses`, `contact_points`, `shops`, `customers`) - identity (imports `users`) -- catalog (imports resources `price_group`, `manufacturer`, `product_category`, `product_prototype`, `product`) +- catalog (imports `price_group`, `manufacturer`, `product_category`, `product_prototype`, `product`) -> NOTE: Resources must be imported in a specific order! +> NOTE: Resources should be imported in a specific order! > Master > Identity > Catalog. > > For the case when importing resources returns "Access denied", one way to fix @@ -41,15 +41,15 @@ to import the data locally or into the production environment. 2. Either export the API-KEY, paste it to an `.env` file or paste as command argument. -2. Generate catalog datasets using [`transform.js script`](./generator/catalog/transform.js): +3. Generate catalog datasets using [`transform.js script`](./generator/catalog/transform.js): - `node ./transform.js` -3. Import datasets using [`import.js script`](./import.js): +4. Import datasets using [`import.js script`](./import.js): - `node ./import.js import -t -d demo-shop -j ` -4. Examples: +5. Examples: - `node ./import.js import -t -d demo-shop -j master` - `node ./import.js import -t -d demo-shop -j identity` @@ -60,10 +60,16 @@ to import the data locally or into the production environment. All flags are optional, and they can be listed by typing `node ./import.js -h`. The API key can be obtained from the [`facade-srv`](https://github.com/restorecommerce/facade-srv/blob/master/cfg/config.json#L21) configuration. The API key is generated during system startup from the `facade-srv` (Check the log message of facade-srv `Bootstrap API Key is`). +In case `facade-srv` is a local container use the following command to extract the `Bootstrap API-KEY`: + +```sh +npm run env:token +``` By default, the GraphQL importer uses the configuration file `config.json` to read data regarding endpoints for retrieving the API key and executing mutations/ queries. + ## Object Importer To import the files, following settings needs to be configured in [config.json](cfg/config.json): @@ -75,10 +81,10 @@ Prerequisite: object importer should be build using `npm run build` command befo ```sh # Run import in production-mode: -npm run import-objects -- --apiKey= -- --NODE_ENV=local +npm run import:demoshop:objects -- --apiKey= -- --NODE_ENV=local # Run import in production-mode: -npm run import-objects -- --apiKey= -- --NODE_ENV=production +npm run import:demoshop:objects -- --apiKey= -- --NODE_ENV=production # or: -npm run import-objects -- --apiKey= # default is development +npm run import:demoshop:objects -- --apiKey= # default is development ``` \ No newline at end of file diff --git a/datasets/demo-shop/data/seed-data/addresses.yaml b/datasets/demo-shop/data/seed-data/addresses.yaml index df2e131..77f0bc1 100644 --- a/datasets/demo-shop/data/seed-data/addresses.yaml +++ b/datasets/demo-shop/data/seed-data/addresses.yaml @@ -5,7 +5,7 @@ street: Ossietzkystraße buildingNumber: '4' region: Stuttgart - countryId: restorecommecre-demo-country-germany + countryId: germany altitude: 0 addressAddition: field1: "" @@ -28,7 +28,7 @@ street: Erste Teststraße buildingNumber: '1' region: Testregion - countryId: restorecommecre-demo-country-germany + countryId: germany altitude: 0 addressAddition: field1: "" @@ -51,7 +51,7 @@ street: Zweite Teststraße buildingNumber: '1' region: Beispielregion - countryId: restorecommecre-demo-country-germany + countryId: germany altitude: 0 addressAddition: field1: "" @@ -74,7 +74,7 @@ street: Zweite Teststraße buildingNumber: '1' region: Beispielregion - countryId: restorecommecre-demo-country-germany + countryId: germany altitude: 0 addressAddition: field1: "" diff --git a/datasets/demo-shop/data/seed-data/customers.yaml b/datasets/demo-shop/data/seed-data/customers.yaml index 808e77d..31008e3 100644 --- a/datasets/demo-shop/data/seed-data/customers.yaml +++ b/datasets/demo-shop/data/seed-data/customers.yaml @@ -1,7 +1,7 @@ --- id: restorecommecre-demo-customer-000 commercial: - organization_id: restorecommecre-demo-customer-000-organization + organizationId: restorecommecre-demo-customer-000-organization meta: modifiedBy: "" owners: @@ -12,8 +12,8 @@ value: restorecommecre-demo-customer-000-organization --- id: restorecommecre-demo-customer-001 - public_sector: - organization_id: restorecommecre-demo-customer-organization-001 + publicSector: + organizationId: restorecommecre-demo-customer-organization-001 meta: modifiedBy: "" owners: @@ -25,8 +25,8 @@ --- id: restorecommecre-demo-customer-002 private: - user_id: restorecommecre-demo-customer-002-user - contact_point_ids: [ + userId: restorecommecre-demo-customer-002-user + contactPointIds: [ restorecommecre-demo-customer-002-contact-point ] meta: @@ -40,8 +40,8 @@ --- id: restorecommecre-demo-customer-unauthenticated private: - user_id: unauthenticated_user - contact_point_ids: [] + userId: unauthenticated_user + contactPointIds: [] meta: modifiedBy: "" owners: diff --git a/datasets/demo-shop/data/seed-data/organizations.yaml b/datasets/demo-shop/data/seed-data/organizations.yaml index c2b9270..d4c2987 100644 --- a/datasets/demo-shop/data/seed-data/organizations.yaml +++ b/datasets/demo-shop/data/seed-data/organizations.yaml @@ -24,9 +24,7 @@ id: restorecommecre-demo-shop-000-organization name: Restorecommerce Demo Shop 000 Organization contactPointIds: - - restorecommecre-demo-shop-000-contact-point-billing - - restorecommecre-demo-shop-000-contact-point-shipping - - restorecommecre-demo-shop-000-contact-point-legal + - restorecommecre-demo-shop-000-contact-point website: https://about.restorecommerce.io/ email: shop000@restorecommerce.io vatId: DE000000000 diff --git a/datasets/demo-shop/generator/catalog/package.json b/datasets/demo-shop/generator/catalog/package.json index 85652d7..2e651b5 100644 --- a/datasets/demo-shop/generator/catalog/package.json +++ b/datasets/demo-shop/generator/catalog/package.json @@ -11,7 +11,6 @@ "dependencies": { "csv-parser": "^2.3.1", "js-yaml": "^3.14.0", - "lodash": "^4.17.21", "object-hash": "^2.0.0", "uuid": "^3.3.3" }, diff --git a/datasets/demo-shop/generator/catalog/transform.js b/datasets/demo-shop/generator/catalog/transform.js index 06ae7c5..bf6f75b 100644 --- a/datasets/demo-shop/generator/catalog/transform.js +++ b/datasets/demo-shop/generator/catalog/transform.js @@ -3,68 +3,45 @@ const fs = require('fs'); const hash = require('object-hash'); const uuid = require('uuid'); const yaml = require('js-yaml'); -const _ = require('lodash'); -const priceGroups = { - 0: { - name: 'PG1', - description: 'Dummy price group 1', - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] +const FILE_SLICE = 500; +const meta = { + modifiedBy: '', + owners: [ + { + id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', + value: 'urn:restorecommerce:acs:model:organization.Organization', + attributes: [ + { + id: 'urn:restorecommerce:acs:names:ownerInstance', + value: 'restorecommecre-demo-shop-000-organization' } ] } + ] +}; + +const priceGroups = [ + { + id: 'PG1', + name: 'PG1', + description: 'Dummy price group 1', + meta }, - 1: { + { + id: 'PG2', name: 'PG2', description: 'Dummy price group 2', - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] - } - ] - } + meta }, - 2: { + { + id: 'PG3', name: 'PG3', description: 'Dummy price group 3', - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] - } - ] - } + meta } -}; // no data available +]; // no data available const prodCategories = {}; const prodPrototypes = {}; @@ -74,23 +51,23 @@ const products = {}; const resources = [ { dataset: priceGroups, - filename: 'price_groups.yaml' + filename: 'price_groups' }, { dataset: manufacturers, - filename: 'manufacturers.yaml' + filename: 'manufacturers' }, { dataset: prodCategories, - filename: 'product_categories.yaml' + filename: 'product_categories' }, { dataset: prodPrototypes, - filename: 'product_prototypes.yaml' + filename: 'product_prototypes' }, { dataset: products, - filename: 'products.yaml' + filename: 'products' } ]; @@ -114,23 +91,10 @@ function parseInputLine(csvLine) { if (!manufacturers[brandHash]) { manufacturers[brandHash] = { + id: brandHash, name: brandEntry, description: 'Dummy description for manufacturer ' + brandEntry, - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] - } - ] - } + meta }; } @@ -171,27 +135,13 @@ function parseInputLine(csvLine) { const categoryLevelHash = hash(categoryTree[index]); if (!prodCategories[categoryLevelHash]) { - const priceGroupIdStr = String(Math.floor(Math.random() * 3)); prodCategories[categoryLevelHash] = { + id: categoryLevelHash, name: categoryTree[index], description: 'Dummy description for category ' + categoryTree[index], image: categoryImgData, - priceGroupId: priceGroupIdStr.toString(), - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] - } - ] - } + priceGroupId: priceGroups[Math.floor(Math.random() * priceGroups.length)]?.id, + meta }; if (lastCategory != null) { prodCategories[categoryLevelHash]['parent'] = { parentId: hash(lastCategory) }; @@ -209,25 +159,12 @@ function parseInputLine(csvLine) { if (!prodPrototypes[prototypeCatHash]) { prodPrototypes[prototypeCatHash] = { + id: prototypeCatHash, name: prototypeCat, description: 'Dummy description for prototype ' + prototypeCat, categoryId: hash(categoryTree[categoryTree.length - 3]), version: 'test', - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] - } - ] - } + meta }; } } @@ -239,8 +176,8 @@ function parseInputLine(csvLine) { // need to create product before adding variant products[productHash] = { + id: productHash, product: { - // id: productHash, name: productEntry, description: 'Dummy description for product ' + productEntry, manufacturerId: brandHash, @@ -248,25 +185,11 @@ function parseInputLine(csvLine) { physical: { variants: [] }, - taxIds: [makeUUID()], + taxIds: ['germany-standard-rate'], gtin: makeUUID() }, - meta: { - modifiedBy: '', - owners: [ - { - id: 'urn:restorecommerce:acs:names:ownerIndicatoryEntity', - value: 'urn:restorecommerce:acs:model:organization.Organization', - attributes: [ - { - id: 'urn:restorecommerce:acs:names:ownerInstance', - value: 'restorecommerce-demo-shop' - } - ] - } - ] - }, - active: Math.random() >= 0.2 + active: Math.random() >= 0.2, + meta }; if (categoryTree.length > 2) { @@ -294,21 +217,6 @@ function parseInputLine(csvLine) { return { id, value, unitCode }; }); - // // values must be of string type, so we replace all numbers with a string - // // to avoid GQL type error. - // for (let i in variantAttributes) { - // let vi = variantAttributes[i]; - // for (let i in vi) { - // let viValues = vi['values']; - // for (let i in viValues) { - // let value = viValues[i]; - // if (isNaN(value) === false) { - // viValues[i] = value + 'm'; - // } - // } - // } - // } - products[productHash].product.physical.variants.push({ id: csvLine['uniq_id'], name: categoryTree[categoryTree.length - 1], @@ -326,28 +234,20 @@ function parseInputLine(csvLine) { } } -function writeYAML(list_meta) { - const outputDir = `${__dirname}/../../data/generated/catalog/`; - const dataset = list_meta.dataset; - const filename = list_meta.filename; - let item_list = []; - for (let datasetIndex in dataset) { - let newObj = { - // add placeholder to replace later with separator between all documents - separator: 'xxx', - id: datasetIndex - }; - let item = dataset[datasetIndex]; - _.merge(newObj, item); - item_list.push(newObj); +function dumpYAMLs(prefix, docs) { + docs = Object.values(docs); + for (let i = 0; i < docs.length / FILE_SLICE; ++i) { + const filename = `${prefix}.${i.toString().padStart(3, '0')}.yaml`; + console.log(filename); + fs.writeFileSync( + filename, + '---\n' + + docs + .slice(i * FILE_SLICE, (i + 1) * FILE_SLICE) + .map((d) => yaml.safeDump(d)) + .join('---\n') + ); } - fs.mkdirSync(outputDir, { recursive: true }); - let filePath = outputDir + filename; - let stream = yaml.safeDump(item_list); - - // replace placeholder with '---' required by yaml-document-stream - stream = stream.replace(/- separator: xxx/g, '---'); - fs.writeFileSync(filePath, stream); } function transform() { @@ -355,8 +255,11 @@ function transform() { .pipe(csv()) .on('data', parseInputLine) .on('end', () => { + const outputDir = `${__dirname}/../../data/generated/catalog/`; + fs.mkdirSync(outputDir, { recursive: true }); for (let resource of resources) { - writeYAML(resource); + const dataset = resource.dataset; + dumpYAMLs(outputDir + resource.filename, dataset); } }); } diff --git a/datasets/demo-shop/jobs/job_catalog.json b/datasets/demo-shop/jobs/job_catalog.json index 47d5231..0a72d03 100644 --- a/datasets/demo-shop/jobs/job_catalog.json +++ b/datasets/demo-shop/jobs/job_catalog.json @@ -1,46 +1,37 @@ { "options": { - "base": "./", + "base": "./datasets/demo-shop/data/generated/catalog/", "concurrency": 1 }, "tasks": [ { "name": "price_groups", "operation": "sync", - "src": "./datasets/demo-shop/data/generated/catalog/", "filter": "price_groups*.yaml", "mutation": "mutation{ catalog { price_group { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "manufacturers", "operation": "sync", - "src": "./datasets/demo-shop/data/generated/catalog/", "filter": "manufacturers*.yaml", - "batchSize": 500, "mutation": "mutation{ catalog { manufacturer { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "product_categories", "operation": "sync", - "src": "./datasets/demo-shop/data/generated/catalog/", "filter": "product_categories*.yaml", - "batchSize": 500, "mutation": "mutation{ catalog { product_category { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "product_prototypes", "operation": "sync", - "src": "./datasets/demo-shop/data/generated/catalog/", "filter": "product_prototypes*.yaml", - "batchSize": 500, "mutation": "mutation{ catalog { product_prototype { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "products", "operation": "sync", - "src": "./datasets/demo-shop/data/generated/catalog/", "filter": "products*.yaml", - "batchSize": 500, "mutation": "mutation{ catalog { product { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" } ] diff --git a/datasets/demo-shop/jobs/job_identity.json b/datasets/demo-shop/jobs/job_identity.json index cb23169..3ca6330 100644 --- a/datasets/demo-shop/jobs/job_identity.json +++ b/datasets/demo-shop/jobs/job_identity.json @@ -1,13 +1,12 @@ { "options": { - "base": "./", + "base": "./datasets/demo-shop/data/seed-data/", "concurrency": 1 }, "tasks": [ { "name": "users", "operation": "sync", - "src": "./datasets/demo-shop/data/seed-data/", "filter": "users*.yaml", "mutation": "mutation mutateUsers($var: [IIoRestorecommerceUserUser!]!){ identity { user { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" } diff --git a/datasets/demo-shop/jobs/job_master.json b/datasets/demo-shop/jobs/job_master.json index 62ba14f..30a64fb 100644 --- a/datasets/demo-shop/jobs/job_master.json +++ b/datasets/demo-shop/jobs/job_master.json @@ -1,34 +1,36 @@ { "options": { - "base": "./", + "base": "./datasets/demo-shop/data/seed-data/", "concurrency": 1 }, "tasks": [ { "name": "addresses", "operation": "sync", - "src": "./datasets/demo-shop/data/seed-data/", "filter": "addresses.yaml", "mutation": "mutation{ master_data { address { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "contact_points", "operation": "sync", - "src": "./datasets/demo-shop/data/seed-data/", "filter": "contact_points.yaml", "mutation": "mutation{ master_data { contact_point { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "organizations", "operation": "sync", - "src": "./datasets/demo-shop/data/seed-data/", "filter": "organizations.yaml", "mutation": "mutation mutateOrganizations($var: [IIoRestorecommerceOrganizationOrganization!]!){ master_data { organization { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, + { + "name": "customers", + "operation": "sync", + "filter": "customers.yaml", + "mutation": "mutation{ master_data { customer { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" + }, { "name": "shops", "operation": "sync", - "src": "./datasets/demo-shop/data/seed-data/", "filter": "shops.yaml", "mutation": "mutation{ master_data { shop { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" } diff --git a/datasets/system/.gitignore b/datasets/system/.gitignore deleted file mode 100644 index 9f7027c..0000000 --- a/datasets/system/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -#data/generated/ -objects/*.js -objects/*.js.map -objects/*.d.ts \ No newline at end of file diff --git a/datasets/system/README.md b/datasets/system/README.md index 49eab29..1fde743 100644 --- a/datasets/system/README.md +++ b/datasets/system/README.md @@ -59,6 +59,11 @@ to import the data locally or into the production environment. All flags are optional, and they can be listed by typing `node ./import.js -h`. The API key can be obtained from the [`facade-srv`](https://github.com/restorecommerce/facade-srv/blob/master/cfg/config.json#L21) configuration. The API key is generated during system startup from the `facade-srv` (Check the log message of facade-srv `Bootstrap API Key is`). +In case `facade-srv` is a local container use the following command to extract the `Bootstrap API-KEY`: + +```sh +npm run env:token +``` By default, the GraphQL importer uses the configuration file `config.json` to read data regarding endpoints for retrieving the API key and executing mutations/ queries. diff --git a/datasets/system/data/seed-data/contact_point_types.yaml b/datasets/system/data/seed-data/contact_point_types.yaml index c012820..82ef9d4 100644 --- a/datasets/system/data/seed-data/contact_point_types.yaml +++ b/datasets/system/data/seed-data/contact_point_types.yaml @@ -21,8 +21,8 @@ - id: urn:restorecommerce:acs:names:ownerInstance value: system --- - id: legal-address - type: legal-address + id: legal + type: legal meta: modifiedBy: "" owners: diff --git a/datasets/system/data/seed-data/taxes.yaml b/datasets/system/data/seed-data/taxes.yaml index 453a19d..fbada20 100644 --- a/datasets/system/data/seed-data/taxes.yaml +++ b/datasets/system/data/seed-data/taxes.yaml @@ -1,156 +1,182 @@ --- id: germany-reduced-rate rate: 0.07 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Reduced rate - country_id: restorecommecre-demo-country-germany - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + countryId: restorecommecre-demo-country-germany + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: germany-standard-rate rate: 0.19 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Standard rate - country_id: restorecommecre-demo-country-germany - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + countryId: restorecommecre-demo-country-germany + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: switzerland-heavily-reduced-rate rate: 0.025 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Heavily reduced rate - country_id: restorecommecre-demo-country-switzerland - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + countryId: restorecommecre-demo-country-switzerland + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: switzerland-reduced-rate rate: 0.038 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Reduced rate - country_id: restorecommecre-demo-country-switzerland - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + countryId: restorecommecre-demo-country-switzerland + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: switzerland-standard-rate rate: 0.08 - country_id: restorecommecre-demo-country-switzerland - type_id: type-value-added-tax + countryId: restorecommecre-demo-country-switzerland + typeId: type-value-added-tax variant: Standard rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: belgium-heavily-reduced-rate - country_id: restorecommecre-demo-country-belgium + countryId: restorecommecre-demo-country-belgium rate: 0.06 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Heavily reduced rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: belgium-reduced-rate - country_id: restorecommecre-demo-country-belgium + countryId: restorecommecre-demo-country-belgium rate: 0.12 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Reduced rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: belgium-standard-rate - country_id: restorecommecre-demo-country-belgium + countryId: restorecommecre-demo-country-belgium rate: 0.21 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Standard rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: belgium-parking-rate - country_id: restorecommecre-demo-country-belgium + countryId: restorecommecre-demo-country-belgium rate: 0.12 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Parking rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: france-heavily-reduced-rate - country_id: restorecommecre-demo-country-france + countryId: restorecommecre-demo-country-france rate: 0.021 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: heavily reduced rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: france-reduced-rate - country_id: restorecommecre-demo-country-france + countryId: restorecommecre-demo-country-france rate: 0.055 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Reduced rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: france-lightly-reduced-rate - country_id: restorecommecre-demo-country-france + countryId: restorecommecre-demo-country-france rate: 0.1 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Lightly reduced rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system --- id: france-standard-rate - country_id: restorecommecre-demo-country-france + countryId: restorecommecre-demo-country-france rate: 0.2 - type_id: type-value-added-tax + typeId: type-value-added-tax variant: Standard rate - owners: - - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity - value: urn:restorecommerce:acs:model:organization.Organization - attributes: - - id: urn:restorecommerce:acs:names:ownerInstance - value: system + meta: + modifiedBy: system + owners: + - id: urn:restorecommerce:acs:names:ownerIndicatoryEntity + value: urn:restorecommerce:acs:model:organization.Organization + attributes: + - id: urn:restorecommerce:acs:names:ownerInstance + value: system diff --git a/datasets/system/jobs/job_extra.json b/datasets/system/jobs/job_extra.json index f243af8..8af64d7 100644 --- a/datasets/system/jobs/job_extra.json +++ b/datasets/system/jobs/job_extra.json @@ -1,14 +1,13 @@ { "options": { - "base": "./", + "base": "./datasets/system/data/generated/", "concurrency": 1 }, "tasks": [ { "name": "unit_codes", "operation": "sync", - "src": "./datasets/system/data/generated/", - "filter": "unit_codes.yaml", + "filter": "unit_codes*.yaml", "batchSize": 100, "mutation": "mutation mutateCodes($var: [IIoRestorecommerceUnitCodeUnitCode!]!){ master_data { unit_code { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" } diff --git a/datasets/system/jobs/job_identity.json b/datasets/system/jobs/job_identity.json index ffccab1..472424e 100644 --- a/datasets/system/jobs/job_identity.json +++ b/datasets/system/jobs/job_identity.json @@ -1,41 +1,36 @@ { "options": { - "base": "./", + "base": "./datasets/system/data/seed-data/", "concurrency": 1 }, "tasks": [ { "name": "rules", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "rules.yaml", "mutation": "mutation mutateRules($var: [IIoRestorecommerceRuleRule!]!){ access_control { rule { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "policies", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "policies.yaml", "mutation": "mutation mutatePolicies($var: [IIoRestorecommercePolicyPolicy!]!){ access_control { policy { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "policy_sets", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "policy_sets.yaml", "mutation": "mutation mutatePolicySets($var: [IIoRestorecommercePolicySetPolicySet!]!){ access_control { policy_set { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "roles", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "roles.yaml", "mutation": "mutation mutateRoles($var: [IIoRestorecommerceRoleRole!]!){ identity { role { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "users", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "users.yaml", "mutation": "mutation mutateUsers($var: [IIoRestorecommerceUserUser!]!){ identity { user { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" } diff --git a/datasets/system/jobs/job_master.json b/datasets/system/jobs/job_master.json index 13ef69f..c00c564 100644 --- a/datasets/system/jobs/job_master.json +++ b/datasets/system/jobs/job_master.json @@ -1,55 +1,54 @@ { "options": { - "base": "./", + "base": "./datasets/system/data/seed-data/", "concurrency": 1 }, "tasks": [ { "name": "countries", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "countries.yaml", "mutation": "mutation{ master_data { country { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "timezones", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "timezones.yaml", "mutation": "mutation{ master_data { timezone { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "locales", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "locales.yaml", "mutation": "mutation{ master_data { locale { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "tax_types", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "tax_types.yaml", "mutation": "mutation{ master_data { tax_type { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, + { + "name": "taxes", + "operation": "sync", + "filter": "taxes.yaml", + "mutation": "mutation{ master_data { tax { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" + }, { "name": "commands", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "commands.yaml", "mutation": "mutation mutateCommands($var: [IIoRestorecommerceCommandCommand!]!){ master_data { command { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message }} } } }}" }, { "name": "contact_point_types", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "contact_point_types.yaml", "mutation": "mutation{ master_data { contact_point_type { Mutate( input: { items: ${resource_list}, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" }, { "name": "organizations", "operation": "sync", - "src": "./datasets/system/data/seed-data/", "filter": "organizations.yaml", "mutation": "mutation mutateOrganizations($var: [IIoRestorecommerceOrganizationOrganization!]!){ master_data { organization { Mutate( input: { items: $var, totalCount: 0, mode: UPSERT } ) { details{items {status {id, code, message}}, operationStatus { code message } } } } }}" } diff --git a/import.js b/import.js index 3d2042d..05e95d6 100644 --- a/import.js +++ b/import.js @@ -11,7 +11,7 @@ const CONFIG_NAME = process.env.CONFIG_NAME ?? '.config.json'; const CONFIG = JSON.parse(fs.readFileSync(CONFIG_NAME).toString()); async function commandDataImport(cmd) { - const dataset = cmd.dataset ?? exitWithError('error: please choose data set'); + const dataset = cmd.dataset ?? exitWithError('error: please select data set'); const accessToken = cmd.token ?? process.env.ACCESS_TOKEN ?? exitWithError('error: please provide an access token'); @@ -29,22 +29,11 @@ async function commandDataImport(cmd) { } }); - const config = { - ...CONFIG - }; - - /* - const protocol = cmd.protocol ?? config.protocol ?? 'http' - const host = cmd.host ?? config.host; - const port = cmd.port ?? config.port; - config.entry = `${protocol}://${host}:${port}${config.endpoint}`; - */ - if (accessToken) { - config.headers = Object.assign({}, config.headers, { 'Authorization': `Bearer ${accessToken}` }); + CONFIG.headers = Object.assign(CONFIG.headers ?? {}, { 'Authorization': `Bearer ${accessToken}` }); } - const gqlProcessor = new GraphQLProcessor(config); + const gqlProcessor = new GraphQLProcessor(CONFIG); /* eslint no-restricted-syntax: ["error", "FunctionExpression", "WithStatement", "BinaryExpression[operator='in']"] */ @@ -54,19 +43,19 @@ async function commandDataImport(cmd) { const jobProcessor = new JobProcessor(job); const jobResult = await jobProcessor.start(null, null, !!cmd.verbose, !!cmd.ignore); jobResult.on('progress', (task) => { - console.log('Progress :', task.basename); + console.log('Progress:', task.basename); }); jobResult.on('done', () => { - console.log('Resources imported successfully'); + console.log('Import successfully'); }); jobResult.on('error', (err) => { - console.error('Error :', err, JSON.stringify(err ?? '', null, 2)); + console.error('Error:', err, JSON.stringify(err ?? '', null, 2)); }); } } function commandListJobs(cmd) { - const dataset = cmd.dataset ?? exitWithError('error: please choose data set'); + const dataset = cmd.dataset ?? exitWithError('Error: please select data set'); const files = fs.readdirSync(path.join(CONFIG['data_directory'], dataset, CONFIG['job_directory'])); const prefix = CONFIG['job_prefix']; files.forEach(file => { diff --git a/package.json b/package.json index f55c9ca..5932815 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "data" ], "dependencies": { + "@restorecommerce/rc-grpc-clients": "^5.1.15", "@restorecommerce/gql-bot": "^1.0.1", "commander": "^9.4.1", "dotenv": "^16.0.3", @@ -23,7 +24,6 @@ }, "devDependencies": { "@restorecommerce/eslint-config-restorecommerce": "^0.1.5", - "@restorecommerce/rc-grpc-clients": "^3.0.0", "@types/node": "^18.11.7", "babel-eslint": "^10.0.3", "eslint": "^8.26.0", @@ -35,7 +35,7 @@ }, "scripts": { "build": "tsc -d -P tsconfig.objects.json", - "env:token": "echo $(docker logs restorecommerce_facade_srv 2>/dev/null | grep API | awk '{print \"\rACCESS_TOKEN=\"$7}') > .env && cat .env", + "env:token": "echo $(docker logs --tail 10000 restorecommerce_facade_srv 2> /dev/null | grep 'Bootstrap API Key is:' | awk '{print \"\rACCESS_TOKEN=\"$7}') > .env && cat .env", "import": "npm-run-all import:system import:demoshop", "import:system": "npm-run-all import:system:master import:system:identity import:system:extra", "import:system:master": "node import.js import -d system -j master", @@ -46,7 +46,8 @@ "import:demoshop:identity": "node import.js import -d demo-shop -j identity", "import:demoshop:catalog": "node import.js import -d demo-shop -j catalog", "import:demoshop:objects": "node ./dataset/demo-shop/objects/object_importer.js", - "transform:demoshop:unitcodes": "node ./dataset/demo-shop/generator/unit_codes/transform.js" + "transform:system:unitcodes": "node ./datasets/system/generator/unit_codes/transform.js", + "transform:demoshop:catalog": "node ./datasets/demo-shop/generator/catalog/transform.js" }, "engines": { "node": ">= 18.0.0"