diff --git a/package.json b/package.json index ba222c9..27b0989 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "web-scraper-chrome-extension", - "version": "0.4.8", + "version": "0.4.9", "description": "Web data extraction tool implemented as chrome extension", "scripts": { "lint": "eslint --ext .js src", diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index e1421cc..9314df9 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -110,6 +110,7 @@ "selector_edit_type_of_html_outer": { "message": "Outer HTML" }, "selector_edit_type_of_html_inner": { "message": "Inner HTML" }, "selector_edit_merge_into_list": { "message": "Merge collected items into list" }, + "selector_edit_dont_flatten": { "message": "Dont flatten results" }, "sitemap_scrape_config_request_interval_label": { "message": "Request interval (ms)" }, "sitemap_scrape_config_request_interval_randomness_label": { "message": "Request interval randomness (ms)" @@ -343,5 +344,12 @@ }, "popup_ws_version": { "message": "Web Scraper version: " - } + }, + + "link_types_for_concept_type": { "message": "Link types for concept type" }, + "concept_types_for_link_type": { "message": "Concept types for link type" }, + "prop_types_for_concept_type": { "message": "Property types for concept type" }, + "prop_types_for_link_type": { "message": "Property types for link type" }, + "all_concept_types": { "message": "All concept types" }, + "incompatible_kb_parent_type": { "message": "There are incompatible parent KB types" } } diff --git a/src/_locales/ru/messages.json b/src/_locales/ru/messages.json index 8d5ea8d..3e1f417 100644 --- a/src/_locales/ru/messages.json +++ b/src/_locales/ru/messages.json @@ -114,6 +114,7 @@ "selector_edit_type_of_html_outer": { "message": "Внешний HTML" }, "selector_edit_type_of_html_inner": { "message": "Внутренний HTML" }, "selector_edit_merge_into_list": { "message": "Объединить собранные элементы в список" }, + "selector_edit_dont_flatten": { "message": "Не приводить данные к плоскому виду" }, "sitemap_scrape_config_requestInterval": { "message": "Интервал между запросами" }, "sitemap_scrape_config_requestIntervalRandomness": { "message": "Случайность между запросами" }, @@ -379,5 +380,14 @@ }, "popup_ws_version": { "message": "Версия Web Scraper: " + }, + + "link_types_for_concept_type": { "message": "Типы связей для типа концепта" }, + "concept_types_for_link_type": { "message": "Типы концептов для типа связи" }, + "prop_types_for_concept_type": { "message": "Типы характеристик для типа концепта" }, + "prop_types_for_link_type": { "message": "Типы характеристик для типа связи" }, + "all_concept_types": { "message": "Все типы концептов" }, + "incompatible_kb_parent_type": { + "message": "Среди селекторов-предков есть несовместимые типы БЗ" } } diff --git a/src/background/background.js b/src/background/background.js index 81d69f0..4147981 100644 --- a/src/background/background.js +++ b/src/background/background.js @@ -140,6 +140,18 @@ browser.runtime.onMessage.addListener(async request => { return store.getSitemapData(Sitemap.sitemapFromObj(request.sitemap)); } + if (request.listAllConceptTypes) { + return store.listAllConceptTypes(); + } + + if (request.getConceptType) { + return store.getConceptType(request.id); + } + + if (request.getLinkType) { + return store.getLinkType(request.id); + } + if (request.scrapeSitemap) { const sitemap = Sitemap.sitemapFromObj(request.sitemap); const queue = new Queue(); diff --git a/src/devtools/views/SelectorEdit.html b/src/devtools/views/SelectorEdit.html index f7757ab..d41a5e7 100644 --- a/src/devtools/views/SelectorEdit.html +++ b/src/devtools/views/SelectorEdit.html @@ -32,6 +32,7 @@ +
@@ -562,6 +563,23 @@ +
+ + +
+
+ +
+
+
+
diff --git a/src/scripts/Controller.js b/src/scripts/Controller.js index 670efa4..93ba448 100644 --- a/src/scripts/Controller.js +++ b/src/scripts/Controller.js @@ -15,6 +15,7 @@ import SelectorList from './SelectorList'; import SelectorTable from './Selector/SelectorTable'; import Model from './Model'; import Translator from './Translator'; +import TalismanKB from './TalismanKB'; export default class SitemapController { constructor(store, templateDir) { @@ -89,6 +90,11 @@ export default class SitemapController { return value; }) .set_sort_objects(true); + + if (store.storageType === 'StoreTalismanApi') { + this.kb = new TalismanKB(store); + } + return this.init(); } @@ -101,10 +107,11 @@ export default class SitemapController { event, selector, (function (selector, event) { - return function () { + return function (...args) { const continueBubbling = controls[selector][event].call( controller, - this + this, + ...args ); if (continueBubbling !== true) { return false; @@ -272,6 +279,7 @@ export default class SitemapController { }, '#edit-selector #selectorId': { 'change:flexdatalist': this.updateCurrentlyEditedSelectorInParentsList, + 'select:flexdatalist': this.selectorIdHintSelected, }, '#selector-tree button[action=add-selector]': { click: this.addSelector, @@ -1193,6 +1201,10 @@ export default class SitemapController { } } + if (this.kb) { + return this.kb.validateParentSelectors(newSelector, sitemap); + } + return true; }.bind(this), }, @@ -1202,9 +1214,9 @@ export default class SitemapController { }); } - editSelector(button) { + async editSelector(button) { const selector = $(button).closest('tr').data('selector'); - this._editSelector(selector); + await this._editSelector(selector); } updateCurrentlyEditedSelectorInParentsList() { @@ -1215,7 +1227,7 @@ export default class SitemapController { $('.currently-edited').val(selector.uuid).text(`${selectorId} - ${selector.uuid}`); } - _editSelector(selector) { + async _editSelector(selector) { const sitemap = this.state.currentSitemap; const selectorIds = sitemap.getPossibleParentSelectorIds(); @@ -1227,19 +1239,6 @@ export default class SitemapController { }); $('#viewport').html($editSelectorForm); - $('#selectorId').flexdatalist({ - init: this.initSelectorValidation(), - textProperty: '{fieldName}', - valueProperty: 'fieldName', - data: [...sitemap.model, { entity: '', field: '', fieldName: selector.id }], - searchIn: ['entity', 'field'], - visibleProperties: ['entity', 'field'], - groupBy: 'entity', - searchContain: true, - noResultsText: '', - minLength: 1, - }); - // mark initially opened selector as currently edited $('#edit-selector #parentSelectors option').each((_, element) => { if ($(element).val() === selector.uuid) { @@ -1267,12 +1266,14 @@ export default class SitemapController { .attr('selected', 'selected'); }); + this.initSelectorValidation(); + this.state.currentSelector = selector; - this.selectorTypeChanged(false); + await this.selectorTypeChanged(false); Translator.translatePage(); } - selectorTypeChanged(changeTrigger) { + async selectorTypeChanged(changeTrigger) { // add this selector to possible parent selector const selector = this.getCurrentlyEditedSelector(); const features = selector.getFeatures(); @@ -1296,6 +1297,48 @@ export default class SitemapController { else { $('#edit-selector #parentSelectors .currently-edited').remove(); } + + await this.initSelectorIdHints(selector); + } + + async initSelectorIdHints(selector) { + const $selectorIdInput = $('#selectorId'); + $selectorIdInput.flexdatalist(); + const idDatalistOptions = { + textProperty: '{fieldName}', + valueProperty: 'fieldName', + searchIn: ['entity', 'field'], + visibleProperties: ['entity', 'field'], + groupBy: 'entity', + searchContain: true, + selectionRequired: false, + noResultsText: '', + minLength: 1, + }; + + const hints = []; + if (this.kb) { + const kbHints = await this.kb.generateIdHints(selector, this.state.currentSitemap); + hints.push(...kbHints); + } else { + hints.push(...this.state.currentSitemap.model); + } + hints.push({ + entity: '', + field: '', + fieldName: selector.id, + }); + + $selectorIdInput.flexdatalist({ + ...idDatalistOptions, + data: hints, + }); + } + + selectorIdHintSelected(selectorIdInput, event, item) { + if (item.kbHint) { + $('#edit-selector [name=dontFlatten]').prop('checked', true); + } } async saveSelector(button) { @@ -1353,6 +1396,7 @@ export default class SitemapController { const delay = $('#edit-selector [name=delay]').val(); const outerHTML = $('#edit-selector [id=outerHTML]').is(':checked'); const mergeIntoList = $('#edit-selector [name=mergeIntoList]').is(':checked'); + const dontFlatten = $('#edit-selector [name=dontFlatten]').is(':checked'); const extractAttribute = $('#edit-selector [name=extractAttribute]').val(); const extractStyle = $('#edit-selector [name=extractStyle]').val(); const value = $('#edit-selector [name=value]').val(); @@ -1376,6 +1420,7 @@ export default class SitemapController { regexgroup: $('#edit-selector [name=regexgroup]').val(), }; const uuid = $('#edit-selector [name=uuid]').val(); + const conceptTypeId = $('#edit-selector [name=conceptTypeId]').val(); $columnHeaders.each(function (i) { const header = $($columnHeaders[i]).val(); @@ -1388,7 +1433,7 @@ export default class SitemapController { }); }); - let options = { + const options = { id, selector: selectorsSelector, tableHeaderRowSelector, @@ -1414,8 +1459,10 @@ export default class SitemapController { textmanipulation, stringReplacement, mergeIntoList, + dontFlatten, outerHTML, uuid, + conceptTypeId, }; return SelectorList.createSelector(options); diff --git a/src/scripts/DataExtractor.js b/src/scripts/DataExtractor.js index 39a47a7..da3380f 100644 --- a/src/scripts/DataExtractor.js +++ b/src/scripts/DataExtractor.js @@ -170,7 +170,11 @@ export default class DataExtractor { newParentElement ); deferredChildCommonData.done(function (data) { - d.resolve(data); + if (selector.dontFlatten) { + d.resolve({ [selector.id]: data }); + } else { + d.resolve(data); + } }); } } else { @@ -214,7 +218,9 @@ export default class DataExtractor { selectorData.forEach( function (element) { - const newCommonData = Object.clone(commonData, true); + const newCommonData = selector.dontFlatten + ? {} + : Object.clone(commonData, true); const childRecordDeferredCall = this.getSelectorTreeData.bind( this, selectors, @@ -231,7 +237,12 @@ export default class DataExtractor { responses.forEach(function (childRecordList) { childRecordList.forEach(function (childRecord) { const rec = {}; - Object.merge(rec, childRecord, true); + if (selector.dontFlatten) { + Object.merge(rec, commonData, true); + Object.merge(rec, { [selector.id]: childRecord }, true); + } else { + Object.merge(rec, childRecord, true); + } resultData.push(rec); }); }); diff --git a/src/scripts/Job.js b/src/scripts/Job.js index 36e6008..f8a0e87 100644 --- a/src/scripts/Job.js +++ b/src/scripts/Job.js @@ -1,5 +1,5 @@ export default class Job { - constructor(url, parentSelector, scraper, parentJob, baseData) { + constructor(url, parentSelector, scraper, parentJob, baseData, baseDataPath) { if (parentJob !== undefined) { this.url = this.combineUrls(parentJob.url, url); } else { @@ -9,6 +9,7 @@ export default class Job { this.scraper = scraper; this.dataItems = []; this.baseData = baseData || {}; + this.baseDataPath = baseDataPath || []; } combineUrls(parentUrl, childUrl) { @@ -69,17 +70,12 @@ export default class Job { this.url, sitemap, this.parentSelector, - function (results) { + results => { // merge data with data from initialization - for (const i in results) { - const result = results[i]; - for (const key in this.baseData) { - if (!(key in result)) { - result[key] = this.baseData[key]; - } - } - this.dataItems.push(result); - } + results.forEach(result => { + const mergedResult = this.mergeWithBaseData(result); + this.dataItems.push(mergedResult); + }); if (sitemap) { // table selector can dynamically add columns (addMissingColumns Feature) @@ -88,11 +84,20 @@ export default class Job { console.log(job); callback(job); - }.bind(this), + }, this ); } + mergeWithBaseData(result) { + const mergedData = structuredClone(this.baseData); + const { _url, _timestamp, ...resultData } = result; + const insertAt = this.baseDataPath.reduce((data, key) => data[key] || {}, mergedData); + Object.assign(mergedData, { _url, _timestamp }); + Object.assign(insertAt, resultData); + return mergedData; + } + getResults() { return this.dataItems; } diff --git a/src/scripts/Scraper.js b/src/scripts/Scraper.js index 395c009..9077fed 100644 --- a/src/scripts/Scraper.js +++ b/src/scripts/Scraper.js @@ -127,38 +127,32 @@ export default class Scraper { const deferredDatamanipulations = []; const records = job.getResults(); - records.forEach( - function (record) { - // var record = JSON.parse(JSON.stringify(rec)); - - deferredDatamanipulations.push(this.saveFile.bind(this, record)); - - // @TODO refactor job exstraction to a seperate method - if (this.recordCanHaveChildJobs(record)) { - // var followSelectorId = record._followSelectorId; - const followURL = record._follow; - const followSelectorId = record._followSelectorId; - delete record._follow; - delete record._followSelectorId; - const newJob = new Job(followURL, followSelectorId, this, job, record); - if (this.queue.canBeAdded(newJob)) { - this.queue.add(newJob); - } - // store already scraped links - else { - console.log('Ignoring next'); - console.log(record); - // scrapedRecords.push(record); - } + records.forEach(record => { + deferredDatamanipulations.push(this.saveFile.bind(this, record)); + + const followEntries = Array.from(this.extractFollowEntries(record)); + + if (!followEntries.length) { + scrapedRecords.push(record); + return; + } + + followEntries.forEach(({ followURL, followSelectorId, recordPath }) => { + const newJob = new Job( + followURL, + followSelectorId, + this, + job, + record, + recordPath + ); + if (this.queue.canBeAdded(newJob)) { + this.queue.add(newJob); } else { - if (record._follow !== undefined) { - delete record._follow; - delete record._followSelectorId; - } - scrapedRecords.push(record); + console.log('Ignoring job', newJob); } - }.bind(this) - ); + }); + }); $.whenCallSequentially(deferredDatamanipulations).done( function () { @@ -186,4 +180,27 @@ export default class Scraper { }.bind(this) ); } + + *extractFollowEntries(record, path = []) { + if (typeof record !== 'object') { + return; + } + + const { _follow, _followSelectorId } = record; + if (_follow !== undefined) { + delete record._follow; + delete record._followSelectorId; + if (this.recordCanHaveChildJobs({ _follow, _followSelectorId })) { + yield { + followURL: _follow, + followSelectorId: _followSelectorId, + recordPath: path, + }; + } + } + + for (const key of Object.keys(record)) { + yield* this.extractFollowEntries(record[key], [...path, key]); + } + } } diff --git a/src/scripts/Selector/SelectorElement.js b/src/scripts/Selector/SelectorElement.js index af00fa7..ca95e1f 100644 --- a/src/scripts/Selector/SelectorElement.js +++ b/src/scripts/Selector/SelectorElement.js @@ -35,6 +35,6 @@ export default class SelectorElement extends Selector { } getFeatures() { - return ['selector', 'multiple', 'delay', 'mergeIntoList']; + return ['selector', 'multiple', 'delay', 'mergeIntoList', 'dontFlatten']; } } diff --git a/src/scripts/Selector/SelectorElementClick.js b/src/scripts/Selector/SelectorElementClick.js index d89b239..0fe05c0 100644 --- a/src/scripts/Selector/SelectorElementClick.js +++ b/src/scripts/Selector/SelectorElementClick.js @@ -144,6 +144,7 @@ export default class SelectorElementClick extends Selector { 'clickElementUniquenessType', 'paginationLimit', 'mergeIntoList', + 'dontFlatten', ]; } } diff --git a/src/scripts/Selector/SelectorElementScroll.js b/src/scripts/Selector/SelectorElementScroll.js index df371ed..2658f06 100644 --- a/src/scripts/Selector/SelectorElementScroll.js +++ b/src/scripts/Selector/SelectorElementScroll.js @@ -101,6 +101,7 @@ export default class SelectorElementScroll extends Selector { 'delay', 'paginationLimit', 'mergeIntoList', + 'dontFlatten', ]; } } diff --git a/src/scripts/SelectorList.js b/src/scripts/SelectorList.js index e02ea41..0ba4e51 100644 --- a/src/scripts/SelectorList.js +++ b/src/scripts/SelectorList.js @@ -290,7 +290,9 @@ export default class SelectorList extends Array { const parentSelectorId = parentSelectorIds[i]; const parentSelector = this.getSelectorByUid(parentSelectorId); if (parentSelector.willReturnElements()) { - CSSSelector = `${parentSelector.selector} ${CSSSelector}`; + if (parentSelector.selector !== '_parent_') { + CSSSelector = `${parentSelector.selector} ${CSSSelector}`; + } } else { break; } diff --git a/src/scripts/StoreTalismanApi.js b/src/scripts/StoreTalismanApi.js index 27a4400..769c6d3 100644 --- a/src/scripts/StoreTalismanApi.js +++ b/src/scripts/StoreTalismanApi.js @@ -1,6 +1,7 @@ import axios from 'axios'; -import StoreRestApi from './StoreRestApi'; import urlJoin from 'url-join'; +import StoreRestApi from './StoreRestApi'; +import 'sugar'; import Sitemap from './Sitemap'; import * as browser from 'webextension-polyfill'; @@ -158,4 +159,71 @@ export default class StoreTalismanApi extends StoreRestApi { sitemapExists ); } + + async listAllConceptTypes() { + return this.axiosInstance + .post('/graphql', { + operationName: 'listConceptTypes', + query: 'query listConceptTypes { listConceptType { id name } }', + }) + .then(response => response.data.data.listConceptType); + } + + async getConceptType(id) { + return this.axiosInstance + .post('/graphql', { + operationName: 'getConceptType', + query: `query getConceptType($id: ID!) { + conceptType(id: $id) { + id + name + listConceptPropertyType { + id + name + } + listConceptLinkType { + id + name + isDirected + conceptFromType { + id + } + conceptToType { + id + } + } + } + }`, + variables: { id }, + }) + .then(response => response.data.data.conceptType); + } + + async getLinkType(id) { + return this.axiosInstance + .post('/graphql', { + operationName: 'getLinkType', + query: `query getLinkType($id: ID!) { + conceptLinkType(id: $id) { + id + name + isDirected + conceptFromType { + id + name + } + conceptToType { + id + name + } + listConceptLinkPropertyType { + id + name + } + } + }`, + variables: { id }, + }) + .then(response => response.data.data.conceptLinkType); + } } diff --git a/src/scripts/TalismanKB.js b/src/scripts/TalismanKB.js new file mode 100644 index 0000000..791d0f0 --- /dev/null +++ b/src/scripts/TalismanKB.js @@ -0,0 +1,279 @@ +import Translator from './Translator'; +import 'sugar'; + +const KB_TYPES = { + CONCEPT: 'c', + LINK: 'l', + CONCEPT_PROPERTY: 'cp', + LINK_PROPERTY: 'lp', +}; + +const LINK_DIRECTION = { + RIGHT: '->', + LEFT: '<-', + NONE: '-', +}; + +/** + * Utilities for integration with Talisman knowledge base + */ +export default class TalismanKB { + constructor(store) { + this.store = store; + } + + makeSelectorId(kbType) { + const { type, id, parentId, direction, childId, name } = kbType; + if (type === KB_TYPES.CONCEPT) { + return `[tlsmn:${type}:${id}] ${name}`; + } + if (type === KB_TYPES.LINK) { + return `[tlsmn:${type}:${id}:${parentId}:${direction}:${childId}] ${name}`; + } + return `[tlsmn:${type}:${id}:${parentId}] ${name}`; + } + + getKBType(selector) { + const match = /^\[tlsmn:(.+?)]/.exec(selector.id); + if (!match) { + return undefined; + } + const [type, id, parentId, direction, childId] = match[1].split(':'); + switch (type) { + case KB_TYPES.CONCEPT: + return { type, id }; + case KB_TYPES.LINK: + return { type, id, parentId, direction, childId }; + case KB_TYPES.CONCEPT_PROPERTY: + case KB_TYPES.LINK_PROPERTY: + return { type, id, parentId }; + default: + return undefined; + } + } + + getParentKBTypes(selector, sitemap) { + const seenSelectors = new Set([sitemap.rootSelector.uuid, selector.uuid]); + const selectorQueue = selector.parentSelectors.filter(uid => !seenSelectors.has(uid)); + const parentKBTypes = []; + while (selectorQueue.length) { + const parentSelector = sitemap.getSelectorByUid(selectorQueue.pop()); + const kbType = this.getKBType(parentSelector); + if (kbType && (kbType.type === KB_TYPES.CONCEPT || kbType.type === KB_TYPES.LINK)) { + parentKBTypes.push(kbType); + } else { + parentSelector.parentSelectors.forEach(uid => { + if (!seenSelectors.has(uid)) { + seenSelectors.add(uid); + selectorQueue.push(uid); + } + }); + } + } + return parentKBTypes; + } + + makeHintField(kbType) { + const field = `#${kbType.id} ${kbType.name}`; + if (kbType.type === KB_TYPES.LINK && kbType.direction !== LINK_DIRECTION.NONE) { + return `${field} (${kbType.direction})`; + } + return field; + } + + makeSelectorIdHint(entity, kbType) { + return { + entity, + field: this.makeHintField(kbType), + fieldName: this.makeSelectorId(kbType), + kbHint: true, + }; + } + + async getPropertyTypeHintsForConceptType(type) { + const conceptType = await this.store.getConceptType(type.id); + const entityPrefix = Translator.getTranslationByKey('prop_types_for_concept_type'); + const entity = `${entityPrefix} ${this.makeHintField({ ...type, ...conceptType })}`; + return conceptType.listConceptPropertyType.map(propertyType => + this.makeSelectorIdHint(entity, { + type: KB_TYPES.CONCEPT_PROPERTY, + id: propertyType.id, + name: propertyType.name, + parentId: conceptType.id, + }) + ); + } + + async getLinkTypeHintsForConceptType(type) { + const conceptType = await this.store.getConceptType(type.id); + const entityPrefix = Translator.getTranslationByKey('link_types_for_concept_type'); + const entity = `${entityPrefix} ${this.makeHintField({ ...type, ...conceptType })}`; + return conceptType.listConceptLinkType.flatMap(linkType => { + const parentId = conceptType.id; + const childId = + parentId === linkType.conceptFromType.id + ? linkType.conceptToType.id + : linkType.conceptFromType.id; + const directions = []; + if (!linkType.isDirected) { + directions.push(LINK_DIRECTION.NONE); + } else { + if (parentId === linkType.conceptFromType.id) { + directions.push(LINK_DIRECTION.RIGHT); + } + if (parentId === linkType.conceptToType.id) { + directions.push(LINK_DIRECTION.LEFT); + } + } + return directions.map(direction => + this.makeSelectorIdHint(entity, { + type: KB_TYPES.LINK, + id: linkType.id, + name: linkType.name, + parentId, + childId, + direction, + }) + ); + }); + } + + async getChildConceptTypeHintForLinkType(type) { + const linkType = await this.store.getLinkType(type.id); + const entityPrefix = Translator.getTranslationByKey('concept_types_for_link_type'); + const entity = `${entityPrefix} ${this.makeHintField({ ...type, ...linkType })}`; + const conceptType = + type.parentId === linkType.conceptFromType.id + ? linkType.conceptToType + : linkType.conceptFromType; + return this.makeSelectorIdHint(entity, { + type: KB_TYPES.CONCEPT, + id: conceptType.id, + name: conceptType.name, + }); + } + + async getPropertyTypeHintsForLinkType(type) { + const linkType = await this.store.getLinkType(type.id); + const entityPrefix = Translator.getTranslationByKey('prop_types_for_link_type'); + const entity = `${entityPrefix} ${this.makeHintField({ ...type, ...linkType })}`; + return linkType.listConceptLinkPropertyType.map(propertyType => + this.makeSelectorIdHint(entity, { + type: KB_TYPES.LINK_PROPERTY, + id: propertyType.id, + name: propertyType.name, + parentId: linkType.id, + }) + ); + } + + async getAllConceptTypeHints() { + const entity = Translator.getTranslationByKey('all_concept_types'); + const conceptTypes = await this.store.listAllConceptTypes(); + return conceptTypes.map(({ id, name }) => + this.makeSelectorIdHint(entity, { type: KB_TYPES.CONCEPT, id, name }) + ); + } + + async generateIdHints(selector, sitemap) { + const parentKBTypes = this.getParentKBTypes(selector, sitemap); + + const parentConceptTypes = parentKBTypes + .filter(({ type }) => type === KB_TYPES.CONCEPT) + .unique('id'); + const parentLinkTypes = parentKBTypes + .filter(({ type }) => type === KB_TYPES.LINK) + .unique('id'); + + const hintsPromises = []; + + if (selector.willReturnElements()) { + // may be concept type or link type + + hintsPromises.push( + ...parentConceptTypes.map(conceptType => + this.getLinkTypeHintsForConceptType(conceptType) + ) + ); + + hintsPromises.push( + ...parentLinkTypes.map(linkType => + this.getChildConceptTypeHintForLinkType(linkType) + ) + ); + + // show hints with all concept types + if (!parentLinkTypes.length) { + hintsPromises.push(this.getAllConceptTypeHints()); + } + } else if (!selector.canCreateNewJobs()) { + // concept or link property + + hintsPromises.push( + ...parentConceptTypes.map(conceptType => + this.getPropertyTypeHintsForConceptType(conceptType) + ) + ); + + hintsPromises.push( + ...parentLinkTypes.map(linkType => this.getPropertyTypeHintsForLinkType(linkType)) + ); + } + + if (!hintsPromises.length) { + return []; + } + + // TODO improve error handling + const hints = await Promise.all(hintsPromises.map(promise => promise.catch(console.error))); + return hints.compact().flatten(); + } + + isCompatibleWithParentType(kbType, parentKbType) { + if (kbType.type === KB_TYPES.CONCEPT && parentKbType.type === KB_TYPES.CONCEPT) { + return true; + } + if (kbType.type === KB_TYPES.CONCEPT && parentKbType.type === KB_TYPES.LINK) { + return kbType.id === parentKbType.childId; + } + if ( + (kbType.type === KB_TYPES.LINK && parentKbType.type === KB_TYPES.CONCEPT) || + (kbType.type === KB_TYPES.CONCEPT_PROPERTY && parentKbType.type === KB_TYPES.CONCEPT) || + (kbType.type === KB_TYPES.LINK_PROPERTY && parentKbType.type === KB_TYPES.LINK) + ) { + return kbType.parentId === parentKbType.id; + } + return false; + } + + isCompatibleWithParentSelector(kbType, parentSelector, sitemap) { + const parentKbType = this.getKBType(parentSelector); + if (parentKbType) { + return this.isCompatibleWithParentType(kbType, parentKbType); + } + return this.getParentKBTypes(parentSelector, sitemap).every(ancestorKbType => + this.isCompatibleWithParentType(kbType, ancestorKbType) + ); + } + + validateParentSelectors(selector, sitemap) { + const kbType = this.getKBType(selector); + if (!kbType) { + return true; + } + for (const parentUid of selector.parentSelectors) { + if (parentUid === sitemap.rootSelector.uuid) { + continue; + } + const parentSelector = sitemap.getSelectorByUid(parentUid); + if (!this.isCompatibleWithParentSelector(kbType, parentSelector, sitemap)) { + // TODO improve validation messages + return { + valid: false, + message: Translator.getTranslationByKey('incompatible_kb_parent_type'), + }; + } + } + return true; + } +} diff --git a/src/scripts/TalismanStoreDevtools.js b/src/scripts/TalismanStoreDevtools.js index a30d5f3..a1d63a4 100644 --- a/src/scripts/TalismanStoreDevtools.js +++ b/src/scripts/TalismanStoreDevtools.js @@ -92,4 +92,16 @@ export default class TalismanStoreDevtools extends StoreDevtools { }; return await browser.runtime.sendMessage(request); } + + async listAllConceptTypes() { + return browser.runtime.sendMessage({ listAllConceptTypes: true }); + } + + async getConceptType(id) { + return browser.runtime.sendMessage({ getConceptType: true, id }); + } + + async getLinkType(id) { + return browser.runtime.sendMessage({ getLinkType: true, id }); + } }