Talisman KB integration

package.json

@@ -21,6 +21,7 @@
 		"@babel/runtime": "^7.10.5",
 		"axios": "^0.21.1",
 		"bootstrap": "3.4.1",
+		"bootstrap-select": "^1.13.18",
 		"d3": "^3.3.8",
 		"icanhaz": "0.10.3",
 		"jquery": "^3.5.0",

src/_locales/en/messages.json

@@ -52,6 +52,7 @@
 	"selectorList_add_selector_button": { "message": "Add new selector" },
 	"selector_edit_id_field": { "message": "ID" },
 	"selector_edit_type_placeholder": { "message": "Type" },
+	"selector_edit_kb_type": { "message": "Attach KB Type" },
 	"selector_edit_selector_css_field": { "message": "Selector" },
 	"selector_edit_data_preview_button": { "message": "Data preview" },
 	"selector_edit_scroll_selector": { "message": "Scroll to selector" },
@@ -110,6 +111,7 @@
 	"selector_edit_type_of_html_outer": { "message": "Outer HTML" },
 	"selector_edit_type_of_html_inner": { "message": "Inner HTML" },
 	"selector_edit_merge_into_list": { "message": "Merge collected items into list" },
+	"selector_edit_dont_flatten": { "message": "Dont flatten results" },
 	"sitemap_scrape_config_request_interval_label": { "message": "Request interval (ms)" },
 	"sitemap_scrape_config_request_interval_randomness_label": {
 		"message": "Request interval randomness (ms)"
@@ -343,5 +345,13 @@
 	},
 	"popup_ws_version": {
 		"message": "Web Scraper version: "
-	}
+	},
+
+	"link_types_for_concept_type": { "message": "Link types for concept type" },
+	"concept_types_for_link_type": { "message": "Concept types for link type" },
+	"prop_types_for_concept_type": { "message": "Property types for concept type" },
+	"prop_types_for_link_type": { "message": "Property types for link type" },
+	"all_concept_types": { "message": "All concept types" },
+	"incompatible_kb_parent_type": { "message": "There are incompatible parent KB types" },
+	"kb_type_none_option": { "message": "None" }
 }

src/_locales/ru/messages.json

@@ -114,6 +114,7 @@
 	"selector_edit_type_of_html_outer": { "message": "Внешний HTML" },
 	"selector_edit_type_of_html_inner": { "message": "Внутренний HTML" },
 	"selector_edit_merge_into_list": { "message": "Объединить собранные элементы в список" },
+	"selector_edit_dont_flatten": { "message": "Не приводить данные к плоскому виду" },
 
 	"sitemap_scrape_config_requestInterval": { "message": "Интервал между запросами" },
 	"sitemap_scrape_config_requestIntervalRandomness": { "message": "Случайность между запросами" },
@@ -379,5 +380,15 @@
 	},
 	"popup_ws_version": {
 		"message": "Версия Web Scraper: "
-	}
+	},
+
+	"link_types_for_concept_type": { "message": "Типы связей для типа концепта" },
+	"concept_types_for_link_type": { "message": "Типы концептов для типа связи" },
+	"prop_types_for_concept_type": { "message": "Типы характеристик для типа концепта" },
+	"prop_types_for_link_type": { "message": "Типы характеристик для типа связи" },
+	"all_concept_types": { "message": "Все типы концептов" },
+	"incompatible_kb_parent_type": {
+		"message": "Среди селекторов-предков есть несовместимые типы БЗ"
+	},
+	"kb_type_none_option": { "message": "Нет" }
 }

src/background/background.js

@@ -140,6 +140,18 @@ browser.runtime.onMessage.addListener(async request => {
 		return store.getSitemapData(Sitemap.sitemapFromObj(request.sitemap));
 	}
 
+	if (request.listAllConceptTypes) {
+		return store.listAllConceptTypes();
+	}
+
+	if (request.getConceptType) {
+		return store.getConceptType(request.id);
+	}
+
+	if (request.getLinkType) {
+		return store.getLinkType(request.id);
+	}
+
 	if (request.scrapeSitemap) {
 		const sitemap = Sitemap.sitemapFromObj(request.sitemap);
 		const queue = new Queue();

src/devtools/panel.css

@@ -182,3 +182,7 @@ select[size] {
 	font-style: normal;
 	font-weight: normal;
 }
+
+.bootstrap-select .dropdown-menu {
+	font-size: 12px;
+}

src/devtools/views/SelectorEdit.html

@@ -1,4 +1,8 @@
 <form class="form-horizontal" role="form" id="edit-selector">
+	<div class="form-group">
+		<input type="hidden" name="uuid" value="{{selector.uuid}}" disabled />
+	</div>
+
 	<div class="form-group">
 		<label
 			for="selectorId"
@@ -32,8 +36,17 @@
 			</select>
 		</div>
 	</div>
-	<div class="form-group">
-		<input type="hidden" name="uuid" value="{{selector.uuid}}" disabled />
+
+	<div class="form-group feature feature-kbType">
+		<label
+			for="kbType"
+			class="col-sm-2 control-label"
+			data-i18n="selector_edit_kb_type"
+		></label>
+
+		<div class="input-group col-sm-8">
+			<select class="form-control" id="kbType" name="kbType"></select>
+		</div>
 	</div>
 
 	<div class="form-group feature feature-selector">
@@ -562,6 +575,23 @@
 		</div>
 	</div>
 
+	<div class="form-group feature feature-dontFlatten">
+		<label
+			for="dontFlatten"
+			class="col-sm-2 control-label"
+			data-i18n="selector_edit_dont_flatten"
+		></label>
+
+		<div class="col-sm-8">
+			<div class="checkbox">
+				<label>
+					<input type="checkbox" id="dontFlatten" name="dontFlatten"
+					{{#selector.dontFlatten}}checked="checked" {{/selector.dontFlatten}}/>
+				</label>
+			</div>
+		</div>
+	</div>
+
 	<div class="form-group feature feature-delay">
 		<label for="delay" class="col-sm-2 control-label" data-i18n="selector_edit_delay"></label>
 

src/scripts/App.js

@@ -1,11 +1,12 @@
 import 'bootstrap/dist/css/bootstrap.css';
 import 'jquery-flexdatalist/jquery.flexdatalist.css';
 import '../libs/jquery.bootstrapvalidator/bootstrapValidator.css';
+import 'bootstrap-select/dist/css/bootstrap-select.css';
 import '../devtools/panel.css';
 import 'bootstrap/dist/js/bootstrap';
+import * as browser from 'webextension-polyfill';
 import StoreDevtools from './StoreDevtools';
 import SitemapController from './Controller';
-import * as browser from 'webextension-polyfill';
 import TalismanStoreDevtools from './TalismanStoreDevtools';
 
 $(async function () {

src/scripts/Controller.js

@@ -8,13 +8,15 @@ import 'jquery-highlight/jquery.highlight';
 import 'jquery-searcher/dist/jquery.searcher.min';
 import 'jquery-flexdatalist/jquery.flexdatalist';
 import '../libs/jquery.bootstrapvalidator/bootstrapValidator';
+import 'bootstrap-select/dist/js/bootstrap-select';
 import getContentScript from './ContentScript';
 import Sitemap from './Sitemap';
 import SelectorGraphv2 from './SelectorGraphv2';
 import SelectorList from './SelectorList';
 import SelectorTable from './Selector/SelectorTable';
 import Model from './Model';
 import Translator from './Translator';
+import TalismanKB from './TalismanKB';
 
 export default class SitemapController {
 	constructor(store, templateDir) {
@@ -89,6 +91,11 @@ export default class SitemapController {
 				return value;
 			})
 			.set_sort_objects(true);
+
+		if (store.storageType === 'StoreTalismanApi') {
+			this.kb = new TalismanKB(store);
+		}
+
 		return this.init();
 	}
 
@@ -101,10 +108,11 @@ export default class SitemapController {
 					event,
 					selector,
 					(function (selector, event) {
-						return function () {
+						return function (...args) {
 							const continueBubbling = controls[selector][event].call(
 								controller,
-								this
+								this,
+								...args
 							);
 							if (continueBubbling !== true) {
 								return false;
@@ -273,6 +281,9 @@ export default class SitemapController {
 			'#edit-selector #selectorId': {
 				'change:flexdatalist': this.updateCurrentlyEditedSelectorInParentsList,
 			},
+			'#edit-selector #kbType': {
+				change: this.selectorKbTypeChanged,
+			},
 			'#selector-tree button[action=add-selector]': {
 				click: this.addSelector,
 			},
@@ -1193,6 +1204,10 @@ export default class SitemapController {
 									}
 								}
 
+								if (this.kb) {
+									return this.kb.validateParentSelectors(newSelector, sitemap);
+								}
+
 								return true;
 							}.bind(this),
 						},
@@ -1202,9 +1217,9 @@ export default class SitemapController {
 		});
 	}
 
-	editSelector(button) {
+	async editSelector(button) {
 		const selector = $(button).closest('tr').data('selector');
-		this._editSelector(selector);
+		await this._editSelector(selector);
 	}
 
 	updateCurrentlyEditedSelectorInParentsList() {
@@ -1215,7 +1230,7 @@ export default class SitemapController {
 		$('.currently-edited').val(selector.uuid).text(`${selectorId} - ${selector.uuid}`);
 	}
 
-	_editSelector(selector) {
+	async _editSelector(selector) {
 		const sitemap = this.state.currentSitemap;
 		const selectorIds = sitemap.getPossibleParentSelectorIds();
 
@@ -1267,12 +1282,28 @@ export default class SitemapController {
 				.attr('selected', 'selected');
 		});
 
+		this.initSelectorValidation();
+
 		this.state.currentSelector = selector;
-		this.selectorTypeChanged(false);
+
+		if (this.kb) {
+			$('#kbType').selectpicker({
+				liveSearch: true,
+				// style: '',
+				// styleBase: 'form-control',
+			});
+		}
+
+		await this.selectorTypeChanged(false);
 		Translator.translatePage();
+
+		if (selector.kbType) {
+			$('#kbType').selectpicker('val', selector.kbType);
+		}
+		$('select:not(#kbType):not([multiple])').selectpicker(); // enforce consistent styles for selects
 	}
 
-	selectorTypeChanged(changeTrigger) {
+	async selectorTypeChanged(changeTrigger) {
 		// add this selector to possible parent selector
 		const selector = this.getCurrentlyEditedSelector();
 		const features = selector.getFeatures();
@@ -1296,6 +1327,49 @@ export default class SitemapController {
 		else {
 			$('#edit-selector #parentSelectors .currently-edited').remove();
 		}
+
+		if (this.kb) {
+			await this.initKbTypeOptions(selector);
+		}
+	}
+
+	async initKbTypeOptions(selector) {
+		const $kbType = $('#kbType');
+		$kbType.empty();
+		$kbType.append(
+			`<option value=''>${Translator.getTranslationByKey('kb_type_none_option')}</option>`
+		);
+
+		const hints = await this.kb.generateIdHints(selector, this.state.currentSitemap);
+		console.log(hints);
+		if (!hints.length) {
+			return;
+		}
+		$('#edit-selector .feature-kbType').show();
+
+		const groupNames = hints.map(hint => hint.entity).unique();
+		const groups = Object.fromEntries(groupNames.map(name => [name, []]));
+		for (const hint of hints) {
+			groups[hint.entity].push({ value: hint.fieldName, label: hint.field });
+		}
+
+		for (const groupName of groupNames) {
+			const $group = $(`<optgroup label='${groupName}'></optgroup>`);
+			for (const hint of groups[groupName]) {
+				$group.append(`<option value='${hint.value}'>${hint.label}</option>`);
+			}
+			$group.appendTo($kbType);
+		}
+
+		if (selector.kbType) {
+			$kbType.selectpicker('val', selector.kbType);
+		}
+
+		$kbType.selectpicker('refresh');
+	}
+
+	selectorKbTypeChanged(selectorIdInput, event, item) {
+		$('#edit-selector [name=dontFlatten]').prop('checked', true);
 	}
 
 	async saveSelector(button) {
@@ -1353,6 +1427,7 @@ export default class SitemapController {
 		const delay = $('#edit-selector [name=delay]').val();
 		const outerHTML = $('#edit-selector [id=outerHTML]').is(':checked');
 		const mergeIntoList = $('#edit-selector [name=mergeIntoList]').is(':checked');
+		const dontFlatten = $('#edit-selector [name=dontFlatten]').is(':checked');
 		const extractAttribute = $('#edit-selector [name=extractAttribute]').val();
 		const extractStyle = $('#edit-selector [name=extractStyle]').val();
 		const value = $('#edit-selector [name=value]').val();
@@ -1376,6 +1451,7 @@ export default class SitemapController {
 			regexgroup: $('#edit-selector [name=regexgroup]').val(),
 		};
 		const uuid = $('#edit-selector [name=uuid]').val();
+		const kbType = $('#edit-selector [name=kbType]').val();
 
 		$columnHeaders.each(function (i) {
 			const header = $($columnHeaders[i]).val();
@@ -1388,7 +1464,7 @@ export default class SitemapController {
 			});
 		});
 
-		let options = {
+		const options = {
 			id,
 			selector: selectorsSelector,
 			tableHeaderRowSelector,
@@ -1414,8 +1490,10 @@ export default class SitemapController {
 			textmanipulation,
 			stringReplacement,
 			mergeIntoList,
+			dontFlatten,
 			outerHTML,
 			uuid,
+			kbType,
 		};
 
 		return SelectorList.createSelector(options);

src/scripts/DataExtractor.js

@@ -170,7 +170,11 @@ export default class DataExtractor {
 							newParentElement
 						);
 						deferredChildCommonData.done(function (data) {
-							d.resolve(data);
+							if (selector.dontFlatten) {
+								d.resolve({ [selector.id]: data });
+							} else {
+								d.resolve(data);
+							}
 						});
 					}
 				} else {
@@ -214,7 +218,9 @@ export default class DataExtractor {
 
 				selectorData.forEach(
 					function (element) {
-						const newCommonData = Object.clone(commonData, true);
+						const newCommonData = selector.dontFlatten
+							? {}
+							: Object.clone(commonData, true);
 						const childRecordDeferredCall = this.getSelectorTreeData.bind(
 							this,
 							selectors,
@@ -231,7 +237,12 @@ export default class DataExtractor {
 					responses.forEach(function (childRecordList) {
 						childRecordList.forEach(function (childRecord) {
 							const rec = {};
-							Object.merge(rec, childRecord, true);
+							if (selector.dontFlatten) {
+								Object.merge(rec, commonData, true);
+								Object.merge(rec, { [selector.id]: childRecord }, true);
+							} else {
+								Object.merge(rec, childRecord, true);
+							}
 							resultData.push(rec);
 						});
 					});