diff --git a/package.json b/package.json index 7f0a9c1..b10b95b 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,9 @@ } }, "dependencies": { + "@vespaiach/axios-fetch-adapter": "^0.3.1", "@babel/runtime": "^7.10.5", - "axios": "^0.21.1", + "axios": "^0.26.1", "bootstrap": "3.4.1", "d3": "^3.3.8", "icanhaz": "0.10.3", @@ -34,7 +35,7 @@ "spark-md5": "^3.0.1", "sugar": "^1.5.0", "url-join": "^5.0.0", - "webextension-polyfill": "^0.7.0" + "webextension-polyfill": "^0.12.0" }, "devDependencies": { "@babel/core": "^7.10.5", diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index b2da4b8..eabaca3 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -378,5 +378,8 @@ }, "popup_ws_version": { "message": "Web Scraper version: " + }, + "searchbar_placeholder_message_for_sitemaps": { + "message": "Sitemap id..." } } diff --git a/src/_locales/ru/messages.json b/src/_locales/ru/messages.json index cade54f..2670e6a 100644 --- a/src/_locales/ru/messages.json +++ b/src/_locales/ru/messages.json @@ -412,5 +412,8 @@ }, "popup_ws_version": { "message": "Версия Web Scraper: " + }, + "searchbar_placeholder_message_for_sitemaps": { + "message": "Id карты обхода..." } } diff --git a/src/background/background.js b/src/background/background.js index 96918d1..c66d055 100644 --- a/src/background/background.js +++ b/src/background/background.js @@ -190,7 +190,7 @@ browser.runtime.onMessage.addListener(async request => { const backgroundScript = getBackgroundScript('BackgroundScript'); // TODO change to promises const deferredResponse = backgroundScript[request.fn](request.request); - deferredResponse.done(resolve).catch(reject); + deferredResponse.then(resolve).catch(reject); }); } }); diff --git a/src/content_script/content_script.js b/src/content_script/content_script.js index c78d2f0..6efa2f8 100644 --- a/src/content_script/content_script.js +++ b/src/content_script/content_script.js @@ -39,7 +39,7 @@ browser.runtime.onMessage.addListener(request => { console.log('received ContentScript request', request); const deferredResponse = contentScript[request.fn](request.request); - deferredResponse.done(function (response) { + deferredResponse.then(function (response) { resolve(response, null); }); diff --git a/src/devtools/panel.css b/src/devtools/panel.css index 02cf403..19f0d17 100644 --- a/src/devtools/panel.css +++ b/src/devtools/panel.css @@ -182,3 +182,14 @@ select[size] { font-style: normal; font-weight: normal; } +.searchbar { + margin: 5px; + &::placeholder { + color: grey; + font-style: italic; + } +} +mark { + margin: 0; + padding: 0; +} diff --git a/src/devtools/views/ProjectList.html b/src/devtools/views/ProjectList.html index 680e670..e1210ff 100644 --- a/src/devtools/views/ProjectList.html +++ b/src/devtools/views/ProjectList.html @@ -1,4 +1,5 @@
+ diff --git a/src/devtools/views/ProjectListItem.html b/src/devtools/views/ProjectListItem.html index 3df96b6..10b8b51 100644 --- a/src/devtools/views/ProjectListItem.html +++ b/src/devtools/views/ProjectListItem.html @@ -1,4 +1,4 @@ - + diff --git a/src/devtools/views/SitemapList.html b/src/devtools/views/SitemapList.html index b8933f0..b4764b8 100644 --- a/src/devtools/views/SitemapList.html +++ b/src/devtools/views/SitemapList.html @@ -1,4 +1,5 @@
+
{{id}}{{title}}{{title}}
diff --git a/src/libs/jquery.whencallsequentially.js b/src/libs/jquery.whencallsequentially.js index 305f4f2..2709e3b 100644 --- a/src/libs/jquery.whencallsequentially.js +++ b/src/libs/jquery.whencallsequentially.js @@ -1,47 +1,14 @@ -/** - * @author Martins Balodis - * - * An alternative version of $.when which can be used to execute asynchronous - * calls sequentially one after another. - * - * @returns $.Deferred().promise() - */ $.whenCallSequentially = function (functionCalls) { - const deferredResonse = $.Deferred(); - const resultData = new Array(); + let promiseChain = Promise.resolve([]); - // nothing to do - if (functionCalls.length === 0) { - return deferredResonse.resolve(resultData).promise(); - } - - let currentDeferred = functionCalls.shift()(); - // execute synchronous calls synchronously - while (currentDeferred.state() === 'resolved') { - currentDeferred.done(function (data) { - resultData.push(data); - }); - if (functionCalls.length === 0) { - return deferredResonse.resolve(resultData).promise(); - } - currentDeferred = functionCalls.shift()(); - } - - // handle async calls - var interval = setInterval(function () { - // handle mixed sync calls - while (currentDeferred.state() === 'resolved') { - currentDeferred.done(function (data) { + functionCalls.forEach(func => { + promiseChain = promiseChain.then(resultData => { + return func().then(data => { resultData.push(data); + return resultData; }); - if (functionCalls.length === 0) { - clearInterval(interval); - deferredResonse.resolve(resultData); - break; - } - currentDeferred = functionCalls.shift()(); - } - }, 10); + }); + }); - return deferredResonse.promise(); + return promiseChain; }; diff --git a/src/manifest.json b/src/manifest.json index de39622..533f5d3 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -1,11 +1,10 @@ { - "manifest_version": 2, + "manifest_version": 3, "version": "", "name": "__MSG_extension_name__", "short_name": "__MSG_extension_short_name__", "description": "__MSG_extension_description__", "permissions": [ - "", "tabs", "notifications", "storage", @@ -13,12 +12,13 @@ "downloads", "webRequest" ], + "host_permissions": [""], "icons": { "16": "icons/icon16.png", "48": "icons/icon48.png", "128": "icons/icon128.png" }, - "browser_action": { + "action": { "default_icon": { "19": "icons/icon19.png", "38": "icons/icon38.png" @@ -33,15 +33,20 @@ }, "devtools_page": "devtools/devtools.html", "background": { - "scripts": ["background/background.js"] + "service_worker": "background/background.js" }, "web_accessible_resources": [ - "icons/icon16.png", - "icons/icon19.png", - "icons/icon38.png", - "icons/icon48.png", - "icons/icon128.png", - "content_script/AttachedToolbar.html" + { + "resources": [ + "icons/icon16.png", + "icons/icon19.png", + "icons/icon38.png", + "icons/icon48.png", + "icons/icon128.png", + "content_script/AttachedToolbar.html" + ], + "matches": ["*://*/*"] + } ], "content_scripts": [ { diff --git a/src/options/options.js b/src/options/options.js index 2901ec9..2ba3f27 100644 --- a/src/options/options.js +++ b/src/options/options.js @@ -1,7 +1,7 @@ +import * as $ from 'jquery'; import 'bootstrap/dist/css/bootstrap.css'; import 'bootstrap/dist/js/bootstrap'; import * as browser from 'webextension-polyfill'; -import * as $ from 'jquery'; import Config from '../scripts/Config'; import Translator from '../scripts/Translator'; diff --git a/src/scripts/BackgroundScript.js b/src/scripts/BackgroundScript.js index 290eddb..28160c5 100644 --- a/src/scripts/BackgroundScript.js +++ b/src/scripts/BackgroundScript.js @@ -5,7 +5,7 @@ import * as browser from 'webextension-polyfill'; */ const BackgroundScript = { dummy() { - return $.Deferred().resolve('dummy').promise(); + return new Promise.resolve('dummy'); }, /** @@ -42,17 +42,9 @@ const BackgroundScript = { fn: request.fn, request: request.request, }; - const deferredResponse = $.Deferred(); - this.getActiveTabId() - .then(tabId => { - browser.tabs - .sendMessage(tabId, reqToContentScript) - .then(deferredResponse.resolve) - .catch(deferredResponse.reject); - }) - .catch(deferredResponse.reject); - - return deferredResponse; + return this.getActiveTabId().then(tabId => + browser.tabs.sendMessage(tabId, reqToContentScript) + ); }, }; @@ -77,15 +69,7 @@ export default function getBackgroundScript(location) { fn: attr, request, }; - - const deferredResponse = $.Deferred(); - - browser.runtime - .sendMessage(reqToBackgroundScript) - .then(deferredResponse.resolve) - .catch(deferredResponse.reject); - - return deferredResponse; + return browser.runtime.sendMessage(reqToBackgroundScript); }; } else { backgroundScript[attr] = BackgroundScript[attr]; diff --git a/src/scripts/ContentScript.js b/src/scripts/ContentScript.js index 146d128..be0db37 100644 --- a/src/scripts/ContentScript.js +++ b/src/scripts/ContentScript.js @@ -1,3 +1,4 @@ +import * as $ from 'jquery'; import getBackgroundScript from './BackgroundScript'; import ContentSelector from './ContentSelector'; @@ -11,10 +12,8 @@ const ContentScript = { * @returns $.Deferred() */ getHTML(request) { - const deferredHTML = $.Deferred(); const html = $(request.CSSSelector).clone().wrap('

').parent().html(); - deferredHTML.resolve(html); - return deferredHTML.promise(); + return Promise.resolve(html); }, /** @@ -22,17 +21,13 @@ const ContentScript = { * @returns $.Deferred() */ removeCurrentContentSelector() { - const deferredResponse = $.Deferred(); const contentSelector = window.cs; if (contentSelector === undefined) { - deferredResponse.resolve(); - } else { - contentSelector.removeGUI(); - window.cs = undefined; - deferredResponse.resolve(); + return Promise.resolve(); } - - return deferredResponse.promise(); + contentSelector.removeGUI(); + window.cs = undefined; + return Promise.resolve(); }, /** @@ -41,34 +36,32 @@ const ContentScript = { * @param request.allowedElements */ selectSelector(request) { - const deferredResponse = $.Deferred(); - - this.removeCurrentContentSelector().done( - function () { - const contentSelector = new ContentSelector({ - parentCSSSelector: request.parentCSSSelector, - allowedElements: request.allowedElements, - }); - window.cs = contentSelector; - - const deferredCSSSelector = contentSelector.getCSSSelector(); - deferredCSSSelector - .done( - function (response) { - this.removeCurrentContentSelector().done(function () { - deferredResponse.resolve(response); - window.cs = undefined; - }); - }.bind(this) - ) - .fail(function (message) { - deferredResponse.reject(message); - window.cs = undefined; + return new Promise((resolve, reject) => { + this.removeCurrentContentSelector().then( + function () { + const contentSelector = new ContentSelector({ + parentCSSSelector: request.parentCSSSelector, + allowedElements: request.allowedElements, }); - }.bind(this) - ); + window.cs = contentSelector; - return deferredResponse.promise(); + const deferredCSSSelector = contentSelector.getCSSSelector(); + deferredCSSSelector + .then( + function (response) { + this.removeCurrentContentSelector().then(function () { + resolve(response); + window.cs = undefined; + }); + }.bind(this) + ) + .catch(function (message) { + reject(message); + window.cs = undefined; + }); + }.bind(this) + ); + }); }, /** @@ -77,26 +70,26 @@ const ContentScript = { * @param request.elementCSSSelector */ previewSelector(request) { - const deferredResponse = $.Deferred(); - this.removeCurrentContentSelector().done(function () { - const contentSelector = new ContentSelector({ - parentCSSSelector: request.parentCSSSelector, - }); - window.cs = contentSelector; - - const deferredSelectorPreview = contentSelector.previewSelector( - request.elementCSSSelector - ); - deferredSelectorPreview - .done(function () { - deferredResponse.resolve(); - }) - .fail(function (message) { - deferredResponse.reject(message); - window.cs = undefined; + return new Promise((resolve, reject) => { + this.removeCurrentContentSelector().then(function () { + const contentSelector = new ContentSelector({ + parentCSSSelector: request.parentCSSSelector, }); + window.cs = contentSelector; + + const deferredSelectorPreview = contentSelector.previewSelector( + request.elementCSSSelector + ); + deferredSelectorPreview + .then(function () { + resolve(); + }) + .catch(function (message) { + reject(message); + window.cs = undefined; + }); + }); }); - return deferredResponse; }, }; diff --git a/src/scripts/Controller.js b/src/scripts/Controller.js index 424efbb..174e438 100644 --- a/src/scripts/Controller.js +++ b/src/scripts/Controller.js @@ -621,7 +621,8 @@ export default class SitemapController { 'sitemap_either_start_urls_and_pattern' ), }; - } else if (Object.hasOwn(sitemap, 'startUrls')) { + } + if (Object.hasOwn(sitemap, 'startUrls')) { if (!Sitemap.validateStartUrls(sitemap.startUrls)) { return { valid: false, @@ -808,6 +809,10 @@ export default class SitemapController { }); $('#viewport').html($projectListPanel); Translator.translatePage(); + + this.initSearchbar('td.projTitle'); + document.querySelector('input.searchbar').placeholder = + Translator.getTranslationByKey('projectName') + '...'; } getCurrentProjectId() { @@ -853,8 +858,36 @@ export default class SitemapController { $('#viewport').html($sitemapListPanel); Translator.translatePage(); } + this.initSearchbar('td.id'); + document.querySelector('input.searchbar').placeholder = Translator.getTranslationByKey( + 'searchbar_placeholder_message_for_sitemaps' + ); } + initSearchbar(rowSelector) { + document.querySelector('.searchbar').addEventListener('input', event => { + let AllRows = []; + const inputText = event.target.value.toLowerCase(); + AllRows = Array.from(document.querySelectorAll(rowSelector)).map( + td => td.parentElement + ); + AllRows.forEach(row => { + const rowText = row.querySelector(rowSelector).innerText; + if (rowText.toLowerCase().includes(inputText)) { + row.style.display = 'table-row'; + let regex = RegExp(inputText, 'gi'); + if (!inputText) { + regex = /$^/; // will never is valid and returns [] + } + row.querySelector(rowSelector).innerHTML = rowText.replace(regex, match => { + return `${match}`; + }); + } else { + row.style.display = 'none'; + } + }); + }); + } getSitemapFromMetadataForm() { const metadata = {}; const $form = $('#viewport form'); @@ -1466,7 +1499,7 @@ export default class SitemapController { }); }); - let options = { + const options = { id, selector: selectorsSelector, tableHeaderRowSelector, @@ -2012,12 +2045,10 @@ export default class SitemapController { currentStateParentSelectorIds ); - const result = await this.contentScript - .selectSelector({ - parentCSSSelector, - allowedElements: selector.getItemCSSSelector(), - }) - .promise(); + const result = await this.contentScript.selectSelector({ + parentCSSSelector, + allowedElements: selector.getItemCSSSelector(), + }); selector = this.getCurrentlyEditedSelector(); await selector.afterSelect(result.CSSSelector, this, input.attr('id')); @@ -2059,12 +2090,10 @@ export default class SitemapController { currentStateParentSelectorIds ); - const result = await this.contentScript - .selectSelector({ - parentCSSSelector, - allowedElements: 'tr', - }) - .promise(); + const result = await this.contentScript.selectSelector({ + parentCSSSelector, + allowedElements: 'tr', + }); const tableHeaderRowSelector = result.CSSSelector; selector.tableHeaderRowSelector = tableHeaderRowSelector; @@ -2089,12 +2118,10 @@ export default class SitemapController { currentStateParentSelectorIds ); - const result = await this.contentScript - .selectSelector({ - parentCSSSelector, - allowedElements: 'tr', - }) - .promise(); + const result = await this.contentScript.selectSelector({ + parentCSSSelector, + allowedElements: 'tr', + }); // update validation for selector field const input = $(button).closest('.form-group').find('input.selector-value'); @@ -2143,7 +2170,7 @@ export default class SitemapController { elementCSSSelector: selector.selector, }); - deferredSelectorPreview.done(function () { + deferredSelectorPreview.then(function () { $(button).addClass('preview'); }); } else { @@ -2172,7 +2199,7 @@ export default class SitemapController { elementCSSSelector: selector[inputName], }); - deferredSelectorPreview.done(function () { + deferredSelectorPreview.then(function () { $(button).addClass('preview'); }); } else { @@ -2202,7 +2229,7 @@ export default class SitemapController { elementCSSSelector: rowSelector, }); - deferredSelectorPreview.done(function () { + deferredSelectorPreview.then(function () { $(button).addClass('preview'); }); } else { @@ -2229,7 +2256,7 @@ export default class SitemapController { elementCSSSelector: selector.selector, }); - deferredSelectorPreview.done(function () { + deferredSelectorPreview.then(function () { $(button).addClass('preview'); }); } else { diff --git a/src/scripts/DataExtractor.js b/src/scripts/DataExtractor.js index 39a47a7..1592a7c 100644 --- a/src/scripts/DataExtractor.js +++ b/src/scripts/DataExtractor.js @@ -129,7 +129,7 @@ export default class DataExtractor { ); const deferredResponse = $.Deferred(); - $.whenCallSequentially(deferredDataCalls).done(function (responses) { + $.whenCallSequentially(deferredDataCalls).then(function (responses) { let commonData = {}; responses.forEach(function (data) { commonData = Object.merge(commonData, data); @@ -159,7 +159,7 @@ export default class DataExtractor { ); }.bind(this) ) - ).done(function (results) { + ).then(function (results) { d.resolve({ [selector.id]: results.flat() }); }); } else { @@ -226,7 +226,7 @@ export default class DataExtractor { }.bind(this) ); - $.whenCallSequentially(deferredDataCalls).done(function (responses) { + $.whenCallSequentially(deferredDataCalls).then(function (responses) { const resultData = []; responses.forEach(function (childRecordList) { childRecordList.forEach(function (childRecord) { @@ -275,7 +275,7 @@ export default class DataExtractor { ); // merge all data records together - $.whenCallSequentially(dataDeferredCalls).done(function (responses) { + $.whenCallSequentially(dataDeferredCalls).then(function (responses) { const resultData = []; responses.forEach(function (childRecords) { childRecords.forEach(function (childRecord) { @@ -356,7 +356,7 @@ export default class DataExtractor { ); const responseDeferred = $.Deferred(); - $.whenCallSequentially(dataDeferredCalls).done( + $.whenCallSequentially(dataDeferredCalls).then( function (responses) { let results = []; responses.forEach(function (dataResults) { diff --git a/src/scripts/Scraper.js b/src/scripts/Scraper.js index adbba10..4a3cce5 100644 --- a/src/scripts/Scraper.js +++ b/src/scripts/Scraper.js @@ -81,10 +81,9 @@ export default class Scraper { * @param record */ saveFile(record) { - const deferredResponse = $.Deferred(); + //const deferredResponse = $.Deferred(); if (!('_attachments' in record)) { - deferredResponse.resolve(); - return deferredResponse.promise(); + return Promise.resolve(); } const downloads = record._attachments.map(async attachment => { @@ -104,12 +103,10 @@ export default class Scraper { return attachment; } }); - Promise.all(downloads).then(attachments => { record._attachments = attachments; - deferredResponse.resolve(); }); - return deferredResponse.promise(); + return Promise.resolve(); } // @TODO remove recursion and add an iterative way to run these jobs. @@ -161,7 +158,7 @@ export default class Scraper { }.bind(this) ); - $.whenCallSequentially(deferredDatamanipulations).done( + $.whenCallSequentially(deferredDatamanipulations).then( function () { this.resultWriter.writeDocs(scrapedRecords).then(() => { const now = new Date().getTime(); diff --git a/src/scripts/StoreRestApi.js b/src/scripts/StoreRestApi.js index 6f7559f..2b786c9 100644 --- a/src/scripts/StoreRestApi.js +++ b/src/scripts/StoreRestApi.js @@ -2,12 +2,14 @@ import axios from 'axios'; import Sitemap from './Sitemap'; import StorePouchDB from './StorePouchDB'; import urlJoin from 'url-join'; +import fetchAdapter from '@vespaiach/axios-fetch-adapter'; export default class StoreRestApi { constructor(config, baseUrl, sitemapsPath = 'sitemaps/') { this.localDataStore = new StorePouchDB(config); this.axiosInstance = axios.create({ baseURL: baseUrl, + adapter: fetchAdapter, }); this.axiosInstance.defaults.headers.post['Content-Type'] = 'application/json'; this.axiosInstance.defaults.headers.put['Content-Type'] = 'application/json'; @@ -17,10 +19,14 @@ export default class StoreRestApi { setAxiosInterceptors() { this.axiosInstance.interceptors.response.use(response => { - const [contentType] = response.headers['content-type'].split(';'); - if (contentType !== 'application/json') { - const error = new Error(`Incorrect response type`); - return Promise.reject(error); + //TODO return it when change axios to fetch + // const [contentType] = response.headers['content-type'].split(';'); + // if (contentType !== 'application/json') { + // const error = new Error(`Incorrect response type`); + // return Promise.reject(error); + // } + if (response.headers['content-type']) { + console.log(response.headers['content-type']); } return response; }); diff --git a/src/scripts/StoreTalismanApi.js b/src/scripts/StoreTalismanApi.js index c5d21b3..fef3017 100644 --- a/src/scripts/StoreTalismanApi.js +++ b/src/scripts/StoreTalismanApi.js @@ -1,4 +1,5 @@ import axios from 'axios'; +import fetchAdapter from '@vespaiach/axios-fetch-adapter'; import StoreRestApi from './StoreRestApi'; import urlJoin from 'url-join'; import * as browser from 'webextension-polyfill'; @@ -35,6 +36,7 @@ export default class StoreTalismanApi extends StoreRestApi { const response = await axios({ method: 'get', url: urlJoin(this.axiosInstance.defaults.baseURL, 'meta.json'), + adapter: fetchAdapter, }); return response.data.APP_NAME; } @@ -50,6 +52,7 @@ export default class StoreTalismanApi extends StoreRestApi { headers: { 'Content-Type': 'multipart/form-data', }, + adapter: fetchAdapter, }) .catch(er => er); if (loginStatus.isAxiosError || loginStatus.data.access_token === undefined) { @@ -74,7 +77,8 @@ export default class StoreTalismanApi extends StoreRestApi { setAxiosInterceptors() { this.axiosInstance.interceptors.response.use(response => { - if (response.request.responseURL.includes('auth')) { + // response.request.responseURL is not returned with current axios config, mb it's return when we change axios to fetch + if (response.request.responseURL && response.request.responseURL.includes('auth')) { browser.runtime.sendMessage({ authError: true, }); @@ -97,6 +101,7 @@ export default class StoreTalismanApi extends StoreRestApi { const response = await axios({ method: 'get', url: `${tUrl}/oauth/token`, + adapter: fetchAdapter, }); try { if (response.data.preferred_username) { @@ -124,7 +129,9 @@ export default class StoreTalismanApi extends StoreRestApi { sortDirection: 'ascending', }, }; - const projects = await this.axiosInstance.post('/graphql', queryData); + const projects = await this.axiosInstance.post('/graphql', queryData, { + adapter: fetchAdapter, + }); return projects.data.data.paginationProject.listProject; } diff --git a/webpack.config.js b/webpack.config.js index 7ea74bb..d970326 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -30,6 +30,7 @@ const config = { resolve: { extensions: ['.js'], }, + devtool: 'source-map', module: { rules: [ { @@ -76,7 +77,7 @@ const config = { 'window.jQuery': 'jquery', }), new webpack.DefinePlugin({ - global: 'window', + global: 'self', }), new MiniCssExtractPlugin({ filename: '[name].css', @@ -106,8 +107,9 @@ const config = { jsonContent.version = version; if (config.mode === 'development') { - jsonContent.content_security_policy = - "script-src 'self' 'unsafe-eval'; object-src 'self'"; + jsonContent.content_security_policy = { + extension_pages: "script-src 'self'; object-src 'self'", + }; } return JSON.stringify(jsonContent, null, 2);