From 7a694375dee872575910922f968bf31898330b51 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 20 May 2021 14:33:02 +1000 Subject: [PATCH 01/77] sequence search : enable call out from container --- backend/scripts/blastServer.py | 35 +++++++++++ backend/scripts/blastn_cont.bash | 26 ++++++++ backend/scripts/blastn_request.bash | 87 ++++++++++++++++++++++++++ backend/scripts/dnaSequenceSearch.bash | 30 +++++++-- 4 files changed, 174 insertions(+), 4 deletions(-) create mode 100755 backend/scripts/blastServer.py create mode 100755 backend/scripts/blastn_cont.bash create mode 100755 backend/scripts/blastn_request.bash diff --git a/backend/scripts/blastServer.py b/backend/scripts/blastServer.py new file mode 100755 index 000000000..fec18ab10 --- /dev/null +++ b/backend/scripts/blastServer.py @@ -0,0 +1,35 @@ +from flask import Flask +from flask_executor import Executor +from flask_shell2http import Shell2HTTP + + +import sys + + +# python's inbuilt logging module +import logging +# get the flask_shell2http logger +logger = logging.getLogger("flask_shell2http") +# create new handler +handler = logging.StreamHandler(sys.stdout) +logger.addHandler(handler) +# log messages of severity DEBUG or lower to the console +logger.setLevel(logging.DEBUG) # this is really important! + + + +# Flask application instance +app = Flask(__name__) + +executor = Executor(app) +executor.init_app(app) +shell2http = Shell2HTTP(app=app, executor=executor, base_url_prefix="/commands/") +shell2http.init_app(app, executor) + +def my_callback_fn(context, future): + # optional user-defined callback function + print(context, future.result()) + +# callback_fn=my_callback_fn, +shell2http.register_command(endpoint="blastn", command_name="/home/ec2-user/scripts/blastn_cont.bash", callback_fn=my_callback_fn, decorators=[]) + diff --git a/backend/scripts/blastn_cont.bash b/backend/scripts/blastn_cont.bash new file mode 100755 index 000000000..75a4fe6ca --- /dev/null +++ b/backend/scripts/blastn_cont.bash @@ -0,0 +1,26 @@ +#!/bin/bash + +B=/mnt/data_blast/blast/GENOME_REFERENCES/190509_RGT_Planet_pseudomolecules_V1 +cd $B || exit $? + +echo $* >> ~/log/blast/blastn_cont +# fileName is e.g. /tmp/tmpo4kfn__8/cbf064c0.query.fasta +fileName=$1 +dbName=$2 +# currently a problem with passing this arg, so it is hard-wired. +if [ -z "$dbName" ] +then + dbName=190509_RGT_Planet_pseudomolecules_V1.fasta +fi + +# blastServer.query.fasta +queryFile=queries/$(basename $fileName) +cp -p "$fileName" "$queryFile" +# cat > + +docker run --rm -v \ + $B:/blast/blastdb \ + ncbi/blast blastn \ + -query /blast/blastdb/"$queryFile" \ + -db /blast/blastdb/$dbName \ + -outfmt '6 std qlen slen' diff --git a/backend/scripts/blastn_request.bash b/backend/scripts/blastn_request.bash new file mode 100755 index 000000000..7ad7e2627 --- /dev/null +++ b/backend/scripts/blastn_request.bash @@ -0,0 +1,87 @@ +#!/bin/bash + +# @param fileName fasta file in $B/queries/ containing the search query +# @param dbName currently the blast db name, e.g. 190509_RGT_Planet_pseudomolecules_V1.fasta +# may change to be the parent / reference name. + +B=/mnt/data_blast/blast/GENOME_REFERENCES/190509_RGT_Planet_pseudomolecules_V1 +if [ -d ~/log ] +then + logDir= ~/log +elif [ -d /log ] || mkdir /log +then + logDir=/log +else + logDir= . +fi +logBlastDir=$logDir/blast +[ -d $logBlastDir ] || mkdir $logBlastDir || exit $? +logFile=$logBlastDir/blastn_request + +set -x +# args : -query "$fileName" -db "$dbName" -outfmt '6 std qlen slen' +fileName="$2" +dbName="$4" +# currently hard-wired in blastn_cont.bash +# outfmt="$6" + +# (pwd; ls -F) >>$logFile +# fileName : dnaSequence is in /, which is the node server cwd. +cd / +# dev testing with : "$B"/queries/"$fileName" +md5=$(md5sum "$fileName" | cut -c-8) + +# $B/queries/ in dev testing +queryDir=$logBlastDir/queries +[ -d "$queryDir" ] || mkdir "$queryDir" || exit +queryFile=$md5.query.fasta +cp -p "$fileName" "$queryDir"/$md5.query.fasta + +# Flask server is on port 4000 +# dev testing : localhost +hostIp=$(ip route show | awk '/default/ {print $3}') +blastnUrl=http://$hostIp:4000/commands/blastn + +# -X POST -H 'Content-Type: application/json' --data-binary +# -d +# refn e.g. Eshaan7_Flask-Shell2HTTP.zip:Flask-Shell2HTTP-master/examples/multiple_files.py, +# and https://flask.palletsprojects.com/en/1.1.x/api/#flask.Request.files +# , \"'"$dbName"'\" +cd "$queryDir" +result_url=$(curl -s -F "$queryFile"=@"$queryFile" -F request_json='{"args": ["'@"$queryFile"'"]}' $blastnUrl | tee -a $logFile | jq ".error // .result_url") +case $result_url in + *' already exists'*) + # 2nd line is "null" + # $blastnUrl contains / etc. + result_url="$blastnUrl?key=$(echo $result_url | sed -n "s/.*future_key \(.*\) already exists.*/\1/p")" + ;; + *) + # Result has surrounding "", so remove these + eval result_url=$result_url + ;; + null) + # error return : result_url=null + exit 1 +esac + +# $B/results/ in dev testing +resultDir=$logBlastDir/results +[ -d "$resultDir" ] || mkdir "$resultDir" || exit +resultFile="$resultDir/$fileName" +for i in 1 2 +do + # Without --raw-output, the .report string is wrapped with "" and has \t + # First result may be : {"key":"60a3ec3c","status":"running"}. length of "running\n" is 8 + if curl -s $result_url | tee "$resultFile".whole | jq --raw-output ".report // .status" > "$resultFile" && [ \! -s "$resultFile" ] || [ "$(ls -ld "$resultFile" | awk ' {print $5;}')" -eq 8 ] ; + then + sleep 5 + else + sed '/^running$/d' "$resultFile" + exit $? + fi +done + +# output +# 127.0.0.1 - - [18/May/2021 06:26:25] "POST /commands/blastn HTTP/1.1" 202 - +# {"key":"ff15c90f","result_url":"http://localhost:4000/commands/blastn?key=ff15c90f","status":"running"} +# {} {'key': 'ff15c90f', 'report': 'BobWhite_c10015_641\tchr2H\t100.000\t50\t0\t0\t1\t50\t139120279\t139120328\t7.70e-18\t93.5\t50\t672273650\n', 'error': '', 'returncode': 0, 'start_time': 1621319185.12147, 'end_time': 1621319208.0651033, 'process_time': 22.943633317947388} diff --git a/backend/scripts/dnaSequenceSearch.bash b/backend/scripts/dnaSequenceSearch.bash index c79d0216c..399b1bb1f 100755 --- a/backend/scripts/dnaSequenceSearch.bash +++ b/backend/scripts/dnaSequenceSearch.bash @@ -1,10 +1,15 @@ #!/bin/bash serverDir=$PWD +# $inContainer is true (0) if running in a container. +[ "$PWD" = / ]; inContainer=$? case $PWD in /) + # container configuration resourcesDir=/app/scripts toolsDev=$resourcesDir + blastn=$resourcesDir/blastn_request.bash + datasetIdDir=$resourcesDir/blast/datasetId ;; *backend) resourcesDir=../resources @@ -15,6 +20,10 @@ case $PWD in esac # Default value of toolsDev, if not set above. unused_var=${toolsDev=$resourcesDir/tools/dev} +unused_var=${blastn=blastn} +unused_var=${datasetIdDir=/mnt/data_blast/blast/datasetId} + + sp=$toolsDev/snps2Dataset.pl logFile=dnaSequenceSearch.log @@ -94,9 +103,22 @@ datasetId=$parent #echo ">BobWhite_c10015_641 # AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" | -fileName=/home/ec2-user/pretzel/"$fileName" +if [ $inContainer -ne 0 ] +then + fileName=/home/ec2-user/pretzel/"$fileName" +fi -datasetIdDir=/mnt/data_blast/blast/datasetId +function datasetId2dbName() +{ + if [ $inContainer -eq 0 ] + then + dbName=$(cat "$datasetId".dbName) + else + dbName="$datasetId".dir/$(cat "$datasetId".dbName) + fi + echo "$dbName" + cd $serverDir +} # Enable this to use dev_blastResult() for dev / loopback test, when blast is not installed. if false @@ -113,10 +135,10 @@ else if ! cd "$datasetIdDir" then echo 1>&3 'Error:' "Blast Database is not configured" - elif ! dbName="$datasetId".dir/$(cat "$datasetId".dbName) + elif ! dbName=$(datasetId2dbName "$datasetId") then echo 1>&3 'Error:' "Blast datasetId is not configured", "$datasetId" - elif ! time blastn -query "$fileName" -db "$dbName" -outfmt '6 std qlen slen' | \ + elif ! time $blastn -query "$fileName" -db "$dbName" -outfmt '6 std qlen slen' | \ ( [ "$addDataset" = true ] && convertSearchResults2Json || cat) | \ ( [ -n "$resultRows" ] && head -n $resultRows || cat) then From 732ea23b51d5e26ab36cbeb59195a55dd617ffd6 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 25 May 2021 10:51:51 +1000 Subject: [PATCH 02/77] sequence-search : catenate outputs from blast child process This handles subsequent chunks arriving after a delay when called via web api : blastn_request.bash -> blastServer.py -> blastn_cont.bash --- backend/common/models/feature.js | 2 +- backend/common/utilities/child-process.js | 26 ++++++++++++++++++++--- backend/scripts/dnaSequenceSearch.bash | 4 ++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/backend/common/models/feature.js b/backend/common/models/feature.js index c4162468e..878b02e32 100644 --- a/backend/common/models/feature.js +++ b/backend/common/models/feature.js @@ -106,7 +106,7 @@ module.exports = function(Feature) { if (true) { let child = childProcess( 'dnaSequenceSearch.bash', - dnaSequence, true, 'dnaSequence', [parent, searchType, resultRows, addDataset, datasetName], searchDataOut, cb); + dnaSequence, true, 'dnaSequence', [parent, searchType, resultRows, addDataset, datasetName], searchDataOut, cb, /*progressive*/ false); } else { let features = dev_blastResult; cb(null, features); diff --git a/backend/common/utilities/child-process.js b/backend/common/utilities/child-process.js index 016808c44..1f22f454f 100644 --- a/backend/common/utilities/child-process.js +++ b/backend/common/utilities/child-process.js @@ -18,9 +18,12 @@ var fs = require('fs'); * child process, after [fileName, useFile] * @param dataOutCb (Buffer chunk, cb) {} * @param cb response node callback + * @param progressive true means pass received data back directly to + * dataOutCb, otherwise catenate it and call dataOutCb just once when + * child closes * @return child */ -exports.childProcess = (scriptName, postData, useFile, fileName, moreParams, dataOutCb, cb) => { +exports.childProcess = (scriptName, postData, useFile, fileName, moreParams, dataOutCb, cb, progressive) => { const fnName = 'childProcess'; /** messages from child via file descriptors 3 and 4 are * collated in these arrays and can be sent back to provide @@ -107,12 +110,21 @@ exports.childProcess = (scriptName, postData, useFile, fileName, moreParams, dat warnings.push(message); }); - child.stdout.on('data', (chunk) => dataOutCb(chunk, cb)); + /** output chunks received from child, if progressive. */ + let outputs = []; + let outCb = progressive ? + (chunk) => dataOutCb(chunk, cb) : + (chunk) => outputs.push(chunk); + child.stdout.on('data', outCb); // since these are streams, you can pipe them elsewhere // child.stderr.pipe(dest); child.on('close', (code) => { console.log('child process exited with code', code); + if (! progressive && outputs.length) { + let combined = Buffer.concat(outputs); + dataOutCb(combined, cb); + } if (code) { const error = Error("Failed processing file '" + fileName + "'."); cb(error); @@ -135,9 +147,15 @@ exports.childProcess = (scriptName, postData, useFile, fileName, moreParams, dat /*----------------------------------------------------------------------------*/ +/* + * childProcess() was factored out of Dataset.upload() in models/dataset.js + * The following is the remainder which was not used in childProcess(), + * i.e. not common, and can replace the use of spawn() there. + */ + /* dataset upload */ function factored(msg, cb) { - exports.childProcess('uploadSpreadsheet.bash', msg.data, true, msg.fileName, dataOutUpload, cb); + exports.childProcess('uploadSpreadsheet.bash', msg.data, true, msg.fileName, dataOutUpload, cb, /*progressive*/ false); } // msg file param from API request {fileName, data, replaceDataset} @@ -178,3 +196,5 @@ let dataOutUpload = (chunk, cb) => { } }); }; + +/*----------------------------------------------------------------------------*/ diff --git a/backend/scripts/dnaSequenceSearch.bash b/backend/scripts/dnaSequenceSearch.bash index 399b1bb1f..1fb35726c 100755 --- a/backend/scripts/dnaSequenceSearch.bash +++ b/backend/scripts/dnaSequenceSearch.bash @@ -120,8 +120,8 @@ function datasetId2dbName() cd $serverDir } -# Enable this to use dev_blastResult() for dev / loopback test, when blast is not installed. -if false +# This directory check enables dev_blastResult() for dev / loopback test, when blast is not installed. +if [ -d ../../pretzel.A1 ] then dev_blastResult | \ ( [ "$addDataset" = true ] && convertSearchResults2Json || cat) | \ From 404ccf3f302aaf539328e2d2e57d3a734357ef48 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 25 May 2021 17:27:53 +1000 Subject: [PATCH 03/77] Add notes for database update for Pretzel v2.7.2 factor Database Configuration out of docker_setup.md to create database-configuration.md and add section for Pretzel 2.7 update. --- README.md | 2 +- doc/notes/database-configuration.md | 100 ++++++++++++++++++++++++++++ doc/notes/docker_setup.md | 31 +-------- 3 files changed, 102 insertions(+), 31 deletions(-) create mode 100644 doc/notes/database-configuration.md diff --git a/README.md b/README.md index b2430268b..f10fc5b83 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ URL=$URL_A uploadData aliases.json ## Database Indexes -Refer to 'Database configuration' in doc/notes/docker_setup.md for indexes added to the Feature and Alias database collections which are required for reasonable performance as datasets grown beyond 1k documents. +Refer to 'Database configuration' in doc/notes/database_configuration.md for indexes added to the Feature and Alias database collections which are required for reasonable performance as datasets grown beyond 1k documents. # Public genetic map references diff --git a/doc/notes/database-configuration.md b/doc/notes/database-configuration.md new file mode 100644 index 000000000..403f69f54 --- /dev/null +++ b/doc/notes/database-configuration.md @@ -0,0 +1,100 @@ +## Database configuration + + +Pretzel now uses (since branch feature/progressive) aggregation pipeline queries in mongoDb, which are available in mongoDb versions after 4. + +Progressive loading of paths via aliases relies on the indices of the Alias collection, and indexes are added to the Feature collection also : + +``` +mongo --quiet admin +db.Feature.getIndexes() +db.Feature.createIndex({"value.0":1},{ partialFilterExpression: {"value.1": {$type: 'number'}} } ) +db.Feature.createIndex({blockId:1} ) + +db.Alias.getIndexes() +db.Alias.createIndex ( {namespace1:1} ) +db.Alias.createIndex ( {namespace2:1} ) +db.Alias.createIndex ( {string1:1} ) +db.Alias.createIndex ( {string2:1} ) + +db.Alias.createIndex ( {namespace1:1, namespace2:1} ) +db.Alias.createIndex ( {string1:1, string2:1} ) + +exit +``` +This is applicable to any of the build methods. +This assumes DB_NAME=admin; substituted e.g. pretzel for admin. + +To check if these aliases are already added : +``` +db.Feature.getIndexes() +db.Alias.getIndexes() +``` + + + +### Pretzel 2.7 update + +From v2.7, Pretzel has support for an added index on the Feature database which improves performance. +There are corresponding changes in the Pretzel server which are enabled by setting use_value_0=1 in the server environment. + +Also these db indexes are added to the recommendation : +``` + db.Alias.createIndex ( {namespace1:1, namespace2:1, string1:1, string2:1} ) + db.Feature.createIndex({blockId:1, value_0:1}) + db.Feature.createIndex({name:1} ) +``` +The above index for Alias can replace all the earlier Alias indexes. + +To update existing Features to add the .value_0 field used by the above index : +``` +db.Feature.updateMany({value : { $type: [ "array" ]} }, +[ {$set : {value_0 : {$arrayElemAt: [ "$value", 0 ] } }}] ) +``` + +For upgrades from early versions of Pretzel, first handle Features with value fields which are no longer standard : + +This will handle Features with .range which is an array [start] or [start,end], instead of .value +``` +db.Feature.updateOne({range : { $type: [ "array" ]}, value : {$exists : false}}, +[ {$set : {value_0 : {$arrayElemAt: [ "$range", 0 ] } }}] ) +``` + +To check if that is required : +``` +db.Feature.find({range : { $type: [ "array" ]} }) +``` + + +This will handle Features with .value which is a single number instead of [start] or [start,end] +``` +db.Feature.updateMany({ value: { $not : { $type: [ "array" ]} , $type: [ "number" ] } }, + [ {$set : {value_0 : "$value" }}]) +``` +To check if that is required : +``` +db.Feature.find({ value: { $not : { $type: [ "array" ]} , $type: [ "number" ] } }) +``` + + +This handles Features which are in the current (v2) format, i.e. .value is [start] or [start,end]. +This is all that is required for installations which don't have datasets from earlier versions. +``` +db.Feature.updateMany({ value_0 : {$exists : false}, value: { $type: [ "array" ]} } , + [ {$set : {value_0 : {$arrayElemAt: [ "$value", 0 ] } }}] ) +``` + + +Check that all Features now have .value_0 +``` +db.Feature.find({ value_0 : {$exists : false}} ) +``` + +As noted above, create the index which uses Feature.value_0 +``` +db.Feature.createIndex({blockId:1, value_0:1}) +``` +This replaces the Feature index with partialFilterExpression, which is no longer required. + +Then enable the use of this in the server: +`export use_value_0=1` diff --git a/doc/notes/docker_setup.md b/doc/notes/docker_setup.md index 542d86ea2..8d12b825e 100644 --- a/doc/notes/docker_setup.md +++ b/doc/notes/docker_setup.md @@ -373,36 +373,7 @@ sudo docker container inspect pretz_80 | jq 'map(.Config.Env)' ## Database configuration -Pretzel now uses (since branch feature/progressive) aggregation pipeline queries in mongoDb, which are available in mongoDb versions after 4. - -Progressive loading of paths via aliases relies on the indices of the Alias collection, and indexes are added to the Feature collection also : - -``` -mongo --quiet admin -db.Feature.getIndexes() -db.Feature.createIndex({"value.0":1},{ partialFilterExpression: {"value.1": {$type: 'number'}} } ) -db.Feature.createIndex({blockId:1} ) - -db.Alias.getIndexes() -db.Alias.createIndex ( {namespace1:1} ) -db.Alias.createIndex ( {namespace2:1} ) -db.Alias.createIndex ( {string1:1} ) -db.Alias.createIndex ( {string2:1} ) - -db.Alias.createIndex ( {namespace1:1, namespace2:1} ) -db.Alias.createIndex ( {string1:1, string2:1} ) - -exit -``` -This is applicable to any of the build methods. -This assumes DB_NAME=admin; substituted e.g. pretzel for admin. - -To check if these aliases are already added : -``` -db.Feature.getIndexes() -db.Alias.getIndexes() -``` - +Refer to 'Database configuration' in doc/notes/database_configuration.md for configuration of database indexes. From 30d8dcd9a499545a4d5fdf34842ffd2923305c53 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 27 May 2021 16:04:47 +1000 Subject: [PATCH 04/77] sequence search : set clientId in the dataset created from the result. feature.js : dnaSequenceSearch() : pass options as param instead of via data, so that it gets optionsFromRequest. for removeExisting(), prefix parent. to datasetName, to align with convertSearchResults2Json() in dnaSequenceSearch.bash. sequence-search.js : dnaSequenceInput() : log err because err.responseJSON was undefined (maybe add promise.catch). unviewDataset() : handle undefined replacedDataset (dataset added from result may not have been received yet via getDatasets()) api-server.js : getDatasets() : handle TaskCancelation, don't set model.datasets list to []. --- backend/common/models/feature.js | 12 ++++++---- .../app/components/panel/sequence-search.js | 24 +++++++++++++------ frontend/app/components/service/api-server.js | 19 ++++++++++++++- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/backend/common/models/feature.js b/backend/common/models/feature.js index 878b02e32..5cfbbeae4 100644 --- a/backend/common/models/feature.js +++ b/backend/common/models/feature.js @@ -62,10 +62,10 @@ module.exports = function(Feature) { * * @param cb node response callback */ - Feature.dnaSequenceSearch = function(data, cb) { + Feature.dnaSequenceSearch = function(data, options, cb) { const models = this.app.models; - let {dnaSequence, parent, searchType, resultRows, addDataset, datasetName, options} = data; + let {dnaSequence, parent, searchType, resultRows, addDataset, datasetName} = data; const fnName = 'dnaSequenceSearch'; console.log(fnName, dnaSequence.length, parent, searchType); @@ -86,8 +86,10 @@ module.exports = function(Feature) { }); if (addDataset) { let jsonFile='tmp/' + datasetName + '.json'; - console.log('before removeExisting "', datasetName, '"', '"', jsonFile, '"'); - upload.removeExisting(models, datasetName, /*replaceDataset*/true, cb, loadAfterDelete); + /** same as convertSearchResults2Json() in dnaSequenceSearch.bash */ + let datasetNameFull=`${parent}.${datasetName}`; + console.log('before removeExisting "', datasetNameFull, '"', '"', jsonFile, '"'); + upload.removeExisting(models, datasetNameFull, /*replaceDataset*/true, cb, loadAfterDelete); function loadAfterDelete(err) { upload.loadAfterDeleteCb( @@ -141,9 +143,9 @@ module.exports = function(Feature) { /* Within data : .dnaSequence, and : {arg: 'parent', type: 'string', required: true}, {arg: 'searchType', type: 'string', required: true}, - {arg: "options", type: "object", http: "optionsFromRequest"} resultRows, addDataset, datasetName */ + {arg: "options", type: "object", http: "optionsFromRequest"} ], // http: {verb: 'post'}, returns: {arg: 'features', type: 'array'}, diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 2ecc981fc..e5147c1f6 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -9,6 +9,7 @@ const dLog = console.debug; export default Component.extend({ auth: service(), + apiServers: service(), /** limit rows in result */ resultRows : 50, @@ -153,6 +154,7 @@ export default Component.extend({ }), dnaSequenceInput(rawText) { + const fnName = 'dnaSequenceInput'; // dLog("dnaSequenceInput", rawText && rawText.length); /** if the user has use paste or newline then .text is defined, * otherwise use jQuery to get it from the textarea. @@ -185,6 +187,10 @@ export default Component.extend({ this.get('newDatasetName'), /*options*/{/*dataEvent : receivedData, closePromise : taskInstance*/}); + promise.catch( + (error) => { + dLog('dnaSequenceInput catch', error, arguments); + }); promise.then( (data) => { dLog('dnaSequenceInput', data.features.length); @@ -195,6 +201,7 @@ export default Component.extend({ }, // copied from data-base.js - could be factored. (err, status) => { + dLog(fnName, 'dnaSequenceSearch reject', err, status); let errobj = err.responseJSON.error; console.log(errobj); let errmsg = null; @@ -206,7 +213,7 @@ export default Component.extend({ errmsg = errobj.name; } this.setError(errmsg); - this.scrollToTop(); + // upload tabs do .scrollToTop(), doesn't seem applicable here. } ); @@ -221,12 +228,15 @@ export default Component.extend({ unviewDataset(datasetName) { let store = this.get('apiServers').get('primaryServer').get('store'), - replacedDataset = store.peekRecord('dataset', datasetName), - viewedBlocks = replacedDataset.get('blocks').toArray().filterBy('isViewed'), - blockService = this.get('blockService'), - blockIds = viewedBlocks.map((b) => b.id); - dLog('unviewDataset', datasetName, blockIds); - blockService.setViewed(blockIds, false); + replacedDataset = store.peekRecord('dataset', datasetName); + if (replacedDataset) { + let + viewedBlocks = replacedDataset.get('blocks').toArray().filterBy('isViewed'), + blockService = this.get('blockService'), + blockIds = viewedBlocks.map((b) => b.id); + dLog('unviewDataset', datasetName, blockIds); + blockService.setViewed(blockIds, false); + } } /*--------------------------------------------------------------------------*/ diff --git a/frontend/app/components/service/api-server.js b/frontend/app/components/service/api-server.js index 5d0daf576..67ac88026 100644 --- a/frontend/app/components/service/api-server.js +++ b/frontend/app/components/service/api-server.js @@ -1,6 +1,8 @@ import EmberObject, { computed } from '@ember/object'; import Component from '@ember/component'; import { inject as service } from '@ember/service'; +import { task, timeout, didCancel } from 'ember-concurrency'; + import { breakPoint } from '../../utils/breakPoint'; @@ -115,11 +117,20 @@ export default EmberObject.extend({ * */ getDatasets : function () { + const fnName = 'getDatasets'; let datasetService = this.get('dataset'); let taskGetList = datasetService.get('taskGetList'); // availableMaps /** server was a param when this function was an attribute of apiServers. */ let server = this; - let datasetsTask = taskGetList.perform(server); + let datasetsTask = taskGetList.perform(server) + .catch((error) => { + // Recognise if the given task error is a TaskCancelation. + if (! didCancel(error)) { + dLog(fnName, ' taskInstance.catch', this.name, error); + throw error; + } + }); + let name = server.get('name'), apiServers = this.get('apiServers'), @@ -137,6 +148,11 @@ export default EmberObject.extend({ { /** change to : apiServers can do .on() of .evented() on task */ let datasetsBlocks = apiServers.get('datasetsBlocks'); + /** if TaskCancelation, no result, so don't replace previous result. */ + if ((! blockValues || ! blockValues.length) && + (datasetsBlocks[datasetsHandle] && datasetsBlocks[datasetsHandle].length)) { + dLog(fnName, 'TaskCancelation datasetsTask.then', blockValues); + } else { datasetsBlocks[datasetsHandle] = blockValues; server.set("datasetsBlocks", blockValues); apiServers.incrementProperty('datasetsBlocksRefresh'); @@ -149,6 +165,7 @@ export default EmberObject.extend({ let ti = this.get('featuresCountAllTaskInstance'); dLog('getDatasets', 'evaluated featuresCountAllTaskInstance', ti); } + } } }); From 2a32f131e46b2e8453d320b48f8155754e4398ef Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 27 May 2021 16:06:04 +1000 Subject: [PATCH 05/77] DocumentTimeline is now available without window. --- frontend/app/services/data/axis-zoom.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/services/data/axis-zoom.js b/frontend/app/services/data/axis-zoom.js index 1eb4df603..cf7c0139f 100644 --- a/frontend/app/services/data/axis-zoom.js +++ b/frontend/app/services/data/axis-zoom.js @@ -18,7 +18,9 @@ export default Service.extend({ let isCurrent; if (this.zoomPan && this.zoomPan.isWheelEvent) { let - documentTimeline = new window.DocumentTimeline(), + /** DocumentTimeline is now available without window., refn + * https://developer.mozilla.org/en-US/docs/Web/API/DocumentTimeline/DocumentTimeline */ + documentTimeline = new (DocumentTimeline || window.DocumentTimeline)(), timeSince = documentTimeline.currentTime - this.zoomPan.timeStamp; isCurrent = timeSince < 1000; // dLog('currentZoomPanIsWheel', timeSince, isCurrent); From 050fc2bdd0f1cd7eb14f7b3a47543d2ae1cac521 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 27 May 2021 16:19:23 +1000 Subject: [PATCH 06/77] enable sequence search in GUI - drop flag options=sequenceSearch this reverses 56ffd65b. --- frontend/app/templates/components/panel/left-panel.hbs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frontend/app/templates/components/panel/left-panel.hbs b/frontend/app/templates/components/panel/left-panel.hbs index a694494fc..3adfc48c1 100644 --- a/frontend/app/templates/components/panel/left-panel.hbs +++ b/frontend/app/templates/components/panel/left-panel.hbs @@ -5,9 +5,7 @@ {{elem/icon-base name="folder-open"}} Explorer {{elem/icon-base name="picture"}} View {{elem/icon-base name="search"}} Feature Search - {{#if this.model.params.parsedOptions.sequenceSearch}} {{elem/icon-base name="search"}} Sequence Search - {{/if}} {{elem/icon-base name="cloud-upload"}} Upload
@@ -21,14 +19,12 @@ }} - {{#if this.model.params.parsedOptions.sequenceSearch}} {{panel/sequence-search datasets=datasets view=view }} - {{/if}} {{panel/manage-explorer From 21e5925f994f078a3dfe367915e15ed88eaa0690 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 27 May 2021 19:13:04 +1000 Subject: [PATCH 07/77] update version of caniuse-lite ... in frontend/package-lock.json, partially via browserslist, but got Cannot find module internal/util/types, so then manual : npm install caniuse-lite --- frontend/package-lock.json | 48 ++++---------------------------------- frontend/package.json | 1 + 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0777e72cb..b7095a504 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,6 +1,6 @@ { "name": "pretzel-frontend", - "version": "2.6.1", + "version": "2.7.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -189,11 +189,6 @@ "node-releases": "^1.1.66" } }, - "caniuse-lite": { - "version": "1.0.30001157", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001157.tgz", - "integrity": "sha512-gOerH9Wz2IRZ2ZPdMfBvyOi3cjaz4O4dgNwPGzx8EhqAs4+2IL/O+fJsbt+znSigujoZG8bVcIAUM/I/E5K3MA==" - }, "electron-to-chromium": { "version": "1.3.593", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.593.tgz", @@ -14487,14 +14482,6 @@ "electron-to-chromium": "^1.3.591", "escalade": "^3.1.1", "node-releases": "^1.1.66" - }, - "dependencies": { - "caniuse-lite": { - "version": "1.0.30001157", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001157.tgz", - "integrity": "sha512-gOerH9Wz2IRZ2ZPdMfBvyOi3cjaz4O4dgNwPGzx8EhqAs4+2IL/O+fJsbt+znSigujoZG8bVcIAUM/I/E5K3MA==", - "dev": true - } } }, "electron-to-chromium": { @@ -14518,9 +14505,9 @@ } }, "caniuse-lite": { - "version": "1.0.30000962", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30000962.tgz", - "integrity": "sha512-WXYsW38HK+6eaj5IZR16Rn91TGhU3OhbwjKZvJ4HN/XBIABLKfbij9Mnd3pM0VEwZSlltWjoWg3I8FQ0DGgNOA==" + "version": "1.0.30001230", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001230.tgz", + "integrity": "sha512-5yBd5nWCBS+jWKTcHOzXwo5xzcj4ePE/yjtkZyUV1BTUmrBaA9MRGC+e7mxnqXSA90CmCA8L3eKLaSUkt099IQ==" }, "capture-exit": { "version": "2.0.0", @@ -15626,11 +15613,6 @@ "node-releases": "^1.1.46" } }, - "caniuse-lite": { - "version": "1.0.30001022", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001022.tgz", - "integrity": "sha512-FjwPPtt/I07KyLPkBQ0g7/XuZg6oUkYBVnPHNj3VHJbOjmmJ/GdSo/GUY6MwINEQvjhP6WZVbX8Tvms8xh0D5A==" - }, "electron-to-chromium": { "version": "1.3.340", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.340.tgz", @@ -16724,12 +16706,6 @@ "electron-to-chromium": "^1.3.47" } }, - "caniuse-lite": { - "version": "1.0.30000933", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30000933.tgz", - "integrity": "sha512-d3QXv7eFTU40DSedSP81dV/ajcGSKpT+GW+uhtWmLvQm9bPk0KK++7i1e2NSW/CXGZhWFt2mFbFtCJ5I5bMuVA==", - "dev": true - }, "electron-to-chromium": { "version": "1.3.109", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.109.tgz", @@ -17931,11 +17907,6 @@ "node-releases": "^1.1.66" } }, - "caniuse-lite": { - "version": "1.0.30001157", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001157.tgz", - "integrity": "sha512-gOerH9Wz2IRZ2ZPdMfBvyOi3cjaz4O4dgNwPGzx8EhqAs4+2IL/O+fJsbt+znSigujoZG8bVcIAUM/I/E5K3MA==" - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -21289,11 +21260,6 @@ "node-releases": "^1.1.66" } }, - "caniuse-lite": { - "version": "1.0.30001157", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001157.tgz", - "integrity": "sha512-gOerH9Wz2IRZ2ZPdMfBvyOi3cjaz4O4dgNwPGzx8EhqAs4+2IL/O+fJsbt+znSigujoZG8bVcIAUM/I/E5K3MA==" - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -26545,12 +26511,6 @@ "node-releases": "^1.1.70" } }, - "caniuse-lite": { - "version": "1.0.30001205", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001205.tgz", - "integrity": "sha512-TL1GrS5V6LElbitPazidkBMD9sa448bQDDLrumDqaggmKFcuU2JW1wTOHJPukAcOMtEmLcmDJEzfRrf+GjM0Og==", - "dev": true - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index fef74f777..6d12abb9b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -80,6 +80,7 @@ }, "private": true, "dependencies": { + "caniuse-lite": "^1.0.30001230", "colresizable": "github:felipedgarcia/colresizable", "ember-array-helper": ">=5", "ember-cli-htmlbars": "^5.3.1", From 5268acd01f52ac995077f30055bd876c6e033eb2 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 28 May 2021 12:54:21 +1000 Subject: [PATCH 08/77] persist the node server API request cache, using file storage Use flat-cache in place of memory-cache. closes #249 --- backend/common/models/block.js | 11 +++++++-- backend/common/utilities/results-cache.js | 29 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 backend/common/utilities/results-cache.js diff --git a/backend/common/models/block.js b/backend/common/models/block.js index f4c544f42..2bf78f71c 100644 --- a/backend/common/models/block.js +++ b/backend/common/models/block.js @@ -18,7 +18,13 @@ const { getAliases } = require('../utilities/localise-aliases'); var ObjectId = require('mongodb').ObjectID -var cache = require('memory-cache'); + +/** results-cache has the same API as 'memory-cache', so that can be + * used instead to avoid the need to setup a cache directory, and + * manage cache accumulation. + */ +const cacheLibraryName = '../utilities/results-cache'; // 'memory-cache'; +var cache = require(cacheLibraryName); var SSE = require('express-sse'); @@ -608,9 +614,10 @@ function blockAddFeatures(db, datasetId, blockId, features, cb) { /** Logically part of reqStream(), but split out so that it can be called * directly or via a promise. */ function pipeStream(sse, intervals, useCache, cacheId, filterFunction, res, cursor) { + console.log('pipeStream', sse, intervals, useCache, cacheId); if (useCache) cursor. - pipe(new pathsStream.CacheWritable(cacheId)); + pipe(new pathsStream.CacheWritable(/*cache,*/ cacheId)); let pipeLine = [cursor]; diff --git a/backend/common/utilities/results-cache.js b/backend/common/utilities/results-cache.js new file mode 100644 index 000000000..db44d0b52 --- /dev/null +++ b/backend/common/utilities/results-cache.js @@ -0,0 +1,29 @@ +let flatCache = require('flat-cache'); + + +/* global require */ + +/* global exports */ + +/*----------------------------------------------------------------------------*/ + + +let cache = flatCache.load('resultsCache'); + +/*----------------------------------------------------------------------------*/ + + +/** Expose the same API as memory-cache, to enable an easy substitution. + */ + +exports.get = (key) => { + let cacheContent = cache.getKey(key); + return cacheContent; +}; + +exports.put = (key, body) => { + cache.setKey(key, body); + cache.save(); +}; + +/*----------------------------------------------------------------------------*/ From 606626953d32d044318ba2c81c27a54d20cf3384 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 28 May 2021 16:44:31 +1000 Subject: [PATCH 09/77] node server API result cache : keep previous (unvisited) values pass noPrune=true to cache.save(). #249 --- backend/common/utilities/results-cache.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/common/utilities/results-cache.js b/backend/common/utilities/results-cache.js index db44d0b52..428e75434 100644 --- a/backend/common/utilities/results-cache.js +++ b/backend/common/utilities/results-cache.js @@ -23,7 +23,11 @@ exports.get = (key) => { exports.put = (key, body) => { cache.setKey(key, body); - cache.save(); + /** https://github.com/royriojas/flat-cache#readme : "Non visited + * keys are removed when cache.save() is called" if noPrune is not + * true + */ + cache.save(/*noPrune*/ true); }; /*----------------------------------------------------------------------------*/ From f442afc173e05e560de9347352a5eb3b610ef459 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 28 May 2021 21:53:32 +1000 Subject: [PATCH 10/77] sequence search : show each result in a new output tab sequence-search.js : add searches, an array of sequence-search-data. past() : delay evaluation of event.target.value. Move promise handling from dnaSequenceInput() and unviewDataset() to blast-results.js. blast-results.js : add showSearch, resultEffect(). factor element ids : sequence-search-output to search.tabId and blast-results-hotable to search.tableId, so that each search result tab and table can have a unique element id. blast-results.hbs : add x-toggle : showSearch. if showSearch, display search.parent, .seq. add sequence-search-data.js, which wraps the search inputs, id and status. --- .../app/components/panel/sequence-search.js | 73 +++++------------- .../components/panel/upload/blast-results.js | 76 +++++++++++++++++-- .../components/panel/sequence-search.hbs | 13 +++- .../components/panel/upload/blast-results.hbs | 26 ++++++- .../app/utils/data/sequence-search-data.js | 35 +++++++++ 5 files changed, 161 insertions(+), 62 deletions(-) create mode 100644 frontend/app/utils/data/sequence-search-data.js diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index e5147c1f6..976103013 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -3,13 +3,15 @@ import { bind, once, later, throttle, debounce } from '@ember/runloop'; import { inject as service } from '@ember/service'; import { observer, computed } from '@ember/object'; import { alias } from '@ember/object/computed'; +import { A as array_A } from '@ember/array'; + +import sequenceSearchData from '../../utils/data/sequence-search-data'; const dLog = console.debug; export default Component.extend({ auth: service(), - apiServers: service(), /** limit rows in result */ resultRows : 50, @@ -18,6 +20,17 @@ export default Component.extend({ classNames: ['col-xs-12'], + /** array of current searches. each one is data for blast-result component. */ + searches : undefined, + + /*--------------------------------------------------------------------------*/ + + init() { + this._super(...arguments); + + this.set('searches', array_A()); + }, + /*--------------------------------------------------------------------------*/ /** copied from data-base.js; may factor or change the approach. */ isProcessing: false, @@ -96,10 +109,11 @@ export default Component.extend({ dLog('inputIsActive'); }, paste: function(event) { - let text = event && (event.target.value || event.originalEvent.target.value); - console.log('paste', event, text.length); + /** text is "" at this time. */ /** this action function is called before jQuery val() is updated. */ later(() => { + let text = event && (event.target.value || event.originalEvent.target.value); + console.log('paste', event, text.length); this.set('text', text); // this.dnaSequenceInput(/*text*/); }, 500); @@ -187,57 +201,12 @@ export default Component.extend({ this.get('newDatasetName'), /*options*/{/*dataEvent : receivedData, closePromise : taskInstance*/}); - promise.catch( - (error) => { - dLog('dnaSequenceInput catch', error, arguments); - }); - promise.then( - (data) => { - dLog('dnaSequenceInput', data.features.length); - this.set('data', data.features); - if (this.get('addDataset') && this.get('replaceDataset')) { - this.unviewDataset(this.get('newDatasetName')); - } - }, - // copied from data-base.js - could be factored. - (err, status) => { - dLog(fnName, 'dnaSequenceSearch reject', err, status); - let errobj = err.responseJSON.error; - console.log(errobj); - let errmsg = null; - if (errobj.message) { - errmsg = errobj.message; - } else if (errobj.errmsg) { - errmsg = errobj.errmsg; - } else if (errobj.name) { - errmsg = errobj.name; - } - this.setError(errmsg); - // upload tabs do .scrollToTop(), doesn't seem applicable here. - } - - ); - } - }, + let searchData = sequenceSearchData.create({promise, seq, parent, searchType}); + this.get('searches').pushObject(searchData); - /*--------------------------------------------------------------------------*/ - /* copied from file-drop-zone.js, can factor if this is retained. */ + } + }, - /** Unview the blocks of the dataset which has been replaced by successful upload. - */ - unviewDataset(datasetName) { - let - store = this.get('apiServers').get('primaryServer').get('store'), - replacedDataset = store.peekRecord('dataset', datasetName); - if (replacedDataset) { - let - viewedBlocks = replacedDataset.get('blocks').toArray().filterBy('isViewed'), - blockService = this.get('blockService'), - blockIds = viewedBlocks.map((b) => b.id); - dLog('unviewDataset', datasetName, blockIds); - blockService.setViewed(blockIds, false); - } - } /*--------------------------------------------------------------------------*/ diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index c1193fe9f..b151db864 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -16,6 +16,12 @@ const dLog = console.debug; * /Feature/dnaSequenceSearch */ export default Component.extend({ + apiServers: service(), + blockService : service('data/block'), + + /** true enables display of the search inputs. */ + showSearch : false, + /** copied from data-base.js, not used yet */ isProcessing: false, @@ -39,6 +45,67 @@ export default Component.extend({ // this.showTable(); }, + /*--------------------------------------------------------------------------*/ + + /** Outcomes of the API search request. + */ + resultEffect : computed('search.promise', function () { + const fnName = 'resultEffect'; + /** auth.dnaSequenceSearch request */ + let promise = this.search.promise; + + promise.catch( + (error) => { + dLog(fnName, 'catch', error, arguments); + }); + promise.then( + (data) => { + dLog(fnName, data.features.length); + this.set('data', data.features); + if (this.get('addDataset') && this.get('replaceDataset')) { + this.unviewDataset(this.get('newDatasetName')); + } + }, + // copied from data-base.js - could be factored. + (err, status) => { + dLog(fnName, 'dnaSequenceSearch reject', err, status); + let errobj = err.responseJSON.error; + console.log(errobj); + let errmsg = null; + if (errobj.message) { + errmsg = errobj.message; + } else if (errobj.errmsg) { + errmsg = errobj.errmsg; + } else if (errobj.name) { + errmsg = errobj.name; + } + this.setError(errmsg); + // upload tabs do .scrollToTop(), doesn't seem applicable here. + } + + ); + + }), + + /*--------------------------------------------------------------------------*/ + /* copied from file-drop-zone.js, can factor if this is retained. */ + + /** Unview the blocks of the dataset which has been replaced by successful upload. + */ + unviewDataset(datasetName) { + let + store = this.get('apiServers').get('primaryServer').get('store'), + replacedDataset = store.peekRecord('dataset', datasetName); + if (replacedDataset) { + let + viewedBlocks = replacedDataset.get('blocks').toArray().filterBy('isViewed'), + blockService = this.get('blockService'), + blockIds = viewedBlocks.map((b) => b.id); + dLog('unviewDataset', datasetName, blockIds); + blockService.setViewed(blockIds, false); + } + }, + /*--------------------------------------------------------------------------*/ /** comments for activeEffect() and shownBsTab() in @see data-csv.js * @desc @@ -74,12 +141,11 @@ export default Component.extend({ const cName = 'upload/blast-results'; const fnName = 'createTable'; dLog('createTable'); - var that = this; - $(function() { - let eltId = 'blast-results-hotable'; + $(() => { + let eltId = this.search.tableId; let hotable = $('#' + eltId)[0]; if (! hotable) { - console.warn(cName, fnName, ' : #', eltId, ' not found', that); + console.warn(cName, fnName, ' : #', eltId, ' not found', this); return; // fail } /** @@ -128,7 +194,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que /* see comment re. handsOnTableLicenseKey in frontend/config/environment.js */ licenseKey: config.handsOnTableLicenseKey }); - that.set('table', table); + this.set('table', table); }); }, diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 9bc8e5223..d0f3c6b0e 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -13,7 +13,10 @@ {{elem/icon-base name="edit"}} Sequence Input - {{elem/icon-base name="arrow-right"}} Blast Output + {{#each this.searches as |search|}} + {{elem/icon-base name="arrow-right"}} Blast Output + {{/each}} +
@@ -114,12 +117,14 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} - + {{#each this.searches as |search|}} + - {{panel/upload/blast-results data=data - active=(bs-eq tab.activeId "sequence-search-output") }} + {{panel/upload/blast-results search=search + active=(bs-eq tab.activeId search.tabId ) }} + {{/each}}
diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index cd6f39a85..4ef1cd497 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -1,6 +1,30 @@ {{dataMatrixEffect}} +{{resultEffect}} {{activeEffect}} +{{!-- --------------------------------------------------------------------- --}} + +{{!-- x-toggle uses flex, so wrap it. --}} +
+
+ {{#x-toggle + theme='light' + value=showSearch + onToggle=(action (mut showSearch)) + as |toggle| + }} + {{toggle.switch}} + {{/x-toggle}} +
+ Show Search Inputs + +
+ + {{#if showSearch}} +
Parent : {{search.parent}}
+
FASTA Sequence Search : {{search.seq}}
+ {{/if}} +
{{!-- --------------------------------------------------------------------- --}} @@ -12,5 +36,5 @@ {{/elem/panel-form}} {{/if}} -
+
diff --git a/frontend/app/utils/data/sequence-search-data.js b/frontend/app/utils/data/sequence-search-data.js new file mode 100644 index 000000000..04bb6017f --- /dev/null +++ b/frontend/app/utils/data/sequence-search-data.js @@ -0,0 +1,35 @@ +const dLog = console.debug; + +import EmberObject, { computed } from '@ember/object'; + +/** Allocate a unique id to each search / blast-result tab, for navigation. + */ +let searchId = 0; + +/** Data which backs each search instance initiated by sequence-search. + * @param promise, seq, parent, searchType + */ +export default EmberObject.extend({ + + init() { + this._super(...arguments); + + /** an alternative is to use concat helper as in + * components/panel/api-server-tab.hbs, but this seems simpler, + * and avoids repeated calculation of a constant value. + */ + let + id = searchId++, + tabId = 'sequence-search-output-' + id; + this.set('tabId', tabId); + this.set('tabHref', '#' + tabId); + /** It would reduce the DOM size to have a single table, shared + * between all searches, but this is simpler, and retains scroll + * position, selection, sorting, etc for each table, which users + * would expect. + */ + this.set('tableId', "blast-results-hotable-" + id); + } + + +}); From d9fdac11e72fb151bf55d6d915ac4cbdd6180447 Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 31 May 2021 11:55:34 +1000 Subject: [PATCH 11/77] sequence search : show tab id : Blast Result (hh:mm:ss) ... 24-hour sequence-search.hbs : nav.item : display search.timeId sequence-search-data.js : init() : set .timeId to HH:MM:SS. --- .../app/templates/components/panel/sequence-search.hbs | 2 +- frontend/app/utils/data/sequence-search-data.js | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index d0f3c6b0e..f3de96f9a 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -14,7 +14,7 @@ {{elem/icon-base name="edit"}} Sequence Input {{#each this.searches as |search|}} - {{elem/icon-base name="arrow-right"}} Blast Output + {{elem/icon-base name="arrow-right"}} Blast Output ({{search.timeId}}) {{/each}} diff --git a/frontend/app/utils/data/sequence-search-data.js b/frontend/app/utils/data/sequence-search-data.js index 04bb6017f..5a08d7a70 100644 --- a/frontend/app/utils/data/sequence-search-data.js +++ b/frontend/app/utils/data/sequence-search-data.js @@ -29,7 +29,13 @@ export default EmberObject.extend({ * would expect. */ this.set('tableId', "blast-results-hotable-" + id); - } + + let startTime = new Date(); + // maybe : this.set('startTime', startTime); + this.set('timeId', startTime.toTimeString().split(' ')[0]); + }, + + }); From 019591b311d7291f652b081b151c062f01e3898c Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 31 May 2021 12:10:42 +1000 Subject: [PATCH 12/77] sequence search : wrap search.seq with auto scroll-bars part of #239 --- .../app/templates/components/panel/upload/blast-results.hbs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 4ef1cd497..d6a326fc7 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -22,7 +22,9 @@ {{#if showSearch}}
Parent : {{search.parent}}
-
FASTA Sequence Search : {{search.seq}}
+
FASTA Sequence Search : +
{{search.seq}}
+
{{/if}}
From b345f82c0a69f826a487dd51cd1ef9dacc267480 Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 31 May 2021 12:56:45 +1000 Subject: [PATCH 13/77] datasets & blocks refresh : handle TaskCancelation in getBlocksLimits() --- frontend/app/services/data/block.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/app/services/data/block.js b/frontend/app/services/data/block.js index 157492441..c0c30c34f 100644 --- a/frontend/app/services/data/block.js +++ b/frontend/app/services/data/block.js @@ -6,7 +6,7 @@ import { alias } from '@ember/object/computed'; import Evented from '@ember/object/evented'; import Ember from 'ember'; import Service, { inject as service } from '@ember/service'; -import { task } from 'ember-concurrency'; +import { task, didCancel } from 'ember-concurrency'; import { keyBy } from 'lodash/collection'; @@ -732,13 +732,22 @@ export default Service.extend(Evented, { * undefined and limits are requested for all blocks. */ getBlocksLimits(blockId) { + const fnName = 'getBlocksLimits'; let taskGet = this.get('taskGetLimits'); console.log("getBlocksLimits", blockId); let p = new Promise(function(resolve, reject){ later(() => { - let blocksTask = taskGet.perform(blockId); - blocksTask.then((result) => resolve(result)); - blocksTask.catch((error) => reject(error)); + let + blocksTask = taskGet.perform(blockId) + .then((result) => resolve(result)) + .catch((error) => { + // Recognise if the given task error is a TaskCancelation. + if (! didCancel(error)) { + dLog(fnName, ' taskInstance.catch', this.name, error); + // throw error; + reject(error); + } + }); }); }); let blocksTask = p; From 3b9e7cebe2bb7ceb4cba2203806d0db9a1f0a45e Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 31 May 2021 15:20:29 +1000 Subject: [PATCH 14/77] sequence search : refreshDatasets after dnaSequenceSearch. sequence-search.js : dnaSequenceInput() : if addDataset, refreshDatasets after dnaSequenceSearch. left-panel.hbs : pass refreshDatasets to sequence-search --- frontend/app/components/panel/sequence-search.js | 7 +++++++ frontend/app/templates/components/panel/left-panel.hbs | 1 + 2 files changed, 8 insertions(+) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 976103013..0d6e5efc9 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -201,6 +201,13 @@ export default Component.extend({ this.get('newDatasetName'), /*options*/{/*dataEvent : receivedData, closePromise : taskInstance*/}); + if (this.get('addDataset')) { + /* On complete, trigger dataset list reload. + * refreshDatasets is passed from controllers/mapview (updateModel ). + */ + promise.then(() => this.get('refreshDatasets')()); + } + let searchData = sequenceSearchData.create({promise, seq, parent, searchType}); this.get('searches').pushObject(searchData); diff --git a/frontend/app/templates/components/panel/left-panel.hbs b/frontend/app/templates/components/panel/left-panel.hbs index 3adfc48c1..b717e3068 100644 --- a/frontend/app/templates/components/panel/left-panel.hbs +++ b/frontend/app/templates/components/panel/left-panel.hbs @@ -23,6 +23,7 @@ {{panel/sequence-search datasets=datasets view=view + refreshDatasets=refreshDatasets }} From 66d59ba05b02f97d5f7b346d0eae778f7ecd850e Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 31 May 2021 17:10:36 +1000 Subject: [PATCH 15/77] sequence search : view added dataset sequence-search.js : dnaSequenceInput() : after refreshDatasets, if viewDatasetFlag then viewDataset. block.js : getCountsForInterval() : handle interval undefined. sequence-search.hbs : add checkbox : viewDatasetFlag. mapview: add viewDataset() and pass to left-panel, which passes it to sequence-search. updateModel() : return datasetsTask. --- .../app/components/panel/sequence-search.js | 17 +++++++++- frontend/app/controllers/mapview.js | 33 +++++++++++++++++++ frontend/app/services/data/block.js | 2 +- .../templates/components/panel/left-panel.hbs | 1 + .../components/panel/sequence-search.hbs | 5 +++ frontend/app/templates/mapview.hbs | 1 + 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 0d6e5efc9..2cf3f7577 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -205,7 +205,22 @@ export default Component.extend({ /* On complete, trigger dataset list reload. * refreshDatasets is passed from controllers/mapview (updateModel ). */ - promise.then(() => this.get('refreshDatasets')()); + promise.then(() => { + const viewDataset = this.get('viewDatasetFlag'); + let refreshed = this.get('refreshDatasets')(); + if (viewDataset) { + refreshed + .then(() => { + /** same as convertSearchResults2Json() in dnaSequenceSearch.bash and + * backend/common/models/feature.js : Feature.dnaSequenceSearch() */ + let + datasetName = this.get('newDatasetName'), + datasetNameFull=`${parent}.${datasetName}`; + dLog(fnName, 'viewDataset', datasetNameFull); + this.get('viewDataset')(datasetNameFull, viewDataset); + }); + } + }); } let searchData = sequenceSearchData.create({promise, seq, parent, searchType}); diff --git a/frontend/app/controllers/mapview.js b/frontend/app/controllers/mapview.js index bc4a5978d..75a0d69d7 100644 --- a/frontend/app/controllers/mapview.js +++ b/frontend/app/controllers/mapview.js @@ -79,6 +79,7 @@ export default Controller.extend(Evented, { /** Change the state of the named block to viewed. * If this block has a parent block, also add the parent. + * This is replaced by loadBlock(). * @param mapName * (named map for consistency, but mapsToView really means block, and "name" is db ID) * Also @see components/record/entry-block.js : action get @@ -238,6 +239,7 @@ export default Controller.extend(Evented, { this.get('block').getBlocksLimits(); }); } + return datasetsTask; } }, @@ -348,6 +350,37 @@ export default Controller.extend(Evented, { } }, + /*--------------------------------------------------------------------------*/ + + /* copied from file-drop-zone.js, can factor if this is retained. */ + /** View/Unview the blocks of the given dataset. + * View is used to view a dataset added by sequence-sequence. + * Unview is used when the datasets has been replaced by successful upload. + * @param datasetName id + * @param view true for view, false for unview + */ + viewDataset(datasetName, view) { + let + store = this.get('apiServers').get('primaryServer').get('store'), + dataset = store.peekRecord('dataset', datasetName); + if (dataset) { + let + blocksToChange = dataset.get('blocks').toArray().filter((b) => b.get('isViewed') !== view), + blockService = this.get('block'), + blockIds = blocksToChange.map((b) => b.id); + dLog('viewDataset', datasetName, view, blockIds); + // blockService.setViewed(blockIds, view); + let loadBlock = this.actions.loadBlock.bind(this); + blocksToChange.forEach((b) => loadBlock(b)); + } else { + dLog('viewDataset', datasetName, 'not found', view); + } + }, + + + /*--------------------------------------------------------------------------*/ + + /** Provide a class for the div which wraps the right panel. * * The class indicates which of the tabs in the right panel is currently diff --git a/frontend/app/services/data/block.js b/frontend/app/services/data/block.js index c0c30c34f..dc5932421 100644 --- a/frontend/app/services/data/block.js +++ b/frontend/app/services/data/block.js @@ -458,7 +458,7 @@ export default Service.extend(Evented, { } let getCountsForInterval = (interval) => { let countsP; - if (interval[0] === interval[1]) { + if (interval && (interval[0] === interval[1])) { dLog('getCountsForInterval', interval); countsP = Promise.resolve([]); } else { diff --git a/frontend/app/templates/components/panel/left-panel.hbs b/frontend/app/templates/components/panel/left-panel.hbs index b717e3068..3ba6eecf8 100644 --- a/frontend/app/templates/components/panel/left-panel.hbs +++ b/frontend/app/templates/components/panel/left-panel.hbs @@ -24,6 +24,7 @@ datasets=datasets view=view refreshDatasets=refreshDatasets + viewDataset=viewDataset }} diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index f3de96f9a..17f31dc97 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -97,6 +97,11 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} + + {{input type="checkbox" name="viewDatasetFlag" checked=viewDatasetFlag }} + + +
{{input id="dataset_new" diff --git a/frontend/app/templates/mapview.hbs b/frontend/app/templates/mapview.hbs index 0fc87f2d1..e225a879f 100644 --- a/frontend/app/templates/mapview.hbs +++ b/frontend/app/templates/mapview.hbs @@ -12,6 +12,7 @@ model=model datasets=datasets refreshDatasets=(action "updateModel") + viewDataset=(action this.viewDataset) selectedBlock=selectedBlock controls=controls isShowUnique=isShowUnique From 9640e0f4965412482438e094a6ae4fc2b7ebd38f Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 31 May 2021 22:22:41 +1000 Subject: [PATCH 16/77] sequence search : button in output tab to add dataset sequence-search.js : declare and comment viewDatasetFlag. blast-results.js : use upload-base and upload-table. add columnsKeyString, c_name, c_chr, c_pos. add services store, auth, classNames:blast-results. add viewDatasetFlag. add didReceiveAttrs() : initialise selectedParent, namespace. add validateData() app.scss : .blast-results : margin. sequence-search.hbs : pass to blast-results : datasets, refreshDatasets, viewDataset. blast-results.hbs : from data-csv : add message fields, and inputs : selectedDataset, selectedParent (currently read-only), namespace. add checkbox viewDatasetFlag, submit button. add upload-base.js, factored from data-base.js add upload-table.js, factored from data-csv.js --- .../app/components/panel/sequence-search.js | 2 + .../components/panel/upload/blast-results.js | 116 ++++++++++++- frontend/app/styles/app.scss | 4 + .../components/panel/sequence-search.hbs | 5 +- .../components/panel/upload/blast-results.hbs | 85 +++++++++ frontend/app/utils/panel/upload-base.js | 79 +++++++++ frontend/app/utils/panel/upload-table.js | 161 ++++++++++++++++++ 7 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 frontend/app/utils/panel/upload-base.js create mode 100644 frontend/app/utils/panel/upload-table.js diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 2cf3f7577..9d68b9962 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -17,6 +17,8 @@ export default Component.extend({ resultRows : 50, /** true means add / upload result to db as a Dataset */ addDataset : false, + /** true means view the blocks of the dataset after it is added. */ + viewDatasetFlag : false, classNames: ['col-xs-12'], diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index b151db864..727c60970 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -1,35 +1,102 @@ import Component from '@ember/component'; import { observer, computed } from '@ember/object'; +import { alias } from '@ember/object/computed'; import { inject as service } from '@ember/service'; import { later as run_later } from '@ember/runloop'; - import config from '../../../config/environment'; +import uploadBase from '../../../utils/panel/upload-base'; +import uploadTable from '../../../utils/panel/upload-table'; + const dLog = console.debug; /* global Handsontable */ /* global $ */ +/*----------------------------------------------------------------------------*/ + +/** + * based on backend/scripts/dnaSequenceSearch.bash : columnsKeyString + * This also aligns with createTable() : colHeaders below. + */ +const columnsKeyString = [ + 'name', 'chr', 'pcIdentity', 'lengthOfHspHit', 'numMismatches', 'numGaps', 'queryStart', 'queryEnd', 'pos', 'end' +]; +const c_name = 0, c_chr = 1, c_pos = 8; + +/*----------------------------------------------------------------------------*/ + + /** Display a table of results from sequence-search API request * /Feature/dnaSequenceSearch */ export default Component.extend({ apiServers: service(), blockService : service('data/block'), + /** Similar comment to data-csv.js applies re. store (user could select server via GUI). + * store is used by upload-table.js : getDatasetId() and submitFile() + */ + store : alias('apiServers.primaryServer.store'), + auth: service('auth'), + + + classNames: ['blast-results'], /** true enables display of the search inputs. */ showSearch : false, + /** true means view the blocks of the dataset after it is added. */ + viewDatasetFlag : false, - /** copied from data-base.js, not used yet */ + /*--------------------------------------------------------------------------*/ + + /** copied from data-base.js. for uploadBase*/ isProcessing: false, successMessage: null, errorMessage: null, warningMessage: null, progressMsg: '', + + setProcessing : uploadBase.setProcessing, + setSuccess : uploadBase.setSuccess, + setError : uploadBase.setError, + setWarning : uploadBase.setWarning, + clearMsgs : uploadBase.clearMsgs, + + /** data-csv.js : scrollToTop() scrolls up #left-panel-upload, but that is not required here. */ + scrollToTop() { + }, + + updateProgress : uploadBase.updateProgress, + + + /*--------------------------------------------------------------------------*/ + + /** copied from data-base.js, for uploadTable */ + + selectedDataset: 'new', + newDatasetName: '', + nameWarning: null, + selectedParent: '', + dataType: 'linear', + namespace: '', + + getDatasetId : uploadTable.getDatasetId, + isDupName : uploadTable.isDupName, + onNameChange : observer('newDatasetName', uploadTable.onNameChange), + onSelectChange : observer('selectedDataset', 'selectedParent', uploadTable.onSelectChange), + + /*--------------------------------------------------------------------------*/ + + actions : { + submitFile : uploadTable.submitFile + }, + + /*--------------------------------------------------------------------------*/ + dataMatrix : computed('data.[]', function () { let cells = this.get('data').map((r) => r.split('\t')); return cells; @@ -45,6 +112,12 @@ export default Component.extend({ // this.showTable(); }, + didReceiveAttrs() { + this._super(...arguments); + this.set('selectedParent', this.get('search.parent')); + this.set('namespace', this.get('search.parent') + ':blast'); + }, + /*--------------------------------------------------------------------------*/ /** Outcomes of the API search request. @@ -205,5 +278,44 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que table.updateSettings({data:[]}); }, + /*--------------------------------------------------------------------------*/ + + /** called by upload-table.js : onSelectChange() + * No validation of user input is required because table content is output from blast process. + */ + checkBlocks() { + }, + + + /** upload-table.js : submitFile() expects this function. + * In blast-results, the data is not user input so validation is not required. + */ + validateData() { + /** based on data-csv.js : validateData(), which uses table.getSourceData(); + * in this case sourceData is based on .dataMatrix instead + * of going via the table. + */ + return new Promise((resolve, reject) => { + let table = this.get('table'); + if (table === null) { + resolve([]); + } + let sourceData = this.get('dataMatrix'); + /** the last row is empty. */ + let validatedData = sourceData + .filter((row) => (row[c_name] !== '') && (row[c_chr])) + .map((row) => + ({ + name: row[c_name], + // blast output chromosome is e.g. 'chr2A'; Pretzel uses simply '2A'. + block: row[c_chr].replace(/^chr/,''), + // Make sure val is a number, not a string. + val: Number(row[c_pos]) + }) ); + resolve(validatedData); + }); + } + + /*--------------------------------------------------------------------------*/ }); diff --git a/frontend/app/styles/app.scss b/frontend/app/styles/app.scss index f7d8df068..f7ed557f2 100644 --- a/frontend/app/styles/app.scss +++ b/frontend/app/styles/app.scss @@ -1485,6 +1485,10 @@ div#left-panel-upload select margin-bottom: 5px; } +.blast-results { + margin: 1em; +} + .feature-list.active-input > div > div > div.from-input > a { background-color : #e8e8f7; } diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 17f31dc97..550e053cc 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -123,9 +123,12 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} {{#each this.searches as |search|}} - + {{panel/upload/blast-results search=search + datasets=datasets + refreshDatasets=refreshDatasets + viewDataset=viewDataset active=(bs-eq tab.activeId search.tabId ) }} diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index d6a326fc7..4ee686cde 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -29,6 +29,16 @@
{{!-- --------------------------------------------------------------------- --}} +{{!-- from data-csv --}} + +{{elem/panel-message + successMessage=successMessage + warningMessage=warningMessage + errorMessage=errorMessage}} +{{#if nameWarning}} +{{elem/panel-message + warningMessage=nameWarning}} +{{/if}} {{#if isProcessing}} {{#elem/panel-form @@ -38,5 +48,80 @@ {{/elem/panel-form}} {{/if}} + + +
+{{#if (eq selectedDataset 'new')}} +
+ {{input + id="dataset_new" + type="text" + value=newDatasetName + class="form-control" + placeholder="New dataset name..." + disabled=isProcessing + }} + +
+ + {{selectedParent}} + {{!-- + + --}} + +
+ + +
+ + {{input id="namespace" type="text" value=namespace disabled=isProcessing }} +
+{{/if}} + +{{!-- --------------------------------------------------------------------- --}} + + + {{input type="checkbox" name="viewDatasetFlag" checked=viewDatasetFlag }} + + + + +
+ + + +
+ +{{!-- --------------------------------------------------------------------- --}} +
diff --git a/frontend/app/utils/panel/upload-base.js b/frontend/app/utils/panel/upload-base.js new file mode 100644 index 000000000..dd7261404 --- /dev/null +++ b/frontend/app/utils/panel/upload-base.js @@ -0,0 +1,79 @@ + +/** + * factored from components/panel/upload/data-base.js + * May evolve this to a decorator or a sub-component. + * + * usage : + * + // file: null, // not required + isProcessing: false, + successMessage: null, + errorMessage: null, + warningMessage: null, + progressMsg: '', + + */ + +export default { + + + setProcessing() { + this.updateProgress(0, 'up'); + this.setProperties({ + isProcessing: true, + successMessage: null, + errorMessage: null, + warningMessage: null + }); + }, + setSuccess(msg) { + let response = msg ? msg : 'Uploaded successfully'; + /** .file is undefined (null) when data is read from table instead of from file */ + let file = this.get('file'); + if (file) + response += ` from file "${file.name}"`; + this.setProperties({ + isProcessing: false, + successMessage: response, + }); + }, + setError(msg) { + this.setProperties({ + isProcessing: false, + errorMessage: msg, + }); + }, + setWarning(msg) { + this.setProperties({ + isProcessing: false, + successMessage: null, + errorMessage: null, + warningMessage: msg, + }); + }, + clearMsgs() { + this.setProperties({ + successMessage: null, + errorMessage: null, + warningMessage: null, + }); + }, + + /** Callback used by data upload, to report progress percent updates */ + updateProgress(percentComplete, direction) { + if (direction === 'up') { + if (percentComplete === 100) { + this.set('progressMsg', 'Please wait. Updating database.'); + } else { + this.set('progressMsg', + 'Please wait. File upload in progress (' + + percentComplete.toFixed(0) + '%)' ); + } + } else { + this.set('progressMsg', + 'Please wait. Receiving result (' + percentComplete.toFixed(0) + '%)' ); + } + }, + + +}; diff --git a/frontend/app/utils/panel/upload-table.js b/frontend/app/utils/panel/upload-table.js new file mode 100644 index 000000000..1f23d13f3 --- /dev/null +++ b/frontend/app/utils/panel/upload-table.js @@ -0,0 +1,161 @@ +import { debounce, later as run_later } from '@ember/runloop'; +import { observer, computed } from '@ember/object'; + + +const dLog = console.debug; + +/*----------------------------------------------------------------------------*/ + + +/** + * factored from components/panel/upload/data-csv.js + * May evolve this to a decorator or a sub-component. + * + * usage / dependencies : object using this defines : + * services : store, auth + * attributes : + selectedDataset: 'new', + newDatasetName: '', + nameWarning: null, + selectedParent: '', + dataType: 'linear', + namespace: '', + + */ + +export default { + + /** Returns a selected dataset name OR + * Attempts to create a new dataset with entered name */ + getDatasetId() { + var that = this; + let datasets = that.get('datasets'); + return new Promise(function(resolve, reject) { + var selectedMap = that.get('selectDataset'); + // If a selected dataset, can simply return it + // If no selectedMap, treat as default, 'new' + if (selectedMap && selectedMap !== 'new') { + resolve(selectedMap); + } else { + var newMap = that.get('newDatasetName'); + // Check if duplicate name + let matched = datasets.findBy('name', newMap); + if(matched){ + reject({ msg: `Dataset name '${newMap}' is already in use` }); + } else { + let newDetails = { + name: newMap, + type: that.get('dataType'), + namespace: that.get('namespace'), + blocks: [] + }; + let parentId = that.get('selectedParent'); + if (parentId && parentId.length > 0) { + newDetails.parentName = parentId; + } + let newDataset = that.get('store').createRecord('Dataset', newDetails); + newDataset.save().then(() => { + resolve(newDataset.id); + }); + } + } + }); + }, + + + /** Checks if entered dataset name is already taken in dataset list + * Debounced call through observer */ + isDupName: function() { + let selectedMap = this.get('selectedDataset'); + if (selectedMap === 'new') { + let newMap = this.get('newDatasetName'); + let datasets = this.get('datasets'); + let matched = datasets.findBy('name', newMap); + if(matched){ + this.set('nameWarning', `Dataset name '${newMap}' is already in use`); + return true; + } + } + this.set('nameWarning', null); + return false; + }, + + onNameChange // : observer('newDatasetName', function), + () { + debounce(this, this.isDupName, 500); + }, + + onSelectChange // : observer('selectedDataset', 'selectedParent', function ), + () { + this.clearMsgs(); + this.isDupName(); + this.checkBlocks(); + }, + + submitFile() { + const fnName = 'submitFile'; + var that = this; + that.clearMsgs(); + that.set('nameWarning', null); + var table = that.get('table'); + // 1. Check data and get cleaned copy + that.validateData() + .then((features) => { + if (features.length > 0) { + // 2. Get new or selected dataset name + that.getDatasetId().then((map_id) => { + var data = { + dataset_id: map_id, + parentName: that.get('selectedParent'), + features: features, + namespace: that.get('namespace'), + }; + that.setProcessing(); + that.scrollToTop(); + // 3. Submit upload to api + that.get('auth').tableUpload(data, that.updateProgress.bind(that)) + .then((res) => { + that.setSuccess(res.status); + that.scrollToTop(); + // On complete, trigger dataset list reload + // through controller-level function + let refreshed = that.get('refreshDatasets')(); + /* as in sequence-search.js : dnaSequenceInput() */ + const viewDataset = this.get('viewDatasetFlag'); + if (viewDataset) { + refreshed + .then(() => { + let + datasetName = map_id; + dLog(fnName, 'viewDataset', datasetName); + this.get('viewDataset')(datasetName, viewDataset); + }); + } + + }, (err, status) => { + console.log(err, status); + that.setError(err.responseJSON.error.message); + that.scrollToTop(); + if(that.get('selectedDataset') === 'new'){ + // If upload failed and here, a new record for new dataset name + // has been created by getDatasetId() and this should be undone + that.get('store') + .findRecord('Dataset', map_id, { reload: true }) + .then((rec) => rec.destroyRecord() + .then(() => rec.unloadRecord()) + ); + } + }); + }, (err) => { + that.setError(err.msg || err.message); + that.scrollToTop(); + }); + } + }, (err) => { + table.selectCell(err.r, err.c); + that.setError(err.msg); + that.scrollToTop(); + }); + }, + +}; From 46f8c0c6a5bb65eb746fa4e02564846f5b98f374 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 12:23:06 +1000 Subject: [PATCH 17/77] sequence search : show table when results are received. blast-results.js : dataMatrixEffect add dependency : table. --- frontend/app/components/panel/upload/blast-results.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 727c60970..478f47a0b 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -101,7 +101,7 @@ export default Component.extend({ let cells = this.get('data').map((r) => r.split('\t')); return cells; }), - dataMatrixEffect : computed('dataMatrix.[]', function () { + dataMatrixEffect : computed('table', 'dataMatrix.[]', function () { let table = this.get('table'); if (table) { table.loadData(this.get('dataMatrix')); From 827d20f74e2393d8859087623c13e39aafad9ae5 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 12:23:40 +1000 Subject: [PATCH 18/77] sequence search and upload/data-csv : remove element ids dataset_new and id=namespace which are common and not required. remove id=dataset_new and id=namespace from sequence-search.hbs, blast-results.hbs, data-csv.hbs, because those ids are not used, and they clash because multiple concurrent elements have the same id. --- frontend/app/templates/components/panel/sequence-search.hbs | 1 - .../app/templates/components/panel/upload/blast-results.hbs | 3 +-- frontend/app/templates/components/panel/upload/data-csv.hbs | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 550e053cc..061c7602d 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -104,7 +104,6 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }}
{{input - id="dataset_new" type="text" value=newDatasetName class="form-control" diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 4ee686cde..0329c691d 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -63,7 +63,6 @@ {{#if (eq selectedDataset 'new')}}
{{input - id="dataset_new" type="text" value=newDatasetName class="form-control" @@ -99,7 +98,7 @@
- {{input id="namespace" type="text" value=namespace disabled=isProcessing }} + {{input type="text" value=namespace disabled=isProcessing }}
{{/if}} diff --git a/frontend/app/templates/components/panel/upload/data-csv.hbs b/frontend/app/templates/components/panel/upload/data-csv.hbs index 31ca707af..6d9ccb0a8 100644 --- a/frontend/app/templates/components/panel/upload/data-csv.hbs +++ b/frontend/app/templates/components/panel/upload/data-csv.hbs @@ -32,7 +32,6 @@ {{#if (eq selectedDataset 'new')}}
{{input - id="dataset_new" type="text" value=newDatasetName class="form-control" @@ -63,7 +62,7 @@
- {{input id="namespace" type="text" value=namespace disabled=isProcessing }} + {{input type="text" value=namespace disabled=isProcessing }}
{{/if}}
From d24584168edc4627a83988934b180ffaa88cf266 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 12:39:19 +1000 Subject: [PATCH 19/77] sequence search : handle initially undefined data blast-results.js : dataMatrix() : return [] while .data is initially undefined. --- frontend/app/components/panel/upload/blast-results.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 478f47a0b..9c2475555 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -98,7 +98,9 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ dataMatrix : computed('data.[]', function () { - let cells = this.get('data').map((r) => r.split('\t')); + let + data = this.get('data'), + cells = data ? data.map((r) => r.split('\t')) : []; return cells; }), dataMatrixEffect : computed('table', 'dataMatrix.[]', function () { From 679858ab36cc5003cee95fc5a5cb9492923bfcc7 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 15:32:30 +1000 Subject: [PATCH 20/77] blast-results : add end position to feature. dataset.js : tableUpload() : if feature.end is defined, append it to value[]. blast-results.js : validateData() : if row[c_end] is defined, pass it as feature.end. A similar change can now be made in data-csv.js : validateData() to support end position : perhaps if row.val contains '-' then split it into [pos start,end]. --- backend/common/models/dataset.js | 10 +++++++- .../components/panel/upload/blast-results.js | 24 ++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/common/models/dataset.js b/backend/common/models/dataset.js index 4d08d3a73..6e3328862 100644 --- a/backend/common/models/dataset.js +++ b/backend/common/models/dataset.js @@ -182,6 +182,10 @@ module.exports = function(Dataset) { } }; + /** + * @param data dataset, with .features with attributes : + * feature.name, .block (blockId), .val, .end (start, end position). + */ Dataset.tableUpload = function(data, options, cb) { var models = this.app.models; var blocks = {}; @@ -232,9 +236,13 @@ module.exports = function(Dataset) { }); var array_features = []; data.features.forEach(function(feature) { + let value = [feature.val]; + if (feature.end !== undefined) { + value.push(feature.end); + } array_features.push({ name: feature.name, - value: [feature.val], + value, value_0: feature.val, blockId: blocks_by_name[feature.block] }); diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 9c2475555..ce8c93d5f 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -24,7 +24,7 @@ const dLog = console.debug; const columnsKeyString = [ 'name', 'chr', 'pcIdentity', 'lengthOfHspHit', 'numMismatches', 'numGaps', 'queryStart', 'queryEnd', 'pos', 'end' ]; -const c_name = 0, c_chr = 1, c_pos = 8; +const c_name = 0, c_chr = 1, c_pos = 8, c_end = 9; /*----------------------------------------------------------------------------*/ @@ -303,17 +303,23 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que resolve([]); } let sourceData = this.get('dataMatrix'); - /** the last row is empty. */ - let validatedData = sourceData - .filter((row) => (row[c_name] !== '') && (row[c_chr])) - .map((row) => - ({ + /** the last row is empty, so it is filtered out. */ + let + validatedData = sourceData + .filter((row) => (row[c_name] !== '') && (row[c_chr])) + .map((row) => { + let feature = { name: row[c_name], - // blast output chromosome is e.g. 'chr2A'; Pretzel uses simply '2A'. - block: row[c_chr].replace(/^chr/,''), + // blast output chromosome has prefix 'chr' e.g. 'chr2A'; Pretzel uses simply '2A'. + block: row[c_chr].replace(/^chr/, ''), // Make sure val is a number, not a string. val: Number(row[c_pos]) - }) ); + }; + if (row[c_end] !== undefined) { + feature.end = Number(row[c_end]); + } + return feature; + }); resolve(validatedData); }); } From 696c304150a8f3575afb9f8cef720416cbd69130 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 16:41:17 +1000 Subject: [PATCH 21/77] sequence search : add a close button to blast-results tab sequence-search.js : add closeResultTab() sequence-search.hbs : in search nav.item, add button closeResultTab --- frontend/app/components/panel/sequence-search.js | 11 +++++++++++ .../templates/components/panel/sequence-search.hbs | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 9d68b9962..bfb8dde8e 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -231,6 +231,17 @@ export default Component.extend({ } }, + closeResultTab(tabId) { + dLog('closeResultTab', tabId); + let searches = this.get('searches'), + tab = searches.find((s) => s.tabId === tabId); + if (tab) { + searches.removeObject(tab); + } else { + dLog('closeResultTab', tabId, tab); + } + + } /*--------------------------------------------------------------------------*/ diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 061c7602d..7b2fe750b 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -14,7 +14,10 @@ {{elem/icon-base name="edit"}} Sequence Input {{#each this.searches as |search|}} - {{elem/icon-base name="arrow-right"}} Blast Output ({{search.timeId}}) + + + {{elem/icon-base name="arrow-right"}} Blast Output ({{search.timeId}}) + {{/each}} From 671432807532180b94a90dddebc5ea16ab742f37 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 19:41:52 +1000 Subject: [PATCH 22/77] blast results : collapse button, put table above the inputs blast-results.js : add tableVisible, and add it as dependency for activeEffect(). showTable() : check rootElement.parentElement to see if tableVisible has removed table DOM. blast-results.hbs : add x-toggle for tableVisible. factor showSearch x-toggle to form added elem/x-toggle. Drop padding to align with other toggle. Move search.tableId from bottom to top, wrapped in #if tableVisible. --- .../components/panel/upload/blast-results.js | 12 +++++-- .../templates/components/elem/x-toggle.hbs | 12 +++++++ .../components/panel/upload/blast-results.hbs | 36 ++++++++++++------- 3 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 frontend/app/templates/components/elem/x-toggle.hbs diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index ce8c93d5f..aa4bc5de8 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -44,6 +44,8 @@ export default Component.extend({ classNames: ['blast-results'], + /** true enables display of the table. */ + tableVisible : true, /** true enables display of the search inputs. */ showSearch : false, @@ -186,10 +188,14 @@ export default Component.extend({ * @desc * active is passed in from parent component sequence-search to indicate if * the tab containing this sub component is active. + * + * @param tableVisible x-toggle value which enables display of the table. + * The user may hide the table for easier viewing of the input + * parameters and button for adding the result as a dataset. */ - activeEffect : computed('active', function () { + activeEffect : computed('active', 'tableVisible', function () { let active = this.get('active'); - if (active) { + if (active && this.get('tableVisible')) { this.shownBsTab(); } }), @@ -202,7 +208,7 @@ export default Component.extend({ showTable() { // Ensure table is created when tab is shown let table = this.get('table'); - if (! table) { + if (! table || ! table.rootElement.parentElement) { this.createTable(); } else { // trigger rerender when tab is shown diff --git a/frontend/app/templates/components/elem/x-toggle.hbs b/frontend/app/templates/components/elem/x-toggle.hbs new file mode 100644 index 000000000..8120099a2 --- /dev/null +++ b/frontend/app/templates/components/elem/x-toggle.hbs @@ -0,0 +1,12 @@ +{{!-- x-toggle uses flex, so wrap it. --}} + +
+ {{#x-toggle + theme='light' + value=@value + onToggle=(action (mut @value)) + as |toggle| + }} + {{toggle.switch}} + {{/x-toggle}} +
diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 0329c691d..92917aa4d 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -4,18 +4,31 @@ {{!-- --------------------------------------------------------------------- --}} -{{!-- x-toggle uses flex, so wrap it. --}} -
-
- {{#x-toggle - theme='light' + +
+ {{elem/x-toggle value=tableVisible }} + {{#unless tableVisible}} + Show Table + {{/unless}} + +
+ + {{#if tableVisible}} +
+
+ {{/if}} +
+ + + + +
+{{!-- --------------------------------------------------------------------- --}} + +
+ {{elem/x-toggle value=showSearch - onToggle=(action (mut showSearch)) - as |toggle| }} - {{toggle.switch}} - {{/x-toggle}} -
Show Search Inputs
@@ -28,6 +41,7 @@ {{/if}}
+
{{!-- --------------------------------------------------------------------- --}} {{!-- from data-csv --}} @@ -122,5 +136,3 @@ {{!-- --------------------------------------------------------------------- --}} -
-
From d99cf5f3eddc983e993b6150a88668ac12b10a45 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 1 Jun 2021 23:32:35 +1000 Subject: [PATCH 23/77] blast results : toggle out table to dialog blast-results : add tableModal --- .../components/panel/upload/blast-results.js | 7 ++-- .../components/panel/upload/blast-results.hbs | 32 +++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index aa4bc5de8..0e6c710be 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -46,6 +46,8 @@ export default Component.extend({ /** true enables display of the table. */ tableVisible : true, + /** true enables display of the table in a modal dialog. */ + tableModal : true, /** true enables display of the search inputs. */ showSearch : false, @@ -114,6 +116,7 @@ export default Component.extend({ didRender() { // this.showTable(); + dLog('didRender', this.get('active'), this.get('tableVisible'), this.get('tableModal')); }, didReceiveAttrs() { @@ -193,7 +196,7 @@ export default Component.extend({ * The user may hide the table for easier viewing of the input * parameters and button for adding the result as a dataset. */ - activeEffect : computed('active', 'tableVisible', function () { + activeEffect : computed('active', 'tableVisible', 'tableModal', function () { let active = this.get('active'); if (active && this.get('tableVisible')) { this.shownBsTab(); @@ -208,7 +211,7 @@ export default Component.extend({ showTable() { // Ensure table is created when tab is shown let table = this.get('table'); - if (! table || ! table.rootElement.parentElement) { + if (! table || ! table.rootElement || ! table.rootElement.isConnected) { this.createTable(); } else { // trigger rerender when tab is shown diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 92917aa4d..48cc0ffb4 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -11,12 +11,38 @@ Show Table {{/unless}} -
+ {{#if this.active}} {{#if tableVisible}} -
-
+ + {{#if this.tableModal}} + {{#ember-modal-dialog title="Blast Results" header-icon='list'}} + + +
+ {{/ember-modal-dialog}} + {{else}} + + {{elem/button-base + classSize='xs' + classColour='default' + click=(action (mut this.tableModal) true ) + icon='new-window'}} + + +
+
+ {{/if}} +
+ + + {{/if}} {{/if}} +
From cdcdcf45126061120532530b01242b21f7936b21 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 2 Jun 2021 00:12:46 +1000 Subject: [PATCH 24/77] blast results : add tooltips for 2 buttons x-toggle.hbs : add yield so that e.g. ember-tooltip is within the toggle. blast-results.hbs : add tooltips for toggle value=tableVisible and button-base tableModal. --- .../app/templates/components/elem/x-toggle.hbs | 1 + .../components/panel/upload/blast-results.hbs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/frontend/app/templates/components/elem/x-toggle.hbs b/frontend/app/templates/components/elem/x-toggle.hbs index 8120099a2..c9879684e 100644 --- a/frontend/app/templates/components/elem/x-toggle.hbs +++ b/frontend/app/templates/components/elem/x-toggle.hbs @@ -8,5 +8,6 @@ as |toggle| }} {{toggle.switch}} + {{yield}} {{/x-toggle}}
diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 48cc0ffb4..e8bb61120 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -6,7 +6,11 @@
- {{elem/x-toggle value=tableVisible }} + {{#elem/x-toggle value=tableVisible }} + {{#ember-tooltip side="right" delay=500}} + Show/hide the table. + {{/ember-tooltip}} + {{/elem/x-toggle}} {{#unless tableVisible}} Show Table {{/unless}} @@ -27,11 +31,17 @@ {{/ember-modal-dialog}} {{else}} - {{elem/button-base + {{#elem/button-base classSize='xs' classColour='default' click=(action (mut this.tableModal) true ) icon='new-window'}} + + {{#ember-tooltip side="left" delay=500}} + Show the table in a modal dialog. + {{/ember-tooltip}} + + {{/elem/button-base}}
From 27e3a5d30e0465cb7b849ba4d6f8dd21e36c38d2 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 2 Jun 2021 20:42:05 +1000 Subject: [PATCH 25/77] blast results : display table rows as feature triangles ... works; MVP for discussing the features. axis-ticks-selected.js : add featureIsTransient(f), and use in featuresOfBlockLookup() : show the features of un-viewed blocks, when block is reference. blast-results.js : change tableModal true -> false (i.e. initially table is in left panel, not in modal dialog). add viewFeaturesFlag, dataFeatures() (factored from validateData()), blockNames(), viewFeaturesEffect(). paths-progressive.js : pushFeature() : default for param flowsService, return record found by peekRecord(). selected.js : toggle() : add param add : if true then add, if false then remove, if undefined then toggle (as before). add transient.js : with pushFeature(), pushData() (based on pathsPro : pushFeature), datasetForSearch(), pushDatasetArgs(), pushBlockArgs(), blocksForSearch(), showFeatures(). blast-results.hbs : use viewFeaturesEffect, add checkbox viewFeaturesFlag. --- .../components/draw/axis-ticks-selected.js | 33 ++++++ .../components/panel/upload/blast-results.js | 94 ++++++++++++--- .../app/services/data/paths-progressive.js | 6 + frontend/app/services/data/selected.js | 29 +++-- frontend/app/services/data/transient.js | 112 ++++++++++++++++++ .../components/panel/upload/blast-results.hbs | 5 + 6 files changed, 253 insertions(+), 26 deletions(-) create mode 100644 frontend/app/services/data/transient.js diff --git a/frontend/app/components/draw/axis-ticks-selected.js b/frontend/app/components/draw/axis-ticks-selected.js index 731cef286..2f7861970 100644 --- a/frontend/app/components/draw/axis-ticks-selected.js +++ b/frontend/app/components/draw/axis-ticks-selected.js @@ -16,6 +16,24 @@ const dLog = console.debug; const CompName = 'components/axis-ticks-selected'; +/*----------------------------------------------------------------------------*/ +/** @return true if feature's block is not viewed and its dataset + * has tag transient. + */ +function featureIsTransient(f) { + let isTransient = ! f.get('blockId.isViewed'); + if (isTransient) { + let d = f.get('blockId.datasetId'); + d = d.get('content') || d; + isTransient = d.hasTag('transient'); + } + return isTransient; +} + +/*----------------------------------------------------------------------------*/ + + + /** Display horizontal ticks on the axis to highlight the position of features * found using Feature Search. * @@ -201,6 +219,21 @@ export default Component.extend(AxisEvents, { if (clickedFeatures && clickedFeatures.length) { features = features.concat(clickedFeatures); } + /** not yet clear whether blast-results transient blocks should be + * viewed - that would introduce complications requiring API + * requests to be blocked when .datasetId.hasTag('transient') + * For the MVP, return the features of un-viewed blocks, when block is reference. + */ + if (! block.get('isData')) { + let clickedFeaturesByAxis = this.get('selected.clickedFeaturesByAxis'), + axisFeatures = clickedFeaturesByAxis && clickedFeaturesByAxis.get(block), + transientFeatures = axisFeatures && axisFeatures + .filter(featureIsTransient); + if (transientFeatures && transientFeatures.length) { + features = features.concat(transientFeatures); + } + } + if (trace) dLog('featuresOfBlockLookup', featuresInBlocks, block, blockId, features); return features; diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 0e6c710be..b3ad06b67 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -40,6 +40,7 @@ export default Component.extend({ */ store : alias('apiServers.primaryServer.store'), auth: service('auth'), + transient : service('data/transient'), classNames: ['blast-results'], @@ -47,12 +48,17 @@ export default Component.extend({ /** true enables display of the table. */ tableVisible : true, /** true enables display of the table in a modal dialog. */ - tableModal : true, + tableModal : false, /** true enables display of the search inputs. */ showSearch : false, - /** true means view the blocks of the dataset after it is added. */ + /** true means view the blocks of the dataset after it is added. + * Used in upload-table.js : submitFile(). + */ viewDatasetFlag : false, + /** true means display the result rows as triangles - clickedFeatures. */ + viewFeaturesFlag : true, + /*--------------------------------------------------------------------------*/ @@ -107,6 +113,38 @@ export default Component.extend({ cells = data ? data.map((r) => r.split('\t')) : []; return cells; }), + dataFeatures : computed('dataMatrix.[]', function () { + let data = this.get('dataMatrix'); + /** the last row is empty, so it is filtered out. */ + let features = + data + .filter((row) => (row[c_name] !== '') && (row[c_chr])) + .map((row) => { + let feature = { + name: row[c_name], + // blast output chromosome has prefix 'chr' e.g. 'chr2A'; Pretzel uses simply '2A'. + block: row[c_chr].replace(/^chr/, ''), + // Make sure val is a number, not a string. + val: Number(row[c_pos]) + }; + if (row[c_end] !== undefined) { + feature.end = Number(row[c_end]); + } + return feature; + }); + dLog('dataFeatures', features.length, features[0]); + return features; + }), + blockNames : computed('dataMatrix.[]', function () { + let data = this.get('dataMatrix'); + /** based on dataFeatures - see comments there. */ + let names = + data + .filter((row) => (row[c_name] !== '') && (row[c_chr])) + .map((row) => row[c_chr].replace(/^chr/, '')); + dLog('blockNames', names.length, names[0]); + return names; + }), dataMatrixEffect : computed('table', 'dataMatrix.[]', function () { let table = this.get('table'); if (table) { @@ -186,6 +224,40 @@ export default Component.extend({ } }, + /*--------------------------------------------------------------------------*/ + + viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', function () { + /** construct feature id from name + val (start position) because + * name is not unique, and in a blast search result, a feature + * name is associated with the sequence string to search for, and + * all features matching that sequence have the same name. + * We may use UUID (e.g. thaume/ember-cli-uuid or ivanvanderbyl/ember-uuid). + */ + let + features = this.get('dataFeatures') + .map((f) => ({ + _id : f.name + '-' + f.val, name : f.name, blockId : f.block, + value : [f.val, f.end]})); + if (features && features.length) { + let viewFeaturesFlag = this.get('viewFeaturesFlag'); + let + transient = this.get('transient'), + datasetName = this.get('newDatasetName') || 'blastResults', + namespace = this.get('namespace'), + dataset = transient.pushDatasetArgs( + datasetName, + this.get('search.parent'), + namespace + ); + let blocks = transient.blocksForSearch( + datasetName, + this.get('blockNames'), + namespace + ); + transient.showFeatures(dataset, blocks, features, viewFeaturesFlag); + } + }), + /*--------------------------------------------------------------------------*/ /** comments for activeEffect() and shownBsTab() in @see data-csv.js * @desc @@ -311,24 +383,8 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que if (table === null) { resolve([]); } - let sourceData = this.get('dataMatrix'); - /** the last row is empty, so it is filtered out. */ let - validatedData = sourceData - .filter((row) => (row[c_name] !== '') && (row[c_chr])) - .map((row) => { - let feature = { - name: row[c_name], - // blast output chromosome has prefix 'chr' e.g. 'chr2A'; Pretzel uses simply '2A'. - block: row[c_chr].replace(/^chr/, ''), - // Make sure val is a number, not a string. - val: Number(row[c_pos]) - }; - if (row[c_end] !== undefined) { - feature.end = Number(row[c_end]); - } - return feature; - }); + validatedData = this.get('dataFeatures'); resolve(validatedData); }); } diff --git a/frontend/app/services/data/paths-progressive.js b/frontend/app/services/data/paths-progressive.js index df7386f2a..875dd7f6e 100644 --- a/frontend/app/services/data/paths-progressive.js +++ b/frontend/app/services/data/paths-progressive.js @@ -281,13 +281,19 @@ export default Service.extend({ return promise; }, + /** Push the feature into the store if it is not already there. + * @return the record handle of the existing or added feature record. + * @param flowsService optional, defaults to this.get('flowsService'). + */ pushFeature(store, f, flowsService) { + flowsService ||= this.get('flowsService'); let c; let fr = store.peekRecord('feature', f._id); if (fr) { let verifyOK = verifyFeatureRecord(fr, f); if (! verifyOK) dLog('peekRecord feature', f._id, f, fr._internalModel.__data, fr); + c = fr; } else { diff --git a/frontend/app/services/data/selected.js b/frontend/app/services/data/selected.js index 9590f956f..b87c52349 100644 --- a/frontend/app/services/data/selected.js +++ b/frontend/app/services/data/selected.js @@ -9,6 +9,8 @@ import { contentOf } from '../../utils/common/promises'; const dLog = console.debug; +const trace = 1; + /** * for #223 : Selections and defining intervals */ @@ -40,16 +42,26 @@ export default Service.extend(Evented, { * This event is published here rather than in the component which receives * the click (axis-tracks : clickTrack) because the feature 'clicked' status * resides here. + * + * @param add undefined or boolean : if true then add, if false then remove */ - toggle(listName, feature) { + toggle(listName, feature, add) { let features = this.get(listName), - i = features.indexOf(feature); - dLog('clickFeature', listName, i, feature, features); - let added = i === -1; - if (added) { + i = features && features.indexOf(feature); + if (! features || trace /*> 1*/) { + dLog('clickFeature', listName, i, feature, features); + } + /** indicates that the feature was initially not in the list. */ + let absent = i === -1; + /** true / false indicates that the feature was added/removed to/from the list. + * undefined indicates no change. + */ + let added; + if (absent && (add !== false)) { features.pushObject(feature); - } else { + added = true; + } else if (! absent && (add !== true)) { features.removeAt(i, 1); if (listName === 'features') { /** clearing from .features is required to clear from the other 2 lists; @@ -62,8 +74,11 @@ export default Service.extend(Evented, { this.labelledFeatures.removeObject(feature); this.shiftClickedFeatures.removeObject(feature); } + added = false; + } + if (added !== undefined) { + this.trigger('toggleFeature', feature, added, listName); } - this.trigger('toggleFeature', feature, added, listName); }, clickFeature(feature) { this.toggle('features', feature); diff --git a/frontend/app/services/data/transient.js b/frontend/app/services/data/transient.js new file mode 100644 index 000000000..0c2908b13 --- /dev/null +++ b/frontend/app/services/data/transient.js @@ -0,0 +1,112 @@ +import { computed } from '@ember/object'; +import Service, { inject as service } from '@ember/service'; + +import { _internalModel_data } from '../../utils/ember-devel'; + + +const dLog = console.debug; + +const trace = 1; + +/** + * Transient data objects (Features / Blocks / Datasets) which are + * created in frontend and not persisted to server & database. May be + * added to store. + * + * Purpose : provide display of Features in similar ways to database + * features, e.g. clickedFeatures as triangles. + */ + +/** + * Related : services/data/selected.js + * + * for #239 : FASTA / DNA sequence search API, display results in frontend + * comment / section : multiple output tabs : + * - after viewing the added dataset : also put them into the feature search so they are highlighted + * - display table rows as Feature triangles + */ +export default Service.extend({ + // push to local store for now; could use primaryServer.store. + store: service(), + pathsPro : service('data/paths-progressive'), + selected : service('data/selected'), + + /*--------------------------------------------------------------------------*/ + + pushFeature(f) { + // pathsPro.pushFeature() will use default flowsService. + return this.get('pathsPro').pushFeature(this.get('store'), f, /*flowsService*/undefined); + }, + + /*--------------------------------------------------------------------------*/ + + pushData(store, modelName, d) { + let c; + let r = store.peekRecord(modelName, d._id); + if (r) { + // this can be a @param verifyFn : if (verifyFn) { verifyFn(d, r); } + if (modelName === 'dataset') { + // if ((r.parent !== d.parent) || (r.namespace !== d.namespace)) + dLog('peekRecord', modelName, d._id, d, r.get(_internalModel_data), r); + dLog(r.parent, d.parent, r.namespace, d.namespace); + } + } + else + { + d.id = d._id; + + // .name is primaryKey of dataset + let n = store.normalize(modelName, d); + c = store.push(n); + + // if (trace > 2) + dLog(c.get('id'), c.get(_internalModel_data)); + } + return c; + }, + + /** + * @param _id datasetName + */ + datasetForSearch(_id, parent, namespace) { + let + d = + { + name : _id, + _id, namespace, parent, + tags : [ 'transient' ], + meta : { paths : false } + }; + return d; + }, + + pushDatasetArgs(_id, parent, namespace) { + let + data = this.datasetForSearch(_id, parent, namespace), + store = this.get('store'), + record = this.pushData(store, 'dataset', data); + return record; + }, + + pushBlockArgs(datasetId, name, namespace) { + let + /** prefix _id with datasetId to make it unique enough. May use UUID. */ + data = {_id : /*datasetId + '-' +*/ name, scope : name, name, namespace, datasetId}, + store = this.get('store'), + record = this.pushData(store, 'block', data); + return record; + }, + blocksForSearch(datasetId, blockNames, namespace) { + let blocks = blockNames.map((name) => this.pushBlockArgs(datasetId, name, namespace)); + return blocks; + }, + + showFeatures(dataset, blocks, features, viewFeaturesFlag) { + let + selected = this.get('selected'), + // may pass dataset, blocks to pushFeature() + stored = features.map((f) => this.pushFeature(f)); + stored.forEach((feature) => selected.toggle('features', feature, viewFeaturesFlag)); + } + +}); diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index e8bb61120..4f03befe3 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -1,6 +1,7 @@ {{dataMatrixEffect}} {{resultEffect}} {{activeEffect}} +{{viewFeaturesEffect}} {{!-- --------------------------------------------------------------------- --}} @@ -159,6 +160,10 @@ + + {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} + +
From 360135488713832c99c6100e2be5104f1c8cff3a Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 2 Jun 2021 20:50:16 +1000 Subject: [PATCH 26/77] blast results : if showing feature triangles, view the parent / reference of the search blast-results.js : viewFeaturesEffect() : if viewFeaturesFlag, viewDataset(parentName) --- frontend/app/components/panel/upload/blast-results.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index b3ad06b67..657ffb441 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -227,6 +227,7 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', function () { + const fnName = 'viewFeaturesEffect'; /** construct feature id from name + val (start position) because * name is not unique, and in a blast search result, a feature * name is associated with the sequence string to search for, and @@ -240,6 +241,11 @@ export default Component.extend({ value : [f.val, f.end]})); if (features && features.length) { let viewFeaturesFlag = this.get('viewFeaturesFlag'); + if (viewFeaturesFlag) { + let parentName = this.get('search.parent'); + dLog(fnName, 'viewDataset', parentName); + this.get('viewDataset')(parentName, true); + } let transient = this.get('transient'), datasetName = this.get('newDatasetName') || 'blastResults', From d5270971ac00b637ef9789e67accc73254a6f3bb Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 2 Jun 2021 21:18:46 +1000 Subject: [PATCH 27/77] blast results : limit the parent axes viewed to those which have results in the blast-results.js : viewFeaturesEffect() : pass .blockNames to viewDataset. mapview.js : viewDataset() : add param blockNames : undefined or an array of names of blocks to affect. --- frontend/app/components/panel/upload/blast-results.js | 2 +- frontend/app/controllers/mapview.js | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 657ffb441..b410be3bc 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -244,7 +244,7 @@ export default Component.extend({ if (viewFeaturesFlag) { let parentName = this.get('search.parent'); dLog(fnName, 'viewDataset', parentName); - this.get('viewDataset')(parentName, true); + this.get('viewDataset')(parentName, true, this.get('blockNames')); } let transient = this.get('transient'), diff --git a/frontend/app/controllers/mapview.js b/frontend/app/controllers/mapview.js index 75a0d69d7..47e8f7e1d 100644 --- a/frontend/app/controllers/mapview.js +++ b/frontend/app/controllers/mapview.js @@ -358,14 +358,17 @@ export default Controller.extend(Evented, { * Unview is used when the datasets has been replaced by successful upload. * @param datasetName id * @param view true for view, false for unview + * @param blockNames undefined or an array of names of blocks to affect. */ - viewDataset(datasetName, view) { + viewDataset(datasetName, view, blockNames) { let store = this.get('apiServers').get('primaryServer').get('store'), dataset = store.peekRecord('dataset', datasetName); if (dataset) { let - blocksToChange = dataset.get('blocks').toArray().filter((b) => b.get('isViewed') !== view), + blocksToChange = dataset.get('blocks').toArray() + .filter((b) => (b.get('isViewed') !== view) && + (! blockNames || blockNames.indexOf(b.get('name')) !== -1) ), blockService = this.get('block'), blockIds = blocksToChange.map((b) => b.id); dLog('viewDataset', datasetName, view, blockIds); From ffc07f10abe6d13c9f62eb3124bb1c6c8ba33de0 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 3 Jun 2021 12:14:33 +1000 Subject: [PATCH 28/77] handle an undefined value draw-map.js : matchParentAndScope() : handle undefined block_. api-server.js : getDatasets() : add blockValuesCurrent to trace. --- frontend/app/components/draw-map.js | 4 ++-- frontend/app/components/service/api-server.js | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/draw-map.js b/frontend/app/components/draw-map.js index 13bbd7fd2..f8a7a2ba8 100644 --- a/frontend/app/components/draw-map.js +++ b/frontend/app/components/draw-map.js @@ -1379,8 +1379,8 @@ export default Component.extend(Evented, { * So adding parentNameMatch, and using b.get('referenceBlock') as fall-back; * this will be replaced anyway (axesBlocks, which uses block.referenceBlock). */ - parentMatch = block_.get('datasetId.content') === dataset.get('parent'), - parentNameMatch = dataset.get('parentName') === get(block_, 'datasetId.id'), + parentMatch = block_ && (block_.get('datasetId.content') === dataset.get('parent')), + parentNameMatch = block_ && (dataset.get('parentName') === get(block_, 'datasetId.id')), match = (block.scope == zd.scope) && (block.dataset.get('name') == parentName); dLog(key, trace_stack ? block : block.dataset.get('name'), match, parentMatch, parentNameMatch); match = match && (parentMatch || parentNameMatch); diff --git a/frontend/app/components/service/api-server.js b/frontend/app/components/service/api-server.js index 67ac88026..375fb2f8d 100644 --- a/frontend/app/components/service/api-server.js +++ b/frontend/app/components/service/api-server.js @@ -148,10 +148,14 @@ export default EmberObject.extend({ { /** change to : apiServers can do .on() of .evented() on task */ let datasetsBlocks = apiServers.get('datasetsBlocks'); - /** if TaskCancelation, no result, so don't replace previous result. */ + /** if TaskCancelation, no result, so don't replace previous result. + * If request failed because of e.g. comms, don't want to repeat so accept the undefined result. + * Can look at separating these 2 cases. + */ + let blockValuesCurrent; if ((! blockValues || ! blockValues.length) && - (datasetsBlocks[datasetsHandle] && datasetsBlocks[datasetsHandle].length)) { - dLog(fnName, 'TaskCancelation datasetsTask.then', blockValues); + ((blockValuesCurrent = datasetsBlocks[datasetsHandle]) && blockValuesCurrent.length)) { + dLog(fnName, 'TaskCancelation datasetsTask.then', blockValues, blockValuesCurrent.length); } else { datasetsBlocks[datasetsHandle] = blockValues; server.set("datasetsBlocks", blockValues); From cd92e0f37aaaf9704d8903ccf6f96d9c9a18f8d3 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 3 Jun 2021 20:17:41 +1000 Subject: [PATCH 29/77] sequence search : show triangles for the blast-results tab that is selected. axis-ticks-selected.js : renderTicksThrottle() : pass immediate=false to throttle() - gives time for clickedFeaturesByAxis to be changed by transient.showFeatures(). blast-results.js : viewFeaturesEffect() : depend on active, mask viewFeaturesFlag with active; wrap showFeatures with nowOrLater(). block.js : getSummary() : summaryTask : interpret task without .state as pending; handle featuresCounts which has no .idWidth, and possibly also length=1 and .min===.max, e.g. [ { _id: { min: 154414057, max: 154414057 }, count: 1 } ] (this occurred for a block with 1 feature, zoomed-out request, so no interval). transient.js : pushData() : return record found by peekRecord(), as done for pushFeature() in 27e3a5d3. ember-devel.js : add nowOrLater() --- .../components/draw/axis-ticks-selected.js | 8 ++++-- .../components/panel/upload/blast-results.js | 28 ++++++++++++++++--- frontend/app/services/auth.js | 9 +++--- frontend/app/services/data/block.js | 12 ++++---- frontend/app/services/data/transient.js | 1 + frontend/app/utils/ember-devel.js | 16 ++++++++++- 6 files changed, 58 insertions(+), 16 deletions(-) diff --git a/frontend/app/components/draw/axis-ticks-selected.js b/frontend/app/components/draw/axis-ticks-selected.js index 2f7861970..8ec71497c 100644 --- a/frontend/app/components/draw/axis-ticks-selected.js +++ b/frontend/app/components/draw/axis-ticks-selected.js @@ -195,8 +195,12 @@ export default Component.extend(AxisEvents, { if (trace) dLog('renderTicksThrottle', axisID); - /* see comments in axis-1d.js : renderTicksThrottle() re. throttle versus debounce */ - throttle(this, this.renderTicks, axisID, 500); + /* see comments in axis-1d.js : renderTicksDebounce() re. throttle versus debounce */ + /* pass immediate=false - gives time for clickedFeaturesByAxis to + * be changed by transient.showFeatures(). Could address this with + * a dependency on selected.clickedFeaturesByAxis + */ + throttle(this, this.renderTicks, axisID, 500, false); }, /** Lookup the given block in featuresInBlocks, and return its features which diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index b410be3bc..39805ace8 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -9,12 +9,14 @@ import config from '../../../config/environment'; import uploadBase from '../../../utils/panel/upload-base'; import uploadTable from '../../../utils/panel/upload-table'; +import { nowOrLater } from '../../../utils/ember-devel'; const dLog = console.debug; /* global Handsontable */ /* global $ */ + /*----------------------------------------------------------------------------*/ /** @@ -226,7 +228,7 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ - viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', function () { + viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', 'active', function () { const fnName = 'viewFeaturesEffect'; /** construct feature id from name + val (start position) because * name is not unique, and in a blast search result, a feature @@ -240,10 +242,11 @@ export default Component.extend({ _id : f.name + '-' + f.val, name : f.name, blockId : f.block, value : [f.val, f.end]})); if (features && features.length) { - let viewFeaturesFlag = this.get('viewFeaturesFlag'); + /** Only view features of the active tab. */ + let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); if (viewFeaturesFlag) { let parentName = this.get('search.parent'); - dLog(fnName, 'viewDataset', parentName); + dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); this.get('viewDataset')(parentName, true, this.get('blockNames')); } let @@ -260,7 +263,24 @@ export default Component.extend({ this.get('blockNames'), namespace ); - transient.showFeatures(dataset, blocks, features, viewFeaturesFlag); + /** When changing between 2 blast-results tabs, this function will be + * called for both. + * + * Ensure that for the tab which is becoming active, + * showFeatures is called after the call for the tab becoming in-active, + * so that the inactive tab's features are removed from selected features + * before the active tab's features are added, so that in the case of + * overlap, the features remain displayed. Perhaps formalise the sequence + * of this transition, as the functionality evolves. + * + * Modifies selected.features, and hence updates clickedFeaturesByAxis, + * which is used by axis-ticks-selected.js : featuresOfBlockLookup(); + * renderTicksThrottle() uses throttle( immediate=false ) to allow time + * for this update. + */ + nowOrLater( + viewFeaturesFlag, + () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag)); } }), diff --git a/frontend/app/services/auth.js b/frontend/app/services/auth.js index 13e21128b..981aa4e53 100644 --- a/frontend/app/services/auth.js +++ b/frontend/app/services/auth.js @@ -538,11 +538,11 @@ function blockIdMap(data, mapFns) { // so manually spread the object : blockA = data.blockA, blockB = data.blockB, - blockIds = data.blockIds, + blockIds = data.blockIds || data.blocks, restFields = Object.keys(data).filter( - (f) => ['blockA', 'blockB', 'blockIds'].indexOf(f) === -1), + (f) => ['blockA', 'blockB', 'blockIds', 'blocks'].indexOf(f) === -1), d = restFields.reduce((result, f) => {result[f] = data[f]; return result;}, {}), - /** ab is true if data contains .blockA,B, false if .blockIds */ + /** ab is true if data contains .blockA,B, false if .blockIds or .blocks */ ab = !!blockA; console.log('blockIdMap', data, blockA, blockB, blockIds, restFields, d, ab); if ((!blockA !== !blockB) || (!blockA === !blockIds)) { @@ -559,7 +559,8 @@ function blockIdMap(data, mapFns) { if (ab) { blockIds = [blockA, blockB]; } - blockIds = blockIds.map((blockId, i) => mapFns[i](blockId)); + /** if blockIds.length > mapFns.length, then mapFns[0] === mapFns[1], so just use mapFns[0]. */ + blockIds = blockIds.map((blockId, i) => (mapFns[i] || mapFns[0])(blockId)); if (ab) [d.blockA, d.blockB] = blockIds; else diff --git a/frontend/app/services/data/block.js b/frontend/app/services/data/block.js index dc5932421..f76cf4eb2 100644 --- a/frontend/app/services/data/block.js +++ b/frontend/app/services/data/block.js @@ -436,7 +436,7 @@ export default Service.extend(Evented, { let taskId = blockId + '_' + nBins + (zoomedDomainText || ''); let summaryTask = this.get('summaryTask'); let p; - if ((p = summaryTask[blockId]) && (p.state() === "pending")) { + if ((p = summaryTask[blockId]) && (! p.state || p.state() === "pending")) { // state() : pending ~ readyState : 1 dLog('getSummary current', blockId, p, p.readyState); } else if ((p = summaryTask[taskId])) { @@ -503,10 +503,12 @@ export default Service.extend(Evented, { /** featuresCounts[0] may be undefined because of delete * featuresCounts[i] ~ outsideBoundaries */ - let binSize = featuresCounts && featuresCounts.length && featuresCounts[0] ? - featuresCounts[0].idWidth[0] : - intervalSize(interval) / nBins, - result = {binSize, nBins, domain : interval, result : featuresCounts}; + let + f0 = featuresCounts && featuresCounts.length && featuresCounts[0], + binSize = f0 ? + (f0.idWidth ? f0.idWidth[0] : ((f0.max - f0.min) || 1)) : + (interval ? (0, intervalSize)(interval) / nBins : 1), + result = {binSize, nBins, domain : interval, result : featuresCounts}; block.featuresCountsResultsMergeOrAppend(result); block.set('featuresCounts', featuresCounts); } diff --git a/frontend/app/services/data/transient.js b/frontend/app/services/data/transient.js index 0c2908b13..7a2ca533c 100644 --- a/frontend/app/services/data/transient.js +++ b/frontend/app/services/data/transient.js @@ -50,6 +50,7 @@ export default Service.extend({ dLog('peekRecord', modelName, d._id, d, r.get(_internalModel_data), r); dLog(r.parent, d.parent, r.namespace, d.namespace); } + c = r; } else { diff --git a/frontend/app/utils/ember-devel.js b/frontend/app/utils/ember-devel.js index 8433009fb..70ab9b2c5 100644 --- a/frontend/app/utils/ember-devel.js +++ b/frontend/app/utils/ember-devel.js @@ -1,3 +1,5 @@ +import { later as run_later } from '@ember/runloop'; + /* global Ember */ /*----------------------------------------------------------------------------*/ @@ -42,7 +44,19 @@ function getAttrOrCP(object, attrName) { const _internalModel_data = '_internalModel._recordData.__data'; +/*----------------------------------------------------------------------------*/ + +/** Run the function now or later. + * @param later if true, then run fn in Ember.run.later() + */ +function nowOrLater(later, fn) { + if (later) { + run_later(fn); + } else { + fn(); + } +} /*----------------------------------------------------------------------------*/ -export { parentOfType, elt0, getAttrOrCP, _internalModel_data }; +export { parentOfType, elt0, getAttrOrCP, _internalModel_data, nowOrLater }; From c0077b4c4a60a73ac4ad626dfd5fcd9629f099b7 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 3 Jun 2021 22:00:07 +1000 Subject: [PATCH 30/77] blast results : show feature name as label axis-ticks-selected.js : factor transientFeaturesLookup() from featuresOfBlockLookup(), to use also in selectedFeaturesOfBlockLookup(). transient.js : showFeatures() : also add features to labelledFeatures. add arrays.js, with arraysConcat(). --- .../components/draw/axis-ticks-selected.js | 44 ++++++++++++++----- frontend/app/services/data/transient.js | 5 ++- frontend/app/utils/common/arrays.js | 15 +++++++ 3 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 frontend/app/utils/common/arrays.js diff --git a/frontend/app/components/draw/axis-ticks-selected.js b/frontend/app/components/draw/axis-ticks-selected.js index 8ec71497c..a88e0a488 100644 --- a/frontend/app/components/draw/axis-ticks-selected.js +++ b/frontend/app/components/draw/axis-ticks-selected.js @@ -8,6 +8,7 @@ import { task, timeout, didCancel } from 'ember-concurrency'; import AxisEvents from '../../utils/draw/axis-events'; import { transitionEndPromise } from '../../utils/draw/d3-svg'; +import { arraysConcat } from '../../utils/common/arrays'; /* global d3 */ @@ -30,6 +31,7 @@ function featureIsTransient(f) { return isTransient; } + /*----------------------------------------------------------------------------*/ @@ -223,26 +225,43 @@ export default Component.extend(AxisEvents, { if (clickedFeatures && clickedFeatures.length) { features = features.concat(clickedFeatures); } + let transientFeatures = this.transientFeaturesLookup(block, 'clickedFeatures'); + features = arraysConcat(features, transientFeatures); + + if (trace) + dLog('featuresOfBlockLookup', featuresInBlocks, block, blockId, features); + return features; + }, + /** If block is a reference block, lookup features of the axis. + * The results of sequence-search : blast-results are currently + * added to selected.features (i.e. clickedFeatures) and + * selected.labelledFeatures and their block is not viewed, so + * associate them with the reference block / axis. + * @return undefined if transientFeatures.length is 0, + * so the caller can do : + * if (transientFeatures) { + * features = features.concat(transientFeatures); + * } + */ + transientFeaturesLookup(block, listName) { + let transientFeatures; /** not yet clear whether blast-results transient blocks should be * viewed - that would introduce complications requiring API * requests to be blocked when .datasetId.hasTag('transient') * For the MVP, return the features of un-viewed blocks, when block is reference. */ if (! block.get('isData')) { - let clickedFeaturesByAxis = this.get('selected.clickedFeaturesByAxis'), - axisFeatures = clickedFeaturesByAxis && clickedFeaturesByAxis.get(block), - transientFeatures = axisFeatures && axisFeatures - .filter(featureIsTransient); - if (transientFeatures && transientFeatures.length) { - features = features.concat(transientFeatures); + let featuresByAxis = this.get('selected.' + listName + 'ByAxis'), + axisFeatures = featuresByAxis && featuresByAxis.get(block); + transientFeatures = axisFeatures && axisFeatures + .filter(featureIsTransient); + if (transientFeatures && ! transientFeatures.length) { + transientFeatures = undefined; } } - - if (trace) - dLog('featuresOfBlockLookup', featuresInBlocks, block, blockId, features); - return features; + return transientFeatures; }, - + /** Lookup selected.labelledFeatures for the given block. * @param listName name of set / selection / group of features : * 'clickedFeatures', 'labelledFeatures', or 'shiftClickedFeatures' @@ -252,6 +271,9 @@ export default Component.extend(AxisEvents, { let map = this.get('selected.' + listName + 'ByBlock'), features = map && map.get(block); + let transientFeatures = this.transientFeaturesLookup(block, listName); + features = arraysConcat(features, transientFeatures); + if (trace) dLog('selectedFeaturesOfBlockLookup', listName, this.featuresInBlocks, block, block.id, features); return features; diff --git a/frontend/app/services/data/transient.js b/frontend/app/services/data/transient.js index 7a2ca533c..c18ae2e84 100644 --- a/frontend/app/services/data/transient.js +++ b/frontend/app/services/data/transient.js @@ -107,7 +107,10 @@ export default Service.extend({ selected = this.get('selected'), // may pass dataset, blocks to pushFeature() stored = features.map((f) => this.pushFeature(f)); - stored.forEach((feature) => selected.toggle('features', feature, viewFeaturesFlag)); + stored.forEach((feature) => { + selected.toggle('features', feature, viewFeaturesFlag); + selected.toggle('labelledFeatures', feature, viewFeaturesFlag); + }); } }); diff --git a/frontend/app/utils/common/arrays.js b/frontend/app/utils/common/arrays.js new file mode 100644 index 000000000..1dba39dda --- /dev/null +++ b/frontend/app/utils/common/arrays.js @@ -0,0 +1,15 @@ +/*----------------------------------------------------------------------------*/ + +/** @return array resulting from concatenating the given arrays a,b + * @param a,b if either is undefined or [], then the result is the other array. + * (this avoids unnecessary re-creation of arrays, and retains the + * array reference where there is no change e.g. for CP result.) + */ +function arraysConcat(a, b) { + let c = (a && a.length) ? ((b && b.length) ? a.concat(b) : a) : b; + return c; +}; + +/*----------------------------------------------------------------------------*/ + +export { arraysConcat }; From d2e81e7bebe79a7582664e0e4a2563157642e423 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 4 Jun 2021 18:58:41 +1000 Subject: [PATCH 31/77] retain param name .blocks auth.js : blockIdMap() : add arrayName to record the param name in the input data for array of blockIds, and use that in the result d. This follows on from change in cd92e0f3 : Add blocks to list of params recognised as blockId; auth.js : blockIdMap() : also use data.blocks for blockIds. --- frontend/app/services/auth.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/app/services/auth.js b/frontend/app/services/auth.js index 981aa4e53..6a6c79e5c 100644 --- a/frontend/app/services/auth.js +++ b/frontend/app/services/auth.js @@ -539,6 +539,7 @@ function blockIdMap(data, mapFns) { blockA = data.blockA, blockB = data.blockB, blockIds = data.blockIds || data.blocks, + arrayName = (data.blockIds && 'blockIds') || (data.blocks && 'blocks'), restFields = Object.keys(data).filter( (f) => ['blockA', 'blockB', 'blockIds', 'blocks'].indexOf(f) === -1), d = restFields.reduce((result, f) => {result[f] = data[f]; return result;}, {}), @@ -563,9 +564,10 @@ function blockIdMap(data, mapFns) { blockIds = blockIds.map((blockId, i) => (mapFns[i] || mapFns[0])(blockId)); if (ab) [d.blockA, d.blockB] = blockIds; - else - d.blockIds = blockIds; - console.log('blockIdMap', d); + else { + d[arrayName] = blockIds; + } + console.log('blockIdMap', d, ab, arrayName); return d; } From ae5be357cf3424cdd9e8c9261aeb2c321a727c97 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 4 Jun 2021 21:21:57 +1000 Subject: [PATCH 32/77] blast-results : split out blast-results-view ... in preparation for adding row selection via checkboxes, etc. Move from blast-results to blast-results-view : columnsKeyString, viewFeaturesFlag, containerStyle (factored from .hbs to a CP), dataMatrix, dataFeatures, blockNames, dataMatrixEffect, didRender(), viewFeaturesEffect, activeEffect, shownBsTab(), showTable(), createTable(), clearTable(), validateData() --- .../panel/upload/blast-results-view.js | 302 ++++++++++++++++++ .../components/panel/upload/blast-results.js | 257 --------------- .../panel/upload/blast-results-view.hbs | 16 + .../components/panel/upload/blast-results.hbs | 18 +- 4 files changed, 326 insertions(+), 267 deletions(-) create mode 100644 frontend/app/components/panel/upload/blast-results-view.js create mode 100644 frontend/app/templates/components/panel/upload/blast-results-view.hbs diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js new file mode 100644 index 000000000..93be0f47b --- /dev/null +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -0,0 +1,302 @@ +import Component from '@ember/component'; +import { observer, computed } from '@ember/object'; +import { inject as service } from '@ember/service'; +import { later as run_later } from '@ember/runloop'; + +import { alias } from '@ember/object/computed'; + +import config from '../../../config/environment'; +import { nowOrLater } from '../../../utils/ember-devel'; + + +/* global Handsontable */ + +/*----------------------------------------------------------------------------*/ + +const dLog = console.debug; + +/*----------------------------------------------------------------------------*/ + +/** + * based on backend/scripts/dnaSequenceSearch.bash : columnsKeyString + * This also aligns with createTable() : colHeaders below. + */ +const columnsKeyString = [ + 'name', 'chr', 'pcIdentity', 'lengthOfHspHit', 'numMismatches', 'numGaps', 'queryStart', 'queryEnd', 'pos', 'end' +]; +const c_name = 0, c_chr = 1, c_pos = 8, c_end = 9; + + +/** Display a table of results from sequence-search API request + * /Feature/dnaSequenceSearch + * @param search search inputs and status + * @param active true if the tab containing this component is active + * @param tableModal true if this component is displayed in a modal dialog + * This enables the full width of the table to be visible. + */ +export default Component.extend({ + + /** Similar comment to data-csv.js applies re. store (user could select server via GUI). + * store is used by upload-table.js : getDatasetId() and submitFile() + */ + store : alias('apiServers.primaryServer.store'), + auth: service('auth'), + transient : service('data/transient'), + + /*--------------------------------------------------------------------------*/ + + /** true means display the result rows as triangles - clickedFeatures. */ + viewFeaturesFlag : true, + + /*--------------------------------------------------------------------------*/ + + /** style of the div which contains the table. + * If this component is in a modal dialog, use most of screen width. + */ + containerStyle : computed('tableModal', function () { + return this.get('tableModal') ? 'overflow-x:hidden; width:70vw' : undefined; + }), + + /*--------------------------------------------------------------------------*/ + + dataMatrix : computed('data.[]', function () { + let + data = this.get('data'), + cells = data ? data.map((r) => r.split('\t')) : []; + return cells; + }), + dataFeatures : computed('dataMatrix.[]', function () { + let data = this.get('dataMatrix'); + /** the last row is empty, so it is filtered out. */ + let features = + data + .filter((row) => (row[c_name] !== '') && (row[c_chr])) + .map((row) => { + let feature = { + name: row[c_name], + // blast output chromosome has prefix 'chr' e.g. 'chr2A'; Pretzel uses simply '2A'. + block: row[c_chr].replace(/^chr/, ''), + // Make sure val is a number, not a string. + val: Number(row[c_pos]) + }; + if (row[c_end] !== undefined) { + feature.end = Number(row[c_end]); + } + return feature; + }); + dLog('dataFeatures', features.length, features[0]); + return features; + }), + blockNames : computed('dataMatrix.[]', function () { + let data = this.get('dataMatrix'); + /** based on dataFeatures - see comments there. */ + let names = + data + .filter((row) => (row[c_name] !== '') && (row[c_chr])) + .map((row) => row[c_chr].replace(/^chr/, '')); + dLog('blockNames', names.length, names[0]); + return names; + }), + dataMatrixEffect : computed('table', 'dataMatrix.[]', function () { + let table = this.get('table'); + if (table) { + table.loadData(this.get('dataMatrix')); + } + }), + + /*--------------------------------------------------------------------------*/ + + didRender() { + // this.showTable(); + dLog('didRender', this.get('active'), this.get('tableVisible'), this.get('tableModal')); + }, + + /*--------------------------------------------------------------------------*/ + + + viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', 'active', function () { + const fnName = 'viewFeaturesEffect'; + /** construct feature id from name + val (start position) because + * name is not unique, and in a blast search result, a feature + * name is associated with the sequence string to search for, and + * all features matching that sequence have the same name. + * We may use UUID (e.g. thaume/ember-cli-uuid or ivanvanderbyl/ember-uuid). + */ + let + features = this.get('dataFeatures') + .map((f) => ({ + _id : f.name + '-' + f.val, name : f.name, blockId : f.block, + value : [f.val, f.end]})); + if (features && features.length) { + /** Only view features of the active tab. */ + let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); + if (viewFeaturesFlag) { + let parentName = this.get('search.parent'); + dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); + this.get('viewDataset')(parentName, true, this.get('blockNames')); + } + let + transient = this.get('transient'), + datasetName = this.get('newDatasetName') || 'blastResults', + namespace = this.get('namespace'), + dataset = transient.pushDatasetArgs( + datasetName, + this.get('search.parent'), + namespace + ); + let blocks = transient.blocksForSearch( + datasetName, + this.get('blockNames'), + namespace + ); + /** When changing between 2 blast-results tabs, this function will be + * called for both. + * + * Ensure that for the tab which is becoming active, + * showFeatures is called after the call for the tab becoming in-active, + * so that the inactive tab's features are removed from selected features + * before the active tab's features are added, so that in the case of + * overlap, the features remain displayed. Perhaps formalise the sequence + * of this transition, as the functionality evolves. + * + * Modifies selected.features, and hence updates clickedFeaturesByAxis, + * which is used by axis-ticks-selected.js : featuresOfBlockLookup(); + * renderTicksThrottle() uses throttle( immediate=false ) to allow time + * for this update. + */ + nowOrLater( + viewFeaturesFlag, + () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag)); + } + }), + + /*--------------------------------------------------------------------------*/ + /** comments for activeEffect() and shownBsTab() in @see data-csv.js + * @desc + * active is passed in from parent component sequence-search to indicate if + * the tab containing this sub component is active. + * + * @param tableVisible x-toggle value which enables display of the table. + * The user may hide the table for easier viewing of the input + * parameters and button for adding the result as a dataset. + */ + activeEffect : computed('active', 'tableVisible', 'tableModal', function () { + let active = this.get('active'); + if (active && this.get('tableVisible')) { + this.shownBsTab(); + } + }), + shownBsTab() { + run_later(() => this.showTable(), 500); + }, + /*--------------------------------------------------------------------------*/ + + + showTable() { + // Ensure table is created when tab is shown + let table = this.get('table'); + if (! table || ! table.rootElement || ! table.rootElement.isConnected) { + this.createTable(); + } else { + // trigger rerender when tab is shown + table.updateSettings({}); + // or .updateSettings({ data : ... }) + table.loadData(this.get('dataMatrix')); + } + }, + + createTable() { + const cName = 'upload/blast-results'; + const fnName = 'createTable'; + dLog('createTable'); + $(() => { + let eltId = this.search.tableId; + let hotable = $('#' + eltId)[0]; + if (! hotable) { + console.warn(cName, fnName, ' : #', eltId, ' not found', this); + return; // fail + } + /** +blast output columns are +query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, query start, query end, subject start, subject end, e-value, score, query length, subject length + */ + var table = new Handsontable(hotable, { + data: [['', '', '', '', '', '', '', '', '', '', '', '', '', '']], + // minRows: 20, + rowHeaders: true, + /* + columns: [ + { + data: 'name', + type: 'text' + }, + { + data: 'block', + type: 'text' + }, + { + data: 'val', + type: 'numeric', + numericFormat: { + pattern: '0,0.*' + } + } + ], + */ + colHeaders: [ + 'query ID', 'subject ID', '% identity', 'length of HSP (hit)', '# mismatches', '# gaps', 'query start', 'query end', 'subject start', 'subject end', 'e-value', 'score', 'query length', 'subject length' + ], + height: 500, + // colWidths: [100, 100, 100], + manualRowResize: true, + manualColumnResize: true, + manualRowMove: true, + manualColumnMove: true, + contextMenu: true, + /* + afterChange: function() { + }, + afterRemoveRow: function() { + }, + */ + /* see comment re. handsOnTableLicenseKey in frontend/config/environment.js */ + licenseKey: config.handsOnTableLicenseKey + }); + this.set('table', table); + + }); + }, + + + clearTable() { + var table = this.get('table'); + table.updateSettings({data:[]}); + }, + + /*--------------------------------------------------------------------------*/ + + + /** upload-table.js : submitFile() expects this function. + * In blast-results, the data is not user input so validation is not required. + */ + validateData() { + /** based on data-csv.js : validateData(), which uses table.getSourceData(); + * in this case sourceData is based on .dataMatrix instead + * of going via the table. + */ + return new Promise((resolve, reject) => { + let table = this.get('table'); + if (table === null) { + resolve([]); + } + let + validatedData = this.get('dataFeatures'); + resolve(validatedData); + }); + } + + + + /*--------------------------------------------------------------------------*/ + +}); diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 39805ace8..cc2d7d327 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -1,33 +1,15 @@ import Component from '@ember/component'; import { observer, computed } from '@ember/object'; -import { alias } from '@ember/object/computed'; import { inject as service } from '@ember/service'; -import { later as run_later } from '@ember/runloop'; - -import config from '../../../config/environment'; import uploadBase from '../../../utils/panel/upload-base'; import uploadTable from '../../../utils/panel/upload-table'; -import { nowOrLater } from '../../../utils/ember-devel'; - const dLog = console.debug; -/* global Handsontable */ /* global $ */ -/*----------------------------------------------------------------------------*/ - -/** - * based on backend/scripts/dnaSequenceSearch.bash : columnsKeyString - * This also aligns with createTable() : colHeaders below. - */ -const columnsKeyString = [ - 'name', 'chr', 'pcIdentity', 'lengthOfHspHit', 'numMismatches', 'numGaps', 'queryStart', 'queryEnd', 'pos', 'end' -]; -const c_name = 0, c_chr = 1, c_pos = 8, c_end = 9; - /*----------------------------------------------------------------------------*/ @@ -37,12 +19,6 @@ const c_name = 0, c_chr = 1, c_pos = 8, c_end = 9; export default Component.extend({ apiServers: service(), blockService : service('data/block'), - /** Similar comment to data-csv.js applies re. store (user could select server via GUI). - * store is used by upload-table.js : getDatasetId() and submitFile() - */ - store : alias('apiServers.primaryServer.store'), - auth: service('auth'), - transient : service('data/transient'), classNames: ['blast-results'], @@ -58,8 +34,6 @@ export default Component.extend({ * Used in upload-table.js : submitFile(). */ viewDatasetFlag : false, - /** true means display the result rows as triangles - clickedFeatures. */ - viewFeaturesFlag : true, /*--------------------------------------------------------------------------*/ @@ -109,56 +83,6 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ - dataMatrix : computed('data.[]', function () { - let - data = this.get('data'), - cells = data ? data.map((r) => r.split('\t')) : []; - return cells; - }), - dataFeatures : computed('dataMatrix.[]', function () { - let data = this.get('dataMatrix'); - /** the last row is empty, so it is filtered out. */ - let features = - data - .filter((row) => (row[c_name] !== '') && (row[c_chr])) - .map((row) => { - let feature = { - name: row[c_name], - // blast output chromosome has prefix 'chr' e.g. 'chr2A'; Pretzel uses simply '2A'. - block: row[c_chr].replace(/^chr/, ''), - // Make sure val is a number, not a string. - val: Number(row[c_pos]) - }; - if (row[c_end] !== undefined) { - feature.end = Number(row[c_end]); - } - return feature; - }); - dLog('dataFeatures', features.length, features[0]); - return features; - }), - blockNames : computed('dataMatrix.[]', function () { - let data = this.get('dataMatrix'); - /** based on dataFeatures - see comments there. */ - let names = - data - .filter((row) => (row[c_name] !== '') && (row[c_chr])) - .map((row) => row[c_chr].replace(/^chr/, '')); - dLog('blockNames', names.length, names[0]); - return names; - }), - dataMatrixEffect : computed('table', 'dataMatrix.[]', function () { - let table = this.get('table'); - if (table) { - table.loadData(this.get('dataMatrix')); - } - }), - - didRender() { - // this.showTable(); - dLog('didRender', this.get('active'), this.get('tableVisible'), this.get('tableModal')); - }, - didReceiveAttrs() { this._super(...arguments); this.set('selectedParent', this.get('search.parent')); @@ -228,193 +152,12 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ - viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', 'active', function () { - const fnName = 'viewFeaturesEffect'; - /** construct feature id from name + val (start position) because - * name is not unique, and in a blast search result, a feature - * name is associated with the sequence string to search for, and - * all features matching that sequence have the same name. - * We may use UUID (e.g. thaume/ember-cli-uuid or ivanvanderbyl/ember-uuid). - */ - let - features = this.get('dataFeatures') - .map((f) => ({ - _id : f.name + '-' + f.val, name : f.name, blockId : f.block, - value : [f.val, f.end]})); - if (features && features.length) { - /** Only view features of the active tab. */ - let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); - if (viewFeaturesFlag) { - let parentName = this.get('search.parent'); - dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); - this.get('viewDataset')(parentName, true, this.get('blockNames')); - } - let - transient = this.get('transient'), - datasetName = this.get('newDatasetName') || 'blastResults', - namespace = this.get('namespace'), - dataset = transient.pushDatasetArgs( - datasetName, - this.get('search.parent'), - namespace - ); - let blocks = transient.blocksForSearch( - datasetName, - this.get('blockNames'), - namespace - ); - /** When changing between 2 blast-results tabs, this function will be - * called for both. - * - * Ensure that for the tab which is becoming active, - * showFeatures is called after the call for the tab becoming in-active, - * so that the inactive tab's features are removed from selected features - * before the active tab's features are added, so that in the case of - * overlap, the features remain displayed. Perhaps formalise the sequence - * of this transition, as the functionality evolves. - * - * Modifies selected.features, and hence updates clickedFeaturesByAxis, - * which is used by axis-ticks-selected.js : featuresOfBlockLookup(); - * renderTicksThrottle() uses throttle( immediate=false ) to allow time - * for this update. - */ - nowOrLater( - viewFeaturesFlag, - () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag)); - } - }), - - /*--------------------------------------------------------------------------*/ - /** comments for activeEffect() and shownBsTab() in @see data-csv.js - * @desc - * active is passed in from parent component sequence-search to indicate if - * the tab containing this sub component is active. - * - * @param tableVisible x-toggle value which enables display of the table. - * The user may hide the table for easier viewing of the input - * parameters and button for adding the result as a dataset. - */ - activeEffect : computed('active', 'tableVisible', 'tableModal', function () { - let active = this.get('active'); - if (active && this.get('tableVisible')) { - this.shownBsTab(); - } - }), - shownBsTab() { - run_later(() => this.showTable(), 500); - }, - /*--------------------------------------------------------------------------*/ - - - showTable() { - // Ensure table is created when tab is shown - let table = this.get('table'); - if (! table || ! table.rootElement || ! table.rootElement.isConnected) { - this.createTable(); - } else { - // trigger rerender when tab is shown - table.updateSettings({}); - // or .updateSettings({ data : ... }) - table.loadData(this.get('dataMatrix')); - } - }, - - createTable() { - const cName = 'upload/blast-results'; - const fnName = 'createTable'; - dLog('createTable'); - $(() => { - let eltId = this.search.tableId; - let hotable = $('#' + eltId)[0]; - if (! hotable) { - console.warn(cName, fnName, ' : #', eltId, ' not found', this); - return; // fail - } - /** -blast output columns are -query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, query start, query end, subject start, subject end, e-value, score, query length, subject length - */ - var table = new Handsontable(hotable, { - data: [['', '', '', '', '', '', '', '', '', '', '', '', '', '']], - // minRows: 20, - rowHeaders: true, - /* - columns: [ - { - data: 'name', - type: 'text' - }, - { - data: 'block', - type: 'text' - }, - { - data: 'val', - type: 'numeric', - numericFormat: { - pattern: '0,0.*' - } - } - ], - */ - colHeaders: [ - 'query ID', 'subject ID', '% identity', 'length of HSP (hit)', '# mismatches', '# gaps', 'query start', 'query end', 'subject start', 'subject end', 'e-value', 'score', 'query length', 'subject length' - ], - height: 500, - // colWidths: [100, 100, 100], - manualRowResize: true, - manualColumnResize: true, - manualRowMove: true, - manualColumnMove: true, - contextMenu: true, - /* - afterChange: function() { - }, - afterRemoveRow: function() { - }, - */ - /* see comment re. handsOnTableLicenseKey in frontend/config/environment.js */ - licenseKey: config.handsOnTableLicenseKey - }); - this.set('table', table); - - }); - }, - - - clearTable() { - var table = this.get('table'); - table.updateSettings({data:[]}); - }, - - /*--------------------------------------------------------------------------*/ - /** called by upload-table.js : onSelectChange() * No validation of user input is required because table content is output from blast process. */ checkBlocks() { }, - - /** upload-table.js : submitFile() expects this function. - * In blast-results, the data is not user input so validation is not required. - */ - validateData() { - /** based on data-csv.js : validateData(), which uses table.getSourceData(); - * in this case sourceData is based on .dataMatrix instead - * of going via the table. - */ - return new Promise((resolve, reject) => { - let table = this.get('table'); - if (table === null) { - resolve([]); - } - let - validatedData = this.get('dataFeatures'); - resolve(validatedData); - }); - } - /*--------------------------------------------------------------------------*/ }); diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs new file mode 100644 index 000000000..6d4890d42 --- /dev/null +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -0,0 +1,16 @@ +{{dataMatrixEffect}} +{{activeEffect}} +{{viewFeaturesEffect}} + +{{!-- --------------------------------------------------------------------- --}} + +
+ +{{!-- --------------------------------------------------------------------- --}} + + + {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} + + + +{{!-- --------------------------------------------------------------------- --}} diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 4f03befe3..09f5c98c7 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -1,7 +1,4 @@ -{{dataMatrixEffect}} {{resultEffect}} -{{activeEffect}} -{{viewFeaturesEffect}} {{!-- --------------------------------------------------------------------- --}} @@ -28,7 +25,10 @@ type="button">x -
+ {{panel/upload/blast-results-view + viewDataset=viewDataset + search=search data=data active=active tableVisible=true tableModal=true + }} {{/ember-modal-dialog}} {{else}} @@ -46,7 +46,10 @@
-
+ {{panel/upload/blast-results-view + viewDataset=viewDataset + search=search data=data active=active tableVisible=tableVisible tableModal=false + }} {{/if}}
@@ -160,11 +163,6 @@ - - {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} - - -
Date: Fri, 4 Jun 2021 21:37:32 +1000 Subject: [PATCH 33/77] blast-results-view : unview features when leaving tab factor body of viewFeaturesEffect() to form dataFeaturesForStore() and viewFeatures(), and call viewFeatures(false) from willDestroyElement(). (this is only required since splitting blast-results-view out of blast-results, in previous commit ae5be357). --- .../panel/upload/blast-results-view.js | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 93be0f47b..2eb954427 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -111,25 +111,41 @@ export default Component.extend({ dLog('didRender', this.get('active'), this.get('tableVisible'), this.get('tableModal')); }, - /*--------------------------------------------------------------------------*/ + willDestroyElement() { + this.viewFeatures(false); + this._super(...arguments); + }, + /*--------------------------------------------------------------------------*/ - viewFeaturesEffect : computed('dataFeatures.[]', 'viewFeaturesFlag', 'active', function () { - const fnName = 'viewFeaturesEffect'; - /** construct feature id from name + val (start position) because + /** Map from .dataFeatures to the format required for store.normalize and .push(). + * + * construct feature id from name + val (start position) because * name is not unique, and in a blast search result, a feature * name is associated with the sequence string to search for, and * all features matching that sequence have the same name. * We may use UUID (e.g. thaume/ember-cli-uuid or ivanvanderbyl/ember-uuid). */ + dataFeaturesForStore : computed('dataFeatures.[]', function () { let features = this.get('dataFeatures') .map((f) => ({ _id : f.name + '-' + f.val, name : f.name, blockId : f.block, value : [f.val, f.end]})); - if (features && features.length) { + return features; + }), + + + viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewFeaturesFlag', 'active', function () { /** Only view features of the active tab. */ let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); + this.viewFeatures(viewFeaturesFlag); + }), + viewFeatures(viewFeaturesFlag) { + const fnName = 'viewFeaturesEffect'; + let + features = this.get('dataFeaturesForStore'); + if (features && features.length) { if (viewFeaturesFlag) { let parentName = this.get('search.parent'); dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); @@ -168,7 +184,7 @@ export default Component.extend({ viewFeaturesFlag, () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag)); } - }), + }, /*--------------------------------------------------------------------------*/ /** comments for activeEffect() and shownBsTab() in @see data-csv.js From 874192bc4c8a957a59a7f26e0772ddefecb2cd91 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 4 Jun 2021 23:42:29 +1000 Subject: [PATCH 34/77] blast-results-view : add a checkbox column to enable showing the feature triangles blast-results-view.js : split dataForTable from dataMatrix, so that it can prepend the view flag column. Add viewAllResultAxesChange() - just a placeholder - not implemented in this commit. prepend column : checkbox : view. Add afterChange() : use transient .pushFeature and .showFeature transient.js : factor showFeature() out of showFeatures(). blast-results-view.hbs : Add checkbox viewAllResultAxesFlag, with click action viewAllResultAxesChange --- .../panel/upload/blast-results-view.js | 88 ++++++++++++++++--- frontend/app/services/data/transient.js | 13 +-- .../panel/upload/blast-results-view.hbs | 7 ++ 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 2eb954427..81e440beb 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -1,7 +1,7 @@ import Component from '@ember/component'; import { observer, computed } from '@ember/object'; import { inject as service } from '@ember/service'; -import { later as run_later } from '@ember/runloop'; +import { later as run_later, bind } from '@ember/runloop'; import { alias } from '@ember/object/computed'; @@ -59,18 +59,24 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + /** Result data, split into columns + */ dataMatrix : computed('data.[]', function () { let data = this.get('data'), - cells = data ? data.map((r) => r.split('\t')) : []; + cells = data ? data + /** the last row is empty, so it is filtered out. */ + .filter((row) => (row !== '')) + .map((r) => r.split('\t')) : + []; return cells; }), + /** Result data formatted for upload-table.js : submitFile() + */ dataFeatures : computed('dataMatrix.[]', function () { let data = this.get('dataMatrix'); - /** the last row is empty, so it is filtered out. */ let features = data - .filter((row) => (row[c_name] !== '') && (row[c_chr])) .map((row) => { let feature = { name: row[c_name], @@ -92,15 +98,24 @@ export default Component.extend({ /** based on dataFeatures - see comments there. */ let names = data - .filter((row) => (row[c_name] !== '') && (row[c_chr])) .map((row) => row[c_chr].replace(/^chr/, '')); dLog('blockNames', names.length, names[0]); return names; }), - dataMatrixEffect : computed('table', 'dataMatrix.[]', function () { + /** Result data formatted for handsontable : loadData() + * Prepend a checkbox column. + */ + dataForTable : computed('dataMatrix.[]', function () { + let + data = this.get('dataMatrix'), + /** prepend with view flag = true. This copies row array. */ + rows = data.map((row) => [true].concat(row) ); + return rows; + }), + dataMatrixEffect : computed('table', 'dataForTable.[]', function () { let table = this.get('table'); if (table) { - table.loadData(this.get('dataMatrix')); + table.loadData(this.get('dataForTable')); } }), @@ -135,6 +150,12 @@ export default Component.extend({ return features; }), + viewAllResultAxesChange(proxy) { + let checked = proxy.target.checked; + /** this value seems to be delayed */ + let viewAll = this.get('viewAllResultAxesFlag'); + dLog('viewAllResultAxesChange', checked, viewAll); + }, viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewFeaturesFlag', 'active', function () { /** Only view features of the active tab. */ @@ -237,11 +258,32 @@ blast output columns are query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, query start, query end, subject start, subject end, e-value, score, query length, subject length */ var table = new Handsontable(hotable, { - data: [['', '', '', '', '', '', '', '', '', '', '', '', '', '']], + data: [[false, '', '', '', '', '', '', '', '', '', '', '', '', '', '']], // minRows: 20, rowHeaders: true, - /* + + /** column field data name is default - array index. */ columns: [ + { + type: 'checkbox', + className: "htCenter" + }, + // remaining columns use default type + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + /* { data: 'name', type: 'text' @@ -257,10 +299,11 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que pattern: '0,0.*' } } + */ ], - */ + colHeaders: [ - 'query ID', 'subject ID', '% identity', 'length of HSP (hit)', '# mismatches', '# gaps', 'query start', 'query end', 'subject start', 'subject end', 'e-value', 'score', 'query length', 'subject length' + 'view', 'query ID', 'subject ID', '% identity', 'length of HSP (hit)', '# mismatches', '# gaps', 'query start', 'query end', 'subject start', 'subject end', 'e-value', 'score', 'query length', 'subject length' ], height: 500, // colWidths: [100, 100, 100], @@ -269,9 +312,8 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que manualRowMove: true, manualColumnMove: true, contextMenu: true, + afterChange: bind(this, this.afterChange), /* - afterChange: function() { - }, afterRemoveRow: function() { }, */ @@ -291,6 +333,26 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que /*--------------------------------------------------------------------------*/ + afterChange(changes, source) { + let + transient = this.get('transient'), + features = this.get('dataFeaturesForStore'); + + if (changes) { + changes.forEach(([row, prop, oldValue, newValue]) => { + dLog('afterChange', row, prop, oldValue, newValue); + /** column 0 is the view checkbox. */ + if (prop === 0) { + let feature = transient.pushFeature(features[row]), + viewFeaturesFlag = newValue; + transient.showFeature(feature, viewFeaturesFlag); + } + }); + } + }, + + /*--------------------------------------------------------------------------*/ + /** upload-table.js : submitFile() expects this function. * In blast-results, the data is not user input so validation is not required. diff --git a/frontend/app/services/data/transient.js b/frontend/app/services/data/transient.js index c18ae2e84..bea4e9586 100644 --- a/frontend/app/services/data/transient.js +++ b/frontend/app/services/data/transient.js @@ -107,10 +107,13 @@ export default Service.extend({ selected = this.get('selected'), // may pass dataset, blocks to pushFeature() stored = features.map((f) => this.pushFeature(f)); - stored.forEach((feature) => { - selected.toggle('features', feature, viewFeaturesFlag); - selected.toggle('labelledFeatures', feature, viewFeaturesFlag); - }); - } + stored.forEach((feature) => this.showFeature(feature, viewFeaturesFlag)); + }, + showFeature(feature, viewFeaturesFlag) { + let + selected = this.get('selected'); + selected.toggle('features', feature, viewFeaturesFlag); + selected.toggle('labelledFeatures', feature, viewFeaturesFlag); + } }); diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index 6d4890d42..ac04259ff 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -8,6 +8,13 @@ {{!-- --------------------------------------------------------------------- --}} + + {{input type="checkbox" name="viewAllResultAxesFlag" checked=viewAllResultAxesFlag + input=(action viewAllResultAxesChange) + }} + + + {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} From 199875a33f9944f18282291abf1375473fef9cdc Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 7 Jun 2021 14:04:54 +1000 Subject: [PATCH 35/77] blast-results-view : implement show / hide - all checkbox blast-results-view.js : add viewRow, use in dataForTable. rename data{Matrix,ForTable}Effect(). blast-results-view.hbs : change span class=filter-group-col to div. --- .../panel/upload/blast-results-view.js | 22 ++++++++++++++----- .../panel/upload/blast-results-view.hbs | 12 +++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 81e440beb..a5938dc1f 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -71,6 +71,14 @@ export default Component.extend({ []; return cells; }), + viewRow : computed('dataMatrix', 'viewFeaturesFlag', function () { + let + data = this.get('dataMatrix'), + viewFeaturesFlag = this.get('viewFeaturesFlag'), + viewRow = data.map((row) => viewFeaturesFlag); + return viewRow; + }), + /** Result data formatted for upload-table.js : submitFile() */ dataFeatures : computed('dataMatrix.[]', function () { @@ -105,14 +113,15 @@ export default Component.extend({ /** Result data formatted for handsontable : loadData() * Prepend a checkbox column. */ - dataForTable : computed('dataMatrix.[]', function () { + dataForTable : computed('dataMatrix.[]', 'viewRow', function () { let data = this.get('dataMatrix'), - /** prepend with view flag = true. This copies row array. */ - rows = data.map((row) => [true].concat(row) ); + viewRow = this.get('viewRow'), + /** prepend with view flag (initially true). Use of [].concat() copies row array (not mutate). */ + rows = data.map((row, i) => [viewRow[i]].concat(row) ); return rows; }), - dataMatrixEffect : computed('table', 'dataForTable.[]', function () { + dataForTableEffect : computed('table', 'dataForTable.[]', function () { let table = this.get('table'); if (table) { table.loadData(this.get('dataForTable')); @@ -151,10 +160,13 @@ export default Component.extend({ }), viewAllResultAxesChange(proxy) { + const fnName = 'viewAllResultAxesChange'; let checked = proxy.target.checked; /** this value seems to be delayed */ let viewAll = this.get('viewAllResultAxesFlag'); - dLog('viewAllResultAxesChange', checked, viewAll); + dLog(fnName, checked, viewAll); + + }, viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewFeaturesFlag', 'active', function () { diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index ac04259ff..d17dbfa4a 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -1,4 +1,4 @@ -{{dataMatrixEffect}} +{{dataForTableEffect}} {{activeEffect}} {{viewFeaturesEffect}} @@ -8,16 +8,16 @@ {{!-- --------------------------------------------------------------------- --}} - +
{{input type="checkbox" name="viewAllResultAxesFlag" checked=viewAllResultAxesFlag input=(action viewAllResultAxesChange) }} - +
- +
{{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} - - + +
{{!-- --------------------------------------------------------------------- --}} From 78a367074e42a3aa2426e4df3e89fbf24ccf7a8f Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 7 Jun 2021 16:48:29 +1000 Subject: [PATCH 36/77] sequence search and blast results : flag out 'add dataset' and 'view (dataset)' add options=searchAddDataset --- frontend/app/components/panel/sequence-search.js | 5 +++++ frontend/app/components/panel/upload/blast-results.js | 5 +++++ frontend/app/templates/components/panel/sequence-search.hbs | 3 +++ .../app/templates/components/panel/upload/blast-results.hbs | 4 +++- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index bfb8dde8e..1d9c65568 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -12,6 +12,11 @@ const dLog = console.debug; export default Component.extend({ auth: service(), + queryParams: service('query-params'), + + urlOptions : alias('queryParams.urlOptions'), + + /*--------------------------------------------------------------------------*/ /** limit rows in result */ resultRows : 50, diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index cc2d7d327..98a69fabc 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -1,6 +1,7 @@ import Component from '@ember/component'; import { observer, computed } from '@ember/object'; import { inject as service } from '@ember/service'; +import { alias } from '@ember/object/computed'; import uploadBase from '../../../utils/panel/upload-base'; import uploadTable from '../../../utils/panel/upload-table'; @@ -19,7 +20,11 @@ const dLog = console.debug; export default Component.extend({ apiServers: service(), blockService : service('data/block'), + queryParams: service('query-params'), + urlOptions : alias('queryParams.urlOptions'), + + /*--------------------------------------------------------------------------*/ classNames: ['blast-results'], diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 7b2fe750b..0c328a382 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -89,6 +89,7 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }}
+ {{#if urlOptions.searchAddDataset}}
  • {{input type="checkbox" name="addDataset" checked=addDataset }} @@ -117,6 +118,8 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} {{/if}}
    • + {{/if}} {{!!-- urlOptions.searchAddDataset --}} + diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 09f5c98c7..e984f2c8f 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -61,7 +61,7 @@ - +{{#if urlOptions.searchAddDataset}}
    {{!-- --------------------------------------------------------------------- --}} @@ -173,5 +173,7 @@
    +{{/if}} {{!-- urlOptions.searchAddDataset --}} + {{!-- --------------------------------------------------------------------- --}} From b7ad90070d9169d14812d5450c4f3e6c9e53085b Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 7 Jun 2021 16:54:57 +1000 Subject: [PATCH 37/77] sequence search : row limit : default 50 -> 500 --- frontend/app/components/panel/sequence-search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 1d9c65568..a695cba70 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -19,7 +19,7 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ /** limit rows in result */ - resultRows : 50, + resultRows : 500, /** true means add / upload result to db as a Dataset */ addDataset : false, /** true means view the blocks of the dataset after it is added. */ From 086ea39ee9cc235d7238365065e07b561571c7e9 Mon Sep 17 00:00:00 2001 From: Don Date: Mon, 7 Jun 2021 19:19:43 +1000 Subject: [PATCH 38/77] sequence search : limit fasta input : <= 1 sequence, total sequence length < 2kb. Check input : require exactly 1 marker line and sequence text, and no other input. sequence-search.js : add searchStringMaxLength. inputIsActive() : save event?.target?.value as .text, so that keyboard : Enter is not required. Add checkTextInput() and use in dnaSequenceInput(). --- .../app/components/panel/sequence-search.js | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index a695cba70..86ae6a2f2 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -7,9 +7,14 @@ import { A as array_A } from '@ember/array'; import sequenceSearchData from '../../utils/data/sequence-search-data'; +/*----------------------------------------------------------------------------*/ const dLog = console.debug; +const searchStringMaxLength = 2000; + +/*----------------------------------------------------------------------------*/ + export default Component.extend({ auth: service(), queryParams: service('query-params'), @@ -112,8 +117,12 @@ export default Component.extend({ // actions actions: { // copied from feature-list, may not be required - inputIsActive() { - dLog('inputIsActive'); + inputIsActive(event) { + dLog('inputIsActive', event?.target); + let text = event?.target?.value; + if (text) { + this.set('text', text); + } }, paste: function(event) { /** text is "" at this time. */ @@ -143,6 +152,15 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + /** Check GUI inputs which are parameters for addDataset : + * - datasetName (newDatasetName) + * - parent name (selectedParent) + * Both are required - check that they have been entered. + * Check that newDatasetName is not a duplicate of an existing dataset. + * Display messages if any checks fail. + * + * @return true if all checks pass. + */ checkInputs() { let ok; this.clearMsgs(); @@ -174,9 +192,49 @@ export default Component.extend({ return bind(this, this.dnaSequenceInput); }), + /** @return a warningMessage if rawText does not meet input requirements, otherwise falsy. + */ + checkTextInput(rawText) { + let warningMessages = []; + let + lines = rawText.split('\n'), + notBases = lines + .filter((l) => ! l.match(/^[ACTGactg]+$/)), + keys = notBases + .filter((maybeKey) => maybeKey.match(/^>[^\n]+$/)), + other = notBases + .filter((maybeKey) => ! maybeKey.match(/^>[^\n]+$/)) + .filter((maybeEmpty) => ! maybeEmpty.match(/^[ \t\n]*$/)); + switch (keys.length) { + case 0: + warningMessages.push('Key line is required : >MarkerName ...'); + break; + case 1: + break; + default: + warningMessages.push('Limit is 1 FASTA search'); + break; + } + let regexpIterator = rawText.matchAll(/\n[ACTGactg]+/g), + sequenceLinesLength = Array.from(regexpIterator).length; + if (sequenceLinesLength === 0) { + warningMessages.push('DNA text is required : e.g. ATCGatcg...'); + } + if (other.length) { + warningMessages.push('Input should be either >MarkerName or DNA text e.g. ATCGatcg...; this input not valid :' + other[0]); + } + + if (rawText.length > searchStringMaxLength) { + warningMessages.push('FASTA search string is limited to ' + searchStringMaxLength); + } + + let warningMessage = warningMessages.length && warningMessages.join('\n'); + return warningMessage; + }, dnaSequenceInput(rawText) { const fnName = 'dnaSequenceInput'; // dLog("dnaSequenceInput", rawText && rawText.length); + let warningMessage; /** if the user has use paste or newline then .text is defined, * otherwise use jQuery to get it from the textarea. */ @@ -185,7 +243,11 @@ export default Component.extend({ /** before textarea is created, .val() will be undefined. */ rawText = text$.val(); } - if (rawText) + if (! rawText) { + this.set('warningMessage', "Please enter search text in the field 'DNA Sequence Input'"); + } else if ((warningMessage = this.checkTextInput(rawText))) { + this.set('warningMessage', warningMessage); + } else { let seq = rawText; From 6219efcb7e2ffaf2b242502821f32f1fe62ac71e Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 12:16:50 +1000 Subject: [PATCH 39/77] fasta input textarea : increase height ... aiming for ~30 rows, but depends on screen size - the warning messages at the bottom of the component may be pushed off-screen. sequence-search.hbs : textarea : change maxLength=15 to rows=16 maxlength=2000 --- frontend/app/templates/components/panel/sequence-search.hbs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 0c328a382..900d32636 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -32,7 +32,8 @@ {{textarea class="form-control" - maxLength=15 + rows=16 + maxlength=2000 input=(action 'inputIsActive') enter=(action 'dnaSequenceInput') insert-newline=(action 'dnaSequenceInput') From 683e3fc11c0c46022627c9d8fd6c98ed0b66fc47 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 13:03:38 +1000 Subject: [PATCH 40/77] blast-results-view : enable table sort on columns ... Disabled row & column insert, and editing functions in contextMenu. blast-results-view.js : contextMenu : limit to non-editing functions. set minSpareRows and minSpareCols to 0 to prevent insert / append rows / cols. enable : sortIndicator, columnSorting. afterChange() : for view column, set warningMessage if row >= features.length. blast-results-view.hbs : add panel-message, to show warningMessage. --- .../panel/upload/blast-results-view.js | 25 ++++++++++++++++--- .../panel/upload/blast-results-view.hbs | 8 ++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index a5938dc1f..313bebc8a 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -24,8 +24,13 @@ const dLog = console.debug; const columnsKeyString = [ 'name', 'chr', 'pcIdentity', 'lengthOfHspHit', 'numMismatches', 'numGaps', 'queryStart', 'queryEnd', 'pos', 'end' ]; +/** Identify the columns of dataFeatures and dataMatrix. + */ const c_name = 0, c_chr = 1, c_pos = 8, c_end = 9; - +/** Identify the columns of dataForTable, which has an additional 'View' column inserted on the left. + * so e.g. t_name would be c_name + 1. + */ +const t_view = 0; /** Display a table of results from sequence-search API request * /Feature/dnaSequenceSearch @@ -273,6 +278,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que data: [[false, '', '', '', '', '', '', '', '', '', '', '', '', '', '']], // minRows: 20, rowHeaders: true, + headerTooltips: true, /** column field data name is default - array index. */ columns: [ @@ -323,7 +329,15 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que manualColumnResize: true, manualRowMove: true, manualColumnMove: true, - contextMenu: true, + contextMenu: ['undo', 'redo', 'readonly', 'alignment', 'copy'], // true + + // prevent insert / append rows / cols + minSpareRows: 0, + minSpareCols: 0, + + sortIndicator: true, + columnSorting: true, + afterChange: bind(this, this.afterChange), /* afterRemoveRow: function() { @@ -353,8 +367,13 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que if (changes) { changes.forEach(([row, prop, oldValue, newValue]) => { dLog('afterChange', row, prop, oldValue, newValue); + /** prop is the property / column index. */ /** column 0 is the view checkbox. */ - if (prop === 0) { + if (prop !== t_view) { + // no action for other columns + } else if (row >= features.length) { + this.set('warningMessage', 'Display of added features not yet supported'); + } else { let feature = transient.pushFeature(features[row]), viewFeaturesFlag = newValue; transient.showFeature(feature, viewFeaturesFlag); diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index d17dbfa4a..cbcdf970b 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -21,3 +21,11 @@
    {{!-- --------------------------------------------------------------------- --}} +
    + + {{elem/panel-message + successMessage=successMessage + warningMessage=warningMessage + errorMessage=errorMessage}} + +{{!-- --------------------------------------------------------------------- --}} From 603485e7d4f56bb507b6b87376fa8bc872292a11 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 15:48:25 +1000 Subject: [PATCH 41/77] sequence search : increase searchStringMaxLength -> 10k sequence-search : change searchStringMaxLength from const to component attribute, use in hbs also, add options=searchStringMaxLength. --- frontend/app/components/panel/sequence-search.js | 13 +++++++++---- .../templates/components/panel/sequence-search.hbs | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 86ae6a2f2..8e7c82817 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -11,8 +11,6 @@ import sequenceSearchData from '../../utils/data/sequence-search-data'; const dLog = console.debug; -const searchStringMaxLength = 2000; - /*----------------------------------------------------------------------------*/ export default Component.extend({ @@ -23,6 +21,8 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + /** length limit for FASTA DNA text search string input. */ + searchStringMaxLength : 10000, /** limit rows in result */ resultRows : 500, /** true means add / upload result to db as a Dataset */ @@ -41,6 +41,11 @@ export default Component.extend({ this._super(...arguments); this.set('searches', array_A()); + + let searchStringMaxLength = this.get('urlOptions.searchStringMaxLength'); + if (searchStringMaxLength) { + this.set('searchStringMaxLength', searchStringMaxLength); + } }, /*--------------------------------------------------------------------------*/ @@ -224,8 +229,8 @@ export default Component.extend({ warningMessages.push('Input should be either >MarkerName or DNA text e.g. ATCGatcg...; this input not valid :' + other[0]); } - if (rawText.length > searchStringMaxLength) { - warningMessages.push('FASTA search string is limited to ' + searchStringMaxLength); + if (rawText.length > this.searchStringMaxLength) { + warningMessages.push('FASTA search string is limited to ' + this.searchStringMaxLength); } let warningMessage = warningMessages.length && warningMessages.join('\n'); diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 900d32636..8106093a7 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -33,7 +33,7 @@ {{textarea class="form-control" rows=16 - maxlength=2000 + maxlength=this.searchStringMaxLength input=(action 'inputIsActive') enter=(action 'dnaSequenceInput') insert-newline=(action 'dnaSequenceInput') From b9851a7749582ed4792a8368ba466be3396bd87a Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 16:24:28 +1000 Subject: [PATCH 42/77] blast-results-view : display statusMessage to indicate if search is in progress, or returned 0 hits. ... handle the case of no result : in that case need to indicate that the search has completed. display Searching... and if result returned, show table, or else No hits found. blast-results-view.js : dataMatrix() : if data, set statusMessage according to cells.length. blast-results-view.hbs : display statusMessage if defined. --- frontend/app/components/panel/upload/blast-results-view.js | 6 ++++++ .../components/panel/upload/blast-results-view.hbs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 313bebc8a..825f4986a 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -53,6 +53,8 @@ export default Component.extend({ /** true means display the result rows as triangles - clickedFeatures. */ viewFeaturesFlag : true, + statusMessage : 'Searching ...', + /*--------------------------------------------------------------------------*/ /** style of the div which contains the table. @@ -74,6 +76,10 @@ export default Component.extend({ .filter((row) => (row !== '')) .map((r) => r.split('\t')) : []; + + if (data) { + this.set('statusMessage', (cells.length ? undefined : 'The search completed and returned 0 hits') ); + } return cells; }), viewRow : computed('dataMatrix', 'viewFeaturesFlag', function () { diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index cbcdf970b..fdc6a37b9 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -4,6 +4,12 @@ {{!-- --------------------------------------------------------------------- --}} +{{#if statusMessage}} +
    {{statusMessage}}
    +{{/if}} + +{{!-- --------------------------------------------------------------------- --}} +
    {{!-- --------------------------------------------------------------------- --}} From 40e77e02a9d32772555e57f93f4e0ca2a588fd86 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 20:21:01 +1000 Subject: [PATCH 43/77] blast results : handle request timeout blast-results-view.js : didReceiveAttrs() : search.promise .catch : set statusMessage : The search did not complete. blast-results.js : resultEffect() promise catch : use err?.responseJSON?.error because 504 timeout causes .responseText to be defined instead of .responseJSON. In that case, show err.status, status, err.statusText in errmsg. blast-results.hbs : re-enable panel-message (was wrapped in #if searchAddDataset). --- .../components/panel/upload/blast-results-view.js | 12 ++++++++++++ .../app/components/panel/upload/blast-results.js | 8 +++++--- .../components/panel/upload/blast-results.hbs | 5 +++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 825f4986a..26f7805de 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -57,6 +57,18 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + didReceiveAttrs() { + this._super(...arguments); + let promise = this.get('search.promise'); + if (promise) { + promise.catch(() => { + this.set('statusMessage', 'The search did not complete'); + }); + } + }, + + /*--------------------------------------------------------------------------*/ + /** style of the div which contains the table. * If this component is in a modal dialog, use most of screen width. */ diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index 98a69fabc..cc74b4d80 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -118,8 +118,8 @@ export default Component.extend({ // copied from data-base.js - could be factored. (err, status) => { dLog(fnName, 'dnaSequenceSearch reject', err, status); - let errobj = err.responseJSON.error; - console.log(errobj); + let errobj = err?.responseJSON?.error || err.statusText; + console.log(fnName, errobj, status, err.status); let errmsg = null; if (errobj.message) { errmsg = errobj.message; @@ -127,7 +127,9 @@ export default Component.extend({ errmsg = errobj.errmsg; } else if (errobj.name) { errmsg = errobj.name; - } + } else if (err.status !== 200) { + errmsg = errobj + ',' + err.status + ',' + status; + } this.setError(errmsg); // upload tabs do .scrollToTop(), doesn't seem applicable here. } diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index e984f2c8f..3323607f4 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -82,6 +82,8 @@
    +{{/if}} {{!-- urlOptions.searchAddDataset --}} + {{!-- --------------------------------------------------------------------- --}} {{!-- from data-csv --}} @@ -89,6 +91,9 @@ successMessage=successMessage warningMessage=warningMessage errorMessage=errorMessage}} + +{{#if urlOptions.searchAddDataset}} + {{#if nameWarning}} {{elem/panel-message warningMessage=nameWarning}} From 2c48b1a325833366588c5c471f6ffb334f8ffd4c Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 22:08:46 +1000 Subject: [PATCH 44/77] sequence search : add range sliders to filter blast results by : length of hit, %id, coverage %. feature.js : Feature.dnaSequenceSearch : searchDataOut () : filter textLines by filterBlastResults(). add attributes : minLengthOfHit, minPercentIdentity, minPercentCoverage, and pass them to auth.dnaSequenceSearch() in dnaSequenceInput(). auth.js : dnaSequenceSearch() : add params minLengthOfHit, minPercentIdentity, minPercentCoverage, sequence-search.hbs : add range sliders for : minLengthOfHit, minPercentIdentity, minPercentCoverage. --- backend/common/models/feature.js | 11 +++- backend/common/utilities/sequence-search.js | 54 +++++++++++++++++++ .../app/components/panel/sequence-search.js | 9 ++++ frontend/app/services/auth.js | 13 ++++- .../components/panel/sequence-search.hbs | 47 ++++++++++++++++ 5 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 backend/common/utilities/sequence-search.js diff --git a/backend/common/models/feature.js b/backend/common/models/feature.js index 5cfbbeae4..3d374a963 100644 --- a/backend/common/models/feature.js +++ b/backend/common/models/feature.js @@ -6,6 +6,7 @@ var acl = require('../utilities/acl') const { childProcess } = require('../utilities/child-process'); var upload = require('../utilities/upload'); +var { filterBlastResults } = require('../utilities/sequence-search'); module.exports = function(Feature) { @@ -58,6 +59,7 @@ module.exports = function(Feature) { * @param resultRows * @param addDataset * @param datasetName + * @param minLengthOfHit, minPercentIdentity, minPercentCoverage : minimum values to filter results * @param options * * @param cb node response callback @@ -65,7 +67,10 @@ module.exports = function(Feature) { Feature.dnaSequenceSearch = function(data, options, cb) { const models = this.app.models; - let {dnaSequence, parent, searchType, resultRows, addDataset, datasetName} = data; + let {dnaSequence, parent, searchType, resultRows, addDataset, datasetName, + minLengthOfHit, minPercentIdentity, minPercentCoverage + } = data; + // data.options : params for streaming result, used later. const fnName = 'dnaSequenceSearch'; console.log(fnName, dnaSequence.length, parent, searchType); @@ -78,7 +83,9 @@ module.exports = function(Feature) { cb(new Error(chunk.toString())); } else { const - textLines = chunk.toString().split('\n'); + textLines = chunk.toString().split('\n') + .filter((textLine) => filterBlastResults( + minLengthOfHit, minPercentIdentity, minPercentCoverage, textLine)); textLines.forEach((textLine) => { if (textLine !== "") { console.log(fnName, 'stdout data', "'", textLine, "'"); diff --git a/backend/common/utilities/sequence-search.js b/backend/common/utilities/sequence-search.js new file mode 100644 index 000000000..e61f7a455 --- /dev/null +++ b/backend/common/utilities/sequence-search.js @@ -0,0 +1,54 @@ +/* global require */ + +/* global exports */ + +/*----------------------------------------------------------------------------*/ + +/* + * Blast Output Columns : +query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, query start, query end, subject start, subject end, e-value, score, query length, subject length + * column names : + 0 name + 1 chr + 2 pcIdentity + 3 lengthOfHspHit + 4 numMismatches + 5 numGaps + 6 queryStart + 7 queryEnd + 8 pos + 9 end + 10 eValue + 11 score + 12 queryLength + 13 subjectLength +*/ + +/** Identify the columns of blast output result + */ +const +c_name = 0, c_chr = 1, c_pcIdentity = 2, c_lengthOfHspHit = 3, +c_pos = 8, c_end = 9, +c_queryLength = 12; + + +/** Filter Blast Search output results. + * + * coverage = length of HSP/query length *100 + */ +exports.filterBlastResults = ( + minLengthOfHit, minPercentIdentity, minPercentCoverage, line) => { + let ok, + cols = line.split('\t'); + if (cols && cols.length > c_end) { + ok = (+cols[c_pcIdentity] >= +minPercentIdentity) && + (+cols[c_lengthOfHspHit] >= +minLengthOfHit) && + (cols[c_queryLength] && (100 * +cols[c_lengthOfHspHit] / +cols[c_queryLength] >= +minPercentCoverage)); + } + console.log('filterBlastResults', ok, minLengthOfHit, minPercentIdentity, minPercentCoverage, line, cols); + return ok; +}; + + + + diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 8e7c82817..160c180a2 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -25,6 +25,12 @@ export default Component.extend({ searchStringMaxLength : 10000, /** limit rows in result */ resultRows : 500, + + /** minimum values for 3 columns, to filter blast output. */ + minLengthOfHit : 0, + minPercentIdentity : 75, + minPercentCoverage : 50, + /** true means add / upload result to db as a Dataset */ addDataset : false, /** true means view the blocks of the dataset after it is added. */ @@ -273,6 +279,9 @@ export default Component.extend({ this.get('resultRows'), this.get('addDataset'), this.get('newDatasetName'), + this.get('minLengthOfHit'), + this.get('minPercentIdentity'), + this.get('minPercentCoverage'), /*options*/{/*dataEvent : receivedData, closePromise : taskInstance*/}); if (this.get('addDataset')) { diff --git a/frontend/app/services/auth.js b/frontend/app/services/auth.js index 6a6c79e5c..331ee6adf 100644 --- a/frontend/app/services/auth.js +++ b/frontend/app/services/auth.js @@ -284,15 +284,24 @@ export default Service.extend({ * @param resultRows limit rows in result * @param addDataset true means add / upload result to db as a Dataset * @param datasetName if addDataset, this value is used to name the added dataset. + * @param minLengthOfHit, minPercentIdentity, minPercentCoverage : minimum values to filter results + * @param options not used yet, probably will be for streaming result */ - dnaSequenceSearch(apiServer, dnaSequence, parent, searchType, resultRows, addDataset, datasetName, options) { + dnaSequenceSearch( + apiServer, dnaSequence, parent, searchType, resultRows, addDataset, datasetName, + minLengthOfHit, minPercentIdentity, minPercentCoverage, + options + ) { dLog('services/auth featureSearch', dnaSequence.length, parent, searchType, resultRows, addDataset, datasetName, options); /** Attach .server to JSON string, instead of using * requestServerAttr (.session.requestServer) * (this can be unwound after adding apiServer as param to ._ajax(), * dropping the new String() ). */ - let data = {dnaSequence, parent, searchType, resultRows, addDataset, datasetName, options}, + let data = { + dnaSequence, parent, searchType, resultRows, addDataset, datasetName, + minLengthOfHit, minPercentIdentity, minPercentCoverage, + options}, dataS = JSON.stringify(data); // new String(); // dataS.server = apiServer; return this._ajax('Features/dnaSequenceSearch', 'POST', dataS, true); diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 8106093a7..662eb0f9a 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -90,6 +90,53 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} + {{!-- --------------------------------------------------------- --}} + +
  • +
    + + {{ minLengthOfHit }} +
    +
    + +
    +
    • + +
  • +
    + + {{ minPercentIdentity }} +
    +
    + +
    +
    • + +
  • +
    + + {{ minPercentCoverage }} +
    +
    + +
    +
    • + + {{!-- --------------------------------------------------------- --}} + + {{#if urlOptions.searchAddDataset}}
  • {{input type="checkbox" name="addDataset" checked=addDataset }} From 522d6ef58fdf719d2e21082739bf66c3a63b1512 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 8 Jun 2021 22:36:11 +1000 Subject: [PATCH 45/77] sequence search : allow minLengthOfHit up to 10kb sequence-search.hbs : minLengthOfHit : change max : 100 -> searchStringMaxLength sequence-search.js : filterBlastResults() : use + before cols[c_queryLength] because '0' is truthy. --- backend/common/utilities/sequence-search.js | 2 +- frontend/app/templates/components/panel/sequence-search.hbs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/common/utilities/sequence-search.js b/backend/common/utilities/sequence-search.js index e61f7a455..e43b24445 100644 --- a/backend/common/utilities/sequence-search.js +++ b/backend/common/utilities/sequence-search.js @@ -43,7 +43,7 @@ exports.filterBlastResults = ( if (cols && cols.length > c_end) { ok = (+cols[c_pcIdentity] >= +minPercentIdentity) && (+cols[c_lengthOfHspHit] >= +minLengthOfHit) && - (cols[c_queryLength] && (100 * +cols[c_lengthOfHspHit] / +cols[c_queryLength] >= +minPercentCoverage)); + (+cols[c_queryLength] && (100 * +cols[c_lengthOfHspHit] / +cols[c_queryLength] >= +minPercentCoverage)); } console.log('filterBlastResults', ok, minLengthOfHit, minPercentIdentity, minPercentCoverage, line, cols); return ok; diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 662eb0f9a..7db8e5228 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -99,7 +99,7 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }}
    From 71f1c13d515d29bfe51dc411f164cf8bfa40e40c Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 9 Jun 2021 13:40:35 +1000 Subject: [PATCH 46/77] blast-results-view : preserve tick box state when switching blast-results-view table between modal dialog and left panel blast-results-view.js : activeEffect() : table.suspendExecution() after tableModal changes, and .resumeExecution() it again in showTable(); this protects handsontable from being confused during move between the two div#blast-results-table-{modal,panel}, without this the table moves to modal ok, but is not shown when moved back. blast-results.js : add tableModalTargetId(). blast-results.hbs : add div#blast-results-table-{modal,panel}. move blast-results-view into ember-wormhole to=tableModalTargetId. 3 tabs -> spaces --- .../panel/upload/blast-results-view.js | 21 ++++++++++++++++--- .../components/panel/upload/blast-results.js | 9 ++++++-- .../components/panel/upload/blast-results.hbs | 14 +++++++------ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 26f7805de..9ddc781bc 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -62,7 +62,7 @@ export default Component.extend({ let promise = this.get('search.promise'); if (promise) { promise.catch(() => { - this.set('statusMessage', 'The search did not complete'); + this.set('statusMessage', 'The search did not complete'); }); } }, @@ -154,7 +154,10 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ didRender() { - // this.showTable(); + // probably not required - renders OK without this. + if (false && this.get('table')) { + this.shownBsTab(); + } dLog('didRender', this.get('active'), this.get('tableVisible'), this.get('tableModal')); }, @@ -255,6 +258,16 @@ export default Component.extend({ activeEffect : computed('active', 'tableVisible', 'tableModal', function () { let active = this.get('active'); if (active && this.get('tableVisible')) { + /** table.suspendExecution() after tableModal changes, and + * .resumeExecution() it again in showTable(); this protects + * handsontable from being confused during move between the two + * div#blast-results-table-{modal,panel}; without this the table + * moves to modal ok, but is not shown when moved back. + */ + let table = this.get('table'); + if (table) { + table.suspendExecution(); + } this.shownBsTab(); } }), @@ -270,10 +283,12 @@ export default Component.extend({ if (! table || ! table.rootElement || ! table.rootElement.isConnected) { this.createTable(); } else { + table.resumeExecution(); // trigger rerender when tab is shown - table.updateSettings({}); + // table.updateSettings({}); // or .updateSettings({ data : ... }) table.loadData(this.get('dataMatrix')); + table.refreshDimensions(); } }, diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index cc74b4d80..a8f01d7df 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -40,6 +40,11 @@ export default Component.extend({ */ viewDatasetFlag : false, + /*--------------------------------------------------------------------------*/ + + tableModalTargetId : computed('tableModal', function () { + return this.get('tableModal') ? 'blast-results-table-modal' : 'blast-results-table-panel'; + }), /*--------------------------------------------------------------------------*/ @@ -128,8 +133,8 @@ export default Component.extend({ } else if (errobj.name) { errmsg = errobj.name; } else if (err.status !== 200) { - errmsg = errobj + ',' + err.status + ',' + status; - } + errmsg = errobj + ',' + err.status + ',' + status; + } this.setError(errmsg); // upload tabs do .scrollToTop(), doesn't seem applicable here. } diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 3323607f4..91eaeb2a6 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -25,10 +25,7 @@ type="button">x - {{panel/upload/blast-results-view - viewDataset=viewDataset - search=search data=data active=active tableVisible=true tableModal=true - }} +
    {{/ember-modal-dialog}} {{else}} @@ -46,11 +43,16 @@
    +
    + {{/if}} + {{#ember-wormhole to=tableModalTargetId }} {{panel/upload/blast-results-view viewDataset=viewDataset - search=search data=data active=active tableVisible=tableVisible tableModal=false + search=search data=data active=active tableVisible=tableVisible tableModal=tableModal }} - {{/if}} +
    Testing
    + {{/ember-wormhole}} +
    From d1bf4867cc26f82323782c553289290a4a764236 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 9 Jun 2021 16:49:04 +1000 Subject: [PATCH 47/77] blast-results-view : set table height when changing tableModal ... this completes the goal of 71f1c13d, which only appeared to work because of an undefined function causing an exception continuously. blast-results-view.js : factor 500 -> tableHeight. activeEffect() : updateSettings({ height : tableHeight }) showTable() : update dataMatrix -> dataForTable for loadData (this should have changed in 874192bc). bower.json : update handsontable ^7 -> ^8, so that {suspend,resume}{Execution,Render}() are defined. --- .../panel/upload/blast-results-view.js | 23 +++++++++++++++++-- .../components/panel/upload/blast-results.hbs | 1 - frontend/bower.json | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 9ddc781bc..b79c30018 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -15,6 +15,8 @@ import { nowOrLater } from '../../../utils/ember-devel'; const dLog = console.debug; +const tableHeight = 500; // pixels + /*----------------------------------------------------------------------------*/ /** @@ -263,10 +265,21 @@ export default Component.extend({ * handsontable from being confused during move between the two * div#blast-results-table-{modal,panel}; without this the table * moves to modal ok, but is not shown when moved back. + * + * Update : updateSettings({ height : ... } ) is required, and possibly sufficient - + * it may not be necessary to suspend/resume Execution/Render. */ let table = this.get('table'); if (table) { + dLog('activeEffect', 'suspendExecution'); table.suspendExecution(); + table.suspendRender(); + run_later(() => { + dLog('activeEffect', 'showTable', 'resumeExecution', this.get('tableModal')); + table.updateSettings({ height : tableHeight }); + table.refreshDimensions(); table.resumeExecution(); table.resumeRender(); + }, 400); + // this runs before showTable() from following shownBsTab(). } this.shownBsTab(); } @@ -283,11 +296,17 @@ export default Component.extend({ if (! table || ! table.rootElement || ! table.rootElement.isConnected) { this.createTable(); } else { + dLog('showTable', table.renderSuspendedCounter); + /* table.resumeExecution(); + table.resumeRender(); + table.render(); + */ + // trigger rerender when tab is shown // table.updateSettings({}); // or .updateSettings({ data : ... }) - table.loadData(this.get('dataMatrix')); + table.loadData(this.get('dataForTable')); table.refreshDimensions(); } }, @@ -356,7 +375,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que colHeaders: [ 'view', 'query ID', 'subject ID', '% identity', 'length of HSP (hit)', '# mismatches', '# gaps', 'query start', 'query end', 'subject start', 'subject end', 'e-value', 'score', 'query length', 'subject length' ], - height: 500, + height: tableHeight, // colWidths: [100, 100, 100], manualRowResize: true, manualColumnResize: true, diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index 91eaeb2a6..e7b7c3a73 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -50,7 +50,6 @@ viewDataset=viewDataset search=search data=data active=active tableVisible=tableVisible tableModal=tableModal }} -
    Testing
    {{/ember-wormhole}}
    diff --git a/frontend/bower.json b/frontend/bower.json index 71e3bcd43..89756c062 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -9,7 +9,7 @@ "d3-tip": "VACLab/d3-tip", "d3": "^4.10.2", "animation-frame": "^0.2.5", - "handsontable": "^7" + "handsontable": "^8" }, "devDependencies": {}, "resolutions": { From 298c77e6d7b8db9126934092c84934a964954c61 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 9 Jun 2021 17:56:08 +1000 Subject: [PATCH 48/77] blast-results-view : delay drawing the table until a result is received blast-results-view.js : showTable() : don't createTable() if ! data?.length, instead search.promise .then( shownBsTab() ). --- .../panel/upload/blast-results-view.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index b79c30018..625a0d528 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -291,9 +291,21 @@ export default Component.extend({ showTable() { + let table; + // delay creation of table until data is received + let data = this.get('data'); + if (! data || ! data.length) { + let p = this.get('search.promise'); + dLog('showTable', p.state && p.state()); + p.then(() => { + dLog('showTable then', this.get('data')?.length); + // alternative : dataForTableEffect() could do this if ! table. + this.shownBsTab(); }); + } else // Ensure table is created when tab is shown - let table = this.get('table'); - if (! table || ! table.rootElement || ! table.rootElement.isConnected) { + if (! (table = this.get('table')) || + ! table.rootElement || + ! table.rootElement.isConnected) { this.createTable(); } else { dLog('showTable', table.renderSuspendedCounter); From 4e9727e5d5aef55c6b7de4215c3b13e24da1dc39 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 9 Jun 2021 18:47:29 +1000 Subject: [PATCH 49/77] combine feature search and blast search into a single panel app.scss : search panel now has multiple divs in div.tab-pane, so only apply margin-bottom: 70px to the last one : add :last-child to selector. left-panel.hbs : move sequence-search into the manage-search tab.pane, and drop Feature from the title -> just Search. pass panel-container to panel-container for the 3 search panels, in manage-search.hbs and sequence-search.hbs. --- frontend/app/styles/app.scss | 2 +- frontend/app/templates/components/panel/left-panel.hbs | 9 ++++----- .../app/templates/components/panel/manage-search.hbs | 4 ++-- .../app/templates/components/panel/sequence-search.hbs | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/frontend/app/styles/app.scss b/frontend/app/styles/app.scss index f7ed557f2..d684bc75d 100644 --- a/frontend/app/styles/app.scss +++ b/frontend/app/styles/app.scss @@ -1434,7 +1434,7 @@ div#left-panel > div > div.tab-content > div.tab-pane overflow-y: auto; /* possibly scrollbar-gutter in future. */ } -#left-panel > div > div.tab-content > div.tab-pane > div { +#left-panel > div > div.tab-content > div.tab-pane > div:last-child { /* this enables the scrollbar to reach the bottom of the div, there is probably a better solution. */ margin-bottom: 70px; } diff --git a/frontend/app/templates/components/panel/left-panel.hbs b/frontend/app/templates/components/panel/left-panel.hbs index 3ba6eecf8..fc847b89a 100644 --- a/frontend/app/templates/components/panel/left-panel.hbs +++ b/frontend/app/templates/components/panel/left-panel.hbs @@ -4,12 +4,11 @@ {{elem/icon-base name="folder-open"}} Explorer {{elem/icon-base name="picture"}} View - {{elem/icon-base name="search"}} Feature Search - {{elem/icon-base name="search"}} Sequence Search + {{elem/icon-base name="search"}}Search {{elem/icon-base name="cloud-upload"}} Upload
    - + {{panel/manage-search view=view loadBlock=(action loadBlock) @@ -17,17 +16,17 @@ selectedFeatures=selectedFeatures updateFeaturesInBlocks="updateFeaturesInBlocks" }} - - {{panel/sequence-search datasets=datasets view=view refreshDatasets=refreshDatasets viewDataset=viewDataset }} + + {{panel/manage-explorer view=view diff --git a/frontend/app/templates/components/panel/manage-search.hbs b/frontend/app/templates/components/panel/manage-search.hbs index 485d64e38..0eed9f87d 100644 --- a/frontend/app/templates/components/panel/manage-search.hbs +++ b/frontend/app/templates/components/panel/manage-search.hbs @@ -1,5 +1,5 @@ -{{#elem/panel-container state="primary" as |panelContainer|}} +{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} {{#elem/panel-heading icon="search" panelContainer=panelContainer}} Feature Search {{/elem/panel-heading}} @@ -15,7 +15,7 @@ {{/if}} {{!-- showComponent --}} {{/elem/panel-container}} -{{#elem/panel-container state="primary" as |panelContainer|}} +{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} {{#elem/panel-heading icon="plane" panelContainer=panelContainer}} External Lookup {{/elem/panel-heading}} diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 7db8e5228..1a32f687b 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -1,5 +1,5 @@ -{{#elem/panel-container state="primary" as |panelContainer|}} +{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} {{!-- https://fontawesome.com/icons/dna Unicode f471 --}} {{#elem/panel-heading icon="search" panelContainer=panelContainer}} DNA Sequence Blast Search From 3bdf49e06de4a74765f9c845f5217a1509643aab Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 10 Jun 2021 13:49:01 +1000 Subject: [PATCH 50/77] search panel order : feature search, blast search, external lookup ... feature and blast can be open by default left-panel.hbs : use search-lookup split search-lookup out of manage-search, so that the order can be changed. --- frontend/app/components/panel/search-lookup.js | 6 ++++++ .../app/templates/components/panel/left-panel.hbs | 5 +++++ .../templates/components/panel/manage-search.hbs | 15 +-------------- .../templates/components/panel/search-lookup.hbs | 13 +++++++++++++ .../components/panel/sequence-search.hbs | 2 +- 5 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 frontend/app/components/panel/search-lookup.js create mode 100644 frontend/app/templates/components/panel/search-lookup.hbs diff --git a/frontend/app/components/panel/search-lookup.js b/frontend/app/components/panel/search-lookup.js new file mode 100644 index 000000000..7706ac890 --- /dev/null +++ b/frontend/app/components/panel/search-lookup.js @@ -0,0 +1,6 @@ +import Component from '@ember/component'; + +export default Component.extend({ + classNames: ['col-xs-12'], + +}); diff --git a/frontend/app/templates/components/panel/left-panel.hbs b/frontend/app/templates/components/panel/left-panel.hbs index fc847b89a..0c53a375a 100644 --- a/frontend/app/templates/components/panel/left-panel.hbs +++ b/frontend/app/templates/components/panel/left-panel.hbs @@ -24,6 +24,11 @@ viewDataset=viewDataset }} + {{panel/search-lookup + selectedFeatures=selectedFeatures + selectedBlock=selectedBlock + }} + diff --git a/frontend/app/templates/components/panel/manage-search.hbs b/frontend/app/templates/components/panel/manage-search.hbs index 0eed9f87d..11e6013d6 100644 --- a/frontend/app/templates/components/panel/manage-search.hbs +++ b/frontend/app/templates/components/panel/manage-search.hbs @@ -1,5 +1,5 @@ -{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} +{{#elem/panel-container state="primary" as |panelContainer|}} {{#elem/panel-heading icon="search" panelContainer=panelContainer}} Feature Search {{/elem/panel-heading}} @@ -15,16 +15,3 @@ {{/if}} {{!-- showComponent --}} {{/elem/panel-container}} -{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} - {{#elem/panel-heading icon="plane" panelContainer=panelContainer}} - External Lookup - {{/elem/panel-heading}} - {{#if panelContainer.showComponent}} - - {{goto-feature drawActions=this class="panel panel-primary" - selectedFeatures=selectedFeatures - selectedBlock=selectedBlock - }} - - {{/if}} {{!-- showComponent --}} -{{/elem/panel-container}} diff --git a/frontend/app/templates/components/panel/search-lookup.hbs b/frontend/app/templates/components/panel/search-lookup.hbs new file mode 100644 index 000000000..33d052bfe --- /dev/null +++ b/frontend/app/templates/components/panel/search-lookup.hbs @@ -0,0 +1,13 @@ +{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} + {{#elem/panel-heading icon="plane" panelContainer=panelContainer}} + External Lookup + {{/elem/panel-heading}} + {{#if panelContainer.showComponent}} + + {{goto-feature class="panel panel-primary" + selectedFeatures=selectedFeatures + selectedBlock=selectedBlock + }} + + {{/if}} {{!-- showComponent --}} +{{/elem/panel-container}} diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 1a32f687b..7db8e5228 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -1,5 +1,5 @@ -{{#elem/panel-container state="primary" showComponent=false as |panelContainer|}} +{{#elem/panel-container state="primary" as |panelContainer|}} {{!-- https://fontawesome.com/icons/dna Unicode f471 --}} {{#elem/panel-heading icon="search" panelContainer=panelContainer}} DNA Sequence Blast Search From 922a918c6b39b00ab1537ce4ff7fe607f82e024e Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 10 Jun 2021 17:13:31 +1000 Subject: [PATCH 51/77] sequence search : move the warning message field ... (e.g. about not selecting a reference) to appear between the search button and the drop down list --- .../app/templates/components/panel/sequence-search.hbs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 7db8e5228..cda5fa999 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -54,6 +54,10 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} {{/elem/button-base}}
    + {{#if nameWarning}} + {{elem/panel-message + warningMessage=nameWarning}} + {{/if}}
      @@ -194,10 +198,6 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} successMessage=successMessage warningMessage=warningMessage errorMessage=errorMessage}} - {{#if nameWarning}} - {{elem/panel-message - warningMessage=nameWarning}} - {{/if}} {{#if isProcessing}} {{#elem/panel-form From e62a521c2e2bc1bef1fb933cf6df19f5e89c21c5 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 10 Jun 2021 20:28:57 +1000 Subject: [PATCH 52/77] blast results : add .values { } this relates to upload-table.js, and to transient.js : showFeatures() -> .pushFeature(), and to backend tableUpload(). dataset.js : tableUpload() : data.features.forEach : include feature.values if defined. blast-results-view.js : extend columnsKeyString to cover all columns (after pos,end). didReceiveAttrs(): use this.registerDataPipe to connect validateData(). dataFeatures() : place the remainder of the columns, after the required/standard columns name/chr/pos/end, into feature.values. blast-results.js : move store,auth here from blast-results-view.js. blast-results.hbs : pass registerDataPipe to blast-results-view. upload-table.js : submitFile() : .validateData() may be defined via .dataPipe. --- backend/common/models/dataset.js | 8 +++- .../panel/upload/blast-results-view.js | 41 ++++++++++++++++--- .../components/panel/upload/blast-results.js | 13 +++++- .../components/panel/upload/blast-results.hbs | 1 + frontend/app/utils/panel/upload-table.js | 4 +- 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/backend/common/models/dataset.js b/backend/common/models/dataset.js index 6e3328862..61ba0e6d6 100644 --- a/backend/common/models/dataset.js +++ b/backend/common/models/dataset.js @@ -240,12 +240,16 @@ module.exports = function(Dataset) { if (feature.end !== undefined) { value.push(feature.end); } - array_features.push({ + let f = { name: feature.name, value, value_0: feature.val, blockId: blocks_by_name[feature.block] - }); + }; + if (feature.values) { + f.values = feature.values; + } + array_features.push(f); }); // create new features return models.Feature.create(array_features); diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 625a0d528..2d7b00810 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -24,7 +24,20 @@ const tableHeight = 500; // pixels * This also aligns with createTable() : colHeaders below. */ const columnsKeyString = [ - 'name', 'chr', 'pcIdentity', 'lengthOfHspHit', 'numMismatches', 'numGaps', 'queryStart', 'queryEnd', 'pos', 'end' + 'name', // query ID + 'chr', // subject ID + 'pcIdentity', // % identity + 'lengthOfHspHit', // length of HSP (hit) + 'numMismatches', // # mismatches + 'numGaps', // # gaps + 'queryStart', // query start + 'queryEnd', // query end + 'pos', // subject start + 'end', // subject end + 'eValue', // e-value + 'score', // score + 'queryLength', // query length + 'subjectLength', // subject length ]; /** Identify the columns of dataFeatures and dataMatrix. */ @@ -43,11 +56,6 @@ const t_view = 0; */ export default Component.extend({ - /** Similar comment to data-csv.js applies re. store (user could select server via GUI). - * store is used by upload-table.js : getDatasetId() and submitFile() - */ - store : alias('apiServers.primaryServer.store'), - auth: service('auth'), transient : service('data/transient'), /*--------------------------------------------------------------------------*/ @@ -61,12 +69,18 @@ export default Component.extend({ didReceiveAttrs() { this._super(...arguments); + let promise = this.get('search.promise'); if (promise) { promise.catch(() => { this.set('statusMessage', 'The search did not complete'); }); } + + /** not clear yet where addDataset functionality will end up, so wire this up for now. */ + this.registerDataPipe({ + validateData: () => this.validateData() + }); }, /*--------------------------------------------------------------------------*/ @@ -121,6 +135,21 @@ export default Component.extend({ if (row[c_end] !== undefined) { feature.end = Number(row[c_end]); } + // place the remainder of the columns into feature.values + feature.values = row.reduce((v, c, i) => { + switch (i) { + case c_name: + case c_chr: + case c_pos: + case c_end: + break; + default: + v[columnsKeyString[i]] = c; + break; + } + return v; + }, {}); + return feature; }); dLog('dataFeatures', features.length, features[0]); diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index a8f01d7df..c33e42b16 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -18,6 +18,17 @@ const dLog = console.debug; * /Feature/dnaSequenceSearch */ export default Component.extend({ + /*--------------------------------------------------------------------------*/ + // support for upload-table + + /** Similar comment to data-csv.js applies re. store (user could select server via GUI). + * store is used by upload-table.js : getDatasetId() and submitFile() + */ + store : alias('apiServers.primaryServer.store'), + auth: service('auth'), // used by upload-table.js : submitFile() + + /*--------------------------------------------------------------------------*/ + apiServers: service(), blockService : service('data/block'), queryParams: service('query-params'), @@ -150,7 +161,7 @@ export default Component.extend({ */ unviewDataset(datasetName) { let - store = this.get('apiServers').get('primaryServer').get('store'), + store = this.get('store'), replacedDataset = store.peekRecord('dataset', datasetName); if (replacedDataset) { let diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index e7b7c3a73..f7685af70 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -49,6 +49,7 @@ {{panel/upload/blast-results-view viewDataset=viewDataset search=search data=data active=active tableVisible=tableVisible tableModal=tableModal + registerDataPipe=(action (mut this.dataPipe)) }} {{/ember-wormhole}} diff --git a/frontend/app/utils/panel/upload-table.js b/frontend/app/utils/panel/upload-table.js index 1f23d13f3..7ff7bfaae 100644 --- a/frontend/app/utils/panel/upload-table.js +++ b/frontend/app/utils/panel/upload-table.js @@ -99,7 +99,9 @@ export default { that.set('nameWarning', null); var table = that.get('table'); // 1. Check data and get cleaned copy - that.validateData() + let validateData = (that.validateData && (() => that.validateData())) || + (that.dataPipe.validateData && (() => that.dataPipe.validateData())); + validateData() .then((features) => { if (features.length > 0) { // 2. Get new or selected dataset name From 7dd687a573ffd026416a0acd71909aaa3af879bf Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 11 Jun 2021 12:15:18 +1000 Subject: [PATCH 53/77] blast-results-view : add just the viewed features blast-results-view.js : validateData() : use getDataAtCol() to get the viewed checkbox values and use them to filter the returned features. --- .../app/components/panel/upload/blast-results-view.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 2d7b00810..49778944b 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -480,6 +480,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que /** upload-table.js : submitFile() expects this function. * In blast-results, the data is not user input so validation is not required. + * Add just those features for which the 'viewed' checkbox is clicked. */ validateData() { /** based on data-csv.js : validateData(), which uses table.getSourceData(); @@ -492,8 +493,11 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que resolve([]); } let - validatedData = this.get('dataFeatures'); - resolve(validatedData); + validatedData = this.get('dataFeatures'), + viewRow = this.get('viewRow'), + view = table.getDataAtCol(t_view), + viewed = validatedData.filter((r,i) => view[i]); + resolve(viewed); }); } From 0eaa8b64899af51a5bb71003a456151eb82ad1c2 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 11 Jun 2021 12:56:50 +1000 Subject: [PATCH 54/77] Handle feature._name in verify paths-progressive.js : verifyFeatureRecord() f may have ._name instead of .name, after push; this change could have been included in 1486120. --- frontend/app/services/data/paths-progressive.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/services/data/paths-progressive.js b/frontend/app/services/data/paths-progressive.js index 875dd7f6e..19d74f655 100644 --- a/frontend/app/services/data/paths-progressive.js +++ b/frontend/app/services/data/paths-progressive.js @@ -50,7 +50,7 @@ function verifyFeatureRecord(fr, f) { same = (fr.id === f._id) && direction && sameDirection && - ((frd ? frd._name : fr.get('name')) === f.name); + ((frd ? frd._name : fr.get('name')) === (f.name || f._name)); return same; } From 28bec64671336ef217ea546d8acab282ab822db9 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 11 Jun 2021 16:30:06 +1000 Subject: [PATCH 55/77] blast-results-view : implement : View all axes with results blast-results-view.js : factor viewParent() out of viewFeatures(). Add parentIsViewedEffect() to update viewAllResultAxesFlag when user un-views an axis of the result. mapview.js : viewDataset() : if ! view, unview the data blocks before the axis / reference block. blast-results-view.hbs : entail parentIsViewedEffect. --- frontend/app/components/draw/axis-1d.js | 2 ++ .../panel/upload/blast-results-view.js | 33 +++++++++++++++---- frontend/app/controllers/mapview.js | 11 +++++-- frontend/app/models/block.js | 5 +++ .../panel/upload/blast-results-view.hbs | 1 + 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/frontend/app/components/draw/axis-1d.js b/frontend/app/components/draw/axis-1d.js index 69728813b..e4116743a 100644 --- a/frontend/app/components/draw/axis-1d.js +++ b/frontend/app/components/draw/axis-1d.js @@ -707,6 +707,8 @@ export default Component.extend(Evented, AxisEvents, AxisPosition, { }), /** viewed blocks on this axis. * For just the data blocks (depends on .hasFeatures), @see dataBlocks() + * @desc + * Related : block : viewedChildBlocks(). */ viewedBlocks : computed('axis', 'blockService.axesViewedBlocks2.[]', function () { let diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 49778944b..384be6dc1 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -55,7 +55,7 @@ const t_view = 0; * This enables the full width of the table to be visible. */ export default Component.extend({ - + block : service('data/block'), transient : service('data/transient'), /*--------------------------------------------------------------------------*/ @@ -222,8 +222,11 @@ export default Component.extend({ /** this value seems to be delayed */ let viewAll = this.get('viewAllResultAxesFlag'); dLog(fnName, checked, viewAll); - - + viewAll = checked; + this.viewFeatures(viewAll); + if (! viewAll) { + this.viewParent(viewAll); + } }, viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewFeaturesFlag', 'active', function () { @@ -231,15 +234,33 @@ export default Component.extend({ let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); this.viewFeatures(viewFeaturesFlag); }), + viewParent(viewFlag) { + const fnName = 'viewParent'; + let parentName = this.get('search.parent'); + dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); + this.get('viewDataset')(parentName, viewFlag, this.get('blockNames')); + }, + /** User may un-view some of the parent axes viewed via viewParent(); + * if so, clear viewAllResultAxesFlag so that they may re-view all + * of them by clicking the toggle. + */ + parentIsViewedEffect : computed('block.viewed.[]', 'blockNames.length', function () { + let parentName = this.get('search.parent'); + /** blocks of parent which are viewed */ + let + viewedBlocks = this.get('block.viewed') + .filter((block) => (block.get('datasetId.id') === parentName)), + allViewed = viewedBlocks.length === this.get('blockNames.length'); + dLog('parentIsViewed', viewedBlocks.length, this.get('blockNames.length')); + this.set('viewAllResultAxesFlag', allViewed); + }), viewFeatures(viewFeaturesFlag) { const fnName = 'viewFeaturesEffect'; let features = this.get('dataFeaturesForStore'); if (features && features.length) { if (viewFeaturesFlag) { - let parentName = this.get('search.parent'); - dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); - this.get('viewDataset')(parentName, true, this.get('blockNames')); + this.viewParent(viewFeaturesFlag); } let transient = this.get('transient'), diff --git a/frontend/app/controllers/mapview.js b/frontend/app/controllers/mapview.js index 47e8f7e1d..25ab8b57b 100644 --- a/frontend/app/controllers/mapview.js +++ b/frontend/app/controllers/mapview.js @@ -372,9 +372,14 @@ export default Controller.extend(Evented, { blockService = this.get('block'), blockIds = blocksToChange.map((b) => b.id); dLog('viewDataset', datasetName, view, blockIds); - // blockService.setViewed(blockIds, view); - let loadBlock = this.actions.loadBlock.bind(this); - blocksToChange.forEach((b) => loadBlock(b)); + if (! view) { + // unview the data blocks before the axis / reference block. + blocksToChange.forEach((b) => b.unViewChildBlocks()); + blockService.setViewed(blockIds, view); + } else { + let loadBlock = this.actions.loadBlock.bind(this); + blocksToChange.forEach((b) => loadBlock(b)); + } } else { dLog('viewDataset', datasetName, 'not found', view); } diff --git a/frontend/app/models/block.js b/frontend/app/models/block.js index 2ab12c292..226ee4667 100644 --- a/frontend/app/models/block.js +++ b/frontend/app/models/block.js @@ -720,6 +720,11 @@ export default Model.extend({ childBlocks = blocksByReference && blocksByReference.get(this); return childBlocks || []; }), + /** @return child blocks of this block which are viewed. + * [] if none. If this block is not a reference then it has no child blocks. + * @desc + * Related : axis-1d:viewedBlocks(), which can be used to verify - should be equivalent. + */ viewedChildBlocks : computed('childBlocks.@each.isViewed', function () { let childBlocks = this.get('childBlocks'), viewedChildBlocks = childBlocks.filterBy('isViewed'); diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index fdc6a37b9..7a32bdefe 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -1,6 +1,7 @@ {{dataForTableEffect}} {{activeEffect}} {{viewFeaturesEffect}} +{{parentIsViewedEffect}} {{!-- --------------------------------------------------------------------- --}} From a8493d4e0a3f38a10e6e1a97f9ce6fbac46548b6 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 11 Jun 2021 18:08:14 +1000 Subject: [PATCH 56/77] sequence search : when child process returns empty file and status 0 (OK), return empty result to client feature.js : dnaSequenceSearch() : searchDataOut() : if ! chunk, reply OK with []. child-process.js : childProcess() : child.on(close ) : if status OK and ! child.killed, dataOutCb(null ). --- backend/common/models/feature.js | 4 ++++ backend/common/utilities/child-process.js | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/common/models/feature.js b/backend/common/models/feature.js index 3d374a963..57cdc90e4 100644 --- a/backend/common/models/feature.js +++ b/backend/common/models/feature.js @@ -76,9 +76,13 @@ module.exports = function(Feature) { /** Receive the results from the Blast. * @param chunk is a Buffer + * null / undefined indicates child process closed with status 0 (OK) and sent no output. * @param cb is cbWrap of cb passed to dnaSequenceSearch(). */ let searchDataOut = (chunk, cb) => { + if (! chunk) { + cb(null, []); + } else if (chunk.asciiSlice(0,6) === 'Error:') { cb(new Error(chunk.toString())); } else { diff --git a/backend/common/utilities/child-process.js b/backend/common/utilities/child-process.js index 1f22f454f..aff3f56f7 100644 --- a/backend/common/utilities/child-process.js +++ b/backend/common/utilities/child-process.js @@ -17,6 +17,8 @@ var fs = require('fs'); * @param moreParams array of params to pass as command-line params to * child process, after [fileName, useFile] * @param dataOutCb (Buffer chunk, cb) {} + * If child process closes with status 0 (OK) and sent no output, then + * dataOutCb will be called with chunk === null * @param cb response node callback * @param progressive true means pass received data back directly to * dataOutCb, otherwise catenate it and call dataOutCb just once when @@ -138,7 +140,12 @@ exports.childProcess = (scriptName, postData, useFile, fileName, moreParams, dat const message = 'Processed file ' + fileName; if (child.killed) { cb(null, message, true); - } // else check again after timeout + } else { // return empty data result + dataOutCb(null, cb); + /* it would be equivalent to do : cb(null, [], true); + * dataOutCb() may have completion actions. + */ + } } }); From 82d9f6185cca15af2f1ca4d041ca5dddc5571d3c Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 11 Jun 2021 19:14:34 +1000 Subject: [PATCH 57/77] sequence search : require minimum of 25 chars in search string. sequence-search.js : add searchStringMinLength : 25, inputsOK() : depend on .text, pass that to checkTextInput() and if warning set .warningMessage. checkTextInput() : add check : rawText.length < this.searchStringMinLength. move empty-input warningMessage from dnaSequenceInput() to checkTextInput(). --- .../app/components/panel/sequence-search.js | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 160c180a2..d200cb0b9 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -21,6 +21,8 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + /** required minimum length for FASTA DNA text search string input. */ + searchStringMinLength : 25, /** length limit for FASTA DNA text search string input. */ searchStringMaxLength : 10000, /** limit rows in result */ @@ -191,8 +193,13 @@ export default Component.extend({ } return ok; }, - inputsOK : computed('selectedParent', 'addDataset', 'newDatasetName', 'datasets.[]', function() { - return this.checkInputs(); + inputsOK : computed('text', 'selectedParent', 'addDataset', 'newDatasetName', 'datasets.[]', function() { + let warningMessage = this.checkTextInput(this.get('text')); + if (warningMessage) { + this.set('warningMessage', warningMessage); + } + /** checkInputs() sets .nameWarning */ + return ! warningMessage && this.checkInputs(); }), searchButtonDisabled : computed('inputsOK', 'isProcessing', function() { return ! this.get('inputsOK') || this.get('isProcessing'); @@ -207,6 +214,9 @@ export default Component.extend({ */ checkTextInput(rawText) { let warningMessages = []; + if (! rawText) { + warningMessages.push("Please enter search text in the field 'DNA Sequence Input'"); + } else { let lines = rawText.split('\n'), notBases = lines @@ -237,6 +247,9 @@ export default Component.extend({ if (rawText.length > this.searchStringMaxLength) { warningMessages.push('FASTA search string is limited to ' + this.searchStringMaxLength); + } else if (rawText.length < this.searchStringMinLength) { + warningMessages.push('FASTA search string should have minimum length ' + this.searchStringMinLength); + } } let warningMessage = warningMessages.length && warningMessages.join('\n'); @@ -254,9 +267,7 @@ export default Component.extend({ /** before textarea is created, .val() will be undefined. */ rawText = text$.val(); } - if (! rawText) { - this.set('warningMessage', "Please enter search text in the field 'DNA Sequence Input'"); - } else if ((warningMessage = this.checkTextInput(rawText))) { + if ((warningMessage = this.checkTextInput(rawText))) { this.set('warningMessage', warningMessage); } else { From 0aced5cb1badcd2f119fe9ba7f52361f92df840a Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 11 Jun 2021 19:45:48 +1000 Subject: [PATCH 58/77] sequence-search : move panel-message above Search button Move panel message from below BsTab into tab.pane #sequence-search-input so it is close to Search; this means it won't be visible when user views a blast-results tab; this is probably OK since the messages it displays will mostly occur in response to actions in sequence-search-input. --- .../app/templates/components/panel/sequence-search.hbs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index cda5fa999..999fbd58e 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -54,6 +54,11 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} {{/elem/button-base}}
    + {{elem/panel-message + successMessage=successMessage + warningMessage=warningMessage + errorMessage=errorMessage}} + {{#if nameWarning}} {{elem/panel-message warningMessage=nameWarning}} @@ -194,11 +199,6 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }} - {{elem/panel-message - successMessage=successMessage - warningMessage=warningMessage - errorMessage=errorMessage}} - {{#if isProcessing}} {{#elem/panel-form name="info" From d56e232dbeeda73e4316dfc1c79c2187413fedf4 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 15 Jun 2021 13:00:11 +1000 Subject: [PATCH 59/77] sequence search : use unique id per session for query string fileName (dnaSequence) ... to handle parallel requests. feature.js : add sessionIds, sessionIndex(). dnaSequenceSearch() : add queryStringFileName : use sessionIndex() to augment file name dnaSequence. (re. comment each user session may have 1 concurrent dnaSequenceSearch; this will be implemented in frontend client) --- backend/common/models/feature.js | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/common/models/feature.js b/backend/common/models/feature.js index 57cdc90e4..6828b0a5b 100644 --- a/backend/common/models/feature.js +++ b/backend/common/models/feature.js @@ -8,6 +8,23 @@ const { childProcess } = require('../utilities/child-process'); var upload = require('../utilities/upload'); var { filterBlastResults } = require('../utilities/sequence-search'); +/*----------------------------------------------------------------------------*/ + +/** ids of sessions which have sent request : dnaSequenceSearch */ +var sessionIds=[]; + +/** Map session ID (accessToken.id) to a small integer index. + */ +function sessionIndex(sessionId) { + let index = sessionIds.indexOf(sessionId); + if (index === -1) { + sessionIds.push(sessionId); + index = sessionIds.length - 1; + } + return index; +} + +/*----------------------------------------------------------------------------*/ module.exports = function(Feature) { Feature.search = function(filter, options, cb) { @@ -72,7 +89,11 @@ module.exports = function(Feature) { } = data; // data.options : params for streaming result, used later. const fnName = 'dnaSequenceSearch'; - console.log(fnName, dnaSequence.length, parent, searchType); + /** each user session may have 1 concurrent dnaSequenceSearch. + * Use session id for a unique index for dnaSequence fileName. */ + let index = sessionIndex(options.accessToken.id), + queryStringFileName = 'dnaSequence.' + index + '.fasta'; + console.log(fnName, dnaSequence.length, parent, searchType, index, queryStringFileName); /** Receive the results from the Blast. * @param chunk is a Buffer @@ -119,7 +140,7 @@ module.exports = function(Feature) { if (true) { let child = childProcess( 'dnaSequenceSearch.bash', - dnaSequence, true, 'dnaSequence', [parent, searchType, resultRows, addDataset, datasetName], searchDataOut, cb, /*progressive*/ false); + dnaSequence, true, queryStringFileName, [parent, searchType, resultRows, addDataset, datasetName], searchDataOut, cb, /*progressive*/ false); } else { let features = dev_blastResult; cb(null, features); From 984d5a1aac886bafa8a674097901b2dd5635272f Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 15 Jun 2021 16:21:00 +1000 Subject: [PATCH 60/77] sequence search : wrap with task to avoid accidental repeat sequence-search.js : rename loading : taskGet to searching : sendRequest (for .isRunning). searchButtonDisabled() : use .searching. split sendRequest() as a task out of dnaSequenceInput(). sequence-search.hbs : if this.searching, display Searching ... --- .../app/components/panel/sequence-search.js | 21 ++++++++++++------- .../components/panel/sequence-search.hbs | 3 +++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index d200cb0b9..2dbd796aa 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -4,6 +4,8 @@ import { inject as service } from '@ember/service'; import { observer, computed } from '@ember/object'; import { alias } from '@ember/object/computed'; import { A as array_A } from '@ember/array'; +import { task, didCancel } from 'ember-concurrency'; + import sequenceSearchData from '../../utils/data/sequence-search-data'; @@ -108,7 +110,7 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ - loading : alias('taskGet.isRunning'), + searching : alias('sendRequest.isRunning'), refreshClassNames : computed('loading', function () { let classNames = "btn btn-info pull-right"; @@ -201,8 +203,8 @@ export default Component.extend({ /** checkInputs() sets .nameWarning */ return ! warningMessage && this.checkInputs(); }), - searchButtonDisabled : computed('inputsOK', 'isProcessing', function() { - return ! this.get('inputsOK') || this.get('isProcessing'); + searchButtonDisabled : computed('searching', 'inputsOK', 'isProcessing', function() { + return this.get('searching') || ! this.get('inputsOK') || this.get('isProcessing'); }), /** throttle depends on constant function */ @@ -269,8 +271,11 @@ export default Component.extend({ } if ((warningMessage = this.checkTextInput(rawText))) { this.set('warningMessage', warningMessage); - } else - { + } else { + let taskInstance = this.get('sendRequest').perform(rawText); + } + }, + sendRequest : task(function* (rawText) { let seq = rawText; /* @@ -299,6 +304,7 @@ export default Component.extend({ /* On complete, trigger dataset list reload. * refreshDatasets is passed from controllers/mapview (updateModel ). */ + promise = promise.then(() => { const viewDataset = this.get('viewDatasetFlag'); let refreshed = this.get('refreshDatasets')(); @@ -320,8 +326,9 @@ export default Component.extend({ let searchData = sequenceSearchData.create({promise, seq, parent, searchType}); this.get('searches').pushObject(searchData); - } - }, + return promise; + }).drop(), + closeResultTab(tabId) { dLog('closeResultTab', tabId); diff --git a/frontend/app/templates/components/panel/sequence-search.hbs b/frontend/app/templates/components/panel/sequence-search.hbs index 999fbd58e..ff4d33896 100644 --- a/frontend/app/templates/components/panel/sequence-search.hbs +++ b/frontend/app/templates/components/panel/sequence-search.hbs @@ -43,6 +43,9 @@ AGCTGGGTGTCGTTGATCTTCAGGTCCTTCTGGATGTACAGCGACGCTCC" }}
    + {{#if this.searching}} + Searching ... + {{/if}} {{#elem/button-base click=(action "search") classNames=refreshClassNames From 93659b26149222eae50744ba1ac9bda2a9f36f95 Mon Sep 17 00:00:00 2001 From: Don Date: Tue, 15 Jun 2021 21:59:39 +1000 Subject: [PATCH 61/77] use upload-table in data-csv, so that it gets the addition of end position data-csv.js : Use these from upload-table.js (uploadTable), which was factored from data-csv.js in 9640e0f4 : getDatasetId(), isDupName(), onNameChange(), onSelectChange(), actions.submitFile. define column index constants : c_name, c_block, c_val, c_end. add viewDatasetFlag. table config : append column : End. validateData() : use table.getData() in place of .getSourceData() (see comment in checkBlocks() related to handsontable version update 7 -> 8). validateData() and checkBlocks() : use array indexes c_name, c_block, c_val, c_end in place of row.name, row.block, row.val. left-panel.hbs : pass viewDataset to upload-data. upload-data.hbs : pass-thru viewDataset to data-csv. data-csv.hbs : add checkbox View : viewDatasetFlag. This commit doesn't enable additional .values columns, that can be done by factoring blast-results-view.js : dataFeatures() --- .../app/components/panel/upload/data-csv.js | 215 +++++++----------- .../templates/components/panel/left-panel.hbs | 3 +- .../components/panel/upload-data.hbs | 1 + .../components/panel/upload/data-csv.hbs | 6 + 4 files changed, 90 insertions(+), 135 deletions(-) diff --git a/frontend/app/components/panel/upload/data-csv.js b/frontend/app/components/panel/upload/data-csv.js index d689c9d56..ae016efa8 100644 --- a/frontend/app/components/panel/upload/data-csv.js +++ b/frontend/app/components/panel/upload/data-csv.js @@ -7,9 +7,23 @@ import { inject as service } from '@ember/service'; const dLog = console.debug; import UploadBase from './data-base'; +import uploadTable from '../../../utils/panel/upload-table'; + import config from '../../../config/environment'; +/*----------------------------------------------------------------------------*/ + +/** Identify the columns of getData(). + * + * Prior to update from handsontable 7 -> 8, the row data was presented as {name, block, val}. + * These column indexes are used as row[c_name], so to use handsontable 7, + * c_name = 'name', c_block = 'block', c_val = 'block'; + */ +const c_name = 0, c_block = 1, c_val = 2, c_end = 3; + +/*----------------------------------------------------------------------------*/ + /* global Handsontable */ /* global FileReader */ @@ -20,6 +34,14 @@ export default UploadBase.extend({ * components/service/api-server.js) */ store : alias('apiServers.primaryServer.store'), + /*--------------------------------------------------------------------------*/ + + /** true means view the blocks of the dataset after it is added. + * Used in upload-table.js : submitFile(). + */ + viewDatasetFlag : false, + + /*--------------------------------------------------------------------------*/ table: null, selectedDataset: 'new', @@ -29,6 +51,16 @@ export default UploadBase.extend({ dataType: 'linear', namespace: '', + /*--------------------------------------------------------------------------*/ + + /** these functions were factored to form upload-table.js */ + getDatasetId : uploadTable.getDatasetId, + isDupName : uploadTable.isDupName, + onNameChange : observer('newDatasetName', uploadTable.onNameChange), + onSelectChange : observer('selectedDataset', 'selectedParent', uploadTable.onSelectChange), + + /*--------------------------------------------------------------------------*/ + didInsertElement() { this._super(...arguments); }, @@ -77,7 +109,7 @@ export default UploadBase.extend({ return; // fail } var table = new Handsontable(hotable, { - data: [['', '', '']], + data: [['', '', '', '']], minRows: 20, rowHeaders: true, columns: [ @@ -95,15 +127,23 @@ export default UploadBase.extend({ numericFormat: { pattern: '0,0.*' } + }, + { + data: 'end', + type: 'numeric', + numericFormat: { + pattern: '0,0.*' + } } ], colHeaders: [ 'Feature', 'Block', - 'Position' + 'Position', + 'End' ], height: 500, - colWidths: [100, 100, 100], + colWidths: [100, 100, 100, 100], manualRowResize: true, manualColumnResize: true, manualRowMove: true, @@ -148,7 +188,13 @@ export default UploadBase.extend({ if (table !== null) { let datasets = that.get('datasets'); if (datasets) { - let data = table.getSourceData(); + /** Previously getSourceData() was used, but that is now + * returning the (empty) initial data. Not sure how it worked + * before (probably change is related to recent update of + * handsOnTable from 7 to 8 (7.4.2 -> 8.2.0). + * getData() seems like the logical function to use, and it works. + */ + let data = table.getData(); let map = null; let parent = null; let selectedMap = that.get('selectedDataset'); @@ -170,9 +216,9 @@ export default UploadBase.extend({ }, {}); // 2. Find blocks duplicated in table data let duplicates = data.reduce((result, row) => { - if (row.block) { - if (row.block in blocks) { - result[row.block] = true; + if (row[c_block]) { + if (row[c_block] in blocks) { + result[row[c_block]] = true; } } return result; @@ -198,9 +244,9 @@ export default UploadBase.extend({ }, {}); // 2. Find table data blocks missing from parent blocks let missing = data.reduce((result, row) => { - if (row.block) { - if (!(row.block in parentBlocks)) { - result[row.block] = true; + if (row[c_block]) { + if (!(row[c_block] in parentBlocks)) { + result[row[c_block]] = true; } } return result; @@ -223,42 +269,6 @@ export default UploadBase.extend({ } }, - /** Returns a selected dataset name OR - * Attempts to create a new dataset with entered name */ - getDatasetId() { - var that = this; - let datasets = that.get('datasets'); - return new Promise(function(resolve, reject) { - var selectedMap = that.get('selectDataset'); - // If a selected dataset, can simply return it - // If no selectedMap, treat as default, 'new' - if (selectedMap && selectedMap !== 'new') { - resolve(selectedMap); - } else { - var newMap = that.get('newDatasetName'); - // Check if duplicate name - let matched = datasets.findBy('name', newMap); - if(matched){ - reject({ msg: `Dataset name '${newMap}' is already in use` }); - } else { - let newDetails = { - name: newMap, - type: that.get('dataType'), - namespace: that.get('namespace'), - blocks: [] - }; - let parentId = that.get('selectedParent'); - if (parentId && parentId.length > 0) { - newDetails.parentName = parentId; - } - let newDataset = that.get('store').createRecord('Dataset', newDetails); - newDataset.save().then(() => { - resolve(newDataset.id); - }); - } - } - }); - }, /** Checks uploaded table data for any missing or invalid elements. * Returns same data, with 'val' cast as numeric */ @@ -269,32 +279,41 @@ export default UploadBase.extend({ if (table === null) { resolve([]); } - let sourceData = table.getSourceData(); + /** was getSourceData() - see comment in checkBlocks(). */ + let sourceData = table.getData(); var validatedData = []; sourceData.every((row, i) => { - if (row.val || row.name || row.block) { - if (!row.val && row.val !== 0) { + if (row[c_val] || row[c_name] || row[c_block]) { + if (!row[c_val] && row[c_val] !== 0) { reject({r: i, c: 'val', msg: `Position required on row ${i+1}`}); return false; } - if (isNaN(row.val)) { + if (isNaN(row[c_val])) { reject({r: i, c: 'val', msg: `Position must be numeric on row ${i+1}`}); return false; } - if (!row.name) { + if ((row[c_end] !== undefined) && isNaN(row[c_end])) { + reject({r: i, c: 'end', msg: `End Position must be numeric on row ${i+1}`}); + return false; + } + if (!row[c_name]) { reject({r: i, c: 'name', msg: `Feature name required on row ${i+1}`}); return false; } - if (!row.block) { + if (!row[c_block]) { reject({r: i, c: 'block', msg: `Block required on row ${i+1}`}); return false; } - validatedData.push({ - name: row.name, - block: row.block, - // Make sure val is a number, not a string. - val: Number(row.val) - }); + let r = { + name: row[c_name], + block: row[c_block], + // Make sure val is a number, not a string. + val: Number(row[c_val]) + }; + if (row[c_end] !== undefined) { + r.end = +row[c_end]; + } + validatedData.push(r); return true; } }); @@ -302,84 +321,10 @@ export default UploadBase.extend({ }); }, - /** Checks if entered dataset name is already taken in dataset list - * Debounced call through observer */ - isDupName: function() { - let selectedMap = this.get('selectedDataset'); - if (selectedMap === 'new') { - let newMap = this.get('newDatasetName'); - let datasets = this.get('datasets'); - let matched = datasets.findBy('name', newMap); - if(matched){ - this.set('nameWarning', `Dataset name '${newMap}' is already in use`); - return true; - } - } - this.set('nameWarning', null); - return false; - }, - onNameChange: observer('newDatasetName', function() { - debounce(this, this.isDupName, 500); - }), - onSelectChange: observer('selectedDataset', 'selectedParent', function() { - this.clearMsgs(); - this.isDupName(); - this.checkBlocks(); - }), + /*--------------------------------------------------------------------------*/ actions: { - submitFile() { - var that = this; - that.clearMsgs(); - that.set('nameWarning', null); - var table = that.get('table'); - // 1. Check data and get cleaned copy - that.validateData() - .then((features) => { - if (features.length > 0) { - // 2. Get new or selected dataset name - that.getDatasetId().then((map_id) => { - var data = { - dataset_id: map_id, - parentName: that.get('selectedParent'), - features: features, - namespace: that.get('namespace'), - }; - that.setProcessing(); - that.scrollToTop(); - // 3. Submit upload to api - that.get('auth').tableUpload(data, that.updateProgress.bind(that)) - .then((res) => { - that.setSuccess(res.status); - that.scrollToTop(); - // On complete, trigger dataset list reload - // through controller-level function - that.get('refreshDatasets')(); - }, (err, status) => { - console.log(err, status); - that.setError(err.responseJSON.error.message); - that.scrollToTop(); - if(that.get('selectedDataset') === 'new'){ - // If upload failed and here, a new record for new dataset name - // has been created by getDatasetId() and this should be undone - that.get('store') - .findRecord('Dataset', map_id, { reload: true }) - .then((rec) => rec.destroyRecord() - .then(() => rec.unloadRecord()) - ); - } - }); - }, (err) => { - that.setError(err.msg); - that.scrollToTop(); - }); - } - }, (err) => { - table.selectCell(err.r, err.c); - that.setError(err.msg); - that.scrollToTop(); - }); - }, + submitFile : uploadTable.submitFile, clearTable() { $("#tableFile").val(''); var table = this.get('table'); @@ -425,4 +370,6 @@ export default UploadBase.extend({ } } } + /*--------------------------------------------------------------------------*/ + }); diff --git a/frontend/app/templates/components/panel/left-panel.hbs b/frontend/app/templates/components/panel/left-panel.hbs index 0c53a375a..892fbf6be 100644 --- a/frontend/app/templates/components/panel/left-panel.hbs +++ b/frontend/app/templates/components/panel/left-panel.hbs @@ -21,7 +21,7 @@ datasets=datasets view=view refreshDatasets=refreshDatasets - viewDataset=viewDataset + viewDataset=viewDataset }} {{panel/search-lookup @@ -75,6 +75,7 @@ {{panel/upload-data datasets=datasets refreshDatasets=refreshDatasets + viewDataset=viewDataset active=(bs-eq tab.activeId "left-panel-upload") }} diff --git a/frontend/app/templates/components/panel/upload-data.hbs b/frontend/app/templates/components/panel/upload-data.hbs index a117f3972..c4c39b2cf 100644 --- a/frontend/app/templates/components/panel/upload-data.hbs +++ b/frontend/app/templates/components/panel/upload-data.hbs @@ -57,6 +57,7 @@ {{panel/upload/data-csv datasets=datasets refreshDatasets=refreshDatasets + viewDataset=viewDataset active=active }} {{else if (compare filter '===' 'json')}} diff --git a/frontend/app/templates/components/panel/upload/data-csv.hbs b/frontend/app/templates/components/panel/upload/data-csv.hbs index 6d9ccb0a8..e924e2288 100644 --- a/frontend/app/templates/components/panel/upload/data-csv.hbs +++ b/frontend/app/templates/components/panel/upload/data-csv.hbs @@ -76,6 +76,12 @@
    + + {{input type="checkbox" name="viewDatasetFlag" checked=viewDatasetFlag }} + + + + Date: Wed, 16 Jun 2021 12:31:09 +1000 Subject: [PATCH 62/77] sequence search : 3 GUI refinements sequence-search.js : default range sliders to 0 blast-results-view.hbs : change text 'Show / Hide all Features (triangles)' to 'Select / Deselect all' blast-results.hbs : don't show 'Show Table' toggle --- frontend/app/components/panel/sequence-search.js | 4 ++-- .../templates/components/panel/upload/blast-results-view.hbs | 2 +- .../app/templates/components/panel/upload/blast-results.hbs | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 2dbd796aa..82525a199 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -32,8 +32,8 @@ export default Component.extend({ /** minimum values for 3 columns, to filter blast output. */ minLengthOfHit : 0, - minPercentIdentity : 75, - minPercentCoverage : 50, + minPercentIdentity : 0, // 75, + minPercentCoverage : 0, // 50, /** true means add / upload result to db as a Dataset */ addDataset : false, diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index 7a32bdefe..b66d7a68c 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -24,7 +24,7 @@
    {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} - +
    {{!-- --------------------------------------------------------------------- --}} diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index f7685af70..a55d74887 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -4,6 +4,7 @@
    + {{#if 0}} {{#elem/x-toggle value=tableVisible }} {{#ember-tooltip side="right" delay=500}} Show/hide the table. @@ -12,6 +13,7 @@ {{#unless tableVisible}} Show Table {{/unless}} + {{/if}} {{#if this.active}} From c89982c9e01492a36f405ffcc9afa0c20f96e7ac Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 16 Jun 2021 16:54:32 +1000 Subject: [PATCH 63/77] blast-results-view : preserve the selected (view flag) column value ... when switching from results-view tab to sequence-search input tab and back again. sequence-search.js : update comment for inputIsActive(), which is actively used since 086ea39e. blast-results-view.js : change CP viewRow() to viewRowInit() which computes initial search.viewRow, called in didReceiveAttrs() if this is the first time that search result has been displayed. viewFeatures() : pass viewRow to showFeatures(). transient.js : showFeatures() : add param view --- .../app/components/panel/sequence-search.js | 6 +++++- .../panel/upload/blast-results-view.js | 21 ++++++++++++++----- frontend/app/services/data/transient.js | 8 +++++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index 82525a199..a618d4c5e 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -131,8 +131,12 @@ export default Component.extend({ // actions actions: { - // copied from feature-list, may not be required + /** called for single-character input to textarea; similar to + * actions.dnaSequenceInput() but that is only called for defined events + * (enter / escape), and actions.paste() (paste). + */ inputIsActive(event) { + // function name and use is copied from feature-list. dLog('inputIsActive', event?.target); let text = event?.target?.value; if (text) { diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 384be6dc1..f5bebb085 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -77,6 +77,13 @@ export default Component.extend({ }); } + /** search.viewRow will be undefined when this is the first + * blast-results-view instance to display this search result. + */ + if (! this.get('viewRow')) { + this.viewRowInit(); + } + /** not clear yet where addDataset functionality will end up, so wire this up for now. */ this.registerDataPipe({ validateData: () => this.validateData() @@ -110,13 +117,15 @@ export default Component.extend({ } return cells; }), - viewRow : computed('dataMatrix', 'viewFeaturesFlag', function () { + viewRowInit() { let data = this.get('dataMatrix'), + /** viewFeaturesFlag is initially true */ viewFeaturesFlag = this.get('viewFeaturesFlag'), viewRow = data.map((row) => viewFeaturesFlag); - return viewRow; - }), + this.set('search.viewRow', viewRow); + }, + viewRow : alias('search.viewRow'), /** Result data formatted for upload-table.js : submitFile() */ @@ -293,7 +302,7 @@ export default Component.extend({ */ nowOrLater( viewFeaturesFlag, - () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag)); + () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag, this.get('viewRow'))); } }, @@ -491,6 +500,8 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que let feature = transient.pushFeature(features[row]), viewFeaturesFlag = newValue; transient.showFeature(feature, viewFeaturesFlag); + let viewRow = this.get('viewRow'); + viewRow[row] = newValue; } }); } @@ -515,7 +526,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que } let validatedData = this.get('dataFeatures'), - viewRow = this.get('viewRow'), + // or view = this.get('viewRow'), view = table.getDataAtCol(t_view), viewed = validatedData.filter((r,i) => view[i]); resolve(viewed); diff --git a/frontend/app/services/data/transient.js b/frontend/app/services/data/transient.js index bea4e9586..c55589ae1 100644 --- a/frontend/app/services/data/transient.js +++ b/frontend/app/services/data/transient.js @@ -102,12 +102,16 @@ export default Service.extend({ return blocks; }, - showFeatures(dataset, blocks, features, viewFeaturesFlag) { + /** + * @param view a flag per-feature to enable display of the feature row; + * from values of the View checkbox column of the results features table. + */ + showFeatures(dataset, blocks, features, viewFeaturesFlag, view) { let selected = this.get('selected'), // may pass dataset, blocks to pushFeature() stored = features.map((f) => this.pushFeature(f)); - stored.forEach((feature) => this.showFeature(feature, viewFeaturesFlag)); + stored.forEach((feature, i) => this.showFeature(feature, viewFeaturesFlag && view[i])); }, showFeature(feature, viewFeaturesFlag) { let From 79d650b2c3135752b0c00f111a1261376924fa80 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 16 Jun 2021 22:11:45 +1000 Subject: [PATCH 64/77] blast-results-view : use the feature view checkbox to enable axis display. blast-results-view.js : declare viewAllResultAxesFlag. didReceiveAttrs() : if viewRow is defined, use it to initialise viewFeaturesFlag. add : viewFeaturesChange(), narrowAxesToViewedEffect(), narrowAxesToViewed(), resultParentBlocks(). viewFeatures() : use narrowAxesToViewed() in place of viewParent(). afterChange() : set viewRow[row] before narrowAxesToViewed and showFeature. use narrowAxesToViewed(). blast-results-view.hbs : replace use of parentIsViewedEffect to narrowAxesToViewedEffect. viewAllResultAxesFlag is now constant true. connect action viewFeaturesChange to checkbox viewFeaturesFlag. --- .../panel/upload/blast-results-view.js | 118 +++++++++++++++++- .../panel/upload/blast-results-view.hbs | 8 +- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index f5bebb085..7474ba4e6 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -55,6 +55,7 @@ const t_view = 0; * This enables the full width of the table to be visible. */ export default Component.extend({ + apiServers: service(), block : service('data/block'), transient : service('data/transient'), @@ -63,6 +64,11 @@ export default Component.extend({ /** true means display the result rows as triangles - clickedFeatures. */ viewFeaturesFlag : true, + /** Control of viewing the axes of the parent blocks corresponding to the result features. + * Originally a toggle to view all / none; now constant true, and narrowAxesToViewed() is used. + */ + viewAllResultAxesFlag : true, + statusMessage : 'Searching ...', /*--------------------------------------------------------------------------*/ @@ -82,6 +88,11 @@ export default Component.extend({ */ if (! this.get('viewRow')) { this.viewRowInit(); + } else { + /** set viewFeaturesFlag false if any of viewRow[*] are false, + * enabling the user to toggle them all on. + */ + this.set('viewFeaturesFlag', ! this.get('viewRow').any((v) => !v)); } /** not clear yet where addDataset functionality will end up, so wire this up for now. */ @@ -238,11 +249,90 @@ export default Component.extend({ } }, + viewFeaturesChange(proxy) { + const fnName = 'viewFeaturesChange'; + let viewFeaturesFlag = proxy.target.checked; + dLog(fnName, viewFeaturesFlag); + // wait for this.viewFeaturesFlag to change, because viewRowInit() reads it. + run_later(() => this.viewRowInit()); + }, viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewFeaturesFlag', 'active', function () { /** Only view features of the active tab. */ let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); this.viewFeatures(viewFeaturesFlag); }), + /** if .viewAllResultAxesFlag, narrowAxesToViewed() + */ + narrowAxesToViewedEffect : computed( + 'viewAllResultAxesFlag', 'dataFeatures', 'blockNames', 'viewRow', + 'resultParentBlocks', + function () { + const fnName = 'narrowAxesToViewedEffect'; + if (this.get('viewAllResultAxesFlag')) { + this.narrowAxesToViewed(); + } + }), + /** unview parent blocks of result features which are de-selected / un-viewed. + * If parent block has other viewed data blocks on it, don't unview it. + */ + narrowAxesToViewed () { + const fnName = 'narrowAxesToViewed'; + let + blockNames = this.get('blockNames'), + /** could use dataFeaturesForStore, or feature record handles from transient.pushFeature() */ + features = this.get('dataFeatures'), + viewRow = this.get('viewRow'), + + /** indexed by block name, true if there are viewed / selected + * features within this parent block in the result. + */ + hasViewedFeatures = features.reduce((result, feature, i) => { + dLog(feature, feature.block, feature.blockId, viewRow[i]); + if (viewRow[i]) { + result[feature.block] = true; + } + return result; }, {}); + + let + blocks = this.get('resultParentBlocks'), + /** split resultParentBlocks() into parentBlocks and resultParentBlocksByName(); array result is not required. */ + blocksByName = blocks.reduce((result, block) => {result[block.get('name')] = block; return result; }, {}); + blocks = blocksByName; + + /** toView[] is an array of blockNames to view / unview (depending on flag). */ + let + toView = blockNames.reduce((result, name) => { + let + block = blocks[name], + /** may be undefined, which is equivalent to false. */ + viewedFeatures = hasViewedFeatures[name] ?? false, + change = block.get('isViewed') !== viewedFeatures; + if (change && (viewedFeatures || ! block.get('viewedChildBlocks.length'))) { + (result[viewedFeatures] ||= []).push(name); + } + return result; }, {}); + + let + parentName = this.get('search.parent'); + dLog(fnName, 'viewDataset', parentName, this.get('search.timeId')); + [true, false].forEach( + (viewFlag) => toView[viewFlag] && this.get('viewDataset')(parentName, viewFlag, toView[viewFlag])); + }, + + resultParentBlocks : computed('search.parent', 'blockNames.[]', function () { + const fnName = 'resultParentBlocks'; + let parentName = this.get('search.parent'); + let blockNames = this.get('blockNames'); + let + store = this.get('apiServers').get('primaryServer').get('store'), + dataset = store.peekRecord('dataset', parentName); + let + blocks = dataset && dataset.get('blocks').toArray() + .filter((b) => (blockNames.indexOf(b.get('name')) !== -1) ); + + dLog('resultParentBlocks', parentName, blockNames, blocks); + return blocks; + }), viewParent(viewFlag) { const fnName = 'viewParent'; let parentName = this.get('search.parent'); @@ -252,6 +342,7 @@ export default Component.extend({ /** User may un-view some of the parent axes viewed via viewParent(); * if so, clear viewAllResultAxesFlag so that they may re-view all * of them by clicking the toggle. + * Probably will be replaced by narrowAxesToViewedEffect */ parentIsViewedEffect : computed('block.viewed.[]', 'blockNames.length', function () { let parentName = this.get('search.parent'); @@ -264,12 +355,17 @@ export default Component.extend({ this.set('viewAllResultAxesFlag', allViewed); }), viewFeatures(viewFeaturesFlag) { - const fnName = 'viewFeaturesEffect'; + const fnName = 'viewFeatures'; let features = this.get('dataFeaturesForStore'); if (features && features.length) { if (viewFeaturesFlag) { - this.viewParent(viewFeaturesFlag); + /* viewParent() could be used initially. + * viewParent() views all of .blockNames in the parent, + * whereas narrowAxesToViewed() takes into account .viewRow + */ + this.narrowAxesToViewed(); + // this.viewParent(viewFeaturesFlag); } let transient = this.get('transient'), @@ -497,14 +593,24 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que } else if (row >= features.length) { this.set('warningMessage', 'Display of added features not yet supported'); } else { - let feature = transient.pushFeature(features[row]), - viewFeaturesFlag = newValue; - transient.showFeature(feature, viewFeaturesFlag); let viewRow = this.get('viewRow'); viewRow[row] = newValue; + + let feature = transient.pushFeature(features[row]), + viewFeaturesFlag = newValue; + if (newValue) { + this.narrowAxesToViewed(); + } + // wait until axis is shown before showing features + nowOrLater( + newValue, + () => transient.showFeature(feature, viewFeaturesFlag)); + if (! newValue) { + this.narrowAxesToViewed(); + } } }); - } + } }, /*--------------------------------------------------------------------------*/ diff --git a/frontend/app/templates/components/panel/upload/blast-results-view.hbs b/frontend/app/templates/components/panel/upload/blast-results-view.hbs index b66d7a68c..3d1ab2064 100644 --- a/frontend/app/templates/components/panel/upload/blast-results-view.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results-view.hbs @@ -1,7 +1,7 @@ {{dataForTableEffect}} {{activeEffect}} {{viewFeaturesEffect}} -{{parentIsViewedEffect}} +{{narrowAxesToViewedEffect}} {{!-- --------------------------------------------------------------------- --}} @@ -15,15 +15,19 @@ {{!-- --------------------------------------------------------------------- --}} +{{#if 0}}
    {{input type="checkbox" name="viewAllResultAxesFlag" checked=viewAllResultAxesFlag input=(action viewAllResultAxesChange) }}
    +{{/if}}
    - {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag }} + {{input type="checkbox" name="viewFeaturesFlag" checked=viewFeaturesFlag + input=(action viewFeaturesChange) + }}
    From 6d9a4f6a93831df5603668ef605af22274a13b09 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 17 Jun 2021 12:53:20 +1000 Subject: [PATCH 65/77] blast-results-view : handle request delay, which was causing view checkboxes to be unticked initially sequence-search.js : didReceiveAttrs() : check viewRow.length - initially viewRow will be initialised from no data (if the request takes a few seconds); calculating viewFeaturesFlag from empty viewRow results in false, and then viewRow=[false,...]. --- frontend/app/components/panel/sequence-search.js | 2 +- frontend/app/components/panel/upload/blast-results-view.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/panel/sequence-search.js b/frontend/app/components/panel/sequence-search.js index a618d4c5e..e0020cc5c 100644 --- a/frontend/app/components/panel/sequence-search.js +++ b/frontend/app/components/panel/sequence-search.js @@ -137,7 +137,7 @@ export default Component.extend({ */ inputIsActive(event) { // function name and use is copied from feature-list. - dLog('inputIsActive', event?.target); + // dLog('inputIsActive', event?.target); let text = event?.target?.value; if (text) { this.set('text', text); diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 7474ba4e6..018008b51 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -86,9 +86,10 @@ export default Component.extend({ /** search.viewRow will be undefined when this is the first * blast-results-view instance to display this search result. */ - if (! this.get('viewRow')) { + if (! this.get('viewRow.length')) { this.viewRowInit(); } else { + dLog('viewRow', this.viewRow, 'viewFeaturesFlag', this.viewFeaturesFlag); /** set viewFeaturesFlag false if any of viewRow[*] are false, * enabling the user to toggle them all on. */ @@ -135,6 +136,7 @@ export default Component.extend({ viewFeaturesFlag = this.get('viewFeaturesFlag'), viewRow = data.map((row) => viewFeaturesFlag); this.set('search.viewRow', viewRow); + dLog('viewRowInit', data.length, viewRow); }, viewRow : alias('search.viewRow'), From 18196f2f390508f37eab9271428c24bf9ebef24c Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 18 Jun 2021 12:22:32 +1000 Subject: [PATCH 66/77] search feature labels : augment name with location in showLabels keyFn This may address stray transition on feature lable - in sequence search result, all feature triangles have the same name, and this would confuse the d3 join and could cause entry/exit transitions. axis-1d.js : FeatureTicks:showLabels() : append location (feature.value[0]) in the keyFn. --- frontend/app/components/draw/axis-1d.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/draw/axis-1d.js b/frontend/app/components/draw/axis-1d.js index e4116743a..0e845988b 100644 --- a/frontend/app/components/draw/axis-1d.js +++ b/frontend/app/components/draw/axis-1d.js @@ -472,7 +472,9 @@ FeatureTicks.prototype.showLabels = function (featuresOfBlockLookup, setupHover, function keyFn (feature) { // here `this` is the parent of the -s, e.g. g.axis - let featureName = getAttrOrCP(feature, 'name'); + let + value = getAttrOrCP(feature, 'value'), + featureName = getAttrOrCP(feature, 'name') + '-' + value.[0]; // dLog('keyFn', feature, featureName); return featureName; }; From 0772e119c5d40c1033e1de3d412830c59a1d7afc Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 18 Jun 2021 13:12:31 +1000 Subject: [PATCH 67/77] blast-results-view : disable cell edit in table blast-results-view.js : narrowAxesToViewed() : handle block undefined - if parent does not have block with name matching result; don't display an axis. resultParentBlocks() : trace when parent has no blocks but search result has. createTable() : for columns after the view checkbox : disable editor . --- .../panel/upload/blast-results-view.js | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 018008b51..3be8426ce 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -296,6 +296,7 @@ export default Component.extend({ return result; }, {}); let + /** it is possible that the parent reference does not have blocks with name matching the results (blockNames) */ blocks = this.get('resultParentBlocks'), /** split resultParentBlocks() into parentBlocks and resultParentBlocksByName(); array result is not required. */ blocksByName = blocks.reduce((result, block) => {result[block.get('name')] = block; return result; }, {}); @@ -305,10 +306,11 @@ export default Component.extend({ let toView = blockNames.reduce((result, name) => { let + /** undefined if parent does not have block with name matching result; don't display an axis. */ block = blocks[name], /** may be undefined, which is equivalent to false. */ viewedFeatures = hasViewedFeatures[name] ?? false, - change = block.get('isViewed') !== viewedFeatures; + change = block && (block.get('isViewed') !== viewedFeatures); if (change && (viewedFeatures || ! block.get('viewedChildBlocks.length'))) { (result[viewedFeatures] ||= []).push(name); } @@ -332,7 +334,9 @@ export default Component.extend({ blocks = dataset && dataset.get('blocks').toArray() .filter((b) => (blockNames.indexOf(b.get('name')) !== -1) ); - dLog('resultParentBlocks', parentName, blockNames, blocks); + if (! blocks.length && blockNames.length) { + dLog('resultParentBlocks', parentName, dataset, blockNames, blocks); + } return blocks; }), viewParent(viewFlag) { @@ -508,20 +512,21 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que className: "htCenter" }, // remaining columns use default type - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, - { }, + // disable editor + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, + { editor: false }, /* { data: 'name', From 5cad00fca6baa84780a8062a60ef33ced5526805 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 18 Jun 2021 17:05:46 +1000 Subject: [PATCH 68/77] getBlockFeaturesInterval : handle task cancellation - no error message required. paths-progressive.js : getBlockFeaturesInterval() : catch task, if didCancel() then return lastSuccessful.value otherwise re-throw. --- .../app/services/data/paths-progressive.js | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/frontend/app/services/data/paths-progressive.js b/frontend/app/services/data/paths-progressive.js index 19d74f655..196e35b07 100644 --- a/frontend/app/services/data/paths-progressive.js +++ b/frontend/app/services/data/paths-progressive.js @@ -4,7 +4,7 @@ import { alias } from '@ember/object/computed'; import Service, { inject as service } from '@ember/service'; -import { task } from 'ember-concurrency'; +import { task, didCancel } from 'ember-concurrency'; import { stacks, Stacked } from '../../utils/stacks'; import { storeFeature } from '../../utils/feature-lookup'; @@ -599,7 +599,7 @@ export default Service.extend({ * are stored in ember data store, as an attribute of block. */ getBlockFeaturesInterval(blockId) { - let fnName = 'getBlockFeaturesInterval'; + const fnName = 'getBlockFeaturesInterval'; if (trace_pathsP) dLog(fnName, blockId); let block = this.get('blockService').peekBlock(blockId); @@ -608,7 +608,25 @@ export default Service.extend({ dLog(fnName, ' not found:', blockId); } else { - features = this.get('getBlockFeaturesIntervalTask').perform(blockId); + let t = this.get('getBlockFeaturesIntervalTask'); + features = t.perform(blockId) + .catch((error) => { + let lastResult; + // Recognise if the given task error is a TaskCancelation. + if (! didCancel(error)) { + dLog(fnName, 'taskInstance.catch', blockId, error); + throw error; + } else { + lastResult = t.get('lastSuccessful.value'); + // .lastRunning seems to be always null. + dLog( + fnName, 'using lastSuccessful.value', lastResult || t.lastSuccessful, + t.get('state'), t.numRunning, t.numQueued, t.lastRunning + ); + } + return lastResult; + }); + features .then(function (features) { if (trace_pathsP) From 437783ef93fced0c5de3b44e94d34445a77b5a16 Mon Sep 17 00:00:00 2001 From: Don Date: Fri, 18 Jun 2021 23:03:00 +1000 Subject: [PATCH 69/77] blast-results-view : retain display of features when changing tab and to/from modal. The role of viewFeaturesFlag has changed in 79d650b2; it is now only used to configure the toggle which sets viewRow[*], it doesn't drive viewFeatures(). feature-list.js : appendSelectedFeatures() : trace added param remove. factor viewFeaturesFlagIntegrate() out of didReceiveAttrs(). viewRowInit() : pass in viewFeaturesFlag so it is not locked to this.viewFeaturesFlag. add viewFeaturesAll(). viewFeaturesChange() : access viewRowInit via viewFeaturesAll() which passes viewFeaturesFlag as param so run_later is not required here. viewFeaturesEffect() : depend on viewRow instead of viewFeaturesFlag. viewFeatures() : rename param viewFeaturesFlag -> active. afterChange() : call .viewFeaturesFlagIntegrate(). transient.js : showFeatures() : rename param viewFeaturesFlag -> active --- frontend/app/components/panel/feature-list.js | 2 +- .../panel/upload/blast-results-view.js | 82 ++++++++++++++----- frontend/app/services/data/transient.js | 7 +- 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/frontend/app/components/panel/feature-list.js b/frontend/app/components/panel/feature-list.js index 4a7abc66c..49cc347b6 100644 --- a/frontend/app/components/panel/feature-list.js +++ b/frontend/app/components/panel/feature-list.js @@ -250,7 +250,7 @@ export default Component.extend({ * remove is true. */ appendSelectedFeatures(selectedFeaturesNames, remove) { - console.log('appendSelectedFeatures', selectedFeaturesNames); + dLog('appendSelectedFeatures', selectedFeaturesNames, remove); /** Append to selectedFeatures to text$, therefore set activeInput true. * (originally : replace instead of append, so activeInput was set false) */ this.set('activeInput', true); diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 3be8426ce..47e7d9a41 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -87,13 +87,10 @@ export default Component.extend({ * blast-results-view instance to display this search result. */ if (! this.get('viewRow.length')) { - this.viewRowInit(); + /** viewFeaturesFlag is initially true, so this will set all viewRow[*] to true. */ + this.viewRowInit(this.get('viewFeaturesFlag')); } else { - dLog('viewRow', this.viewRow, 'viewFeaturesFlag', this.viewFeaturesFlag); - /** set viewFeaturesFlag false if any of viewRow[*] are false, - * enabling the user to toggle them all on. - */ - this.set('viewFeaturesFlag', ! this.get('viewRow').any((v) => !v)); + this.viewFeaturesFlagIntegrate(); } /** not clear yet where addDataset functionality will end up, so wire this up for now. */ @@ -104,6 +101,21 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + /** set viewFeaturesFlag false if any of viewRow[*] are false, + * enabling the user to toggle them all on. + * The user is able to change the view flag for individual features, + * and to change all with the viewFeaturesFlag toggle; these changes are + * propagated both ways : + * viewRow --[viewFeaturesFlagIntegrate()]--> viewFeaturesFlag + * <--[viewRowInit()]-- + */ + viewFeaturesFlagIntegrate() { + dLog('viewRow', this.viewRow, 'viewFeaturesFlag', this.viewFeaturesFlag); + this.set('viewFeaturesFlag', ! this.get('viewRow').any((v) => !v)); + }, + + /*--------------------------------------------------------------------------*/ + /** style of the div which contains the table. * If this component is in a modal dialog, use most of screen width. */ @@ -129,11 +141,9 @@ export default Component.extend({ } return cells; }), - viewRowInit() { + viewRowInit(viewFeaturesFlag) { let data = this.get('dataMatrix'), - /** viewFeaturesFlag is initially true */ - viewFeaturesFlag = this.get('viewFeaturesFlag'), viewRow = data.map((row) => viewFeaturesFlag); this.set('search.viewRow', viewRow); dLog('viewRowInit', data.length, viewRow); @@ -241,27 +251,40 @@ export default Component.extend({ viewAllResultAxesChange(proxy) { const fnName = 'viewAllResultAxesChange'; let checked = proxy.target.checked; - /** this value seems to be delayed */ + /** this action function is called before .viewAllResultAxesFlag is changed. */ let viewAll = this.get('viewAllResultAxesFlag'); dLog(fnName, checked, viewAll); viewAll = checked; + // narrowAxesToViewedEffect() will call narrowAxesToViewed() if checked + /* Potentially viewAllResultAxesFlag===false could mean that axes are not + * automatically viewed/unviewed in response to user viewing/unviewing features. + * From user / stakeholder feedback it seems viewAllResultAxesFlag===true will + * be suitable, and hence viewAllResultAxesFlag===false is not required, + * so these can be dropped : + * viewAllResultAxesChange(), viewAllResultAxesFlag, parentIsViewedEffect(). + * hbs : checkbox name="viewAllResultAxesFlag" + * this.viewFeatures(viewAll); if (! viewAll) { this.viewParent(viewAll); } + */ + }, + /** View/Unview all features in the result set. + */ + viewFeaturesAll(viewAll) { + this.viewRowInit(viewAll); + this.viewFeatures(); }, - viewFeaturesChange(proxy) { const fnName = 'viewFeaturesChange'; let viewFeaturesFlag = proxy.target.checked; dLog(fnName, viewFeaturesFlag); - // wait for this.viewFeaturesFlag to change, because viewRowInit() reads it. - run_later(() => this.viewRowInit()); + this.viewFeaturesAll(viewFeaturesFlag); }, - viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewFeaturesFlag', 'active', function () { - /** Only view features of the active tab. */ - let viewFeaturesFlag = this.get('viewFeaturesFlag') && this.get('active'); - this.viewFeatures(viewFeaturesFlag); + viewFeaturesEffect : computed('dataFeaturesForStore.[]', 'viewRow', 'active', function () { + // viewFeatures() uses the dependencies : dataFeaturesForStore, viewRow, active. + this.viewFeatures(); }), /** if .viewAllResultAxesFlag, narrowAxesToViewed() */ @@ -339,6 +362,9 @@ export default Component.extend({ } return blocks; }), + /** View the blocks of the parent which are identifeid by .blockNames. + * @param viewFlag true/false for view/unview + */ viewParent(viewFlag) { const fnName = 'viewParent'; let parentName = this.get('search.parent'); @@ -360,12 +386,19 @@ export default Component.extend({ dLog('parentIsViewed', viewedBlocks.length, this.get('blockNames.length')); this.set('viewAllResultAxesFlag', allViewed); }), - viewFeatures(viewFeaturesFlag) { + /** + * @param active willDestroyElement passes false, otherwise default to this.active. + */ + viewFeatures(active) { const fnName = 'viewFeatures'; + /** Only view features of the active tab. */ + if (active === undefined) { + active = this.get('active'); + } let features = this.get('dataFeaturesForStore'); if (features && features.length) { - if (viewFeaturesFlag) { + if (active) { /* viewParent() could be used initially. * viewParent() views all of .blockNames in the parent, * whereas narrowAxesToViewed() takes into account .viewRow @@ -403,8 +436,10 @@ export default Component.extend({ * for this update. */ nowOrLater( - viewFeaturesFlag, - () => transient.showFeatures(dataset, blocks, features, viewFeaturesFlag, this.get('viewRow'))); + active, + /* when switching tabs got : this.isDestroyed===true, this.viewRow and this.get('viewRow') undefined + * but this.search.viewRow OK */ + () => transient.showFeatures(dataset, blocks, features, active, this.get('viewRow') || this.search.viewRow)); } }, @@ -589,6 +624,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que let transient = this.get('transient'), features = this.get('dataFeaturesForStore'); + let viewChangeCount = 0; if (changes) { changes.forEach(([row, prop, oldValue, newValue]) => { @@ -600,6 +636,7 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que } else if (row >= features.length) { this.set('warningMessage', 'Display of added features not yet supported'); } else { + viewChangeCount++; let viewRow = this.get('viewRow'); viewRow[row] = newValue; @@ -617,6 +654,9 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que } } }); + if (viewChangeCount) { + this.viewFeaturesFlagIntegrate(); + } } }, diff --git a/frontend/app/services/data/transient.js b/frontend/app/services/data/transient.js index c55589ae1..62c40809d 100644 --- a/frontend/app/services/data/transient.js +++ b/frontend/app/services/data/transient.js @@ -103,15 +103,18 @@ export default Service.extend({ }, /** + * @param active true if the tab containing these results is (becoming) active + * Features are displayed only while the tab is active, to separate + * viewing of distinct result sets. * @param view a flag per-feature to enable display of the feature row; * from values of the View checkbox column of the results features table. */ - showFeatures(dataset, blocks, features, viewFeaturesFlag, view) { + showFeatures(dataset, blocks, features, active, view) { let selected = this.get('selected'), // may pass dataset, blocks to pushFeature() stored = features.map((f) => this.pushFeature(f)); - stored.forEach((feature, i) => this.showFeature(feature, viewFeaturesFlag && view[i])); + stored.forEach((feature, i) => this.showFeature(feature, active && view[i])); }, showFeature(feature, viewFeaturesFlag) { let From 69e526b5a69280a75c7b2129411542313a16d35a Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 23 Jun 2021 16:11:48 +1000 Subject: [PATCH 70/77] sequence-search result display : use value[0] in keyFn of showTickLocations() also. axis-1d.js : FeatureTicks:showTickLocations() : keyFn() : append value[0] to key, because features created from blast search results will all have the same name (same change as applied to .showLabels() in 18196f2f). 0a78044 48368 Jun 19 01:06 frontend/app/components/draw/axis-1d.js --- frontend/app/components/draw/axis-1d.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/draw/axis-1d.js b/frontend/app/components/draw/axis-1d.js index 0e845988b..9dead2055 100644 --- a/frontend/app/components/draw/axis-1d.js +++ b/frontend/app/components/draw/axis-1d.js @@ -274,8 +274,13 @@ FeatureTicks.prototype.showTickLocations = function (featuresOfBlockLookup, setu * The function getAttrOrCP() will use .get if defined, otherwise .name (via ['name']). * This comment applies to use of 'feature.'{name,range,value} in * inRange() (above), and keyFn(), pathFn(), hoverTextFn() below. + * + * The features created from blast search results will all have the same name, + * so for better d3 join, append location to the key. */ - let featureName = getAttrOrCP(feature, 'name'); + let + value = getAttrOrCP(feature, 'value'), + featureName = getAttrOrCP(feature, 'name') + '-' + value[0]; // dLog('keyFn', feature, featureName); return featureName; }; @@ -474,7 +479,7 @@ FeatureTicks.prototype.showLabels = function (featuresOfBlockLookup, setupHover, // here `this` is the parent of the -s, e.g. g.axis let value = getAttrOrCP(feature, 'value'), - featureName = getAttrOrCP(feature, 'name') + '-' + value.[0]; + featureName = getAttrOrCP(feature, 'name') + '-' + value[0]; // dLog('keyFn', feature, featureName); return featureName; }; From ec333c13f8be3c6670721dfbcb8b9beb59be7b0e Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 23 Jun 2021 17:07:53 +1000 Subject: [PATCH 71/77] blast-results-view : avoid duplication of table elements when switching to modal. blast-results-view.js : tableModal : width:70vw -> 90vw, wide enough to expose all columns; otherwise get 'The provided element is not a child of the top overlay' in getRelativeCellPosition() from setupHandlePosition(); possible alternative work-around noted in comment. showTable() : handle search.promise undefined; log and stop if table defined but not connected (these may not be needed) createTable() : remove : $(() => ); when switching to modal this appears to cause duplication of the div.handsontable.{ht_master,ht_clone_{top,bottom,left,top_left_corner} and doesn't seem required - copied from data-csv.js:createTable() where it is probably also not required. add param data (this.dataForTable) so that initial data is not required. 5a7f36b1 23319 Jun 22 11:19 frontend/app/components/panel/upload/blast-results-view.js (on dev) --- .../panel/upload/blast-results-view.js | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results-view.js b/frontend/app/components/panel/upload/blast-results-view.js index 47e7d9a41..43e6df3de 100644 --- a/frontend/app/components/panel/upload/blast-results-view.js +++ b/frontend/app/components/panel/upload/blast-results-view.js @@ -120,7 +120,13 @@ export default Component.extend({ * If this component is in a modal dialog, use most of screen width. */ containerStyle : computed('tableModal', function () { - return this.get('tableModal') ? 'overflow-x:hidden; width:70vw' : undefined; + /** 80vw is wide enough to expose all columns; otherwise get + * 'The provided element is not a child of the top overlay' in getRelativeCellPosition() + * from setupHandlePosition(). + * If the table is wider than the screen, it could be scrolled right then + * left initially to expose all columns so they get initialised. + */ + return this.get('tableModal') ? 'overflow-x:hidden; width:90vw' : undefined; }), /*--------------------------------------------------------------------------*/ @@ -492,8 +498,8 @@ export default Component.extend({ let data = this.get('data'); if (! data || ! data.length) { let p = this.get('search.promise'); - dLog('showTable', p.state && p.state()); - p.then(() => { + dLog('showTable', p && p.state && p.state()); + p && p.then(() => { dLog('showTable then', this.get('data')?.length); // alternative : dataForTableEffect() could do this if ! table. this.shownBsTab(); }); @@ -502,7 +508,11 @@ export default Component.extend({ if (! (table = this.get('table')) || ! table.rootElement || ! table.rootElement.isConnected) { - this.createTable(); + if (table) { + dLog('showTable', table, table.rootElement); + debugger; + } + this.createTable(this.get('dataForTable')); } else { dLog('showTable', table.renderSuspendedCounter); /* @@ -519,11 +529,11 @@ export default Component.extend({ } }, - createTable() { + createTable(data) { const cName = 'upload/blast-results'; const fnName = 'createTable'; dLog('createTable'); - $(() => { + { let eltId = this.search.tableId; let hotable = $('#' + eltId)[0]; if (! hotable) { @@ -535,7 +545,7 @@ blast output columns are query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, query start, query end, subject start, subject end, e-value, score, query length, subject length */ var table = new Handsontable(hotable, { - data: [[false, '', '', '', '', '', '', '', '', '', '', '', '', '', '']], + data, // minRows: 20, rowHeaders: true, headerTooltips: true, @@ -609,7 +619,8 @@ query ID, subject ID, % identity, length of HSP (hit), # mismatches, # gaps, que }); this.set('table', table); - }); + } + }, From 46298e9721a91d5d579e022ba8c13f99d08fd8ce Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 23 Jun 2021 21:31:27 +1000 Subject: [PATCH 72/77] result features : change transition on labels to match triangles. axis-1d.js : showLabels() : Change pSM to pS because pSE is handled separately. Use transition.call directly instead of transitionFn() which wraps in run_bind() (labelsTransitionPerform() in axis-ticks-selected.js, axisScaleEffect()) - seems to effect timing. --- frontend/app/components/draw/axis-1d.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/draw/axis-1d.js b/frontend/app/components/draw/axis-1d.js index 9dead2055..d24251468 100644 --- a/frontend/app/components/draw/axis-1d.js +++ b/frontend/app/components/draw/axis-1d.js @@ -527,11 +527,12 @@ FeatureTicks.prototype.showLabels = function (featuresOfBlockLookup, setupHover, let attrY_featureY = this.attrY_featureY.bind(this); pSE.call(attrY_featureY); - let transition = this.selectionToTransition(pSM); - if (transition === pSM) { - pSM.call(attrY_featureY); + let transition = this.selectionToTransition(pS); + if (transition === pS) { + pS.call(attrY_featureY); } else { - transitionFn(transition, attrY_featureY); + transition.call(attrY_featureY); + // transitionFn(transition, attrY_featureY); } } } From f64085df1da793aef1eb29d691611e7406807e52 Mon Sep 17 00:00:00 2001 From: Don Date: Wed, 23 Jun 2021 23:06:16 +1000 Subject: [PATCH 73/77] result features : label entry after transition axis-1d.js : if transition is used, set y (attrY_featureY) for .enter() after the transition time; also set text then because default y is (0) near the top of the axis. d3-svg.js : add nowOrAfterTransition() --- frontend/app/components/draw/axis-1d.js | 12 +++++++++--- frontend/app/utils/draw/d3-svg.js | 20 +++++++++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/draw/axis-1d.js b/frontend/app/components/draw/axis-1d.js index d24251468..a3346b3a0 100644 --- a/frontend/app/components/draw/axis-1d.js +++ b/frontend/app/components/draw/axis-1d.js @@ -38,7 +38,7 @@ import { dragTransition } from '../../utils/stacks-drag'; import { selectAxis } from '../../utils/draw/stacksAxes'; -import { selectGroup } from '../../utils/draw/d3-svg'; +import { selectGroup, nowOrAfterTransition } from '../../utils/draw/d3-svg'; import { breakPoint } from '../../utils/breakPoint'; import { configureHover } from '../../utils/hover'; import { getAttrOrCP } from '../../utils/ember-devel'; @@ -520,14 +520,20 @@ FeatureTicks.prototype.showLabels = function (featuresOfBlockLookup, setupHover, * For showTickLocations / , the d updates, so pSM is used */ pSE - .text(textFn) // positioned just left of the base of the triangles. inherits text-anchor from axis; .attr('x', '-30px'); let attrY_featureY = this.attrY_featureY.bind(this); - pSE.call(attrY_featureY); let transition = this.selectionToTransition(pS); + /** pass in the delay time, because transition has no duration if empty(). */ + nowOrAfterTransition( + transition, () => { + return pSE.call(attrY_featureY) + .text(textFn); + }, + this.axis1d.transitionTime); + if (transition === pS) { pS.call(attrY_featureY); } else { diff --git a/frontend/app/utils/draw/d3-svg.js b/frontend/app/utils/draw/d3-svg.js index 439d66370..260648207 100644 --- a/frontend/app/utils/draw/d3-svg.js +++ b/frontend/app/utils/draw/d3-svg.js @@ -1,3 +1,5 @@ +import { later as run_later } from '@ember/runloop'; + /*----------------------------------------------------------------------------*/ const trace = 0; @@ -73,6 +75,22 @@ function transitionEndPromise(transition) { return transitionEnd; } +/** If selection is a d3 transition, run fn after transitionTime or the transition.duration(), + * otherwise run it now. + */ +function nowOrAfterTransition(selection, fn, transitionTime) { + let isTransition = !!selection.duration; + if (isTransition) { + /** if selection is empty then selection.node() is null, and + * transition_duration() uses get$1(this.node() ) which will get an + * exception on node.__transition; + */ + transitionTime ??= ! selection.empty() && selection.duration(); + run_later(fn, transitionTime); + } else { + fn(); + } +} /*----------------------------------------------------------------------------*/ @@ -116,4 +134,4 @@ function ensureSvgDefs(svg) /*----------------------------------------------------------------------------*/ -export { I, selectGroup, transitionEndPromise, ensureSvgDefs }; +export { I, selectGroup, transitionEndPromise, nowOrAfterTransition, ensureSvgDefs }; From 78c5087f9dd36b1f1c7d9276b41781c1f43c5ec8 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 24 Jun 2021 12:50:11 +1000 Subject: [PATCH 74/77] handle warning re. tableModal update, seen in testing blast-results.{js,hbs} : add setTableModal() to set tableModal in later(), to avoid warning : attempted to update tableModal, but it had already been used previously in the same computation. --- frontend/app/components/panel/upload/blast-results.js | 4 ++++ .../app/templates/components/panel/upload/blast-results.hbs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/panel/upload/blast-results.js b/frontend/app/components/panel/upload/blast-results.js index c33e42b16..6d22e9e5e 100644 --- a/frontend/app/components/panel/upload/blast-results.js +++ b/frontend/app/components/panel/upload/blast-results.js @@ -2,6 +2,7 @@ import Component from '@ember/component'; import { observer, computed } from '@ember/object'; import { inject as service } from '@ember/service'; import { alias } from '@ember/object/computed'; +import { later as run_later } from '@ember/runloop'; import uploadBase from '../../../utils/panel/upload-base'; import uploadTable from '../../../utils/panel/upload-table'; @@ -53,6 +54,9 @@ export default Component.extend({ /*--------------------------------------------------------------------------*/ + setTableModal(isModal) { + run_later(() => this.set('tableModal', isModal)); + }, tableModalTargetId : computed('tableModal', function () { return this.get('tableModal') ? 'blast-results-table-modal' : 'blast-results-table-panel'; }), diff --git a/frontend/app/templates/components/panel/upload/blast-results.hbs b/frontend/app/templates/components/panel/upload/blast-results.hbs index a55d74887..1d1dee2b0 100644 --- a/frontend/app/templates/components/panel/upload/blast-results.hbs +++ b/frontend/app/templates/components/panel/upload/blast-results.hbs @@ -22,7 +22,7 @@ {{#if this.tableModal}} {{#ember-modal-dialog title="Blast Results" header-icon='list'}} @@ -34,7 +34,7 @@ {{#elem/button-base classSize='xs' classColour='default' - click=(action (mut this.tableModal) true ) + click=(action this.setTableModal true ) icon='new-window'}} {{#ember-tooltip side="left" delay=500}} From 4f62dffd3a12769d36bb2a3aa5db66f0a8d2fc16 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 24 Jun 2021 12:50:26 +1000 Subject: [PATCH 75/77] feature search results : show entering triangles after axis has transitioned axis-1d.js: showTickLocations() : instead of using .merge() (pSM), show .enter() elements (pSE) (at their final posiiton) after the pS elements have transitioned to their final position, using nowOrAfterTransition(). factor to form pathAndColour() so it can be called separately for pS and pSE. --- frontend/app/components/draw/axis-1d.js | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/draw/axis-1d.js b/frontend/app/components/draw/axis-1d.js index a3346b3a0..4686fc5fc 100644 --- a/frontend/app/components/draw/axis-1d.js +++ b/frontend/app/components/draw/axis-1d.js @@ -249,15 +249,29 @@ FeatureTicks.prototype.showTickLocations = function (featuresOfBlockLookup, setu pS.exit() .remove(); - let pSM = pSE.merge(pS); + /** Instead of using .merge(), show .enter() elements (at their + * final posiiton) after the pS elements have transitioned to + * their final position. + let pSM = pSE.merge(pS); + */ /* update attr d in a transition if one was given. */ let p1 = // (t === undefined) ? pSM : - this.selectionToTransition(pSM) - p1.attr("d", pathFn) + this.selectionToTransition(pS); + + /** similar comment re. transitionTime as in showLabels() */ + nowOrAfterTransition( + p1, () => pSE.call(pathAndColour), + this.axis1d.transitionTime); + + p1.call(pathAndColour); + function pathAndColour(selection) { + selection + .attr("d", pathFn) .attr('stroke', featurePathStroke) .attr('fill', featurePathStroke) ; + } } } From 4f1abcb3104b8d87c7490ca3952935274b696e6b Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 24 Jun 2021 13:30:49 +1000 Subject: [PATCH 76/77] update version -> 2.8.0 --- frontend/package.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 6d12abb9b..61fbe735c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pretzel-frontend", - "version": "2.7.2", + "version": "2.8.0", "description": "Frontend code for Pretzel", "repository": "", "license": "MIT", diff --git a/package.json b/package.json index c9e34ceff..3fe745eed 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "pretzel", "private": true, - "version": "2.7.2", + "version": "2.8.0", "dependencies": { }, "repository" : From fce5df12b847de64ee93ec3a70b5b5b646cd2818 Mon Sep 17 00:00:00 2001 From: Don Date: Thu, 24 Jun 2021 13:37:19 +1000 Subject: [PATCH 77/77] dev test support dev_blastResult() : add param test file name. Add devResultDir to identify location of test files. Commented-out sleep 10 is useful to simulating the delay of a typical blast query. --- backend/scripts/dnaSequenceSearch.bash | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/scripts/dnaSequenceSearch.bash b/backend/scripts/dnaSequenceSearch.bash index 1fb35726c..9b0a118c8 100755 --- a/backend/scripts/dnaSequenceSearch.bash +++ b/backend/scripts/dnaSequenceSearch.bash @@ -56,12 +56,21 @@ fi #------------------------------------------------------------------------------- function dev_blastResult() { + devResultDir=${pA-${HOME-/home/don}/new/projects/agribio}/data/wheat/fasta/blast_result + if [ $# -eq 1 -a -d $devResultDir ] + then + cat $devResultDir/$1 + else # Convert spaces to \t # unexpand does not alter single spaces, so use sed to map those. +# lines 3-4 are modified copies of 1-2, to provide multiple features on an axis. (start +1 for unique location) unexpand -t 8 <<\EOF | sed "s/ /\t/" BobWhite_c10015_641 chr2A 100.000 50 0 0 1 50 154414057 154414008 2.36e-17 93.5 50 780798557 BobWhite_c10015_641 chr2B 98.000 50 1 0 1 50 207600007 207600056 1.10e-15 87.9 50 801256715 +BobWhite_c10015_641 chr2A 100.000 50 0 0 1 50 207600008 207600056 2.36e-17 93.5 50 780798557 +BobWhite_c10015_641 chr2B 98.000 50 1 0 1 50 154414058 154414008 1.10e-15 87.9 50 801256715 EOF +fi } #------------------------------------------------------------------------------- @@ -123,6 +132,8 @@ function datasetId2dbName() # This directory check enables dev_blastResult() for dev / loopback test, when blast is not installed. if [ -d ../../pretzel.A1 ] then + # sleep 10 + # FJ039903.1, DQ146423.1 dev_blastResult | \ ( [ "$addDataset" = true ] && convertSearchResults2Json || cat) | \ ( [ -n "$resultRows" ] && head -n $resultRows || cat)