Skip to content

Commit

Permalink
Merge pull request #169 from plantinformatics/feature/axisFinishSplit
Browse files Browse the repository at this point in the history
Master release v1.3.0
  • Loading branch information
Gabriel Keeble-Gagnere authored Feb 25, 2020
2 parents 2e6b050 + 1d6437a commit 1adbff2
Show file tree
Hide file tree
Showing 48 changed files with 4,216 additions and 1,165 deletions.
5 changes: 3 additions & 2 deletions backend/common/models/block.js
Original file line number Diff line number Diff line change
Expand Up @@ -474,10 +474,10 @@ Block.blockNamespace = async function(blockIds) {
* @param blockId block
* @param nBins number of bins to partition the block's features into
*/
Block.blockFeaturesCounts = function(blockId, nBins, options, res, cb) {
Block.blockFeaturesCounts = function(blockId, interval, nBins, options, res, cb) {
let db = this.dataSource.connector;
let cursor =
blockFeatures.blockFeaturesCounts(db, blockId, nBins);
blockFeatures.blockFeaturesCounts(db, blockId, interval, nBins);
cursor.toArray()
.then(function(featureCounts) {
cb(null, featureCounts);
Expand Down Expand Up @@ -633,6 +633,7 @@ Block.blockNamespace = async function(blockIds) {
Block.remoteMethod('blockFeaturesCounts', {
accepts: [
{arg: 'block', type: 'string', required: true},
{arg: 'interval', type: 'array', required: false},
{arg: 'nBins', type: 'number', required: false},
{arg: "options", type: "object", http: "optionsFromRequest"},
{arg: 'res', type: 'object', 'http': {source: 'res'}},
Expand Down
103 changes: 97 additions & 6 deletions backend/common/utilities/block-features.js
Original file line number Diff line number Diff line change
Expand Up @@ -49,35 +49,118 @@ exports.blockFeaturesCount = function(db, blockIds) {

/*----------------------------------------------------------------------------*/

/** Calculate the bin size for even-sized bins to span the given interval.
* The bin size is rounded to be a multiple of a power of 10, only the first 1-2
* digits are non-zero.
* Used in @see binBoundaries().
* @return lengthRounded
*/
function binEvenLengthRound(interval, nBins) {
let lengthRounded;
if (interval && (interval.length === 2) && (nBins > 0)) {
/* if (interval[1] < interval[0])
interval = interval.sort(); */
let intervalLength = interval[1] - interval[0],
binLength = intervalLength / nBins,
digits = Math.floor(Math.log10(binLength)),
eN1 = Math.exp(digits * Math.log(10)),
mantissa = binLength / eN1,
/** choose 1 2 or 5 as the first digit of the bin size. */
m1 = mantissa > 5 ? 5 : (mantissa > 2 ? 2 : 1);
lengthRounded = Math.round(m1 * eN1);

console.log('binEvenLengthRound', interval, nBins, intervalLength, binLength, digits, eN1, mantissa, m1, lengthRounded);
}
return lengthRounded;
};
/** Generate an array of even-sized bins to span the given interval.
* Used for mongo aggregation pipeline : $bucket : boundaries.
*/
function binBoundaries(interval, lengthRounded) {
let b;
if (lengthRounded) {
let
start = interval[0],
intervalLength = interval[1] - interval[0],
direction = Math.sign(intervalLength),
forward = (direction > 0) ?
function (a,b) {return a < b; }
: function (a,b) {return a > b; };

let location = Math.floor(start / lengthRounded) * lengthRounded;
b = [location];
do {
location += lengthRounded;
b.push(location);
}
while (forward(location, interval[1]));
console.log('binBoundaries', direction, b.length, location, b[0], b[b.length-1]);
}
return b;
};




/** Count features of the given block in bins.
*
* @param blockCollection dataSource collection
* @param blockId id of data block
* @param interval if given then use $bucket with boundaries in this range,
* otherwise use $bucketAuto.
* @param nBins number of bins to group block's features into
*
* @return cursor : binned feature counts
* { "_id" : { "min" : 4000000, "max" : 160000000 }, "count" : 22 }
* { "_id" : { "min" : 160000000, "max" : 400000000 }, "count" : 21 }
*/
exports.blockFeaturesCounts = function(db, blockId, nBins = 10) {
exports.blockFeaturesCounts = function(db, blockId, interval, nBins = 10) {
// initial draft based on blockFeaturesCount()
let featureCollection = db.collection("Feature");
/** The requirement (so far) is for even-size boundaries on even numbers,
* e.g. 1Mb bins, with each bin boundary a multiple of 1e6.
*
* $bucketAuto doesn't require the boundaries to be defined, but there may not
* be a way to get it to use even-sized boundaries which are multiples of 1eN.
* By default it defines bins which have even numbers of features, i.e. the
* bin length will vary. If the parameter 'granularity' is given, the bin
* boundaries are multiples of 1e4 at the start and 1e7 near the end (in a
* dataset [0, 800M]; the bin length increases in an exponential progression.
*
* So $bucket is used instead, and the boundaries are given explicitly.
* This requires interval; if it is not passed, $bucketAuto is used, without granularity.
*/
const useBucketAuto = ! (interval && interval.length === 2);
if (trace_block)
console.log('blockFeaturesCount', blockId, nBins);
console.log('blockFeaturesCounts', blockId, interval, nBins);
let ObjectId = ObjectID;

let lengthRounded, boundaries;
if (! useBucketAuto) {
lengthRounded = binEvenLengthRound(interval, nBins),
boundaries = binBoundaries(interval, lengthRounded);
}

let
matchBlock =
[
{$match : {blockId : ObjectId(blockId)}},
{ $bucketAuto: { groupBy: {$arrayElemAt : ['$value', 0]}, buckets: Number(nBins), granularity : 'R5'} }
useBucketAuto ?
{ $bucketAuto : { groupBy: {$arrayElemAt : ['$value', 0]}, buckets: Number(nBins)} } // , granularity : 'R5'
: { $bucket :
{
groupBy: {$arrayElemAt : ['$value', 0]}, boundaries,
output: {
count: { $sum: 1 },
idWidth : {$addToSet : lengthRounded }
}
}
}
],

pipeline = matchBlock;

if (trace_block)
console.log('blockFeaturesCount', pipeline);
console.log('blockFeaturesCounts', pipeline);
if (trace_block > 1)
console.dir(pipeline, { depth: null });

Expand Down Expand Up @@ -114,10 +197,18 @@ exports.blockFeatureLimits = function(db, blockId) {
console.log('blockFeatureLimits', blockId);
let ObjectId = ObjectID;

/** unwind the values array of Features, and group by blockId, with the min &
* max values within the Block.
* value may be [from, to, anyValue], so use slice to extract just [from, to],
* and $match $ne null to ignore to === undefined.
* The version prior to adding this comment assumed value was just [from, to] (optional to);
* so we can revert to that code if we separate additional values from the [from,to] location range.
*/
let
group = [
{$project : {_id : 1, name: 1, blockId : 1, value : 1 }},
{$project : {_id : 1, name: 1, blockId : 1, value : {$slice : ['$value', 2]} }},
{$unwind : '$value'},
{$match: { $or: [ { value: { $ne: null } } ] } },
{$group : {
_id : '$blockId' ,
featureCount : { $sum: 1 },
Expand Down
2 changes: 1 addition & 1 deletion backend/common/utilities/paths-aggr.js
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ function keyValue (k,v) { let r = {}; r[k] = v; return r; };
function valueBound(intervals, b, l) {
let r = keyValue(
l ? '$lte' : '$gte',
[ keyValue('$arrayElemAt', ['$value', l ? -1 : 0]),
[ keyValue('$arrayElemAt', ['$value', l ? 1 : 0]),
+intervals.axes[b].domain[l]]
);
return r;
Expand Down
27 changes: 27 additions & 0 deletions doc/notes/d3_devel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# debugging techniques for d3

## Web Inspector : Console, Elements tab, Ember Inspector

- nodes(), node(), data(), .__data__

- click in Elements tab to select an element, then in console can inspect the element details with
$0, $0.__data__

- d3 stores .data(), .datum() on element.__data__, so $0.__data__ shows the d3 datum

- in console, selection .node() shows the element as it appears in the Elements panel;
right-click on this -> view in Elements panel

- when single-stepping through d3 attribute changes, note that in a transition the change won't take place until later, so commenting out the .transition().duration() line makes it easy to observe element changes while stepping through code

- Elements panel highlights attribute changes for about 1 second, so have this displayed and continue (eg. to the next breakpoint)

- use ember inspector to get component handle, from there get element and datum

- d3 selection in console : can be used to inspect the app at any time - no need to breakpoint the app

- Chrome and Firefox Web Inspector are similar, each has at least 1 thing which the other doesn't do or isn't as easy to use

- more :
https://developers.google.com/web/tools/chrome-devtools/console/utilities
http://anti-code.com/devtools-cheatsheet/
81 changes: 74 additions & 7 deletions frontend/app/components/axis-2d.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import AxisEvents from '../utils/draw/axis-events';

/* global d3 */

const dLog = console.debug;

const axisTransitionTime = 750;


Expand Down Expand Up @@ -35,6 +37,10 @@ export default Ember.Component.extend(Ember.Evented, AxisEvents, {
console.log('axis1d', axis1d, axesPCount);
return axis1d;
}),
/** @return the blocks of this axis.
* Result type is [] of (stack.js) Block.
* For the Ember data Object blocks, @see dataBlocks().
*/
blocks : Ember.computed('axis', function() {
let axis = this.get('axis');
return axis && axis.blocks;
Expand Down Expand Up @@ -69,6 +75,17 @@ export default Ember.Component.extend(Ember.Evented, AxisEvents, {
return dataBlocks;
}),

/** @return blocks which are viewedChartable, and whose axis is this axis.
*/
viewedChartable : Ember.computed('blockService.viewedChartable.[]', 'axisID',
function () {
let
id = this.get('axisID'),
viewedChartable = this.get('blockService.viewedChartable')
.filter((b) => { let axis = b.get('axis'); return axis && axis.axisName === id; });
console.log('viewedChartable', id, viewedChartable);
return viewedChartable;
}),

/*--------------------------------------------------------------------------*/

Expand Down Expand Up @@ -153,12 +170,6 @@ export default Ember.Component.extend(Ember.Evented, AxisEvents, {
remove: function(){
this.remove();
console.log("components/axis-2d remove()");
},
/**
* @param componentName e.g. 'axis-tracks'
*/
contentWidth : function (componentName, axisID, width) {
this.contentWidth(componentName, axisID, width);
}
},

Expand All @@ -177,7 +188,53 @@ export default Ember.Component.extend(Ember.Evented, AxisEvents, {
&& (currentWidth = use_data[0]);
return currentWidth;
},
childWidths : {},
/** width of sub-components within this axis-2d. Indexed by componentName.
* For each sub-component, [min, max] : the minimum required width, and the
* maximum useful width, i.e. the maximum width that the component can fill
* with content.
* Measured nominally in pixels, but space may be allocated proportional to the width allocated to this axis-2d.
*/
childWidths : undefined,
/** Allocate the available width among the children listed in .childWidths
* @return [horizontal start offset, width] for each child.
* The key of the result is the same as the input .childWidths
*/
allocatedWidths : Ember.computed('[email protected]', 'childWidths.chart.1', function () {
let allocatedWidths,
childWidths = this.get('childWidths'),
groupNames = Object.keys(childWidths),
requested =
groupNames.reduce((result, groupName) => {
let cw = childWidths[groupName];
result[0] += cw[0]; // min
result[1] += cw[1]; // max
}, [0, 0]);
/** Calculate the spare width after each child is assigned its requested
* minimum width, and apportion the spare width among them.
* If spare < 0 then each child will get < min, but not <0.
*/
let
startWidth = this.get('startWidth'),
available = (this.get('axisUse') && this.rectWidth()) || startWidth || 120,
/** spare and share may be -ve */
spare = available - requested[0],
share = 0;
if (groupNames.length > 0) {
share = spare / groupNames.length;
}
/** horizontal offset to the start (left) of the child. */
let offset = 0;
allocatedWidths = groupNames.map((groupName) => {
let w = childWidths[groupName][0] + share;
if (w < 0)
w = 0;
let allocated = [offset, w];
offset += w;
return allocated;
});
dLog('allocatedWidths', allocatedWidths, childWidths);
return allocatedWidths;
}),
contentWidth : function (componentName, axisID, width) {
let
childWidths = this.get('childWidths'),
Expand All @@ -200,6 +257,16 @@ export default Ember.Component.extend(Ember.Evented, AxisEvents, {
Ember.run.later(call_setWidth);
},

init() {
this._super(...arguments);
this.set('childWidths', Ember.Object.create());
},

/*--------------------------------------------------------------------------*/

resizeEffect : Ember.computed.alias('drawMap.resizeEffect'),

/*--------------------------------------------------------------------------*/

didInsertElement() {
let oa = this.get('data'),
Expand Down
Loading

0 comments on commit 1adbff2

Please sign in to comment.