Skip to content

Commit

Permalink
Merge pull request #13 from ISG-ICS/async-tweet-load
Browse files Browse the repository at this point in the history 
Async tweet load
  • Loading branch information
@Yicong-Huang
Yicong-Huang authored Oct 14, 2019
2 parents dfd4590 + 003ef57 commit da43e7c
Show file tree
Hide file tree
Showing 6 changed files with 128 additions and 64 deletions.
45 changes: 44 additions & 1 deletion backend/web/router/tweet_router.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,23 @@ def send_live_tweet():
return resp


@bp.route("/tweet-count")
def send_tweet_count_data():
"""
This func gives all historical tweets objects with id
:returns: a list of tweet objects, each with time, lat, long, id
"""
resp = make_response(
jsonify({date.isoformat(): count for date, count in
Connection().sql_execute(
"select m.t_date, count(*) from "
"(select r.create_at::timestamp::date as t_date from records r,locations l where r.id=l.id group by(r.create_at)) "
"as m group by m.t_date order by m.t_date")}))

return resp


@bp.route("/fire-tweet")
def send_fire_tweet_data():
"""
Expand All @@ -69,7 +86,7 @@ def send_fire_tweet_data():
jsonify([{"create_at": time.isoformat(), "long": lon, "lat": lat, "id": str(id)} for time, lon, lat, _, _, id in
Connection().sql_execute(
"select r.create_at, l.top_left_long, l.top_left_lat, l.bottom_right_long, l.bottom_right_lat, r.id "
"from records r,locations l where r.id=l.id")]))
"from records r,locations l where r.id=l.id AND create_at>now()-interval '30 day'")]))
return resp


Expand Down Expand Up @@ -146,6 +163,32 @@ def region_tweet():
return resp


@bp.route('/tweet-by-date')
def tweet_by_date():
"""
tweet count within specific date range
@:param start-date: ISO string
@:param end-date: ISO string
:return: [ {create_at, id, lat, lon}, ... ]
"""
start_date_str = flask_request.args.get('start-date').split('.')[0][:-3]
end_date_str = flask_request.args.get('end-date').split('.')[0][:-3]

query = f'''
select r.create_at, r.id, top_left_long, top_left_lat, bottom_right_long, bottom_right_lat
from records r, locations l where r.id = l.id and r.create_at < to_timestamp({end_date_str}) and r.create_at > to_timestamp({start_date_str})
'''

resp = make_response(
jsonify(
[{'create_at':create_at, 'id': id, 'lat': (top_left_lat + bottom_right_lat) / 2, 'long': (top_left_long + bottom_right_long) / 2}
for create_at, id, top_left_long, top_left_lat, bottom_right_long, bottom_right_lat in
Connection.sql_execute(query)]))

return resp


@bp.route("/tweet-from-id", methods=['GET'])
def tweet_from_id():
"""
Expand Down
37 changes: 0 additions & 37 deletions frontend/src/app/map/layers/FireTweetLayer.ts
View file

This file was deleted.

31 changes: 23 additions & 8 deletions frontend/src/app/map/layers/fire.tweet.layer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ export class FireTweetLayer {

constructor(private mainControl, private mapService: MapService, private map, private timeService: TimeService) {
// This is the overlay controller defined in constructor
// rewrite
this.mapService.getFireTweetData().subscribe(this.tweetDataHandler);

this.map.on('overlayadd', (event) => {
if (event.name === 'Fire tweet') {
this.map.on('mousemove', e => this.onMapMouseMove(e));
Expand Down Expand Up @@ -155,6 +157,7 @@ export class FireTweetLayer {
* @param {Object} list of tweet object with geolocation, time of tweet, id of tweet
*/
this.tweetData = tweets;
console.log(this.tweetData);
this.tweetLayer = L.TileLayer.maskCanvas({
radius: 10,
useAbsoluteRadius: true,
Expand Down Expand Up @@ -182,17 +185,29 @@ export class FireTweetLayer {
* Filter out tweets which not satisfy the time range selection;
* store qualified tweets with their id in list 'tempDataWithID' for later content display usage
*/
const tempData = [];
this.tempDataWithID = [];
const [startDateInMs, endDateInMs] = this.timeService.getRangeDate();
this.tweetData.forEach(tweet => {
const time = new Date(tweet.create_at).getTime();
if (time > startDateInMs && time < endDateInMs) {
tempData.push([tweet.lat, tweet.long]);
this.tempDataWithID.push([tweet.lat, tweet.long, tweet.id]);
this.timeService.getTweetByDate(startDateInMs, endDateInMs).subscribe(tweets => {
const tempData = [];
tweets.forEach(tweet => {
tempData.push([tweet.lat, tweet.long]);
}
);
this.tweetLayer.setData(tempData);
}
});
this.tweetLayer.setData(tempData);
);
// console.log(startDateInMs);
// console.log(this.timeService.getTweetByDate(startDateInMs, endDateInMs));

// this.tweetData.forEach(tweet => {
// const time = new Date(tweet.create_at).getTime();
// if (time > startDateInMs && time < endDateInMs) {
// tempData.push([tweet.lat, tweet.long]);
// this.tempDataWithID.push([tweet.lat, tweet.long, tweet.id]);
// }
// });
// console.log(tempData);
// this.tweetLayer.setData(tempData); //draw on twittermap
}

idOverPoint(x, y) {
Expand Down
61 changes: 44 additions & 17 deletions frontend/src/app/map/time-series/time-series.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export class TimeSeriesComponent implements OnInit {

ngOnInit() {
/** Subscribe tweet data related to wildfire in service. */
this.mapService.getFireTweetData().subscribe(data => this.drawTimeSeries(data));
this.mapService.getDateCountData().subscribe(data => this.drawTimeSeries(data));
}

/**
Expand All @@ -43,25 +43,34 @@ export class TimeSeriesComponent implements OnInit {
* @param tweets tweet data crawled using tweet api
*
*/
drawTimeSeries = (tweets: Tweet[]) => {
drawTimeSeries = (dateAndCount) => {
/** replace */
/**
* Refine tweet data to count related to 'wildfire' in each DAY,
* storing in charData.
*/
const chartData = [];
const dailyCount = {};
for (const tweet of tweets) {
const createAt = tweet.create_at.split('T')[0];
if (dailyCount.hasOwnProperty(createAt)) {
dailyCount[createAt]++;
} else {
dailyCount[createAt] = 1;
}
}
Object.keys(dailyCount).sort().forEach(key => {
chartData.push([new Date(key).getTime(), dailyCount[key]]);
});
Object.keys(dateAndCount).forEach(key => {
chartData.push([new Date(key).getTime(), dateAndCount[key]]);
});
console.log(chartData);
// const chartData = [];
// const dailyCount = {};
// for (const tweet of tweets) {
// const createAt = tweet.create_at.split('T')[0];
// if (dailyCount.hasOwnProperty(createAt)) {
// dailyCount[createAt]++;
// } else {
// dailyCount[createAt] = 1;
// }
// }
// Object.keys(dailyCount).sort().forEach(key => {
// chartData.push([new Date(key).getTime(), dailyCount[key]]);
// });


/** Plotting format of time-series. */
console.log("xxxxx");
const timeseries = Highcharts.stockChart('timebar-container', {
chart: {
height: 150,
Expand Down Expand Up @@ -123,7 +132,6 @@ export class TimeSeriesComponent implements OnInit {
this.hasPlotBand = false;
this.timeService.setCurrentDate(undefined);
}

},
}
},
Expand Down Expand Up @@ -161,12 +169,16 @@ export class TimeSeriesComponent implements OnInit {
* updating information of date.
*/
setExtremes: (event) => {
this.timeService.setRangeDate(event.min + this.halfUnit, event.max);
console.log(event.min);
console.log(event.max);
this.timeService.setRangeDate(event.min, event.max);
/** this.timeService.getTweetByDate(event.min, event.max).subscribe((data)=>{});*/


$('#report').html('Date Range => ' +
'Start: ' + Highcharts.dateFormat('%Y-%m-%d', event.min) +
', End: ' + Highcharts.dateFormat('%Y-%m-%d', event.max));
$(window).trigger('timeRangeChange');

}
}
},
Expand All @@ -187,6 +199,7 @@ export class TimeSeriesComponent implements OnInit {
* time series click event.
*/
closestTickNearClick(eventAxis): [number, number, number, any] {
console.log("yyyyy");
const halfUnitDistance = 43200000;
const xAxis = eventAxis.axis;
const dateClickedInMs = eventAxis.value;
Expand Down Expand Up @@ -217,6 +230,20 @@ export class TimeSeriesComponent implements OnInit {
distanceToTheRight = (xAxis.ordinalPositions[minKey + 1] - xAxis.ordinalPositions[minKey]) / 2;
}
}
console.log(minValue - distanceToTheLeft);
console.log(minValue);
console.log(distanceToTheRight + minValue);
// this.timeService.setRangeDate(minValue - distanceToTheLeft, distanceToTheRight + minValue);
// $('#report').html('Date Range => ' +
// 'Start: ' + Highcharts.dateFormat('%Y-%m-%d', minValue - distanceToTheLeft) +
// ', End: ' + Highcharts.dateFormat('%Y-%m-%d', distanceToTheRight + minValue));
// $(window).trigger('timeRangeChange');
this.timeService.setRangeDate(minValue, (2 * distanceToTheRight) + minValue);
/**this.timeService.getTweetByDate(minValue, (2 * distanceToTheRight) + minValue);*/
$('#report').html('Date Range => ' +
'Start: ' + Highcharts.dateFormat('%Y-%m-%d', minValue) +
', End: ' + Highcharts.dateFormat('%Y-%m-%d', 2 * distanceToTheRight + minValue));
$(window).trigger('timeRangeChange');
return [minValue - distanceToTheLeft, minValue, distanceToTheRight + minValue, xAxis.ticks[minValue]];
}
}
5 changes: 5 additions & 0 deletions frontend/src/app/services/map-service/map.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ export class MapService {
return this.http.get<Tweet[]>(`http://${environment.host}:${environment.port}/tweet/fire-tweet`);
}

getDateCountData(): Observable<Tweet[]> {
return this.http.get<Tweet[]>(`http://${environment.host}:${environment.port}/tweet/tweet-count`);
}


getWildfirePredictionData(northEastBoundaries, southWestBoundaries, start, end): Observable<any> {
return this.http.post(`http://${environment.host}:${environment.port}/wildfire-prediction`, JSON.stringify({
northEast: northEastBoundaries,
Expand Down
13 changes: 12 additions & 1 deletion frontend/src/app/services/time/time.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@
*/

import {Injectable} from '@angular/core';
import {Observable} from 'rxjs';
import {environment} from '../../../environments/environment';
import {HttpClient, HttpParams} from '@angular/common/http';
import {Tweet} from "../../models/tweet.model";

// import {Observable, Subject, BehaviorSubject} from 'rxjs';
// import {HttpClient, HttpParams} from '@angular/common/http';

Expand All @@ -27,7 +32,8 @@ export class TimeService {
private rangeStartDateInMS = new Date().getTime() - 6 * 30 * 24 * 3600 * 1000;
private rangeEndDateInMS = new Date().getTime();

constructor() {
constructor(private http: HttpClient) {

}

setCurrentDate(dateInYMD: string): void {
Expand All @@ -46,5 +52,10 @@ export class TimeService {
getRangeDate(): [number, number] {
return [this.rangeStartDateInMS, this.rangeEndDateInMS];
}

getTweetByDate(startDate, endDate): Observable<Tweet[]> {
return this.http.get<Tweet[]>(`http://${environment.host}:${environment.port}/tweet/tweet-by-date`,
{params: new HttpParams().set('start-date', startDate).set('end-date', endDate)});
}
}

0 comments on commit da43e7c

Please sign in to comment.