From 1ad7e2474e978d5f15541633c6bde945638f8218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Prod=27homme?= Date: Fri, 11 Dec 2020 16:30:15 +0100 Subject: [PATCH] Save tiles to S3 when not pregenerated --- .env.default | 5 ++ .github/workflows/production.yml | 18 ++---- .github/workflows/staging.yml | 10 ++- api/tiles/land-cover.js | 19 +++++- api/tiles/pregenerated-tile.js | 30 ++++++--- api/tiles/save-tile.js | 29 +++++++++ api/tiles/soc-experimental-change.js | 36 ++++++----- api/tiles/soc-experimental-timeseries.js | 34 +++++----- api/tiles/soc-stock-future-change.js | 36 ++++++----- api/tiles/soc-stock-future-period.js | 36 ++++++----- api/tiles/soc-stock-historic-change.js | 34 +++++----- api/tiles/soc-stock-historic-period.js | 36 ++++++----- api/tiles/soc-stock-recent-change.js | 34 +++++----- api/tiles/soc-stock-recent-timeseries.js | 34 +++++----- index.js | 5 ++ package.json | 1 + yarn.lock | 81 +++++++++++++++++++++--- 17 files changed, 318 insertions(+), 160 deletions(-) create mode 100644 api/tiles/save-tile.js diff --git a/.env.default b/.env.default index 727086e..7c1e2ce 100644 --- a/.env.default +++ b/.env.default @@ -4,3 +4,8 @@ API_URL= ANALYSIS_API_URL= DEPLOYMENT_KEY= GOOGLE_ANALYTICS_KEY= +AWS_REGION= +AWS_BUCKET_NAME= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_MAX_Z_TILE_STORAGE= diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 8a61d88..c7bce60 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -20,13 +20,17 @@ jobs: ANALYSIS_API_URL: https://soilsrevealed.org/api/v1/analysis GEE_KEY: ${{ secrets.GEE_KEY }} GOOGLE_ANALYTICS_KEY: UA-179817360-1 - TILES_URL: https://soils-revealed.s3.amazonaws.com/tiles + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_MAX_Z_TILE_STORAGE: 5 with: host: ${{ secrets.SSH_HOST }} key: ${{ secrets.SSH_KEY }} port: ${{ secrets.SSH_PORT }} username: ${{ secrets.SSH_USER }} - envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,TILES_URL + envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,AWS_REGION,AWS_BUCKET_NAME,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_MAX_Z_TILE_STORAGE script: | echo '> Source nvm' export NVM_DIR=~/.nvm @@ -35,20 +39,12 @@ jobs: export PATH=$PATH:/home/ubuntu/.nvm/versions/node/v12.16.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin echo '> Kill previous node server' pm2 stop production - echo '> Open production folder' - cd ~/soils-revealed - echo '> Pull from git' - git fetch origin master - git reset --hard origin/master - echo '> Use correct node version' - nvm install - nvm use echo '> Install dependencies' yarn install --frozen-lockfile echo '> Generate a deployment key' DEPLOYMENT_KEY=`date +%s` echo '> Create .env file' - echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nTILES_URL=$TILES_URL" > .env + echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nAWS_REGION=$AWS_REGION\nAWS_BUCKET_NAME=$AWS_BUCKET_NAME\nAWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID\nAWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY\nAWS_MAX_Z_TILE_STORAGE=$AWS_MAX_Z_TILE_STORAGE" > .env echo '> Create gee.key.json file' echo $GEE_KEY > gee.key.json echo '> Build the app' diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 5b866d9..b102368 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -20,13 +20,17 @@ jobs: ANALYSIS_API_URL: https://soilsrevealed.org/api/v1/analysis GEE_KEY: ${{ secrets.GEE_KEY }} GOOGLE_ANALYTICS_KEY: - TILES_URL: https://soils-revealed.s3.amazonaws.com/tiles + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_MAX_Z_TILE_STORAGE: 5 with: host: ${{ secrets.SSH_HOST }} key: ${{ secrets.SSH_KEY }} port: ${{ secrets.SSH_PORT }} username: ${{ secrets.SSH_USER }} - envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,TILES_URL + envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,AWS_REGION,AWS_BUCKET_NAME,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_MAX_Z_TILE_STORAGE script: | echo '> Source nvm' export NVM_DIR=~/.nvm @@ -48,7 +52,7 @@ jobs: echo '> Generate a deployment key' DEPLOYMENT_KEY=`date +%s` echo '> Create .env file' - echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nTILES_URL=$TILES_URL" > .env + echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nAWS_REGION=$AWS_REGION\nAWS_BUCKET_NAME=$AWS_BUCKET_NAME\nAWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID\nAWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY\nAWS_MAX_Z_TILE_STORAGE=$AWS_MAX_Z_TILE_STORAGE" > .env echo '> Create gee.key.json file' echo $GEE_KEY > gee.key.json echo '> Build the app' diff --git a/api/tiles/land-cover.js b/api/tiles/land-cover.js index 88cc4dd..898df7e 100644 --- a/api/tiles/land-cover.js +++ b/api/tiles/land-cover.js @@ -2,6 +2,7 @@ const ee = require('@google/earthengine'); const axios = require('axios').default; const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const RAMP = ` @@ -49,7 +50,7 @@ const RAMP = ` const sendImage = (res, data) => { res.set('Content-Type', 'image/png'); - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (year, x, y, z) => { @@ -77,13 +78,25 @@ const getOnTheFlyTile = async (year, x, y, z) => { }; module.exports = async ({ params: { year, x, y, z } }, res) => { + const S3Path = `land-cover/${year}/${z}/${x}/${y}`; + try { - const image = await getPregeneratedTile(['land-cover', year, z, x, y]); + const image = await getPregeneratedTile(S3Path); sendImage(res, image); } catch (e) { try { const image = await getOnTheFlyTile(year, x, y, z); - sendImage(res, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/pregenerated-tile.js b/api/tiles/pregenerated-tile.js index e14e458..c5b83c1 100644 --- a/api/tiles/pregenerated-tile.js +++ b/api/tiles/pregenerated-tile.js @@ -1,12 +1,26 @@ -const axios = require('axios').default; +const AWS = require('aws-sdk'); -module.exports = async params => { - const url = encodeURI(`${process.env.TILES_URL}/${params.join('/')}.png`); +const s3 = new AWS.S3({ apiVersion: '2006-03-01' }); - const { data } = await axios.get(url, { - headers: { Accept: 'image/*' }, - responseType: 'arraybuffer', - }); +/** + * Get a pregenerated tile based on its path + * @param {string} path Path of the tile + * @returns {Promise} + */ +module.exports = path => { + const bucketParams = { + Bucket: process.env.AWS_BUCKET_NAME, + Key: `tiles/${path}.png`, + }; + + return new Promise((resolve, reject) => { + s3.getObject(bucketParams, function(err, data) { + if (err) { + reject(err); + return; + } - return data; + resolve(data.Body); + }); + }); }; diff --git a/api/tiles/save-tile.js b/api/tiles/save-tile.js new file mode 100644 index 0000000..b129968 --- /dev/null +++ b/api/tiles/save-tile.js @@ -0,0 +1,29 @@ +const AWS = require('aws-sdk'); + +const s3 = new AWS.S3({ apiVersion: '2006-03-01' }); + +/** + * Save a tile in S3 + * @param {string} path Path of the tile + * @param {Buffer} file Tile + */ +module.exports = (path, file) => { + return new Promise((resolve, reject) => { + s3.putObject( + { + Key: `tiles/${path}.png`, + Bucket: process.env.AWS_BUCKET_NAME, + ContentType: 'image/png', + Body: file, + }, + function(err) { + if (err) { + reject(err); + return; + } + + resolve(); + } + ); + }); +}; diff --git a/api/tiles/soc-experimental-change.js b/api/tiles/soc-experimental-change.js index 00426c9..ba2290f 100644 --- a/api/tiles/soc-experimental-change.js +++ b/api/tiles/soc-experimental-change.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const STOCK_RAMP = ` @@ -38,7 +39,7 @@ const CONCENTRATION_RAMP = ` const sendImage = (res, data) => { res.set('Content-Type', 'image/png'); - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (type, depth, year1, year2, x, y, z) => { @@ -88,26 +89,29 @@ const getOnTheFlyTile = async (type, depth, year1, year2, x, y, z) => { }; module.exports = async ({ params: { type, depth, year1, year2, x, y, z } }, res) => { - try { - const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options - .find(option => option.value === type) - .settings.depth.options[depth].label.replace(/\scm/, ''); - - const image = await getPregeneratedTile([ - 'soc-experimental-change', - type, - depthValue, - `${year2}-${year1}`, - z, - x, - y, - ]); + const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options + .find(option => option.value === type) + .settings.depth.options[depth].label.replace(/\scm/, ''); + + const S3Path = `soc-experimental-change/${type}/${depthValue}/${year2}-${year1}/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, image); } catch (e) { try { const image = await getOnTheFlyTile(type, depth, year1, year2, x, y, z); - sendImage(res, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-experimental-timeseries.js b/api/tiles/soc-experimental-timeseries.js index d93b979..7bcdb31 100644 --- a/api/tiles/soc-experimental-timeseries.js +++ b/api/tiles/soc-experimental-timeseries.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const STOCK_RAMP = ` @@ -37,7 +38,7 @@ const sendImage = (res, z, data) => { if (+z <= 5) { res.set('Cache-Control', 'public,max-age=604800'); } - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (type, depth, year, x, y, z) => { @@ -82,26 +83,29 @@ const getOnTheFlyTile = async (type, depth, year, x, y, z) => { }; module.exports = async ({ params: { type, depth, year, x, y, z } }, res) => { - try { - const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options - .find(option => option.value === type) - .settings.depth.options[depth].label.replace(/\scm/, ''); + const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options + .find(option => option.value === type) + .settings.depth.options[depth].label.replace(/\scm/, ''); - const image = await getPregeneratedTile([ - 'soc-experimental-timeseries', - type, - depthValue, - year, - z, - x, - y, - ]); + const S3Path = `soc-experimental-timeseries/${type}/${depthValue}/${year}/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, z, image); } catch (e) { try { const image = await getOnTheFlyTile(type, depth, year, x, y, z); - sendImage(res, z, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, z, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-stock-future-change.js b/api/tiles/soc-stock-future-change.js index 234fee4..01638dd 100644 --- a/api/tiles/soc-stock-future-change.js +++ b/api/tiles/soc-stock-future-change.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const SCENARIOS = { '00': 'crop_MGI', @@ -50,7 +51,7 @@ const DEGRADATION_RAMP = ` const sendImage = (res, data) => { res.set('Content-Type', 'image/png'); - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (scenario, year, x, y, z) => { @@ -89,27 +90,28 @@ const getOnTheFlyTile = async (scenario, year, x, y, z) => { }; module.exports = async ({ params: { scenario, year, x, y, z } }, res) => { - try { - const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options - .find(option => option.value === 'future') - .settings.depth.options[0].label.replace(/\scm/, ''); - - const image = await getPregeneratedTile([ - 'soc-stock-future-change', - SCENARIOS[scenario], - 'stock', - depth, - `${year}-2018`, - z, - x, - y, - ]); + const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options + .find(option => option.value === 'future') + .settings.depth.options[0].label.replace(/\scm/, ''); + const S3Path = `soc-stock-future-change/${SCENARIOS[scenario]}/stock/${depth}/${year}-2018/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, image); } catch (e) { try { const image = await getOnTheFlyTile(scenario, year, x, y, z); - sendImage(res, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-stock-future-period.js b/api/tiles/soc-stock-future-period.js index 9c033fd..d044bd0 100644 --- a/api/tiles/soc-stock-future-period.js +++ b/api/tiles/soc-stock-future-period.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const SCENARIOS = { '00': 'crop_MGI', @@ -35,7 +36,7 @@ const sendImage = (res, z, data) => { if (+z <= 5) { res.set('Cache-Control', 'public,max-age=604800'); } - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (scenario, year, x, y, z) => { @@ -72,27 +73,28 @@ const getOnTheFlyTile = async (scenario, year, x, y, z) => { }; module.exports = async ({ params: { scenario, year, x, y, z } }, res) => { - try { - const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options - .find(option => option.value === 'future') - .settings.depth.options[0].label.replace(/\scm/, ''); - - const image = await getPregeneratedTile([ - 'soc-stock-future-period', - SCENARIOS[scenario], - 'stock', - depth, - year, - z, - x, - y, - ]); + const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options + .find(option => option.value === 'future') + .settings.depth.options[0].label.replace(/\scm/, ''); + const S3Path = `soc-stock-future-period/${SCENARIOS[scenario]}/stock/${depth}/${year}/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, z, image); } catch (e) { try { const image = await getOnTheFlyTile(scenario, year, x, y, z); - sendImage(res, z, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, z, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-stock-historic-change.js b/api/tiles/soc-stock-historic-change.js index 5cff0ef..b46c45c 100644 --- a/api/tiles/soc-stock-historic-change.js +++ b/api/tiles/soc-stock-historic-change.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const IMAGE = { 0: { @@ -69,7 +70,7 @@ const RAMP = { const sendImage = (res, data) => { res.set('Content-Type', 'image/png'); - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (depth, x, y, z) => { @@ -93,26 +94,29 @@ const getOnTheFlyTile = async (depth, x, y, z) => { }; module.exports = async ({ params: { depth, x, y, z } }, res) => { - try { - const depthValue = LAYERS['soc-stock'].paramsConfig.settings.type.options - .find(option => option.value === 'historic') - .settings.depth.options[depth].label.replace(/\scm/, ''); + const depthValue = LAYERS['soc-stock'].paramsConfig.settings.type.options + .find(option => option.value === 'historic') + .settings.depth.options[depth].label.replace(/\scm/, ''); - const image = await getPregeneratedTile([ - 'soc-stock-historic-change', - 'stock', - depthValue, - '2010-NoLU', - z, - x, - y, - ]); + const S3Path = `soc-stock-historic-change/stock/${depthValue}/2010-NoLU/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, image); } catch (e) { try { const image = await getOnTheFlyTile(depth, x, y, z); - sendImage(res, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-stock-historic-period.js b/api/tiles/soc-stock-historic-period.js index 13b71b0..c432fb9 100644 --- a/api/tiles/soc-stock-historic-period.js +++ b/api/tiles/soc-stock-historic-period.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const IMAGE = { 0: { @@ -66,7 +67,7 @@ const sendImage = (res, z, data) => { if (+z <= 5) { res.set('Cache-Control', 'public,max-age=604800'); } - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (depth, period, x, y, z) => { @@ -87,26 +88,31 @@ const getOnTheFlyTile = async (depth, period, x, y, z) => { }; module.exports = async ({ params: { depth, period, x, y, z } }, res) => { - try { - const depthValue = LAYERS['soc-stock'].paramsConfig.settings.type.options - .find(option => option.value === 'historic') - .settings.depth.options[depth].label.replace(/\scm/, ''); + const depthValue = LAYERS['soc-stock'].paramsConfig.settings.type.options + .find(option => option.value === 'historic') + .settings.depth.options[depth].label.replace(/\scm/, ''); - const image = await getPregeneratedTile([ - 'soc-stock-historic-period', - 'stock', - depthValue, - period === 'historic' ? 'NoLU' : '2010', - z, - x, - y, - ]); + const S3Path = `soc-stock-historic-period/stock/${depthValue}/${ + period === 'historic' ? 'NoLU' : '2010' + }/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, z, image); } catch (e) { try { const image = await getOnTheFlyTile(depth, period, x, y, z); - sendImage(res, z, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, z, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-stock-recent-change.js b/api/tiles/soc-stock-recent-change.js index 5a327f9..50c5fb6 100644 --- a/api/tiles/soc-stock-recent-change.js +++ b/api/tiles/soc-stock-recent-change.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const RAMP = ` @@ -22,7 +23,7 @@ const RAMP = ` const sendImage = (res, data) => { res.set('Content-Type', 'image/png'); - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (year1, year2, x, y, z) => { @@ -49,26 +50,29 @@ const getOnTheFlyTile = async (year1, year2, x, y, z) => { }; module.exports = async ({ params: { year1, year2, x, y, z } }, res) => { - try { - const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options - .find(option => option.value === 'recent') - .settings.depth.options[0].label.replace(/\scm/, ''); + const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options + .find(option => option.value === 'recent') + .settings.depth.options[0].label.replace(/\scm/, ''); - const image = await getPregeneratedTile([ - 'soc-stock-recent-change', - 'stock', - depth, - `${year2}-${year1}`, - z, - x, - y, - ]); + const S3Path = `soc-stock-recent-change/stock/${depth}/${year2}-${year1}/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, image); } catch (e) { try { const image = await getOnTheFlyTile(year1, year2, x, y, z); - sendImage(res, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/api/tiles/soc-stock-recent-timeseries.js b/api/tiles/soc-stock-recent-timeseries.js index 60aaffa..af83f41 100644 --- a/api/tiles/soc-stock-recent-timeseries.js +++ b/api/tiles/soc-stock-recent-timeseries.js @@ -3,6 +3,7 @@ const axios = require('axios').default; const { LAYERS } = require('../../components/map/constants'); const getPregeneratedTile = require('./pregenerated-tile'); +const saveTile = require('./save-tile'); const RAMP = ` @@ -23,7 +24,7 @@ const sendImage = (res, z, data) => { if (+z <= 5) { res.set('Cache-Control', 'public,max-age=604800'); } - return res.send(Buffer.from(data)); + return res.send(data); }; const getOnTheFlyTile = async (year, x, y, z) => { @@ -51,26 +52,29 @@ const getOnTheFlyTile = async (year, x, y, z) => { }; module.exports = async ({ params: { year, x, y, z } }, res) => { - try { - const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options - .find(option => option.value === 'recent') - .settings.depth.options[0].label.replace(/\scm/, ''); + const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options + .find(option => option.value === 'recent') + .settings.depth.options[0].label.replace(/\scm/, ''); - const image = await getPregeneratedTile([ - 'soc-stock-recent-timeseries', - 'stock', - depth, - year, - z, - x, - y, - ]); + const S3Path = `soc-stock-recent-timeseries/stock/${depth}/${year}/${z}/${x}/${y}`; + try { + const image = await getPregeneratedTile(S3Path); sendImage(res, z, image); } catch (e) { try { const image = await getOnTheFlyTile(year, x, y, z); - sendImage(res, z, image); + + // We save the data to the S3 bucket + if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) { + try { + await saveTile(S3Path, image); + } catch (e) { + console.log(`> Unable to save tile in S3 (${S3Path})`); + } + } + + sendImage(res, z, Buffer.from(image)); } catch (e) { res.status(404).end(); } diff --git a/index.js b/index.js index c561247..66b4c51 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,7 @@ const express = require('express'); const next = require('next'); const ee = require('@google/earthengine'); +const AWS = require('aws-sdk'); require('dotenv').config(); @@ -32,6 +33,7 @@ const isDev = process.env.NODE_ENV !== 'production'; const app = next({ dev: isDev }); const handle = app.getRequestHandler(); +// Initialize GEE let geePrivateKey; try { geePrivateKey = require('./gee.key.json'); @@ -50,6 +52,9 @@ try { console.log('> GEE private key missing. GEE services disabled.'); } +// Initialize AWS +AWS.config.update({ region: process.env.AWS_REGION }); + app.prepare().then(() => { const server = express(); diff --git a/package.json b/package.json index 1fd8334..8faaec6 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@turf/bbox": "^6.0.1", "@zeit/next-sass": "^1.0.1", "axios": "^0.19.2", + "aws-sdk": "^2.808.0", "bootstrap": "^4.4.1", "classnames": "^2.2.6", "dayjs": "^1.9.6", diff --git a/yarn.lock b/yarn.lock index e43b6bc..40df4e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2074,6 +2074,21 @@ atob@^2.1.2: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== +aws-sdk@^2.808.0: + version "2.808.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.808.0.tgz#ab535c09f1ca607e41feaf37e61e96c2d87a0a23" + integrity sha512-RJpQ2PyQ8fM+PV9NeDlgA77D1B0wVNkqe/pxu9lZ8zqnYy3DvqYYHmK8gwA9nmTB0OLHFo8FAIKMB/5fvm0AfQ== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -2490,15 +2505,7 @@ buffer-xor@^1.0.3: resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@5.6.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" - integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw== - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - -buffer@^4.3.0: +buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -2507,6 +2514,14 @@ buffer@^4.3.0: ieee754 "^1.1.4" isarray "^1.0.0" +buffer@5.6.0: + version "5.6.0" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" + integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw== + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -4103,6 +4118,11 @@ event-target-shim@^5.0.0: resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + events@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/events/-/events-3.1.0.tgz#84279af1b34cb75aa88bf5ff291f6d0bd9b31a59" @@ -5089,7 +5109,7 @@ icss-utils@^4.0.0, icss-utils@^4.1.1: dependencies: postcss "^7.0.14" -ieee754@^1.1.12, ieee754@^1.1.4: +ieee754@1.1.13, ieee754@^1.1.12, ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== @@ -5970,6 +5990,11 @@ jest@^25.1.0: import-local "^3.0.2" jest-cli "^25.1.0" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + joi@^17.1.1: version "17.1.1" resolved "https://registry.yarnpkg.com/joi/-/joi-17.1.1.tgz#b7339e8a509510a04ce9cdeccb35a6e188080287" @@ -9006,6 +9031,16 @@ sass-loader@6.0.6: lodash.tail "^4.1.1" pify "^3.0.0" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + saxes@^3.1.9: version "3.1.11" resolved "https://registry.yarnpkg.com/saxes/-/saxes-3.1.11.tgz#d59d1fd332ec92ad98a2e0b2ee644702384b1c5b" @@ -10285,6 +10320,14 @@ url-template@^2.0.8: resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" integrity sha1-/FZaPMy/93MMd19WQflVV5FDnyE= +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -10339,6 +10382,11 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -10718,6 +10766,19 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xmlchars@^2.1.1: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"