Skip to content

Commit

Permalink
Save tiles to S3 when not pregenerated
Browse files Browse the repository at this point in the history
  • Loading branch information
@clementprdhomme
clementprdhomme committed Dec 11, 2020
1 parent 0f7affe commit a32f8ff
Show file tree
Hide file tree
Showing 17 changed files with 316 additions and 160 deletions.
5 changes: 5 additions & 0 deletions .env.default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,8 @@ API_URL=
ANALYSIS_API_URL=
DEPLOYMENT_KEY=
GOOGLE_ANALYTICS_KEY=
AWS_REGION=
AWS_BUCKET_NAME=
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_MAX_Z_TILE_STORAGE=
17 changes: 6 additions & 11 deletions .github/workflows/production.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@ jobs:
ANALYSIS_API_URL: https://soilsrevealed.org/api/v1/analysis
GEE_KEY: ${{ secrets.GEE_KEY }}
GOOGLE_ANALYTICS_KEY: UA-179817360-1
TILES_URL: https://soils-revealed.s3.amazonaws.com/tiles
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
with:
host: ${{ secrets.SSH_HOST }}
key: ${{ secrets.SSH_KEY }}
port: ${{ secrets.SSH_PORT }}
username: ${{ secrets.SSH_USER }}
envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,TILES_URL
envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,AWS_REGION,AWS_BUCKET_NAME,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_MAX_Z_TILE_STORAGE
script: |
echo '> Source nvm'
export NVM_DIR=~/.nvm
Expand All @@ -35,20 +38,12 @@ jobs:
export PATH=$PATH:/home/ubuntu/.nvm/versions/node/v12.16.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin
echo '> Kill previous node server'
pm2 stop production
echo '> Open production folder'
cd ~/soils-revealed
echo '> Pull from git'
git fetch origin master
git reset --hard origin/master
echo '> Use correct node version'
nvm install
nvm use
echo '> Install dependencies'
yarn install --frozen-lockfile
echo '> Generate a deployment key'
DEPLOYMENT_KEY=`date +%s`
echo '> Create .env file'
echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nTILES_URL=$TILES_URL" > .env
echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nAWS_REGION=$AWS_REGION\nAWS_BUCKET_NAME=$AWS_BUCKET_NAME\nAWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID\nAWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY\nAWS_MAX_Z_TILE_STORAGE=$AWS_MAX_Z_TILE_STORAGE" > .env
echo '> Create gee.key.json file'
echo $GEE_KEY > gee.key.json
echo '> Build the app'
Expand Down
9 changes: 6 additions & 3 deletions .github/workflows/staging.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@ jobs:
ANALYSIS_API_URL: https://soilsrevealed.org/api/v1/analysis
GEE_KEY: ${{ secrets.GEE_KEY }}
GOOGLE_ANALYTICS_KEY:
TILES_URL: https://soils-revealed.s3.amazonaws.com/tiles
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
with:
host: ${{ secrets.SSH_HOST }}
key: ${{ secrets.SSH_KEY }}
port: ${{ secrets.SSH_PORT }}
username: ${{ secrets.SSH_USER }}
envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,TILES_URL
envs: NODE_PORT,MAPBOX_API_KEY,API_URL,ANALYSIS_API_URL,GEE_KEY,GOOGLE_ANALYTICS_KEY,AWS_REGION,AWS_BUCKET_NAME,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_MAX_Z_TILE_STORAGE
script: |
echo '> Source nvm'
export NVM_DIR=~/.nvm
Expand All @@ -48,7 +51,7 @@ jobs:
echo '> Generate a deployment key'
DEPLOYMENT_KEY=`date +%s`
echo '> Create .env file'
echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nTILES_URL=$TILES_URL" > .env
echo -e "PORT=$NODE_PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nDEPLOYMENT_KEY=$DEPLOYMENT_KEY\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nAWS_REGION=$AWS_REGION\nAWS_BUCKET_NAME=$AWS_BUCKET_NAME\nAWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID\nAWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY\nAWS_MAX_Z_TILE_STORAGE=$AWS_MAX_Z_TILE_STORAGE" > .env
echo '> Create gee.key.json file'
echo $GEE_KEY > gee.key.json
echo '> Build the app'
Expand Down
19 changes: 16 additions & 3 deletions api/tiles/land-cover.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ const ee = require('@google/earthengine');
const axios = require('axios').default;

const getPregeneratedTile = require('./pregenerated-tile');
const saveTile = require('./save-tile');

const RAMP = `
<RasterSymbolizer>
Expand Down Expand Up @@ -49,7 +50,7 @@ const RAMP = `

const sendImage = (res, data) => {
res.set('Content-Type', 'image/png');
return res.send(Buffer.from(data));
return res.send(data);
};

const getOnTheFlyTile = async (year, x, y, z) => {
Expand Down Expand Up @@ -77,13 +78,25 @@ const getOnTheFlyTile = async (year, x, y, z) => {
};

module.exports = async ({ params: { year, x, y, z } }, res) => {
const S3Path = `land-cover/${year}/${z}/${x}/${y}`;

try {
const image = await getPregeneratedTile(['land-cover', year, z, x, y]);
const image = await getPregeneratedTile(S3Path);
sendImage(res, image);
} catch (e) {
try {
const image = await getOnTheFlyTile(year, x, y, z);
sendImage(res, image);

// We save the data to the S3 bucket
if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) {
try {
await saveTile(S3Path, image);
} catch (e) {
console.log(`> Unable to save tile in S3 (${S3Path})`);
}
}

sendImage(res, Buffer.from(image));
} catch (e) {
res.status(404).end();
}
Expand Down
30 changes: 22 additions & 8 deletions api/tiles/pregenerated-tile.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,26 @@
const axios = require('axios').default;
const AWS = require('aws-sdk');

module.exports = async params => {
const url = encodeURI(`${process.env.TILES_URL}/${params.join('/')}.png`);
const s3 = new AWS.S3({ apiVersion: '2006-03-01' });

const { data } = await axios.get(url, {
headers: { Accept: 'image/*' },
responseType: 'arraybuffer',
});
/**
* Get a pregenerated tile based on its path
* @param {string} path Path of the tile
* @returns {Promise<AWS.S3.Body>}
*/
module.exports = path => {
const bucketParams = {
Bucket: process.env.AWS_BUCKET_NAME,
Key: `tiles/${path}.png`,
};

return new Promise((resolve, reject) => {
s3.getObject(bucketParams, function(err, data) {
if (err) {
reject(err);
return;
}

return data;
resolve(data.Body);
});
});
};
29 changes: 29 additions & 0 deletions api/tiles/save-tile.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
const AWS = require('aws-sdk');

const s3 = new AWS.S3({ apiVersion: '2006-03-01' });

/**
* Save a tile in S3
* @param {string} path Path of the tile
* @param {Buffer} file Tile
*/
module.exports = (path, file) => {
return new Promise((resolve, reject) => {
s3.putObject(
{
Key: `tiles/${path}.png`,
Bucket: process.env.AWS_BUCKET_NAME,
ContentType: 'image/png',
Body: file,
},
function(err) {
if (err) {
reject(err);
return;
}

resolve();
}
);
});
};
36 changes: 20 additions & 16 deletions api/tiles/soc-experimental-change.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const axios = require('axios').default;

const { LAYERS } = require('../../components/map/constants');
const getPregeneratedTile = require('./pregenerated-tile');
const saveTile = require('./save-tile');

const STOCK_RAMP = `
<RasterSymbolizer>
Expand Down Expand Up @@ -38,7 +39,7 @@ const CONCENTRATION_RAMP = `

const sendImage = (res, data) => {
res.set('Content-Type', 'image/png');
return res.send(Buffer.from(data));
return res.send(data);
};

const getOnTheFlyTile = async (type, depth, year1, year2, x, y, z) => {
Expand Down Expand Up @@ -88,26 +89,29 @@ const getOnTheFlyTile = async (type, depth, year1, year2, x, y, z) => {
};

module.exports = async ({ params: { type, depth, year1, year2, x, y, z } }, res) => {
try {
const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options
.find(option => option.value === type)
.settings.depth.options[depth].label.replace(/\scm/, '');

const image = await getPregeneratedTile([
'soc-experimental-change',
type,
depthValue,
`${year2}-${year1}`,
z,
x,
y,
]);
const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options
.find(option => option.value === type)
.settings.depth.options[depth].label.replace(/\scm/, '');

const S3Path = `soc-experimental-change/${type}/${depthValue}/${year2}-${year1}/${z}/${x}/${y}`;

try {
const image = await getPregeneratedTile(S3Path);
sendImage(res, image);
} catch (e) {
try {
const image = await getOnTheFlyTile(type, depth, year1, year2, x, y, z);
sendImage(res, image);

// We save the data to the S3 bucket
if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) {
try {
await saveTile(S3Path, image);
} catch (e) {
console.log(`> Unable to save tile in S3 (${S3Path})`);
}
}

sendImage(res, Buffer.from(image));
} catch (e) {
res.status(404).end();
}
Expand Down
34 changes: 19 additions & 15 deletions api/tiles/soc-experimental-timeseries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const axios = require('axios').default;

const { LAYERS } = require('../../components/map/constants');
const getPregeneratedTile = require('./pregenerated-tile');
const saveTile = require('./save-tile');

const STOCK_RAMP = `
<RasterSymbolizer>
Expand Down Expand Up @@ -37,7 +38,7 @@ const sendImage = (res, z, data) => {
if (+z <= 5) {
res.set('Cache-Control', 'public,max-age=604800');
}
return res.send(Buffer.from(data));
return res.send(data);
};

const getOnTheFlyTile = async (type, depth, year, x, y, z) => {
Expand Down Expand Up @@ -82,26 +83,29 @@ const getOnTheFlyTile = async (type, depth, year, x, y, z) => {
};

module.exports = async ({ params: { type, depth, year, x, y, z } }, res) => {
try {
const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options
.find(option => option.value === type)
.settings.depth.options[depth].label.replace(/\scm/, '');
const depthValue = LAYERS['soc-experimental'].paramsConfig.settings.type.options
.find(option => option.value === type)
.settings.depth.options[depth].label.replace(/\scm/, '');

const image = await getPregeneratedTile([
'soc-experimental-timeseries',
type,
depthValue,
year,
z,
x,
y,
]);
const S3Path = `soc-experimental-timeseries/${type}/${depthValue}/${year}/${z}/${x}/${y}`;

try {
const image = await getPregeneratedTile(S3Path);
sendImage(res, z, image);
} catch (e) {
try {
const image = await getOnTheFlyTile(type, depth, year, x, y, z);
sendImage(res, z, image);

// We save the data to the S3 bucket
if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) {
try {
await saveTile(S3Path, image);
} catch (e) {
console.log(`> Unable to save tile in S3 (${S3Path})`);
}
}

sendImage(res, z, Buffer.from(image));
} catch (e) {
res.status(404).end();
}
Expand Down
36 changes: 19 additions & 17 deletions api/tiles/soc-stock-future-change.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const axios = require('axios').default;

const { LAYERS } = require('../../components/map/constants');
const getPregeneratedTile = require('./pregenerated-tile');
const saveTile = require('./save-tile');

const SCENARIOS = {
'00': 'crop_MGI',
Expand Down Expand Up @@ -50,7 +51,7 @@ const DEGRADATION_RAMP = `

const sendImage = (res, data) => {
res.set('Content-Type', 'image/png');
return res.send(Buffer.from(data));
return res.send(data);
};

const getOnTheFlyTile = async (scenario, year, x, y, z) => {
Expand Down Expand Up @@ -89,27 +90,28 @@ const getOnTheFlyTile = async (scenario, year, x, y, z) => {
};

module.exports = async ({ params: { scenario, year, x, y, z } }, res) => {
try {
const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options
.find(option => option.value === 'future')
.settings.depth.options[0].label.replace(/\scm/, '');

const image = await getPregeneratedTile([
'soc-stock-future-change',
SCENARIOS[scenario],
'stock',
depth,
`${year}-2018`,
z,
x,
y,
]);
const depth = LAYERS['soc-stock'].paramsConfig.settings.type.options
.find(option => option.value === 'future')
.settings.depth.options[0].label.replace(/\scm/, '');

const S3Path = `soc-stock-future-change/${SCENARIOS[scenario]}/stock/${depth}/${year}-2018/${z}/${x}/${y}`;
try {
const image = await getPregeneratedTile(S3Path);
sendImage(res, image);
} catch (e) {
try {
const image = await getOnTheFlyTile(scenario, year, x, y, z);
sendImage(res, image);

// We save the data to the S3 bucket
if (+z <= +process.env.AWS_MAX_Z_TILE_STORAGE) {
try {
await saveTile(S3Path, image);
} catch (e) {
console.log(`> Unable to save tile in S3 (${S3Path})`);
}
}

sendImage(res, Buffer.from(image));
} catch (e) {
res.status(404).end();
}
Expand Down
Loading

0 comments on commit a32f8ff

Please sign in to comment.