From dddf1bd0ef66c7e20be5102c340ed3fd4d3a0f81 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 21 Sep 2023 21:05:26 -0700 Subject: [PATCH 1/7] feat: enable date-based partitioning and add more fields to the out and value structs to enable metrics prototyping --- stacks/firehose-stack.js | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index 8ff7fd4f..ff42b3a7 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -114,7 +114,7 @@ export function UcanFirehoseStack({ stack, app }) { bufferingHints: { intervalInSeconds: 60 }, - // makes easier to run hight performance, cost efficient analytics with Athena + // makes easier to run high performance, cost efficient analytics with Athena dynamicPartitioningConfiguration: { enabled: true }, @@ -126,7 +126,7 @@ export function UcanFirehoseStack({ stack, app }) { parameters: [ { parameterName: 'MetadataExtractionQuery', - parameterValue: '{issuer: .value.iss}', + parameterValue: '{day: (.ts/1000) | strftime("%Y-%m-%d")}', }, { parameterName: 'JsonParsingEngine', @@ -146,7 +146,7 @@ export function UcanFirehoseStack({ stack, app }) { ], }, // See https://docs.aws.amazon.com/athena/latest/ug/partition-projection-kinesis-firehose-example.html - prefix: 'logs/issuer=!{partitionKeyFromQuery:issuer}/', + prefix: 'logs/!{partitionKeyFromQuery:day}/', errorOutputPrefix: 'error' } }) @@ -169,20 +169,32 @@ export function UcanFirehoseStack({ stack, app }) { databaseName, tableInput: { name: tableName, - partitionKeys: [], + partitionKeys: [ + { name: 'day', type: 'date' }, + ], + parameters: { + classification: "json", + typeOfData: "file", + "projection.enabled": "true", + "projection.day.type": "date", + "projection.day.format": "yyyy-MM-dd", + "projection.day.range": "2023-01-01,NOW", + "projection.day.interval": "1", + "projection.day.interval.unit": "DAYS", + "storage.location.template": `s3://${streamLogBucket.bucketName}/logs/\${day}/` + }, storageDescriptor: { location: `s3://${streamLogBucket.bucketName}/logs`, columns: [ { name: 'carcid', type: 'string' }, { name: 'type', type: 'string' }, - { name: 'value', type: 'STRUCT>,iss:STRING,aud:STRING>' } + // STRUCT here refers to the Apache Hive STRUCT datatype + { name: 'value', type: 'STRUCT>>,iss:STRING,aud:STRING>' }, + { name: "out", type: "STRUCT,ok:STRUCT>" }, + { name: "ts", type: "timestamp" } ], inputFormat: 'org.apache.hadoop.mapred.TextInputFormat', outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - parameters: { - "classification": "json", - "typeOfData": "file" - }, serdeInfo: { serializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', parameters: { From bf54ee7d4ce08157e656ecbb9ff5e8ef373c9fed Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 21 Sep 2023 21:20:52 -0700 Subject: [PATCH 2/7] feat: make it possible to query the actual CID thanks to the "forbidden characters" section of https://aws.amazon.com/blogs/big-data/create-tables-in-amazon-athena-from-nested-json-and-mappings-using-jsonserde/ also remove the paths parameter because it does nothing!! https://stackoverflow.com/questions/42170339/what-does-with-serdeproperties-paths-key1-key2-key3-really-do-in-h?noredirect=1&lq=1 --- stacks/firehose-stack.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index ff42b3a7..f2788430 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -189,7 +189,7 @@ export function UcanFirehoseStack({ stack, app }) { { name: 'carcid', type: 'string' }, { name: 'type', type: 'string' }, // STRUCT here refers to the Apache Hive STRUCT datatype - { name: 'value', type: 'STRUCT>>,iss:STRING,aud:STRING>' }, + { name: 'value', type: 'STRUCT,consumer:STRING>>>,iss:STRING,aud:STRING>' }, { name: "out", type: "STRUCT,ok:STRUCT>" }, { name: "ts", type: "timestamp" } ], @@ -198,7 +198,8 @@ export function UcanFirehoseStack({ stack, app }) { serdeInfo: { serializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', parameters: { - paths: 'carCid,invocationCid,out,ts,type,value' + // see https://aws.amazon.com/blogs/big-data/create-tables-in-amazon-athena-from-nested-json-and-mappings-using-jsonserde/ + 'mapping._cid_slash': '/' } } } From 75f3ea981f34b799026fc58eee1fb5070ede8052 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 22 Sep 2023 00:18:51 -0700 Subject: [PATCH 3/7] feat: add fields to ok struct --- stacks/firehose-stack.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index f2788430..5b278166 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -190,7 +190,7 @@ export function UcanFirehoseStack({ stack, app }) { { name: 'type', type: 'string' }, // STRUCT here refers to the Apache Hive STRUCT datatype { name: 'value', type: 'STRUCT,consumer:STRING>>>,iss:STRING,aud:STRING>' }, - { name: "out", type: "STRUCT,ok:STRUCT>" }, + { name: "out", type: "STRUCT,ok:STRUCT>" }, { name: "ts", type: "timestamp" } ], inputFormat: 'org.apache.hadoop.mapred.TextInputFormat', From 9c7c942bd072e97fbcc481ce6f08996f28984f95 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 22 Sep 2023 11:03:16 -0700 Subject: [PATCH 4/7] chore: add useful comments and links --- stacks/firehose-stack.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index 5b278166..dc1f7a5f 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -126,6 +126,7 @@ export function UcanFirehoseStack({ stack, app }) { parameters: [ { parameterName: 'MetadataExtractionQuery', + // extract yyyy-MM-dd formatted current date from millisecond epoch timestamp "ts" using jq syntax parameterValue: '{day: (.ts/1000) | strftime("%Y-%m-%d")}', }, { @@ -145,7 +146,10 @@ export function UcanFirehoseStack({ stack, app }) { }, ], }, - // See https://docs.aws.amazon.com/athena/latest/ug/partition-projection-kinesis-firehose-example.html + // See https://docs.aws.amazon.com/athena/latest/ug/partition-projection-kinesis-firehose-example.html for general information on partitioning. + // Daily partitions seem right (https://www.upsolver.com/blog/partitioning-data-s3-improve-performance-athena-presto) + // "A rough rule of thumb is that each 100 partitions scanned adds about 1 second of latency to your query in Amazon Athena. This is why minutely or hourly + // partitions are rarely used – typically you would choose between daily, weekly, and monthly partitions, depending on the nature of your queries." prefix: 'logs/!{partitionKeyFromQuery:day}/', errorOutputPrefix: 'error' } @@ -162,7 +166,6 @@ export function UcanFirehoseStack({ stack, app }) { } }) - // TODO: See https://catalog.us-east-1.prod.workshops.aws/workshops/fad47f62-3d06-430b-ad32-8588b74fe16f/en-US/lab-5-athena/55-athena-best-practices const tableName = getCdkNames('ucan-stream-delivery-table', app.stage) const glueTable = new glue.CfnTable(stack, tableName, { catalogId: Aws.ACCOUNT_ID, @@ -175,6 +178,8 @@ export function UcanFirehoseStack({ stack, app }) { parameters: { classification: "json", typeOfData: "file", + // See See https://docs.aws.amazon.com/athena/latest/ug/partition-projection-kinesis-firehose-example.html for more information on projection + // configuration - this should match the "day" parameter and S3 prefix configured in the delivery stream "projection.enabled": "true", "projection.day.type": "date", "projection.day.format": "yyyy-MM-dd", @@ -188,7 +193,7 @@ export function UcanFirehoseStack({ stack, app }) { columns: [ { name: 'carcid', type: 'string' }, { name: 'type', type: 'string' }, - // STRUCT here refers to the Apache Hive STRUCT datatype + // STRUCT here refers to the Apache Hive STRUCT datatype - see https://aws.amazon.com/blogs/big-data/create-tables-in-amazon-athena-from-nested-json-and-mappings-using-jsonserde/ { name: 'value', type: 'STRUCT,consumer:STRING>>>,iss:STRING,aud:STRING>' }, { name: "out", type: "STRUCT,ok:STRUCT>" }, { name: "ts", type: "timestamp" } From 49fc03b209e9a17ef228dd23ebfe3e82d0f1e952 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 22 Sep 2023 14:48:42 -0700 Subject: [PATCH 5/7] feat: add preload queries based on the queries I've put together this week --- package-lock.json | 18 +++++- package.json | 4 +- stacks/firehose-stack.js | 119 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 137 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index ebd5fe19..f95bc8a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,8 @@ ], "dependencies": { "@serverless-stack/node": "^1.18.0", - "aws-cdk-lib": "2.50.0" + "aws-cdk-lib": "2.50.0", + "cdk-athena": "^3.0.0" }, "devDependencies": { "@ipld/car": "^5.1.0", @@ -6033,6 +6034,15 @@ "cborg": "cli.js" } }, + "node_modules/cdk-athena": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cdk-athena/-/cdk-athena-3.0.0.tgz", + "integrity": "sha512-Eyxna+t3Jj8hC+QzDUrlwZ6Nd6813/+rhgD1uO4gwK76mU+PUvUPyV7FkOMjlP4BPWA6nR6Rhu5rJbhGHx6cUQ==", + "peerDependencies": { + "aws-cdk-lib": "^2.0.0", + "constructs": "^10.0.0" + } + }, "node_modules/chalk": { "version": "4.1.2", "dev": true, @@ -18702,6 +18712,12 @@ "cborg": { "version": "1.10.2" }, + "cdk-athena": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cdk-athena/-/cdk-athena-3.0.0.tgz", + "integrity": "sha512-Eyxna+t3Jj8hC+QzDUrlwZ6Nd6813/+rhgD1uO4gwK76mU+PUvUPyV7FkOMjlP4BPWA6nR6Rhu5rJbhGHx6cUQ==", + "requires": {} + }, "chalk": { "version": "4.1.2", "dev": true, diff --git a/package.json b/package.json index 33f7963e..94bbba19 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,6 @@ "test-integration": "ava --verbose --timeout=60s test/*.test.js", "fetch-metrics-for-space": "npm run fetch-metrics-for-space -w tools", "d1-dynamo-migration": "npm run d1-dynamo-migration -w tools" - }, "devDependencies": { "@ipld/car": "^5.1.0", @@ -47,7 +46,8 @@ }, "dependencies": { "@serverless-stack/node": "^1.18.0", - "aws-cdk-lib": "2.50.0" + "aws-cdk-lib": "2.50.0", + "cdk-athena": "^3.0.0" }, "simple-git-hooks": { "pre-commit": "npx lint-staged" diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index dc1f7a5f..6aca01fb 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -7,6 +7,7 @@ import { aws_kinesisfirehose as firehose, aws_glue as glue } from 'aws-cdk-lib' +import { NamedQuery } from 'cdk-athena' import { UcanInvocationStack } from './ucan-invocation-stack.js' @@ -19,7 +20,7 @@ import { /** * @param {import('@serverless-stack/resources').StackContext} properties */ -export function UcanFirehoseStack({ stack, app }) { +export function UcanFirehoseStack ({ stack, app }) { stack.setDefaultFunctionProps({ srcPath: 'ucan-firehose' }) @@ -212,4 +213,120 @@ export function UcanFirehoseStack({ stack, app }) { }) glueTable.addDependsOn(glueDatabase) + + const inputOutputQueryName = getCdkNames('input-output-query', app.stage) + new NamedQuery(stack, inputOutputQueryName, { + name: "Inputs and Outputs, last 24 hours", + desc: "(w3up preloaded)", + database: databaseName, + queryString: `SELECT + value.att[1] as "in", + out +FROM "${tableName}" +WHERE type = 'receipt' + AND day > (CURRENT_DATE - INTERVAL '1' DAY) +` + }) + + const dataStoredQueryName = getCdkNames('data-stored-query', app.stage) + new NamedQuery(stack, dataStoredQueryName, { + name: "Data stored by space, last week", + desc: "(w3up preloaded)", + database: databaseName, + queryString: `SELECT + SUM(value.att[1].nb.size) AS size, + value.att[1]."with" AS space +FROM "${tableName}" +WHERE value.att[1].can='store/add' + AND out.ok IS NOT NULL + AND type='receipt' + AND day > (CURRENT_DATE - INTERVAL '7' DAY) +GROUP BY value.att[1]."with" +` + }) + + const storesBySpaceQueryName = getCdkNames('stores-by-space-query', app.stage) + new NamedQuery(stack, storesBySpaceQueryName, { + name: "Stores by space, last week", + desc: "(w3up preloaded)", + database: databaseName, + queryString: `SELECT + value.att[1].nb.size AS size, + value.att[1]."with" AS space, + ts +FROM "${tableName}" +WHERE value.att[1].can='store/add' + AND day > (CURRENT_DATE - INTERVAL '2' DAY) + AND out.ok IS NOT NULL + AND type='receipt' +` + }) + + const uploadsQueryName = getCdkNames('uploads-query', app.stage) + new NamedQuery(stack, uploadsQueryName, { + name: "Uploads, last week", + desc: "(w3up preloaded)", + database: databaseName, + queryString: `SELECT + value.att[1].nb.root._cid_slash AS cid, + value.att[1]."with" AS space, + ts +FROM "${tableName}" +WHERE value.att[1].can='upload/add' + AND day > (CURRENT_DATE - INTERVAL '7' DAY) + AND out.ok IS NOT NULL + AND type='receipt' +ORDER BY ts +` + }) + + const spacesByAccountQueryName = getCdkNames('spaces-by-account-query', app.stage) + new NamedQuery(stack, spacesByAccountQueryName, { + name: "Spaces by account", + desc: "(w3up preloaded)", + database: databaseName, + queryString: `SELECT + value.att[1].nb.consumer AS space, + value.att[1]."with" AS account +FROM "${tableName}" +WHERE value.att[1].can='provider/add' + AND out.ok IS NOT NULL + AND type='receipt' +` + }) + + const uploadsByAccountQueryName = getCdkNames('uploads-by-account-query', app.stage) + new NamedQuery(stack, uploadsByAccountQueryName, { + name: "Uploads by account", + desc: "(w3up preloaded)", + database: databaseName, + queryString: `WITH +spaces AS ( + SELECT value.att[1].nb.consumer AS did, + value.att[1]."with" AS account + FROM "${tableName}" + WHERE value.att[1].can='provider/add' + AND out.ok IS NOT NULL + AND type='receipt' +), +uploads AS ( + SELECT value.att[1].nb.root._cid_slash AS cid, + value.att[1]."with" AS space, + ts + FROM "${tableName}" + WHERE value.att[1].can='upload/add' + AND out.ok IS NOT NULL + AND type='receipt' +), +uploads_by_account AS ( + SELECT spaces.did AS space, + spaces.account AS account, + uploads.cid AS cid, + uploads.ts AS "timestamp" + FROM uploads LEFT JOIN spaces on spaces.did = uploads.space +) SELECT * + FROM uploads_by_account + ORDER BY timestamp +` + }) } From e3833a06f91d4c6dcf2f09c715237f3a4e4f8d04 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 22 Sep 2023 16:33:10 -0700 Subject: [PATCH 6/7] feat: get rid of cdk-athena, use core library support this seems to behave better with deleting stacks --- package-lock.json | 39 ++++++++------------------------------- package.json | 3 +-- stacks/firehose-stack.js | 35 +++++++++++++++++++++-------------- 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/package-lock.json b/package-lock.json index f95bc8a5..50d1a357 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,8 +19,7 @@ ], "dependencies": { "@serverless-stack/node": "^1.18.0", - "aws-cdk-lib": "2.50.0", - "cdk-athena": "^3.0.0" + "aws-cdk-lib": "2.50.0" }, "devDependencies": { "@ipld/car": "^5.1.0", @@ -5170,7 +5169,6 @@ }, "node_modules/at-least-node": { "version": "1.0.0", - "dev": true, "license": "ISC", "engines": { "node": ">= 4.0.0" @@ -5445,6 +5443,8 @@ }, "node_modules/aws-cdk-lib": { "version": "2.50.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.50.0.tgz", + "integrity": "sha512-deDbZTI7oyu3rqUyqjwhP6tnUO8MD70lE98yR65xiYty4yXBpsWKbeH3s1wNLpLAWS3hWJYyMtjZ4ZfC35NtVg==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -5456,7 +5456,6 @@ "semver", "yaml" ], - "license": "Apache-2.0", "dependencies": { "@balena/dockerignore": "^1.0.2", "case": "1.6.3", @@ -6034,15 +6033,6 @@ "cborg": "cli.js" } }, - "node_modules/cdk-athena": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cdk-athena/-/cdk-athena-3.0.0.tgz", - "integrity": "sha512-Eyxna+t3Jj8hC+QzDUrlwZ6Nd6813/+rhgD1uO4gwK76mU+PUvUPyV7FkOMjlP4BPWA6nR6Rhu5rJbhGHx6cUQ==", - "peerDependencies": { - "aws-cdk-lib": "^2.0.0", - "constructs": "^10.0.0" - } - }, "node_modules/chalk": { "version": "4.1.2", "dev": true, @@ -8293,7 +8283,6 @@ }, "node_modules/fs-extra": { "version": "9.1.0", - "dev": true, "license": "MIT", "dependencies": { "at-least-node": "^1.0.0", @@ -8616,7 +8605,6 @@ }, "node_modules/graceful-fs": { "version": "4.2.11", - "dev": true, "license": "ISC" }, "node_modules/graphemer": { @@ -9801,7 +9789,6 @@ }, "node_modules/jsonfile": { "version": "6.1.0", - "dev": true, "license": "MIT", "dependencies": { "universalify": "^2.0.0" @@ -13503,7 +13490,6 @@ }, "node_modules/universalify": { "version": "2.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 10.0.0" @@ -18168,8 +18154,7 @@ } }, "at-least-node": { - "version": "1.0.0", - "dev": true + "version": "1.0.0" }, "atomically": { "version": "1.7.0" @@ -18348,6 +18333,8 @@ }, "aws-cdk-lib": { "version": "2.50.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.50.0.tgz", + "integrity": "sha512-deDbZTI7oyu3rqUyqjwhP6tnUO8MD70lE98yR65xiYty4yXBpsWKbeH3s1wNLpLAWS3hWJYyMtjZ4ZfC35NtVg==", "requires": { "@balena/dockerignore": "^1.0.2", "case": "1.6.3", @@ -18712,12 +18699,6 @@ "cborg": { "version": "1.10.2" }, - "cdk-athena": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cdk-athena/-/cdk-athena-3.0.0.tgz", - "integrity": "sha512-Eyxna+t3Jj8hC+QzDUrlwZ6Nd6813/+rhgD1uO4gwK76mU+PUvUPyV7FkOMjlP4BPWA6nR6Rhu5rJbhGHx6cUQ==", - "requires": {} - }, "chalk": { "version": "4.1.2", "dev": true, @@ -20155,7 +20136,6 @@ }, "fs-extra": { "version": "9.1.0", - "dev": true, "requires": { "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", @@ -20354,8 +20334,7 @@ } }, "graceful-fs": { - "version": "4.2.11", - "dev": true + "version": "4.2.11" }, "graphemer": { "version": "1.4.0", @@ -21019,7 +20998,6 @@ }, "jsonfile": { "version": "6.1.0", - "dev": true, "requires": { "graceful-fs": "^4.1.6", "universalify": "^2.0.0" @@ -23310,8 +23288,7 @@ } }, "universalify": { - "version": "2.0.0", - "dev": true + "version": "2.0.0" }, "unpipe": { "version": "1.0.0", diff --git a/package.json b/package.json index 94bbba19..d393aac2 100644 --- a/package.json +++ b/package.json @@ -46,8 +46,7 @@ }, "dependencies": { "@serverless-stack/node": "^1.18.0", - "aws-cdk-lib": "2.50.0", - "cdk-athena": "^3.0.0" + "aws-cdk-lib": "2.50.0" }, "simple-git-hooks": { "pre-commit": "npx lint-staged" diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index 6aca01fb..acc47cb4 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -5,9 +5,9 @@ import { aws_iam as iam, aws_logs as logs, aws_kinesisfirehose as firehose, - aws_glue as glue + aws_glue as glue, + aws_athena as athena } from 'aws-cdk-lib' -import { NamedQuery } from 'cdk-athena' import { UcanInvocationStack } from './ucan-invocation-stack.js' @@ -215,9 +215,9 @@ export function UcanFirehoseStack ({ stack, app }) { glueTable.addDependsOn(glueDatabase) const inputOutputQueryName = getCdkNames('input-output-query', app.stage) - new NamedQuery(stack, inputOutputQueryName, { + const inputOutputQuery = new athena.CfnNamedQuery(stack, inputOutputQueryName, { name: "Inputs and Outputs, last 24 hours", - desc: "(w3up preloaded)", + description: "(w3up preloaded)", database: databaseName, queryString: `SELECT value.att[1] as "in", @@ -227,11 +227,12 @@ WHERE type = 'receipt' AND day > (CURRENT_DATE - INTERVAL '1' DAY) ` }) + inputOutputQuery.addDependsOn(glueTable) const dataStoredQueryName = getCdkNames('data-stored-query', app.stage) - new NamedQuery(stack, dataStoredQueryName, { + const dataStoredQuery = new athena.CfnNamedQuery(stack, dataStoredQueryName, { name: "Data stored by space, last week", - desc: "(w3up preloaded)", + description: "(w3up preloaded)", database: databaseName, queryString: `SELECT SUM(value.att[1].nb.size) AS size, @@ -244,11 +245,12 @@ WHERE value.att[1].can='store/add' GROUP BY value.att[1]."with" ` }) + dataStoredQuery.addDependsOn(glueTable) const storesBySpaceQueryName = getCdkNames('stores-by-space-query', app.stage) - new NamedQuery(stack, storesBySpaceQueryName, { + const storesBySpaceQuery = new athena.CfnNamedQuery(stack, storesBySpaceQueryName, { name: "Stores by space, last week", - desc: "(w3up preloaded)", + description: "(w3up preloaded)", database: databaseName, queryString: `SELECT value.att[1].nb.size AS size, @@ -261,11 +263,12 @@ WHERE value.att[1].can='store/add' AND type='receipt' ` }) + storesBySpaceQuery.addDependsOn(glueTable) const uploadsQueryName = getCdkNames('uploads-query', app.stage) - new NamedQuery(stack, uploadsQueryName, { + const uploadsQuery = new athena.CfnNamedQuery(stack, uploadsQueryName, { name: "Uploads, last week", - desc: "(w3up preloaded)", + description: "(w3up preloaded)", database: databaseName, queryString: `SELECT value.att[1].nb.root._cid_slash AS cid, @@ -279,11 +282,12 @@ WHERE value.att[1].can='upload/add' ORDER BY ts ` }) + uploadsQuery.addDependsOn(glueTable) const spacesByAccountQueryName = getCdkNames('spaces-by-account-query', app.stage) - new NamedQuery(stack, spacesByAccountQueryName, { + const spacesByAccountQuery = new athena.CfnNamedQuery(stack, spacesByAccountQueryName, { name: "Spaces by account", - desc: "(w3up preloaded)", + description: "(w3up preloaded)", database: databaseName, queryString: `SELECT value.att[1].nb.consumer AS space, @@ -294,11 +298,12 @@ WHERE value.att[1].can='provider/add' AND type='receipt' ` }) + spacesByAccountQuery.addDependsOn(glueTable) const uploadsByAccountQueryName = getCdkNames('uploads-by-account-query', app.stage) - new NamedQuery(stack, uploadsByAccountQueryName, { + const uploadsByAccountQuery = new athena.CfnNamedQuery(stack, uploadsByAccountQueryName, { name: "Uploads by account", - desc: "(w3up preloaded)", + description: "(w3up preloaded)", database: databaseName, queryString: `WITH spaces AS ( @@ -329,4 +334,6 @@ uploads_by_account AS ( ORDER BY timestamp ` }) + uploadsByAccountQuery.addDependsOn(glueTable) + } From 6b8504a5af70b6e84fe794b8ca3e569584219b1f Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Mon, 25 Sep 2023 18:17:32 -0700 Subject: [PATCH 7/7] feat: fix typo --- stacks/firehose-stack.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/firehose-stack.js b/stacks/firehose-stack.js index acc47cb4..00959f4f 100644 --- a/stacks/firehose-stack.js +++ b/stacks/firehose-stack.js @@ -179,7 +179,7 @@ export function UcanFirehoseStack ({ stack, app }) { parameters: { classification: "json", typeOfData: "file", - // See See https://docs.aws.amazon.com/athena/latest/ug/partition-projection-kinesis-firehose-example.html for more information on projection + // @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-kinesis-firehose-example.html for more information on projection // configuration - this should match the "day" parameter and S3 prefix configured in the delivery stream "projection.enabled": "true", "projection.day.type": "date",