diff --git a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml index d4634f38de..b5a9fcf4d1 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml index faf22139cc..99b149e96b 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml index 4690e3b788..3591f216bf 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml index 28c51b8ace..c293590e51 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml index 96a473ccdc..1accc20e81 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml index 2c7feeea6e..af881e985a 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml index 2616bf1aac..3d6a19484f 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml index d3cce60c7b..a7f190c239 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml index 80e3206cca..04c1110e96 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml index 04875be997..d13b5ed3ad 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml index d121a50cc5..41674e812f 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml index c8a8681636..63cf56f61c 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU= + stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml index b1abd6db78..afac799af1 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml index f8e56a1b75..430a9b6ed9 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml index f1c8d7a954..43ab792fa8 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml index d68261f5ce..960212c6d8 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml index d485f62506..bb650d12bc 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml index adb5ff874b..a8a6dcbda6 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml index 71a3dd88f7..2abc68bab8 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml index c330b417fb..fdd4337658 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml index 661889f435..63a287cfae 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml index 9c79a4a79b..4c8f8c5cdb 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml index a157d515f7..ad9e5db172 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml index 53f5f4dc24..7c886c8fa5 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA= + stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml index e4c0885e48..76f2d33b1f 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml index 4a1be92a3f..f917588e71 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU= + stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml index 9bc3d1776f..91bdd5e8ea 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml index 360c6d3e42..c2316172ad 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: ohqyd8i40N1OhA-WN_ifFolSeJokKiFPJd4m-8X3ybA= + stack.formance.cloud/volumes-hash: N51Q5ybfoJgUQ5aJzKW0j9antWOEQn2YykvrD29h3no= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml index eb22c15698..01e166ba7e 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml index 138f43b459..f57a47856c 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: ohqyd8i40N1OhA-WN_ifFolSeJokKiFPJd4m-8X3ybA= + stack.formance.cloud/volumes-hash: N51Q5ybfoJgUQ5aJzKW0j9antWOEQn2YykvrD29h3no= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml index c7305a8af8..fa206d4090 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml index 662dd1bf4a..2bb9ca02b9 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU= + stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml index 3291204469..9cc26077e0 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml index 760038605b..84d06824f4 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU= + stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml index b597216161..a090836f24 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: \"\\\"\" + item.value.string() + \"\\\"\" },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml index 18227f2696..ebb5fb4cb0 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU= + stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos diff --git a/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml b/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml index 598c436e4e..7136fc918c 100644 --- a/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml +++ b/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml @@ -118,7 +118,7 @@ pipeline: "upsert": { "data": { "address": this.payload.targetId, - "metadata": { item.key: "\"" + item.value.string() + "\"" }, + "metadata": { item.key: item.value.string() }, "ledger": this.payload.ledger }, "indexed": {