From 5d4eaebeb391d88a614e5182b73824e01b58c811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Bern=C3=A1rdez?= Date: Thu, 4 Apr 2024 09:14:41 +0200 Subject: [PATCH 1/3] Fix prefix for athena externa table if the table belongs to a namespace --- .../data/core/services/athena/AthenaAction.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala b/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala index cd6ad39..51c695f 100644 --- a/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala +++ b/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala @@ -15,12 +15,10 @@ class AthenaAction extends AfterAction with Logging { val options = config.environment - val s3Path = config.sink.asInstanceOf[FileSink].path - .replace("version=1/", "") + val region = options.region val dbName = options.dbName - val prefix = ConfigUtilsService.getTablePrefix(options.namespaces, s3Path) - val tableName = prefix+ConfigUtilsService.getTableName(config) + val athena = new AthenaCatalogueService()(region) @@ -30,10 +28,10 @@ class AthenaAction extends AfterAction with Logging { case IOFormat.DELTA => logger.info(f"After Action $name: Creating Delta Table for ${config.name}") - - athena.dropView(dbName, tableName) - val s3Path = sink.path.replaceAll("version=\\d+", "") + val prefix = ConfigUtilsService.getTablePrefix(options.namespaces, s3Path) + val tableName = prefix + ConfigUtilsService.getTableName(config) + athena.dropView(dbName, tableName) athena.createDeltaTable(dbName, tableName, s3Path) case _ => From cc9d792a9325f8fb5cc15f51c6c54085f89f4afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Bern=C3=A1rdez?= Date: Thu, 4 Apr 2024 10:31:42 +0200 Subject: [PATCH 2/3] emd --- .gitignore | 8 ++++++++ .../core/services/athena/AthenaAction.scala | 18 +++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 60257be..69ea798 100644 --- a/.gitignore +++ b/.gitignore @@ -307,3 +307,11 @@ gradle-app.setting terraform/.terraform old_envs/* + +### VSCode ### +.bloop/ +.metals/ +.vscode/ +project/.bloop/ +project/metals.sbt +project/project/ \ No newline at end of file diff --git a/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala b/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala index 51c695f..53fc902 100644 --- a/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala +++ b/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala @@ -15,30 +15,34 @@ class AthenaAction extends AfterAction with Logging { val options = config.environment - val region = options.region val dbName = options.dbName - val athena = new AthenaCatalogueService()(region) config.sink match { case sink: FileSink => sink.format match { case IOFormat.DELTA => - - logger.info(f"After Action $name: Creating Delta Table for ${config.name}") + logger.info( + f"After Action $name: Creating Delta Table for ${config.name}" + ) val s3Path = sink.path.replaceAll("version=\\d+", "") - val prefix = ConfigUtilsService.getTablePrefix(options.namespaces, s3Path) + val prefix = + ConfigUtilsService.getTablePrefix(options.namespaces, s3Path) val tableName = prefix + ConfigUtilsService.getTableName(config) athena.dropView(dbName, tableName) athena.createDeltaTable(dbName, tableName, s3Path) case _ => - logger.warn(f"After Action: Skipping $name for ${config.name} as it is not a DeltaSink") + logger.warn( + f"After Action: Skipping $name for ${config.name} as it is not a DeltaSink" + ) } case _ => - logger.warn(f"After Action: Skipping $name for ${config.name} as it is not a FileSink") + logger.warn( + f"After Action: Skipping $name for ${config.name} as it is not a FileSink" + ) } } From 9bbb08d260dee840d27076bc29474ab29e426e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Bern=C3=A1rdez?= Date: Fri, 5 Apr 2024 09:39:10 +0200 Subject: [PATCH 3/3] emd --- .../data/core/services/athena/AthenaAction.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala b/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala index 53fc902..0d5e386 100644 --- a/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala +++ b/src/main/scala/com/metabolic/data/core/services/athena/AthenaAction.scala @@ -24,9 +24,7 @@ class AthenaAction extends AfterAction with Logging { case sink: FileSink => sink.format match { case IOFormat.DELTA => - logger.info( - f"After Action $name: Creating Delta Table for ${config.name}" - ) + logger.info(f"After Action $name: Creating Delta Table for ${config.name}") val s3Path = sink.path.replaceAll("version=\\d+", "") val prefix = ConfigUtilsService.getTablePrefix(options.namespaces, s3Path) @@ -35,14 +33,10 @@ class AthenaAction extends AfterAction with Logging { athena.createDeltaTable(dbName, tableName, s3Path) case _ => - logger.warn( - f"After Action: Skipping $name for ${config.name} as it is not a DeltaSink" - ) + logger.warn(f"After Action: Skipping $name for ${config.name} as it is not a DeltaSink") } case _ => - logger.warn( - f"After Action: Skipping $name for ${config.name} as it is not a FileSink" - ) + logger.warn(f"After Action: Skipping $name for ${config.name} as it is not a FileSink") } }