Merge pull request #31 from metabolicdata/hotfix/glue_crawler

Incomptaible SchemaChanges mode
metabolicdata · Mar 25, 2024 · c5de1b1 · c5de1b1
2 parents 680f893 + 6617896
commit c5de1b1
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/main/scala/com/metabolic/data/core/services/glue/GlueCrawlerService.scala b/src/main/scala/com/metabolic/data/core/services/glue/GlueCrawlerService.scala
@@ -66,11 +66,11 @@ class GlueCrawlerService(implicit val region: Regions) extends Logging {
       .withS3Targets(s3TargetList:_*)
 
     val recrawlPolicy = new RecrawlPolicy()
-      .withRecrawlBehavior(RecrawlBehavior.CRAWL_NEW_FOLDERS_ONLY)
+      .withRecrawlBehavior(RecrawlBehavior.CRAWL_EVERYTHING)
 
     val schemaChangePolicy = new SchemaChangePolicy()
       .withUpdateBehavior(UpdateBehavior.UPDATE_IN_DATABASE)
-      .withDeleteBehavior(DeleteBehavior.DEPRECATE_IN_DATABASE)
+      .withDeleteBehavior(DeleteBehavior.DELETE_FROM_DATABASE)
 
     val crawlerCreateRequest = new CreateCrawlerRequest()
       .withDatabaseName(dbName)