chore: AzureSQL - Introduce the estimation of number_of_batches in the unconditional masking to optimize number of masking API calls

CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+# 0.0.38
+* Added number of batches parameter to `dcsazure_AzureSQL_to_AzureSQL_mask_pl` in unconditional masking.
+
 # 0.0.37
 * Added number of batches parameter to `dcsazure_AzureSQL_MI_to_AzureSQL_MI_mask_pl` in unconditional masking.
 * Increased TimeOut for the masking API call in `dcsazure_AzureSQL_MI_to_AzureSQL_MI_mask_pl` to 300 seconds.

VERSION.md

@@ -1 +1 @@
-0.0.37
+0.0.38

dcsazure_AzureSQL_to_AzureSQL_mask_pl/README.md

@@ -108,7 +108,8 @@ have customized your metadata store, then these variables may need editing.
 * `CONDITIONAL_MASKING_RESERVED_CHARACTER` - This is a string (preferably a character) reserved as for shorthand for
   when referring to the key column when defining filter conditions, in the pipeline this will be expanded out to use the
   ADF syntax for referencing the key column (default `%`)
-* `TARGET_BATCH_SIZE` - This is the target number of rows per batch (default `2000`)
+* `TARGET_BATCH_SIZE` - This is the target number of rows per batch (default `2000`) use for conditional masking
+* `DF_NUMBER_OF_BATCHES` - This is the target number of rows per batch (default `100`) use for unconditional masking
 * `METADATA_EVENT_PROCEDURE_NAME` - This is the name of the procedure used to capture pipeline information in the metadata data store and sets the masked and mapping states on the items processed during execution (default `insert_adf_masking_event`).
 * `METADATA_MASKING_PARAMS_PROCEDURE_NAME` - This is the name of the stored procedure used to generate masking parameters for both conditional and non-conditional masking scenarios (default `generate_masking_parameters`).
 * `METADATA_CONSTRAINT_TABLE` - This table is used to store the foreign key constraints for each table in the sink schema that are required for masking. The table is queried to drop and recreate those constraints back.

dcsazure_AzureSQL_to_AzureSQL_mask_pl/dcsazure_AzureSQL_to_AzureSQL_mask_pl.json

@@ -215,6 +215,10 @@
                               "DF_COLUMNS_TO_CAST_BACK_TO_TIMESTAMP": {
                                 "value": "@activity('Lookup Masking Parameters').output.firstRow.ColumnsToCastBackToTimestamp",
                                 "type": "Expression"
+                              },
+                              "DF_NUMBER_OF_BATCHES": {
+                                "value": "@activity('Lookup Masking Parameters').output.firstRow.NumberOfBatches",
+                                "type": "Expression"
                               }
                             },
                             "datasetParameters": {
@@ -2666,7 +2670,7 @@
           "name": "dcsazure_AzureSQL_to_AzureSQL"
         },
         "annotations": [],
-        "lastPublishTime": "2025-09-11T07:04:14Z"
+        "lastPublishTime": "2025-10-08T11:34:30Z"
       },
       "dependsOn": [
         "[concat(variables('factoryId'), '/datasets/dcsazure_AzureSQL_to_AzureSQL_mask_metadata_ds')]",
@@ -3394,7 +3398,7 @@
           "name": "dcsazure_AzureSQL_to_AzureSQL"
         },
         "annotations": [],
-        "lastPublishTime": "2025-09-11T06:59:43Z"
+        "lastPublishTime": "2025-10-08T06:38:20Z"
       },
       "dependsOn": [
         "[concat(variables('factoryId'), '/datasets/dcsazure_AzureSQL_to_AzureSQL_mask_sink_ds')]",
@@ -3824,7 +3828,7 @@
           "name": "dcsazure_AzureSQL_to_AzureSQL"
         },
         "annotations": [],
-        "lastPublishTime": "2025-09-11T07:05:00Z"
+        "lastPublishTime": "2025-10-08T06:38:20Z"
       },
       "dependsOn": [
         "[concat(variables('factoryId'), '/datasets/dcsazure_AzureSQL_to_AzureSQL_mask_sink_ds')]",
@@ -3989,7 +3993,8 @@
             "     DF_COLUMNS_TO_CAST_BACK_TO_FLOAT as string[] ([\"\"]),",
             "     DF_COLUMNS_TO_CAST_BACK_TO_INTEGER as string[] ([\"\"]),",
             "     DF_COLUMNS_TO_CAST_BACK_TO_LONG as string[] ([\"\"]),",
-            "     DF_COLUMNS_TO_CAST_BACK_TO_TIMESTAMP as string[] ([\"\"])",
+            "     DF_COLUMNS_TO_CAST_BACK_TO_TIMESTAMP as string[] ([\"\"]),",
+            "     DF_NUMBER_OF_BATCHES as integer (100)",
             "}",
             "source(allowSchemaDrift: true,",
             "     validateSchema: false,",
@@ -4035,7 +4040,7 @@
             "     ),",
             "     skipDuplicateMapInputs: true,",
             "     skipDuplicateMapOutputs: true) ~> SelectColumnsUnmasked",
-            "WrapValuesInArray aggregate(groupBy(DELPHIX_COMPLIANCE_SERVICE_BATCH_GROUP = toInteger(ceil(DELPHIX_COMPLIANCE_SERVICE_BATCH_ID/$DF_NUMBER_OF_ROWS_PER_BATCH))),",
+            "WrapValuesInArray aggregate(groupBy(DELPHIX_COMPLIANCE_SERVICE_BATCH_GROUP = DELPHIX_COMPLIANCE_SERVICE_BATCH_ID%$DF_NUMBER_OF_BATCHES),",
             "     each(match((startsWith(name,'x')&&not(contains($DF_COLUMNS_TO_MASK,#item==name)))||(name==\"DELPHIX_COMPLIANCE_SERVICE_BATCH_ID\")), $$ = collect($$))) ~> AggregateColumnsByBatch",
             "AggregateColumnsByBatch derive(each(match(startsWith(name,'x')||(name==\"DELPHIX_COMPLIANCE_SERVICE_BATCH_ID\")), $$ = flatten($$))) ~> FlattenValuesOutOfArray",
             "DCSForAzureAPI assert(expectTrue(toInteger(regexExtract(status, '(\\\\d+)', 1)) == 200, false, 'Failed_request', null, iif(isNull(body.message), status, concatWS(', ', 'timestamp: ' + toString(body.timestamp), 'status: ' + body.status, 'message: ' + body.message, 'trace_id: ' + body.trace_id))),",