From 64f8d9cc28d3f3061185998c18c911bf32a225a8 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Wed, 11 Sep 2024 15:31:18 +0100 Subject: [PATCH] Add modules for datalake backup and restore (#150) Signed-off-by: Jim Enright --- plugins/modules/datalake_backup.py | 632 ++++++++++++++++++++++++ plugins/modules/datalake_backup_info.py | 207 ++++++++ 2 files changed, 839 insertions(+) create mode 100644 plugins/modules/datalake_backup.py create mode 100644 plugins/modules/datalake_backup_info.py diff --git a/plugins/modules/datalake_backup.py b/plugins/modules/datalake_backup.py new file mode 100644 index 0000000..9c400ec --- /dev/null +++ b/plugins/modules/datalake_backup.py @@ -0,0 +1,632 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + +DOCUMENTATION = r""" +--- +module: cloudera.cloud.datalake_backup +short_description: Create a backup of a datalake +description: + - Create a backup of a datalake + - Optionally wait for the backup to complete +author: + - "Jim Enright (@jimright)" +options: + datalake_name: + description: + - The name of the datalake to backup + required: true + type: str + backup_name: + description: + - The name of the backup + - If I(state=backup) this is the name of the backup to create + - If I(state=restore) this is the name of the backup to restore + required: false + type: str + backup_id: + description: + - The Id of the backup to restore + - Only applicable when I(state=restore) + required: false + type: str + backup_location: + description: + - The location of the backup to use during the restore + - When not specified the location used will be the backup storage of the environment + - If provided the I(backup_id) parameter is required + - Only applicable when I(state=restore) + required: false + type: str + skip_atlas_indexes: + description: + - Skips the restore of the Atlas indexes + - Only applicable when I(state=restore) + required: false + type: bool + skip_atlas_metadata: + description: + - Skips the restore of the Atlas metadata + - Only applicable when I(state=restore) + required: false + type: bool + skip_ranger_audits: + description: + - Skips the restore of the Ranger audits + - Only applicable when I(state=restore) + required: false + type: bool + skip_ranger_hms_metadata: + description: + - Skips the restore of the databases backing HMS/Ranger services + - Only applicable when I(state=restore) + required: false + type: bool + skip_validation: + description: + - Skips the validation steps that run prior to the restore + - Only applicable when I(state=restore) + required: false + type: bool + state: + description: + - The declarative state of the datalake backup. + type: str + required: False + default: backup + choices: + - backup + - restore + wait: + description: + - Whether to wait for the backup to complete + required: false + type: bool +""" + +EXAMPLES = r""" +# Note: These examples do not set authentication details. + +- name: Create a datalake backup + cloudera.cloud.datalake_backup: + datalake_name: "datalake" + backup_name: "my_backup" + state: "backup" + register: backup_result + +- name: Create a datalake backup and wait for it to complete + cloudera.cloud.datalake_backup: + datalake_name: "datalake" + state: "backup" + wait: true + register: backup_result + +- name: Restore a named datalake backup wait for it to complete + cloudera.cloud.datalake_backup: + datalake_name: "datalake" + backup_name: "my_backup" + state: "restore" + wait: true +""" + +RETURN = r""" +backup: + description: The details of the backup or restore operation + type: list + elements: dict + returned: always + contains: + backupName: + description: The name of the backup + type: str + accountId: + description: The account id + type: str + userCrn: + description: The crn of the user that initiated the backup operation + type: str + restoreId: + description: + - The restore id + - Only returned when I(state=restore) + type: str + backupId: + description: The backup id + type: str + internalState: + description: The internal state of each backup or restore stage + type: str + status: + description: The overall status of the backup or restore operation + type: str + startTime: + description: The start time + type: str + endTime: + description: The end time + type: str + backupLocation: + description: The backup location + type: str + operationStates: + description: Object representing the state of each service running a backup or restore + type: dict + contains: + adminOperations: + description: The state of Cloudera Manager admin operations + type: dict + contains: + stopServices: + description: Details of the stop services admin operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + startServices: + description: Details of the start services admin operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + precheckStoragePermission: + description: Details of the pre check storage permission admin operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + rangerAuditCollectionValidation: + description: Details of the Ranger Audit Collection Validation admin operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + dryRunValidation: + description: Details of the dry run validation admin operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + hbase: + description: The state of each HBase backup/restore operation + type: dict + contains: + atlasEntityAuditEventTable: + description: Details of the Atlas entity audit event table HBase operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + atlasJanusTable: + description: Details of the Atlas Janus HBase operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + solr: + description: The state of each Solr backup/restore operation + type: dict + contains: + edgeIndexCollection: + description: Details of the edge index collection Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + fulltextIndexCollection: + description: Details of the full text index collection Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + rangerAuditsCollection: + description: Details of the ranger audits collection Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + vertexIndexCollection: + description: Details of the vertex index collection Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + edgeIndexCollectionDelete: + description: Details of the edge index collection delete Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + fulltextIndexCollectionDelete: + description: Details of the full text index collection delete Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + rangerAuditsCollectionDelete: + description: Details of the ranger audits collection delete Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + vertexIndexCollectionDelete: + description: Details of the vertex index collection delete Solr operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + database: + description: The state of each database backup/restore operation + type: dict + contains: + database: + description: Details of the database operation + type: dict + contains: + status: + description: The status of the backup or restore operation + type: str + failureReason: + description: The failure reason if the operation was not successful + type: str + durationInMinutes: + description: The duration of the operation, in minutes + type: str + predictedDurationInMinutes: + description: Predicted duration of the operation, in minutes + type: str + runtimeVersion: + description: Datalake runtime version + type: str +""" + + +class DatalakeBackup(CdpModule): + def __init__(self, module): + super(DatalakeBackup, self).__init__(module) + + # Set Variables + self.datalake_name = self._get_param("datalake_name") + self.backup_name = self._get_param("backup_name") + self.state = self._get_param("state").lower() + self.wait = self._get_param("wait", False) + # ...variables for restore only + self.backup_id = self._get_param("backup_id") + self.backup_location = self._get_param("backup_location") + self.skip_atlas_indexes = self._get_param("skip_atlas_indexes") + self.skip_atlas_metadata = self._get_param("skip_atlas_metadata") + self.skip_ranger_audits = self._get_param("skip_ranger_audits") + self.skip_ranger_hms_metadata = self._get_param("skip_ranger_hms_metadata") + self.skip_validation = self._get_param("skip_validation") + + # Initialize the return values + self.output = dict() + self.changed = False + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + + # Check parameters that should only specified with state=restore + if self.state == "backup" and ( + self.backup_id + or self.backup_location + or self.skip_atlas_indexes + or self.skip_atlas_metadata + or self.skip_ranger_audits + or self.skip_ranger_hms_metadata + or self.skip_validation + ): + self.module.fail_json( + msg="Unable to use 'state=backup' with args 'backup_id', 'backup_location', 'skip_atlas_indexes', 'skip_atlas_metadata', 'skip_ranger_audits', 'skip_ranger_hms_metadata' or 'skip_validation'" + ) + + # Confirm datalake exists + datalake_info = self.cdpy.datalake.describe_datalake(self.datalake_name) + + if datalake_info is None: + self.module.fail_json( + msg="Datalake {0} does not exist".format(self.datalake_name) + ) + else: + if self.state == "backup": + + backup = self.cdpy.datalake.create_datalake_backup( + datalake_name=self.datalake_name, backup_name=self.backup_name + ) + + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.datalake.check_datalake_backup_status, + params=dict( + datalake_name=self.datalake_name, + backup_id=backup["backupId"], + ), + state=["SUCCESSFUL"], + ) + + datalake_backups = self.cdpy.datalake.list_datalake_backups( + datalake_name=self.datalake_name + ) + self.output = [ + item + for item in datalake_backups["backups"] + if item["backupId"] == backup["backupId"] + ] + self.changed = True + + elif self.state == "restore": + # If specified confirm that backup (name or id) exists + if self.backup_location is None and any( + bk is not None for bk in [self.backup_name, self.backup_id] + ): + datalake_backups = self.cdpy.datalake.list_datalake_backups( + datalake_name=self.datalake_name + ) + if ( + len( + [ + item + for item in datalake_backups["backups"] + if item["backupName"] == self.backup_name + or item["backupId"] == self.backup_id + ] + ) + == 0 + ): + self.module.fail_json( + msg="Specified backup {0} does not exist for datalake {1}".format( + next( + bk + for bk in [self.backup_name, self.backup_id] + if bk is not None + ), + self.datalake_name, + ) + ) + + restore = self.cdpy.datalake.restore_datalake_backup( + datalake_name=self.datalake_name, + backup_name=self.backup_name, + backup_id=self.backup_id, + backup_location_override=self.backup_location, + skip_atlas_indexes=self.skip_atlas_indexes, + skip_atlas_metadata=self.skip_atlas_metadata, + skip_ranger_audits=self.skip_ranger_audits, + skip_ranger_hms_metadata=self.skip_ranger_hms_metadata, + skip_validation=self.skip_validation, + ) + + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.datalake.check_datalake_restore_status, + params=dict( + datalake_name=self.datalake_name, + restore_id=restore["restoreId"], + ), + state=["SUCCESSFUL"], + ) + restore = self.cdpy.datalake.check_datalake_restore_status( + datalake_name=self.datalake_name, + restore_id=restore["restoreId"], + ) + + self.output = restore + self.changed = True + else: + self.module.fail_json(msg="Invalid state: %s" % self.state) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + datalake_name=dict(required=True, type="str", aliases=["name"]), + backup_name=dict(required=False, type="str"), + state=dict( + required=False, + type="str", + choices=["backup", "restore"], + default="backup", + ), + wait=dict(required=False, type="bool"), + backup_id=dict(required=False, type="str"), + backup_location=dict(required=False, type="str"), + skip_atlas_indexes=dict(required=False, type="bool"), + skip_atlas_metadata=dict(required=False, type="bool"), + skip_ranger_audits=dict(required=False, type="bool"), + skip_ranger_hms_metadata=dict(required=False, type="bool"), + skip_validation=dict(required=False, type="bool"), + ), + required_by={ + "backup_location": ("backup_id"), + }, + mutually_exclusive=[ + ["backup_name", "backup_id"], + ], + supports_check_mode=True, + ) + + result = DatalakeBackup(module) + + output = dict(changed=result.changed, backup=result.output) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == "__main__": + main() diff --git a/plugins/modules/datalake_backup_info.py b/plugins/modules/datalake_backup_info.py new file mode 100644 index 0000000..b61c5fd --- /dev/null +++ b/plugins/modules/datalake_backup_info.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + +DOCUMENTATION = r""" +--- +module: cloudera.cloud.datalake_backup_info +short_description: Gather information about a Datalake backup +description: + - Gather information about a Datalake backup + - Optionally filter by backup name or backup id +author: + - "Jim Enright (@jimright)" +options: + datalake_name: + description: + - The name of the datalake to backup + required: true + type: str + backup_name: + description: + - The name of the backup + required: false + type: str + backup_id: + description: + - The id of the backup + required: false + type: str +""" + +EXAMPLES = r""" +# Note: These examples do not set authentication details. + +- name: Gather information about all backups for a Datalake + cloudera.cloud.datalake_backup_info: + datalake_name: "datalake" + register: backup_result + +- name: Gather information about a specific backup name for a Datalake + cloudera.cloud.datalake_backup_info: + datalake_name: "datalake" + backup_name: "backup" + register: backup_result + +- name: Gather information about a specific backup id for a Datalake + cloudera.cloud.datalake_backup_info: + datalake_name: "datalake" + backup_id: "backup_id" + register: backup_result +""" + +RETURN = r""" +backups: + description: Information about the backup + type: list + elements: dict + returned: always + contains: + backupName: + description: The name of the backup + type: str + returned: always + accountId: + description: The account id + type: str + returned: always + userCrn: + description: The user crn + type: str + returned: always + backupId: + description: The backup id + type: str + returned: always + internalState: + description: The state of each internal backup components + type: str + returned: always + status: + description: The status of the backup + type: str + returned: always + startTime: + description: The start time of the backup + type: str + returned: always + endTime: + description: The end time of the backup + type: str + returned: always + backupLocation: + description: The location of the backup + type: str + returned: always +""" + + +class DatalakeBackupInfo(CdpModule): + def __init__(self, module): + super(DatalakeBackupInfo, self).__init__(module) + + # Set Variables + self.datalake_name = self._get_param("datalake_name") + self.backup_name = self._get_param("backup_name") + self.backup_id = self._get_param("backup_id") + + # Initialize the return values + self.output = [] + self.changed = False + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + + # Confirm datalake exists + datalake_info = self.cdpy.datalake.describe_datalake(self.datalake_name) + + if datalake_info is None: + self.module.warn("Datalake {0} not found".format(self.datalake_name)) + else: + datalake_backups = self.cdpy.datalake.list_datalake_backups( + datalake_name=self.datalake_name + ) + + # Filter for backup name or backup id if specified + if self.backup_name is not None: + named_backups = [ + item + for item in datalake_backups["backups"] + if item["backupName"] == self.backup_name + ] + if len(named_backups) == 0: + self.module.warn( + "Backup name {0} not found for Datalake {1}".format( + self.backup_name, self.datalake_name + ) + ) + + self.output = named_backups + + elif self.backup_id is not None: + single_backup = [ + item + for item in datalake_backups["backups"] + if item["backupId"] == self.backup_id + ] + + if len(single_backup) == 0: + self.module.warn( + "Backup id {0} not found for Datalake {1}".format( + self.backup_id, self.datalake_name + ) + ) + + self.output = single_backup + else: + self.output = datalake_backups["backups"] + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + datalake_name=dict(required=True, type="str", aliases=["name"]), + backup_name=dict(required=False, type="str"), + backup_id=dict(required=False, type="str"), + ), + mutually_exclusive=[["backup_name", "backup_id"]], + supports_check_mode=True, + ) + + result = DatalakeBackupInfo(module) + + output = dict(changed=False, backups=result.output) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == "__main__": + main()