Databricks External Location Terraform module

Terraform module for creation Databricks External Location

Usage

# Prerequisite resources

# Databricks Workspace with Premium SKU
data "azurerm_databricks_workspace" "example" {
  name                = "example-workspace"
  resource_group_name = "example-rg"
}

resource "azurerm_databricks_access_connector" "example" {
  name                = "example-resource"
  resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location

  identity {
    type = "SystemAssigned"
  }
}

# Databricks Provider configuration
provider "databricks" {
  alias                       = "main"
  host                        = data.azurerm_databricks_workspace.example.workspace_url
  azure_workspace_resource_id = data.azurerm_databricks_workspace.example.id
}

locals {
  storage_credentials = {
    prefix                    = "example"
    azure_access_connector_id = azurerm_databricks_access_connector.example.id
    permissions               = [{ principal = "ALL_PRIVILEGES_GROUP", privileges = ["ALL_PRIVILEGES"] }]
  }
  
  external_locations = 
    {
      name        = "adls-example"
      url         = "abfss://container@storageaccount.dfs.core.windows.net"
      permissions = [
        { principal = "ALL_PRIVILEGES_GROUP", privileges = ["ALL_PRIVILEGES"] },
        { principaprincipal = "EXAMPLE_PERMISSION_GROUP", privileges = ["CREATE_EXTERNAL_TABLE", "READ_FILES"] }
      ]    
      owner           = "username@domain.com"
      skip_validation = true
      read_only       = false
      comment         = "example_comment"
    }    
}

# Databricks External Location 
module "databricks_locations" {
  count  = var.databricks_configure ? (module.databricks_workspace.sku == "premium" ? 1 : 0) : 0

  source  = "data-platform-hq/external-location/databricks"
  version  = "~> 1.0"

  project            = "datahq"
  env                = "example"
  location           = "eastus"
  storage_credential = local.storage_credentials
  external_locations = local.external_locations

  providers = {
    databricks = databricks.workspace
  }
}

Requirements

Name	Version
terraform	~>1.3
databricks	~>1.0

Providers

Name	Version
databricks	~>1.0

Modules

No modules.

Resources

Name	Type
databricks_external_location.this	resource
databricks_grants.credential	resource
databricks_grants.locations	resource
databricks_storage_credential.this	resource

Inputs

Name	Description	Type	Default	Required
cloud	Cloud (azure, aws or gcp)	string	n/a	yes
external_locations	List of object with external location configuration attributes	list(object({ index = string # Index of instance, for example short name, used later to access exact external location in output map name = string # Custom whole name of resource url = string # Path URL in cloud storage credentials_name = optional(string) # If storage_credential.create_storage_credential is set to false, provide id of existing storage credential here owner = optional(string) # Owner of resource skip_validation = optional(bool, true) # Suppress validation errors if any & force save the external location read_only = optional(bool, false) # Indicates whether the external location is read-only. force_destroy = optional(bool, true) force_update = optional(bool, true) comment = optional(string, "External location provisioned by Terraform") permissions = optional(set(object({ principal = string privileges = list(string) })), []) isolation_mode = optional(string, "ISOLATION_MODE_OPEN") }))	[]	no
storage_credential	Object with storage credentials configuration attributes	object({ azure_access_connector_id = optional(string, null) # Azure Databricks Access Connector Id name = optional(string, null) # Custom whole name of resource owner = optional(string) # Owner of resource force_destroy = optional(bool, true) comment = optional(string, "Managed identity credential provisioned by Terraform") create_storage_credential = optional(bool, true) # "Boolean flag that determines whether to create storage credential or use the existing one" permissions = optional(set(object({ principal = string privileges = list(string) })), []) isolation_mode = optional(string, "ISOLATION_MODE_OPEN") })	n/a	yes

Outputs

Name	Description
databricks_gcp_service_account	The email of the GCP service account created, to be granted access to relevant buckets
external_locations	Map of objects with External Location parameters, like name, credentials name and url of target storage
storage_credential_metastore_id	Storage Credential metastore id
storage_credential_name	Storage Credential name

License

Apache 2 Licensed. For more information please see LICENSE