-
Notifications
You must be signed in to change notification settings - Fork 5
/
datalake_helper.py
41 lines (32 loc) · 1.28 KB
/
datalake_helper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import time
import json
import os, uuid
import config
from azure.storage.filedatalake import (DataLakeServiceClient,)
from enum import Enum
storage_account_name = config.STORAGE_ACCOUNT_NAME
storage_account_key = config.STORAGE_ACCOUNT_KEY
storage_filesystem_name = config.STORAGE_FILESYSTEM_NAME
class FileType(Enum):
Level1 = 1
Level2 = 2
TimeSale = 3
def get_filename(fileType):
epoch_time = str(time.time())
fileType = FileType(fileType)
return "{}-{}.json".format(epoch_time, fileType.name)
def write_file(msg, folder_name, file_name):
# set up the service client with the credentials from the environment variables
service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format(
"https",
storage_account_name
), credential=storage_account_key)
# get the filesystem
file_system_client = service_client.get_file_system_client(file_system=storage_filesystem_name)
directory_client = file_system_client.get_directory_client(folder_name)
if not directory_client.exists():
directory_client.create_directory()
file_client = directory_client.create_file(file_name)
content = json.dumps(msg)
file_client.append_data(content, 0, len(content))
file_client.flush_data(len(content))