From 9179f3e3d43ba0e3e9a457c38ac008a85581ec52 Mon Sep 17 00:00:00 2001 From: Kieras Date: Tue, 9 Jan 2018 11:56:37 -0200 Subject: [PATCH 1/4] Export feature. --- .gitignore | 2 ++ gcdu/__init__.py | 2 +- gcdu/__main__.py | 2 ++ gcdu/__version__.py | 3 ++- gcdu/cli.py | 5 ++-- gcdu/export_kind.py | 62 +++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 7 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 gcdu/export_kind.py diff --git a/.gitignore b/.gitignore index d636710..b56215e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Created by https://www.gitignore.io/api/node,java,linux,macos,maven,python,eclipse,windows,virtualenv,intellij+all +data/ + ### Eclipse ### .metadata diff --git a/gcdu/__init__.py b/gcdu/__init__.py index 8b13789..40a96af 100644 --- a/gcdu/__init__.py +++ b/gcdu/__init__.py @@ -1 +1 @@ - +# -*- coding: utf-8 -*- diff --git a/gcdu/__main__.py b/gcdu/__main__.py index d92dc90..199fc4b 100644 --- a/gcdu/__main__.py +++ b/gcdu/__main__.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from cli import main main() diff --git a/gcdu/__version__.py b/gcdu/__version__.py index 5a62164..0cecb61 100644 --- a/gcdu/__version__.py +++ b/gcdu/__version__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # ____ ____ ____ _ _ # / ___| / ___| | _ \ | | | | # | | _ | | | | | | | | | | @@ -7,4 +8,4 @@ # # Google Cloud Datastore Utils -__version__ = '0.0.1' +__version__ = '0.0.2' diff --git a/gcdu/cli.py b/gcdu/cli.py index a8c8a6b..4373890 100644 --- a/gcdu/cli.py +++ b/gcdu/cli.py @@ -1,4 +1,6 @@ +# -*- coding: utf-8 -*- import click +import export_kind from __version__ import __version__ as version @@ -56,8 +58,7 @@ def migrate(from_project, from_namespace, to_project, to_namespace, skip_export, with click.progressbar(kinds_list, label='Exporting', show_eta=False, item_show_func=show_progressbar_item) as bar: for kind in bar: - # TODO: Implement export feature. - pass + export_kind.run(from_project, from_namespace, to_project, to_namespace, export_data_dir, kind) if not skip_import: with click.progressbar(kinds_list, label='Importing', diff --git a/gcdu/export_kind.py b/gcdu/export_kind.py new file mode 100644 index 0000000..9c948b4 --- /dev/null +++ b/gcdu/export_kind.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +"""The export module.""" +import io +import os +import json +import googleapiclient.discovery + + +def run(from_project, from_namespace, to_project, to_namespace, export_data_dir, kind): + datastore = googleapiclient.discovery.build('datastore', 'v1') + + export_request_body = { + 'partitionId': { + 'projectId': from_project, + 'namespaceId': from_namespace + }, + 'query': { + 'kind': [ + { + 'name': kind + } + ] + } + } + + export_response = datastore.projects() \ + .runQuery(projectId=from_project, body=export_request_body) \ + .execute() + + entities = extract_entities(export_response) + + updated_entities = update_entities_to_new_project_and_namespace(entities, from_namespace, from_project, + to_namespace, to_project) + + save(updated_entities, kind, export_data_dir) + + +def extract_entities(export_response): + entities = [] + for entityResult in export_response.get('batch').get('entityResults'): + entity = entityResult.get('entity') + entities.append(entity) + return entities + + +def update_entities_to_new_project_and_namespace(entities, from_namespace, from_project, to_namespace, to_project): + entities_str = json.dumps(entities) + entities_str = entities_str.replace('"projectId": "{}"'.format(from_project), + '"projectId": "{}"'.format(to_project)) + entities_str = entities_str.replace('"namespaceId": "{}"'.format(from_namespace), + '"namespaceId": "{}"'.format(to_namespace)) + updated_entities = json.loads(entities_str) + return updated_entities + + +def save(updated_entities, kind, export_data_dir): + if not os.path.exists(export_data_dir): + os.makedirs(export_data_dir) + + with io.open('{}/{}.json'.format(export_data_dir, kind), 'w', encoding='utf-8') as export_file: + export_file.write( + json.dumps(updated_entities, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))) diff --git a/requirements.txt b/requirements.txt index b7f445a..0c71150 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ google-auth-httplib2==0.0.3 click==6.7 tox==2.9.1 pytest==3.3.2 -pytest-cov==2.5.1 \ No newline at end of file +pytest-cov==2.5.1 From a35c45848c645c1ccbf009a9e4f2d0d1a8d23a1e Mon Sep 17 00:00:00 2001 From: Kieras Date: Tue, 9 Jan 2018 18:25:40 -0200 Subject: [PATCH 2/4] Isolating export/import commands. --- gcdu/cli.py | 66 ++-------------------------- gcdu/commands/__init__.py | 0 gcdu/commands/export.py | 91 +++++++++++++++++++++++++++++++++++++++ gcdu/commands/utils.py | 18 ++++++++ gcdu/export_kind.py | 62 -------------------------- 5 files changed, 112 insertions(+), 125 deletions(-) create mode 100644 gcdu/commands/__init__.py create mode 100644 gcdu/commands/export.py create mode 100644 gcdu/commands/utils.py delete mode 100644 gcdu/export_kind.py diff --git a/gcdu/cli.py b/gcdu/cli.py index 4373890..2c3a0b2 100644 --- a/gcdu/cli.py +++ b/gcdu/cli.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import click -import export_kind + from __version__ import __version__ as version +from commands.export import export @click.group() @@ -10,65 +11,4 @@ def main(): """Utilities for Google Cloud Datastore.""" pass - -@main.command() -@click.option('--from-project', '-fp', - help='Origin GCP Project.', - required=True) -@click.option('--from-namespace', '-fn', - help='Origin Datastore namespace.', - required=True) -@click.option('--to-project', '-tp', - help='Destination GCP Project.', - required=True) -@click.option('--to-namespace', '-tn', - help='Destination Datastore namespace.', - required=True) -@click.option('--skip-export', - help='Skip the export process from origin database.', - required=True, - is_flag=True, - default=False, - show_default=True) -@click.option('--skip-import', - help='Skip the import process to destination database', - required=True, - is_flag=True, - default=False, - show_default=True) -@click.option('--export-data-dir', help='Directory used to store exported data files.', - required=True, - default='./data', - show_default=True) -@click.option('--kinds', - help='Comma separated list of Datastore Kinds to use.', - required=True) -def migrate(from_project, from_namespace, to_project, to_namespace, skip_export, skip_import, export_data_dir, kinds): - """Migrate data from one namespace to another.""" - kinds_list = kinds.split(',') - - click.echo("Executing migration from '{}.{}' to '{}.{}'. Kinds: '{}'" - .format(from_project, from_namespace, to_project, to_namespace, kinds)) - click.echo("Storing export data in '{}' directory." - .format(export_data_dir)) - - click.echo("Starting...") - - if not skip_export: - with click.progressbar(kinds_list, label='Exporting', - show_eta=False, item_show_func=show_progressbar_item) as bar: - for kind in bar: - export_kind.run(from_project, from_namespace, to_project, to_namespace, export_data_dir, kind) - - if not skip_import: - with click.progressbar(kinds_list, label='Importing', - show_eta=False, item_show_func=show_progressbar_item) as bar: - for kind in bar: - # TODO: Implement import feature. - pass - - click.echo("Finished!") - - -def show_progressbar_item(item): - return 'Kind: {}'.format(item) +main.add_command(export) diff --git a/gcdu/commands/__init__.py b/gcdu/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gcdu/commands/export.py b/gcdu/commands/export.py new file mode 100644 index 0000000..5ff0981 --- /dev/null +++ b/gcdu/commands/export.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +"""The export command.""" +import io +import os +import json +import click +import googleapiclient.discovery + +from .utils import show_progressbar_item, replace_with_placeholders + + +@click.command() +@click.option('--project', '-p', + help='GCP project.', + required=True) +@click.option('--namespace', '-n', + help='Datastore namespace.', + required=True) +@click.option('--data-dir', + help='Directory to be used to store exported files.', + required=True, + default='./data', + show_default=True) +@click.option('--project-placeholder', '-pp', + help='Placeholder value to replace the project value in exported files.', + required=True, + default='___PROJECT___', + show_default=True) +@click.option('--namespace-placeholder', '-np', + help='Placeholder value to replace the namespace value in exported files.', + required=True, + default='___NAMESPACE___', + show_default=True) +@click.option('--kinds', '-k', + help='Comma separated list of Datastore Kinds to export.', + required=True) +def export(project, namespace, data_dir, project_placeholder, namespace_placeholder, kinds): + """Export data from database.""" + kinds_list = kinds.split(',') + click.echo("Executing export. Project={}, Namespace={}, Kinds={}.".format(project, namespace, kinds_list)) + with click.progressbar(kinds_list, label='Exporting', show_eta=True, + item_show_func=show_progressbar_item) as bar: + for kind in bar: + execute_export(project, namespace, data_dir, project_placeholder, namespace_placeholder, kind) + + +def execute_export(project, namespace, data_dir, project_placeholder, namespace_placeholder, kind): + datastore = googleapiclient.discovery.build('datastore', 'v1') + + request_body = { + 'partitionId': { + 'projectId': project, + 'namespaceId': namespace + }, + 'query': { + 'kind': [ + { + 'name': kind + } + ] + } + } + + response = datastore.projects() \ + .runQuery(projectId=project, body=request_body) \ + .execute() + + entities = extract_entities(response) + entities_json = json.dumps(entities) + entities_replaced_json = replace_with_placeholders(entities_json, project, project_placeholder, namespace, + namespace_placeholder) + entities_replaced = json.loads(entities_replaced_json) + + save(entities_replaced, kind, data_dir) + + +def extract_entities(response): + entities = [] + for entityResult in response.get('batch').get('entityResults'): + entity = entityResult.get('entity') + entities.append(entity) + return entities + + +def save(entities, kind, data_dir): + if not os.path.exists(data_dir): + os.makedirs(data_dir) + + with io.open('{}/{}.json'.format(data_dir, kind), 'w', encoding='utf-8') as export_file: + export_file.write( + json.dumps(entities, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))) diff --git a/gcdu/commands/utils.py b/gcdu/commands/utils.py new file mode 100644 index 0000000..f0a2259 --- /dev/null +++ b/gcdu/commands/utils.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +"""Utilities.""" + + +def show_progressbar_item(item): + if item is None: + return 'Done!' + return "Kind: {}".format(item) + + +def replace_with_placeholders(entities_json, from_project, to_project, from_namespace, to_namespace): + result = entities_json.replace('"projectId": "{}"'.format(from_project), + '"projectId": "{}"'.format(to_project)) + result = result.replace('"namespaceId": "{}"'.format(from_namespace), + '"namespaceId": "{}"'.format(to_namespace)) + return result + + diff --git a/gcdu/export_kind.py b/gcdu/export_kind.py deleted file mode 100644 index 9c948b4..0000000 --- a/gcdu/export_kind.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- -"""The export module.""" -import io -import os -import json -import googleapiclient.discovery - - -def run(from_project, from_namespace, to_project, to_namespace, export_data_dir, kind): - datastore = googleapiclient.discovery.build('datastore', 'v1') - - export_request_body = { - 'partitionId': { - 'projectId': from_project, - 'namespaceId': from_namespace - }, - 'query': { - 'kind': [ - { - 'name': kind - } - ] - } - } - - export_response = datastore.projects() \ - .runQuery(projectId=from_project, body=export_request_body) \ - .execute() - - entities = extract_entities(export_response) - - updated_entities = update_entities_to_new_project_and_namespace(entities, from_namespace, from_project, - to_namespace, to_project) - - save(updated_entities, kind, export_data_dir) - - -def extract_entities(export_response): - entities = [] - for entityResult in export_response.get('batch').get('entityResults'): - entity = entityResult.get('entity') - entities.append(entity) - return entities - - -def update_entities_to_new_project_and_namespace(entities, from_namespace, from_project, to_namespace, to_project): - entities_str = json.dumps(entities) - entities_str = entities_str.replace('"projectId": "{}"'.format(from_project), - '"projectId": "{}"'.format(to_project)) - entities_str = entities_str.replace('"namespaceId": "{}"'.format(from_namespace), - '"namespaceId": "{}"'.format(to_namespace)) - updated_entities = json.loads(entities_str) - return updated_entities - - -def save(updated_entities, kind, export_data_dir): - if not os.path.exists(export_data_dir): - os.makedirs(export_data_dir) - - with io.open('{}/{}.json'.format(export_data_dir, kind), 'w', encoding='utf-8') as export_file: - export_file.write( - json.dumps(updated_entities, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))) From f02502fbd4a332309e8b871ef323042b5ce56b51 Mon Sep 17 00:00:00 2001 From: Kieras Date: Tue, 9 Jan 2018 22:13:01 -0200 Subject: [PATCH 3/4] Import command implementation. --- gcdu/cli.py | 2 ++ gcdu/commands/export.py | 20 +++-------- gcdu/commands/import_cmd.py | 67 +++++++++++++++++++++++++++++++++++++ gcdu/commands/utils.py | 27 ++++++++++++++- 4 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 gcdu/commands/import_cmd.py diff --git a/gcdu/cli.py b/gcdu/cli.py index 2c3a0b2..a0e61b2 100644 --- a/gcdu/cli.py +++ b/gcdu/cli.py @@ -3,6 +3,7 @@ from __version__ import __version__ as version from commands.export import export +from commands.import_cmd import import_cmd @click.group() @@ -12,3 +13,4 @@ def main(): pass main.add_command(export) +main.add_command(import_cmd) diff --git a/gcdu/commands/export.py b/gcdu/commands/export.py index 5ff0981..cdebdbe 100644 --- a/gcdu/commands/export.py +++ b/gcdu/commands/export.py @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- """The export command.""" -import io -import os import json import click -import googleapiclient.discovery -from .utils import show_progressbar_item, replace_with_placeholders +from .utils import get_datastore_api, get_kinds_list, show_progressbar_item, partition_replace, save @click.command() @@ -36,7 +33,7 @@ required=True) def export(project, namespace, data_dir, project_placeholder, namespace_placeholder, kinds): """Export data from database.""" - kinds_list = kinds.split(',') + kinds_list = get_kinds_list(kinds) click.echo("Executing export. Project={}, Namespace={}, Kinds={}.".format(project, namespace, kinds_list)) with click.progressbar(kinds_list, label='Exporting', show_eta=True, item_show_func=show_progressbar_item) as bar: @@ -45,7 +42,7 @@ def export(project, namespace, data_dir, project_placeholder, namespace_placehol def execute_export(project, namespace, data_dir, project_placeholder, namespace_placeholder, kind): - datastore = googleapiclient.discovery.build('datastore', 'v1') + datastore = get_datastore_api() request_body = { 'partitionId': { @@ -67,7 +64,7 @@ def execute_export(project, namespace, data_dir, project_placeholder, namespace_ entities = extract_entities(response) entities_json = json.dumps(entities) - entities_replaced_json = replace_with_placeholders(entities_json, project, project_placeholder, namespace, + entities_replaced_json = partition_replace(entities_json, project, project_placeholder, namespace, namespace_placeholder) entities_replaced = json.loads(entities_replaced_json) @@ -80,12 +77,3 @@ def extract_entities(response): entity = entityResult.get('entity') entities.append(entity) return entities - - -def save(entities, kind, data_dir): - if not os.path.exists(data_dir): - os.makedirs(data_dir) - - with io.open('{}/{}.json'.format(data_dir, kind), 'w', encoding='utf-8') as export_file: - export_file.write( - json.dumps(entities, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))) diff --git a/gcdu/commands/import_cmd.py b/gcdu/commands/import_cmd.py new file mode 100644 index 0000000..561e472 --- /dev/null +++ b/gcdu/commands/import_cmd.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +"""The import command.""" +import json +import click + +from .utils import get_datastore_api, get_kinds_list, show_progressbar_item, partition_replace, load + + +@click.command('import') +@click.option('--project', '-p', + help='GCP project.', + required=True) +@click.option('--namespace', '-n', + help='Datastore namespace.', + required=True) +@click.option('--data-dir', + help='Directory to be used to store exported files.', + required=True, + default='./data', + show_default=True) +@click.option('--project-placeholder', '-pp', + help='Placeholder value to replace the project value in previously exported files.', + required=True, + default='___PROJECT___', + show_default=True) +@click.option('--namespace-placeholder', '-np', + help='Placeholder value to replace the namespace value in previously exported files.', + required=True, + default='___NAMESPACE___', + show_default=True) +@click.option('--kinds', '-k', + help='Comma separated list of Datastore Kinds to import.', + required=True) +def import_cmd(project, namespace, data_dir, project_placeholder, namespace_placeholder, kinds): + """Import data to database using previously exported data as input.""" + kinds_list = get_kinds_list(kinds) + click.echo("Executing import. Project={}, Namespace={}, Kinds={}.".format(project, namespace, kinds_list)) + with click.progressbar(kinds_list, label='Importing', show_eta=True, + item_show_func=show_progressbar_item) as bar: + for kind in bar: + execute_import(project, namespace, data_dir, project_placeholder, namespace_placeholder, kind) + + +def execute_import(project, namespace, data_dir, project_placeholder, namespace_placeholder, kind): + datastore = get_datastore_api() + + entities = load(kind, data_dir) + entities_json = json.dumps(entities) + entities_replaced_json = partition_replace(entities_json, project_placeholder, project, + namespace_placeholder, namespace) + entities_replaced = json.loads(entities_replaced_json) + + inserts = [] + for entity in entities_replaced: + insert = { + 'insert': entity + } + inserts.append(insert) + + request_body = { + "mutations": inserts, + "mode": "NON_TRANSACTIONAL" + } + + datastore.projects() \ + .commit(projectId=project, body=request_body) \ + .execute() diff --git a/gcdu/commands/utils.py b/gcdu/commands/utils.py index f0a2259..c1c6fe8 100644 --- a/gcdu/commands/utils.py +++ b/gcdu/commands/utils.py @@ -1,5 +1,17 @@ # -*- coding: utf-8 -*- """Utilities.""" +import io +import os +import json +import googleapiclient.discovery + + +def get_datastore_api(): + return googleapiclient.discovery.build('datastore', 'v1') + + +def get_kinds_list(kinds): + return kinds.split(',') def show_progressbar_item(item): @@ -8,7 +20,7 @@ def show_progressbar_item(item): return "Kind: {}".format(item) -def replace_with_placeholders(entities_json, from_project, to_project, from_namespace, to_namespace): +def partition_replace(entities_json, from_project, to_project, from_namespace, to_namespace): result = entities_json.replace('"projectId": "{}"'.format(from_project), '"projectId": "{}"'.format(to_project)) result = result.replace('"namespaceId": "{}"'.format(from_namespace), @@ -16,3 +28,16 @@ def replace_with_placeholders(entities_json, from_project, to_project, from_name return result +def save(entities, kind, data_dir): + if not os.path.exists(data_dir): + os.makedirs(data_dir) + + with io.open('{}/{}.json'.format(data_dir, kind), 'w', encoding='utf-8') as export_file: + export_file.write( + json.dumps(entities, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))) + + +def load(kind, data_dir): + with io.open('{}/{}.json'.format(data_dir, kind), 'r', encoding='utf-8') as export_file: + entities = json.load(export_file) + return entities From d6935957c286d18de804ec21fbefc5336b5f86ce Mon Sep 17 00:00:00 2001 From: Kieras Date: Tue, 9 Jan 2018 22:27:25 -0200 Subject: [PATCH 4/4] New version. --- README.md | 5 +---- gcdu/__version__.py | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3d567b4..798c2c8 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,9 @@ Utilities for Google Cloud Datastore. # Installation -If you don't use `pipsi`, you're missing out. -Here are [installation instructions](https://github.com/mitsuhiko/pipsi#readme). - Simply run: - $ pipsi install . + $ pipsi install gcdu # Usage diff --git a/gcdu/__version__.py b/gcdu/__version__.py index 0cecb61..a85d36d 100644 --- a/gcdu/__version__.py +++ b/gcdu/__version__.py @@ -8,4 +8,4 @@ # # Google Cloud Datastore Utils -__version__ = '0.0.2' +__version__ = '0.0.3' diff --git a/setup.py b/setup.py index 5919925..024218e 100644 --- a/setup.py +++ b/setup.py @@ -108,8 +108,8 @@ def run(self): # Trove classifiers # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers # 'Development Status :: 1 - Planning', - 'Development Status :: 2 - Pre-Alpha', - # 'Development Status :: 3 - Alpha', + # 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 3 - Alpha', # 'Development Status :: 4 - Beta', # 'Development Status :: 5 - Production/Stable', # 'Development Status :: 6 - Mature',