Update README, add linter, add GHA

.github/workflows/test.yaml

@@ -0,0 +1,34 @@
+name: 'Test'
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      # Installing python directly due to poetry issue with upstream python
+      # https://github.com/python-poetry/poetry/issues/7343
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.4.2
+          virtualenvs-create: false
+
+      - name: Install dependencies
+        run: make install
+
+      - name: Run lint
+        run: make lint
+
+      - name: Run test
+        run: make test

Makefile

@@ -13,13 +13,23 @@ ifeq ($(VERBOSE),1)
 endif
 
 export:
-	python3 metabase_export.py ${FLAGS} all
+	poetry run metabase_export.py ${FLAGS} all
 
 import:
-	python3 metabase_import.py ${FLAGS} all ${COLLECTION}
+	poetry run metabase_import.py ${FLAGS} all ${COLLECTION}
 
 clean:
 	rm -rf ${MB_DATA_DIR}
 
+format:
+	poetry run isort .
+
+lint: format
+	poetry run flake8 --max-line-length=140 .
+	git diff --quiet --exit-code
+
 test:
-	python3 -m pytest --verbose -v -s -k $(or ${TEST_FUNC},'') .
+	poetry run pytest --verbose -v -s -k $(or ${TEST_FUNC},'') .
+
+install:
+	poetry install --verbose

README.md

@@ -0,0 +1,134 @@
+# Metabase Export/Import
+
+This python library allows to export and import a community version instance of Metabase
+
+## Example Scripts
+
+Two scripts are provided to import and export fields, cards and dashboards of a specific database configuration of metabase:
+
+```shell
+# Export each component separately
+python3 metabase_export.py [cards|fields|metrics|dashboards]
+
+# Export everything together
+python3 metabase_export.py all
+
+# Export everything together and store the raw JSON without parsing it
+python3 metabase_export.py --raw all
+```
+
+The script produces 3 files for each exported elements (the name of the database is user as prefix) : `my_database_fields_exported.csv`, `my_database_cards_exported.json` and `my_database_dashboard_exported.json`
+
+```shell
+# Import each component separately
+python3 metabase_import.py [cards|fields|metrics|dashboards]
+
+# Export everything together
+python3 metabase_import.py all 
+```
+
+The script imports from 3 files, one for each elements : `my_database_fields_forimport.csv`, `my_database_cards_forimport.json` and `my_database_dashboard_forimport.json`
+
+## Configuration
+
+Available flags for the scripts:
+
+| flag      | description                                     |
+|-----------|-------------------------------------------------|
+| --verbose | increase output verbosity of each command       |
+| --dry-run | run the script without making POST/PUT requests |
+| --raw     | store the raw JSON without parsing it           |
+
+It is recommended to predefine all the following environment variables in a `.env` file:
+
+| env var              | description                                               |
+|----------------------|-----------------------------------------------------------|
+| `MB_DATA_DIR`        | Directory to use for the export/import operations         |
+| `MB_EXPORT_HOST`     | Source Metabase instance (`https://<url>/api/`)           |
+| `MB_EXPORT_USERNAME` | Admin username                                            |
+| `MB_EXPORT_PASSWORD` | Admin username password                                   |
+| `MB_EXPORT_DB`       | The database name to export                               |
+| `MB_IMPORT_HOST`     | Destination Metabase instance (`https://<url>/api/`)      |
+| `MB_IMPORT_USERNAME` | Admin username                                            |
+| `MB_IMPORT_PASSWORD` | Admin username password                                   |
+| `MB_IMPORT_DB`       | The name of the same database in the destination instance |
+
+## Library calls
+
+### database creation/deletion
+
+```python
+import metabase
+
+# connect to metabase
+ametabase = metabase.MetabaseApi("http://localhost:3000/api/", "metabase_username", "metabase_password")
+
+# add a sqlite database located at /path/to/database.sqlite. The metabase associated name is my_database
+ametabase.create_database("my_database", 'sqlite', {"db":"/path/to/database.sqlite"})
+
+#ametabase.delete_database('my_database')
+```
+
+
+### users and permisssions
+
+```python
+ametabase.create_user("[email protected]", "the_password", {'first_name': 'John', 'last_name': 'Doe'})
+
+# Add a group and associate it with our new user
+ametabase.membership_add('[email protected]', 'a_group')
+
+# allow read data and create interraction with my_database for users members of our new group (a_group)
+ametabase.permission_set_database('a_group', 'my_database', True, True)
+```
+
+### collections and permissions
+
+```python
+# create a collection and its sub collection
+ametabase.create_collection('sub_collection', 'main_collection')
+
+# allow write right on the new collections to the membres of a_group
+ametabase.permission_set_collection('main_collection', 'a_group', 'write')
+ametabase.permission_set_collection('sub_collection', 'a_group', 'write')
+```
+
+
+### schema
+
+```python
+# export and import the schema of fields
+ametabase.export_fields_to_csv('my_database', 'my_database_fields.csv')
+ametabase.import_fields_from_csv('my_database', 'my_database_fields.csv')
+```
+
+### cards and dashboards
+
+```python
+ametabase.export_cards_to_json('my_database', 'my_database_cards.json')
+ametabase.export_dashboards_to_json('my_database', 'my_database_dashboard.json')
+
+ametabase.import_cards_from_json('my_database', 'my_database_cards.json')
+ametabase.import_dashboards_from_json('my_database', 'my_database_dashboard.json')
+```
+
+## Development
+
+Development of this repository is done with [Poetry](https://python-poetry.org/docs/).
+
+### Install dependencies
+
+```shell
+make install
+```
+
+### Test
+
+The tests are running with pytest
+
+```shell
+make test
+
+# Run a tests matching an expression TEST_FUNC=<expression>
+TEST_FUNC="invalid" make test
+```

metabase.py

@@ -1,11 +1,12 @@
-import requests
-import json
 import csv
 import datetime
+import json
 import os
+import re
 from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
 from rich.progress import Progress
-import re
 
 
 class MetabaseApi:
@@ -108,7 +109,7 @@ def create_session(self):
 
     def create_session_if_needed(self):
         if self.metabase_session:
-            return;
+            return
         self.create_session()
 
     def get_databases(self, full_info=False):
@@ -168,7 +169,7 @@ def get_table(self, database_name, table_name):
         for t in self.get_tables_of_database(database_name):
             if t['name'] == table_name:
                 return t
-        table = {}
+        return {}
 
     def get_field(self, database_name, table_name, field_name):
         table = self.get_table(database_name, table_name)
@@ -365,7 +366,7 @@ def database_name2id(self, database_name):
         return None
 
     def get_snippets(self, database_name):
-        database_id = self.database_name2id(database_name)
+        _ = self.database_name2id(database_name)
         return self.query('GET', 'native-query-snippet')
 
     def get_cards(self, database_name):
@@ -880,7 +881,7 @@ def get_users(self):
         users = self.query('GET', 'user?status=all')
         try:
             return users['data']
-        except:
+        except KeyError:
             return None
 
     def user_email2id(self, user_email):
@@ -914,11 +915,7 @@ def create_group(self, group_name):
 
     def get_groups(self):
         self.create_session_if_needed()
-        groups = self.query('GET', 'permissions/group')
-        try:
-            return groups
-        except:
-            return None
+        return self.query('GET', 'permissions/group')
 
     def group_name2id(self, group_name):
         for g in self.get_groups():
@@ -977,7 +974,7 @@ def permission_get_collection(self):
         return self.query('GET', 'collection/graph')
 
     def permission_set_collection(self, group_name, collection_name, right):
-        if not right in ['read', 'write', 'none']:
+        if right not in ['read', 'write', 'none']:
             raise ValueError('right not read/write/none')
         if group_name == 'all':
             group_id = '1'

metabase_export.py

@@ -1,9 +1,10 @@
-import metabase
 from pathlib import Path
 
-from typer import Typer, Option
+from typer import Option, Typer
 from typing_extensions import Annotated
 
+import metabase
+
 app = Typer()
 
 db_name: str

metabase_full_import.py

@@ -1,6 +1,7 @@
-import metabase
 import sys
 
+import metabase
+
 metabase_apiurl = sys.argv[1]
 metabase_username = sys.argv[2]
 metabase_password = sys.argv[3]
@@ -11,9 +12,9 @@
 pass_to_create = sys.argv[8]
 
 ametabase = metabase.MetabaseApi(metabase_apiurl, metabase_username, metabase_password)
-#ametabase.debug = True
+# ametabase.debug = True
 
-#ametabase.delete_database('base')
+# ametabase.delete_database('base')
 #
 ametabase.create_database(metabase_basename, 'sqlite', {'db': sqlite_database_path_to_create})
 

metabase_import.py

@@ -1,10 +1,11 @@
-import metabase
 from pathlib import Path
-
-from typer import Typer, Option, Argument
 from typing import Optional
+
+from typer import Argument, Option, Typer
 from typing_extensions import Annotated
 
+import metabase
+
 app = Typer()
 
 db_name: str