From 9128183366df334fc223303988f39e1b575d8533 Mon Sep 17 00:00:00 2001 From: Seva D Date: Sun, 30 Jun 2019 20:47:28 +0200 Subject: [PATCH 1/4] Initial changes --- .DS_Store | Bin 0 -> 6148 bytes docs/ANDROID.md | 17 -- docs/DRIVERS_ROADMAP.md | 16 -- docs/DRIVER_PACKAGE_STRUCTURE.md | 45 ---- docs/EXTRAS.md | 9 - docs/INDEX.md | 99 --------- docs/Makefile | 19 -- docs/RULES.md | 25 --- docs/make.bat | 35 --- docs/source/conf.py | 66 ------ docs/source/contents.rst | 20 -- docs/source/index.rst | 19 -- docs/source/metadrive.rst | 110 ---------- docs/source/modules.rst | 7 - metadrive/__init__.py | 157 -------------- metadrive/_requests.py | 97 --------- metadrive/_selenium.py | 301 -------------------------- metadrive/_xarray.py | 39 ---- metadrive/auth.py | 62 ------ metadrive/cli.py | 151 ------------- metadrive/config.py | 157 -------------- metadrive/{tests => core}/__init__.py | 0 metadrive/core/driver.py | 32 +++ metadrive/{mnt.py => core/mount.py} | 43 +--- metadrive/core/runner.py | 62 ++++++ metadrive/drivers.py | 153 ------------- metadrive/drives.py | 109 ---------- metadrive/entrypoints/__init__.py | 0 metadrive/entrypoints/connect.py | 48 ++++ metadrive/helpers.py | 36 --- metadrive/mixins.py | 63 ------ metadrive/settings.py | 7 + metadrive/tests/config.stub | 18 -- metadrive/tests/sample.csv | 3 - metadrive/tests/test_wrapper.py | 19 -- metadrive/utils.py | 273 ----------------------- setup.py | 15 +- 37 files changed, 159 insertions(+), 2173 deletions(-) create mode 100644 .DS_Store delete mode 100644 docs/ANDROID.md delete mode 100644 docs/DRIVERS_ROADMAP.md delete mode 100644 docs/DRIVER_PACKAGE_STRUCTURE.md delete mode 100644 docs/EXTRAS.md delete mode 100644 docs/INDEX.md delete mode 100644 docs/Makefile delete mode 100644 docs/RULES.md delete mode 100644 docs/make.bat delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/contents.rst delete mode 100644 docs/source/index.rst delete mode 100644 docs/source/metadrive.rst delete mode 100644 docs/source/modules.rst delete mode 100644 metadrive/_requests.py delete mode 100644 metadrive/_selenium.py delete mode 100644 metadrive/_xarray.py delete mode 100644 metadrive/auth.py delete mode 100644 metadrive/cli.py delete mode 100644 metadrive/config.py rename metadrive/{tests => core}/__init__.py (100%) create mode 100644 metadrive/core/driver.py rename metadrive/{mnt.py => core/mount.py} (70%) create mode 100644 metadrive/core/runner.py delete mode 100644 metadrive/drivers.py delete mode 100644 metadrive/drives.py create mode 100644 metadrive/entrypoints/__init__.py create mode 100644 metadrive/entrypoints/connect.py delete mode 100644 metadrive/helpers.py delete mode 100644 metadrive/mixins.py create mode 100644 metadrive/settings.py delete mode 100644 metadrive/tests/config.stub delete mode 100644 metadrive/tests/sample.csv delete mode 100644 metadrive/tests/test_wrapper.py delete mode 100644 metadrive/utils.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..89c8aa197a6725be5bd1fcec69c3948fe6763e49 GIT binary patch literal 6148 zcmeH~JqiLr422WjLa?#4oW@r027~AcynuyZBP$46&(VGPQE;^ukr$YJlWelfzG7!1 zBD%gFRw6SInP3@ZVQyfGSY>(JUHa3xKYH{i8RaHwaF4C|@yyQuE!L;`f6&5^3Q&PRQ$V}T zZnMTq<=Oi7dY0d3*7gkx^m4eDp8y1Q6wk5TaJ|?DtjQKc2F4!&mw}E7d{u!v^cD`f literal 0 HcmV?d00001 diff --git a/docs/ANDROID.md b/docs/ANDROID.md deleted file mode 100644 index 2faf63e..0000000 --- a/docs/ANDROID.md +++ /dev/null @@ -1,17 +0,0 @@ -# Android - -If installed on Termux (Android), needs: - -``` -pkg i clang -pkg i make -pkg i python-dev -pkg i libcrypt-dev -pkg i libffi-dev -pkg i openssl -pkg i openssl-dev -pkg i openssl-tool -pkg i libjpeg-turbo-dev -LDFLAGS="-L/system/lib/" CFLAGS="-I/data/data/com.termux/files/usr/include/" pip install Pillow -OR LIBRARY_PATH="/system/lib" CPATH="$PREFIX/include" pip install pillow -``` diff --git a/docs/DRIVERS_ROADMAP.md b/docs/DRIVERS_ROADMAP.md deleted file mode 100644 index 106bdec..0000000 --- a/docs/DRIVERS_ROADMAP.md +++ /dev/null @@ -1,16 +0,0 @@ -# TODO - -## Web information services - -**Industrial and Medical Equipment** -**Metal printers** ([https://www.aniwaa.com/best-of/3d-printers/best-metal-3d-printer/#The_best_metal_3D_printers_in_2018](https://www.aniwaa.com/best-of/3d-printers/best-metal-3d-printer/#The_best_metal_3D_printers_in_2018)), **CNC Machines** ( [https://github.com/Nikolay-Kha/PyCNC#readme](https://github.com/Nikolay-Kha/PyCNC#readme), [https://mmi-direct.com/machines/search/?make_id=&page=brand](https://mmi-direct.com/machines/search/?make_id=&page=brand)). - -**Products** -Taobao, 天猫, Alibaba, Amazon, EBay,... - -**Business data** - -Flights ( flightradar24.com ), Skyscanner ( skyscanner.com ), Weather ( windy.com ), Human ( biodigital.com ), Ships ( marinetraffic.com ), Deaths ( https://www.cdc.gov/nchs/data_access/vitalstatsonline.htm ), Companies ( opencorporates.com, etc.), Oil Miners ( http://aleph.openoil.net/ ),... - -**Common services** -Gmail API ( get all your mails ), LinkedIn, Google Plus, Twitter, Weibo, Telegram, WeChat, Kik, KakoTalk, Line, WhatsApp, Quora, Kr36, MeetUp, 知乎, Huodongxing, YouTube, YouKu, Vimeo,... diff --git a/docs/DRIVER_PACKAGE_STRUCTURE.md b/docs/DRIVER_PACKAGE_STRUCTURE.md deleted file mode 100644 index 340d358..0000000 --- a/docs/DRIVER_PACKAGE_STRUCTURE.md +++ /dev/null @@ -1,45 +0,0 @@ -# Driver package structure - -``` -. -├── driver_name -│ ├── __init__.py # _login(), and an items generator function _harvest() -│ └── api.py # classes, that define methods _get() and _filter() generators. -├── README.md -└── setup.py -``` - -## Defualt files structure - -``` -__init__.py file: -===================== - _login(): authentication function - - _harvest(): default downloading function - -api.py file: -============ - Classes represent data types available in data source of driver package. - Methods represent way to query for objects in the data source. - - @classmethod - _filter(): Returns a generator of the objects of the class. - - @classmethod - _get(): Returns a method to retrieve a single object. - - @classmethod - _update(): A method to update or delete the object in source by ID. -``` - -1. Publish drivers on `PyPI`. - -2. Reference them on `-` wikis on GitHub (example: [https://github.com/mindey/-/wiki/topic#halfbakery](https://github.com/mindey/-/wiki/topic#halfbakery). - -3. Use, like `harvest https://github.com/mindey/-/wiki/topic#halfbakery -o my_data`. - -Alternatively, to database: -`harvest https://github.com/user/-/wiki/concept\#source --db mongodb://username:password@hostname:27017/db_name/collection` - -More advanced usage will be covered in the future. diff --git a/docs/EXTRAS.md b/docs/EXTRAS.md deleted file mode 100644 index 83dedf2..0000000 --- a/docs/EXTRAS.md +++ /dev/null @@ -1,9 +0,0 @@ -# Extras - -As a plugin, data normalization package is available, to use it, install: - -``` -pip install -U --extra-index-url https://pypi.wefindx.io/ metaform --no-cache -``` - -then, pass `?normalize=true` as URL parameter as part of `POST` requests. The data `results` key will be normalized. diff --git a/docs/INDEX.md b/docs/INDEX.md deleted file mode 100644 index 950a505..0000000 --- a/docs/INDEX.md +++ /dev/null @@ -1,99 +0,0 @@ -## Development -Add `~/.pypirc` file: - -``` -[distutils] -index-servers = - pypi - internal - -[pypi] -username: -password: - -[internal] -repository: https://pypi.wefindx.io -username: -password: -``` - -Then, use: -`python setup.py sdist upload -r internal` - -Or also, use: -`pip install -i https://pypi.wefindx.io metadrive` - - -And then, `requirements.txt` may look like so: -` -metadir==0.0.1 ---extra-index-url https://@ypi.wefindx.io// -metadrive==0.4.0 -` - -Use `pip install --editable .` to preview changes to the commands. - -# Structure - -Metadrive structure is intended to be self-explanator. That said, to operate on every specific machine or site, we use specific repositories, that depend on `metadrive`, and each of them has to have the following structure: - -``` -driver - __init__.py - api.py -``` - -- Each content type of source is represented by a class in `api.py`, which has two default methods - `_get()` and `_filter()`, which returns generators for content type. -- The `__init__.py` contains two default functions - `_login()` and `_harvest()`, which orgnanizes continuous dumping of all site with filters (generators). - -## The `__init__` files provide: - -Variables -`__site_url__` - a variable that specifies the site url, that provides site's UI (user interface). The reason why we need it, is because, e.g., if we open a site via `"data browser"` relying metadrive, we can just enter the familiar url of the service. -`__base_url__`, - a variable that specifies the site's API (application programming interface), if available. - -Functions -`_login()` - a function that provides a way to sign-in to resource. -`_harvest()` - a function that implements one-off non-stop continual crawling of the whole resource (but also accepts filters). - -## The `api.py` files provides: - -The internal structure of the objects within the service. For example: - -``` -class ThingA: - namespace = '::someone/thing#1' - def method1(): - ... - def method2(): - ... - -class ThingB: - namespace = 'https://github.com/someone/-/wiki/thing#N' - def method1(): - ... - def method2(): - ... -``` - -For example, if Mindey decided to crawl Wikipedia topics in his own way, it may be something like: - -``` -class Topic: - namespace = '::mindey/topic#wikipedia' - def method1(): - ... - def method2(): - ... -``` - -The `search()` and `generate()` are generators, that must produce data records that follow metaformat ([MFT-1](https://book.mindey.com/metaformat/0002-data-object-format/0002-data-object-format.html)) specifications, which means, records **must** have `*` field, that specifies the url of schema, and **may** have `-`, `+`, `^` fields, that specify location, authentication and permissions, and author intent urls. - -To understand the links to `namespace` specified in the objects in the `api.py`, we have shorthands the metawiki (`pip install metawiki`) provides mapping [map.py](https://github.com/mindey/metawiki/blob/master/metawiki/map.py) for `namespaces` for MFT-1. Under metawiki namespaces, the `-:` provides links to folders, while `::` provides links to wiki of a GitHub repo. For example, if the GitHub username is `someone`, then `metawiki.name_to_url('::someone/transaction#service_name')` results in `https://github.com/someone/-/wiki/transaction#service_name'`. (Everyone can create concept definitions under `-` repo wiki of their Github user, or just provide namespace URL.) - -This is useful for records, that also have authentication data url in `+` field, that makes it possible to call methods on data records, no matter where the data is. The authentication data is stored on-line, on `+` folder on `-` repository. For example, `https://github.com/mindey/-/tree/master/+` (or, for example `metawiki.name_to_url('-:mindey/+/test.md')`), encrypted with GPG (e.g., `pip install gpgrecord`, and `gpgrecord.encrypt_data({'key': 'value'}, ['fingerprint1', 'fingerprint2',...])`.) - -The sessions are stored locally in `sessions` folder, under `~/.metadrive` dot-folder, where the repos `-` is checked out to the `~/.metadrive/-` of the user specified via automation in the `config.py`, that generates `~/.metadrive/config` file, that specifies the default `GITHUB` username, and `GPG` key of the local user. - -The schema and authentication/permissions data could be stored anywhere by providing arbitrary URLs, where we store the data, but then, the automation here would have to reviewed a little. - diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 69fe55e..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/RULES.md b/docs/RULES.md deleted file mode 100644 index 9ec9fa4..0000000 --- a/docs/RULES.md +++ /dev/null @@ -1,25 +0,0 @@ -# RULES 0.1 - -## Terminology - -- `resource`: any addressable system with I/O capabilities. -- `driver`: a package, that implements the below `Package Rules`. -- `drive`: a pair of driver package name and session data, encoded and stored as an encapsuled unit in isolated location. -- `subtool`: an property of metadrive, that implements a middleware and extension of a generic package, like selenium, ansible, etc., by providing (MUST) `get_drive` method, that returns a `drive` ([example](https://github.com/wefindx/metadrive/blob/master/metadrive/_requests.py#L12)), that binds session data with driver. Subtool examples `metadrive._requests`, `metadrive._selenium`, `metadrive._ansible`, etc. -- `types`: term describing groups of classes of things, that can have instances. (For example, term: user, class: AirBnbUser, instance: Joe) - -## Package Rules - -In order to make your software package available as a metadrive API module, the package MUST be a driver dedicated for interacting with an addressable web `resource`, that is: - -1. It MUST have `__site_url__` property as the property of imported package name. Additionally: -2. It MUST have `_login` function as property of imported package, that returns an instance of `drive` defined in the driver package that uses generic `get_drive` function from a `subtool` of metadrive. -3. It MUST have `api` attribute as property of imported package, that has at least one `class` corresponding with to a `type` of objects available in the `resource`, such that: - - Each `class` MUST inherit from a `dict` subtype defined in [metatype](https://github.com/wefindx/metatype/) package, - - and MUST have `-` or `url` key specifying their *data item origin location*, as per [MFT-1](https://book.mindey.com/metaformat/0002-data-object-format/0002-data-object-format.html) specification. - - and MAY have `*`, `+`, `^` keys specifying their *schema*, *keys and permissions*, and *itention* respectively as per [MFT-1](https://book.mindey.com/metaformat/0002-data-object-format/0002-data-object-format.html) specification. - - Each `class` MUST have `@classmethod` method named `_filter`, that returns a generator for class instances. - - Each `class` MAY have `@classmethod` method named `_get`, that returns an instance of class. - - Each `class` MAY have `@classmethod` method named `_update`, that allows creating/modifying/deleting of class instances in `resource`. - - Each class method MUST return instance of classes that inherit from a `Dict` class specified in [metatype](https://github.com/wefindx/metatype/) package, which wraps various [LinkedData](https://en.wikipedia.org/wiki/Linked_data) formats. -4. It MAY have `_harvest` function as the property of imported package, that returns a generator, made for dumping all data (full crawl) from resource. diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 543c6b1..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index cfbc31a..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,66 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - -import os -import shutil -import sys -from os.path import dirname, join - -sys.path.insert(0, os.path.abspath('../..')) - - -# -- Project information ----------------------------------------------------- - -project = 'Metadrive' -copyright = '2019, Mindey Indriūnas' -author = 'Mindey Indriūnas' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -metadrive = '{}/.metadrive'.format(os.getenv('HOME')) -config = '{}/config'.format(metadrive) -if not os.path.exists(config): - if not os.path.isdir(metadrive): - os.mkdir(metadrive) - - src = join(dirname(__file__), '../../metadrive/tests/config.stub') - dst = join(metadrive, 'config') - shutil.copyfile(src, dst) diff --git a/docs/source/contents.rst b/docs/source/contents.rst deleted file mode 100644 index 475705e..0000000 --- a/docs/source/contents.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. Metadrive documentation master file, created by - sphinx-quickstart on Sun Apr 28 22:26:41 2019. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Metadrive's documentation! -===================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 5d873cd..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -Metadrive: Unified API for working with different kinds of resources -==================================================================== - -**Metadrive** a unified API for gathering information from and performing operations on different kinds of resources. - -.. note:: **Metadrive** requires at least Python 3.6. Python 2 is not supported. - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/metadrive.rst b/docs/source/metadrive.rst deleted file mode 100644 index e0d0fe8..0000000 --- a/docs/source/metadrive.rst +++ /dev/null @@ -1,110 +0,0 @@ -metadrive package -================= - -Submodules ----------- - -metadrive.api module --------------------- - -.. automodule:: metadrive.api - :members: - :undoc-members: - :show-inheritance: - -metadrive.auth module ---------------------- - -.. automodule:: metadrive.auth - :members: - :undoc-members: - :show-inheritance: - -metadrive.cli module --------------------- - -.. automodule:: metadrive.cli - :members: - :undoc-members: - :show-inheritance: - -metadrive.config module ------------------------ - -.. automodule:: metadrive.config - :members: - :undoc-members: - :show-inheritance: - -metadrive.console module ------------------------- - -.. automodule:: metadrive.console - :members: - :undoc-members: - :show-inheritance: - -metadrive.drivers module ------------------------- - -.. automodule:: metadrive.drivers - :members: - :undoc-members: - :show-inheritance: - -metadrive.drives module ------------------------ - -.. automodule:: metadrive.drives - :members: - :undoc-members: - :show-inheritance: - -metadrive.helpers module ------------------------- - -.. automodule:: metadrive.helpers - :members: - :undoc-members: - :show-inheritance: - -metadrive.mixins module ------------------------ - -.. automodule:: metadrive.mixins - :members: - :undoc-members: - :show-inheritance: - -metadrive.tasks module ----------------------- - -.. automodule:: metadrive.tasks - :members: - :undoc-members: - :show-inheritance: - -metadrive.ui module -------------------- - -.. automodule:: metadrive.ui - :members: - :undoc-members: - :show-inheritance: - -metadrive.utils module ----------------------- - -.. automodule:: metadrive.utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: metadrive - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/modules.rst b/docs/source/modules.rst deleted file mode 100644 index 3edf335..0000000 --- a/docs/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -metadrive -========= - -.. toctree:: - :maxdepth: 4 - - metadrive diff --git a/metadrive/__init__.py b/metadrive/__init__.py index 12ebb5a..e69de29 100644 --- a/metadrive/__init__.py +++ b/metadrive/__init__.py @@ -1,157 +0,0 @@ -import inspect -import os -import pkgutil - -from metaform import get_schema - -from metadrive import drives, utils, config - -__all__ = [] -for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): - __all__.append(module_name) - _module = loader.find_module(module_name).load_module(module_name) - globals()[module_name] = _module - - -if not os.path.exists(config.DEFAULT_LOCATION): - os.makedirs(config.DEFAULT_LOCATION) - - -def load(data, iterator=False): - ''' - takes: saved data item - returns: item instnace of the class from driver - ''' - - from metatype import Dict - - if isinstance(data, str): - if os.path.isfile(data): - return instantiate(next(Dict.readin(data))) - - return [instantiate(item) for item in Dict.readin(data)] - - elif isinstance(data, dict): - return instantiate(data) - - -def search(source, features: dict): - ''' - Object discovery by features in source. - - Accepts: - source: drive and class, e.g. linkedin-driver:default.api.Post - features: object dictionary of serialized object - - Returns: - generator of objects. - - Note: url itself is a feature, too. - ''' - raise NotImplementedError - - -def create(source, features: dict): - ''' - Object creation by features in source. - - Accepts: - source: drive and class, e.g. linkedin-driver:default.api.Post - features: dictionary of serialized object. - - Returns: - object's address, and success status and/or errors. - ''' - raise NotImplementedError - - -def read(term, limit=None): - ''' - calls '_:emitter' - - Reads term as source, where there is '_:emitter' attribute. - - The attribute has to specify one or more functions, that are generators of Dict objects. - - Examples: - >>> read('::mindey/topic#halfbakery') - >>> read('https://github.com/mindey/-/wiki/topic#halfbakery') - ''' - template = get_schema(term) - - readers = template.get('_:emitter') - - if not readers: - raise Exception('Readers not found in template.') - - if isinstance(readers, list): - - for i, reader in enumerate(readers): - print(i + 1, reader) - - reader_id = input("Choose reader [1] ") - - if not reader_id: - reader_id = 1 - else: - reader_id = int(reader_id) - - if reader_id not in range(1, len(readers) + 1): - raise Exception("The choice does not exist.") - - reader_id -= 1 - reader = readers[reader_id] - - elif isinstance(readers, str): - reader = readers - else: - raise Exception("Reader defined as anything other than string or list is not supported.") - - package = utils.ensure_driver_installed(reader) - - module = __import__(package) - - # Get method and package: - namespace = reader.split('.', 1)[-1] - - method = module - for name in namespace.split('.'): - method = getattr(method, name) - - method_args = inspect.getfullargspec(method).args - - if 'limit' in method_args: - return method(limit=limit) - else: - return method() - - -def instantiate(data): - _id = data.get('-') - _drive = data.get('@') - - if _id is not None and _drive is not None: - - # parsing '@' field: - # sample: PyPI::halfbakery_driver==0.0.1:default.api.Topic - - # TBD: refactor by importing from metatype - # packman = _drive.split('::', 1)[0] # sample: PyPI (Conan, NPM, Paket, etc.) - drivespec = _drive.split('::', 1)[-1] # sample: halfbakery_driver==0.0.1:default.api.Topic - driver_name_version = drivespec.split(':', 1)[0] # sample: halfbakery_driver==0.0.1 - driver_name = driver_name_version.split('==', 1)[0] # sample: halfbakery_driver - # driver_version = driver_name_version.rsplit('==', 1)[-1] # sample: 0.0.1 - profile_name_pkg_path = drivespec.rsplit(':', 1)[-1] # sample: default.api.Topic - profile_name = profile_name_pkg_path.split('.', 1)[0] # sample: default - # pkg_path = profile_name_pkg_path.split('.', 1)[-1] # sample: api.Topic - - # TBD: refactor by reusing metadrive/api.py# around 90 line - ndriver = driver_name.replace('-', '_') - # module = __import__(ndriver) - api = __import__('{}.api'.format(ndriver), fromlist=[ndriver]) - classname = _drive.rsplit('.', 1)[-1] - Klass = getattr(api, classname) - - item = Klass(data) - item.drive = drives.get('{}:{}'.format(driver_name, profile_name)) - return item diff --git a/metadrive/_requests.py b/metadrive/_requests.py deleted file mode 100644 index 6fe27f7..0000000 --- a/metadrive/_requests.py +++ /dev/null @@ -1,97 +0,0 @@ -import inspect -import os - -import requests - -from metadrive import config, mixins, utils - -SUBTOOL = os.path.basename(__file__).split('.py')[0] - - -def get_session(*args, **kwargs): - return requests.Session(*args, **kwargs) - - -class RequestsDrive(requests.Session): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.response = None - self.desired_capabilities = {} - self.metaname = '' - - def get(self, *args, **kwargs): - - proxy = self.desired_capabilities.get('proxy') - if isinstance(proxy, dict): - socks = proxy.get('socksProxy') - if isinstance(socks, str): - proxies = { - 'http': 'socks5h://{}'.format(socks), - 'https': 'socks5h://{}'.format(socks)} - kwargs.update({'proxies': proxies}) - - self.response = super().get(*args, **kwargs) - - if hasattr(self, 'profile'): - session_data = requests.utils.dict_from_cookiejar(self.cookies) - - session_prefix_file = os.path.join( - SUBTOOL, '{drive_id}/cookies.json'.format( - drive_id=self.profile - )) - - utils.save_session_data(session_prefix_file, session_data) - - def quit(self): - del self - - -def get_drive( - profile='default', - porfiles_dir='.metadrive/sessions/_requests', - recreate_profile=False, - proxies='default'): - - # ----------- TO MOVE TO MIXIN --------------- # - proxy = mixins.set_proxies(proxies) - local = mixins.init_profile(profile, porfiles_dir, recreate_profile) - - # Instantiating Drive - - if local: - drive = RequestsDrive() - drive.subtool = SUBTOOL - drive.profile = profile - else: - drive = None - - if drive is not None: - - session_prefix_file = os.path.join( - drive.subtool, '{drive_id}/cookies.json'.format( - drive_id=profile - )) - - if os.path.exists(os.path.join(config.SESSIONS_DIR, session_prefix_file)): - session_data = utils.load_session_data(session_prefix_file) - else: - session_data = requests.utils.dict_from_cookiejar(drive.cookies) - # utils.save_session_data(session_prefix_file, session_data) - - if session_data: - drive.cookies.update( - requests.utils.cookiejar_from_dict( - session_data - ) - ) - - if proxy is not None: - drive.desired_capabilities.update( - {'proxy': proxy} - ) - - # LATER MOVE TO MIXIN - drive.caller_module = inspect.getmodule(inspect.currentframe().f_back) - - return drive diff --git a/metadrive/_selenium.py b/metadrive/_selenium.py deleted file mode 100644 index b0d4d15..0000000 --- a/metadrive/_selenium.py +++ /dev/null @@ -1,301 +0,0 @@ -''' -Provides a function to get a new browser with session in specific directory. - -get_drive(profile='default', profiles_dir='.chrome-profile', local=DEVELOPMENT) - -# To create selenium driver may use something like: -docker run -d -p 4444:4444 selenium/standalone-chrome:3.7.1-beryllium -''' -import inspect -import os -import pathlib - -from deprecated import deprecated -from selenium import webdriver -from selenium.webdriver.chrome.options import Options - -from metadrive import config - -# from selenium.webdriver.support.wait import WebDriverWait - - -class TabsMixin: - - def open_tab(self, name, url=None): - if name not in self.tabs: - if url is not None: - self.execute_script("window.open('{}', '_blank');".format(url)) - self.tabs[name] = self.window_handles[-1] - else: - raise Exception('Tab not found, and url not provided.') - else: - self.switch_to.window(self.tabs[name]) - - def current_tab(self): - return next(filter(lambda x: x[1] == self.current_window_handle, self.tabs.items())) - - def switch_tab(self, name): - self.open_tab(name) - - def close_tab(self, name): - self.switch_tab(name) - self.close() - - -class Chrome(webdriver.Chrome, TabsMixin): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.tabs = {'default': self.current_window_handle} - self.metaname = '' - - -class Remote(webdriver.Remote, TabsMixin): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.tabs = {'default': self.current_window_handle} - self.metaname = '' - - -def get_drive( - driver_location=config.CHROME_DRIVER, - profile='default', - porfiles_dir='.metadrive/sessions/_selenium', - headless=config.SELENIUM.get('headless') == 'True' or False, - load_images=True, - load_adblocker=True, - recreate_profile=False, - download_to='', - proxies='default', -): - ''' - Gets a new browser, with session in specific directory. - - proxies = { - 'httpProxy': None, - 'sslProxy': None, - 'socksProxy': '127.0.0.1:9999', - } - - Can use requests-like proxy specification, e.g.: - - proxies= { - 'http': 'socks5h://127.0.0.1:9999', - 'https': 'socks5h://127.0.0.1:9999' - } - - ''' - if proxies == 'default': - proxies = config.ENSURE_PROXIES() - - if not proxies: - proxies = { - 'httpProxy': None, - 'sslProxy': None, - 'socksProxy': None - } - - # ------------- PROXIES SECTION ------------ # - proxy = {'proxyType': 'MANUAL'} - for key in proxies: - if proxies[key] is not None: - if key == 'http_proxy': - Key = 'httpProxy' - elif key == 'ssl_proxy': - Key = 'sslProxy' - elif key == 'socks_proxy': - Key = 'socksProxy' - else: - Key = key - - if Key in ['http', 'https']: - Value = proxies[key].rsplit('//', 1)[-1] - proxy['socksProxy'] = Value - else: - proxy[Key] = proxies[key] - - if len(proxy) <= 1: - proxy = None - - # ------------- OPTIONS SECTION ------------ # - OPTIONS = Options() - - OPTIONS.add_argument("--window-size=1600,900") - OPTIONS.add_argument("--disable-infobars") - OPTIONS.add_argument('--no-sandbox') - OPTIONS.add_argument('--disable-dev-shm-usage') - - if isinstance(proxy, dict): - - if proxy.get('socksProxy') is not None: - OPTIONS.add_argument( - '--proxy-server=socks5://{}'.format( - proxy['socksProxy'])) - elif proxy.get('sslProxy') is not None: - OPTIONS.add_argument( - '--proxy-server=https://{}'.format( - proxy['sslProxy'])) - elif proxy.get('httpProxy') is not None: - OPTIONS.add_argument( - '--proxy-server=http://{}'.format( - proxy['httpProxy'])) - - -# OPTIONS.add_argument('--ignore-ssl-errors=yes') -# OPTIONS.add_argument('--ssl-protocol=any') -# OPTIONS.add_argument('--web-security=no') - - PREFERENCES = {} - # OPTIONS.experimental_options["prefs"] = PREFERENCES - - if headless: - OPTIONS.add_argument('--headless') - - if load_images: - PREFERENCES["profile.default_content_settings"] = {"images": 0} - PREFERENCES["profile.managed_default_content_settings"] = {"images": 0} - else: - PREFERENCES["profile.default_content_settings"] = {"images": 2} - PREFERENCES["profile.managed_default_content_settings"] = {"images": 2} - - if download_to: - PREFERENCES.update( - {'download.default_directory': download_to, - 'download.prompt_for_download': False, - 'download.directory_upgrade': True, - 'safebrowsing.enabled': False, - 'safebrowsing.disable_download_protection': True}) - - OPTIONS.add_experimental_option('prefs', PREFERENCES) - - if load_adblocker: - try: - # OPTIONS.add_extension(os.path.join(os.getcwd(), 'subtools/extensions/ghostery.crx')) - OPTIONS.add_extension( - os.path.join(os.getcwd(), 'subtools/selenium/extensions/ublock_origin.crx')) - except Exception: - pass - - # ------------- INITIALIZATION SECTION ------------ # - if driver_location.startswith('http'): - # e.g.: http://0.0.0.0:4444/wd/hub - SELENIUM_HUB_URL = driver_location - CHROME_DRIVER_LOCATION = '' - local = False - else: - SELENIUM_HUB_URL = '' - CHROME_DRIVER_LOCATION = driver_location - local = True - - if local: - profile_path = os.path.join( - str(pathlib.Path.home()), - os.path.join(porfiles_dir, profile)) - else: - profile_path = None - - if profile_path is not None: - - OPTIONS.add_argument("--user-data-dir={}".format(profile_path)) - - if not profile_path: - os.makedirs(profile_path) - else: - if recreate_profile: - import shutil - shutil.rmtree(profile_path) - - if local: - if CHROME_DRIVER_LOCATION: - browser = Chrome( - CHROME_DRIVER_LOCATION, - chrome_options=OPTIONS) - else: - browser = Chrome( - chrome_options=OPTIONS) - else: - browser = Remote( - SELENIUM_HUB_URL, - dict( - webdriver.DesiredCapabilities.CHROME, - **OPTIONS.to_capabilities() - ) - ) - - if download_to: - - browser.command_executor._commands["send_command"] = ( - "POST", '/session/$sessionId/chromium/send_command') - - browser.desired_capabilities['browserName'] = 'ur mum' - - browser.execute( - "send_command", { - 'cmd': 'Page.setDownloadBehavior', - 'params': { - 'behavior': 'allow', - 'downloadPath': download_to}}) - - if proxy is not None: - browser.desired_capabilities.update( - {'proxy': proxy} - ) - - browser.profile = profile - browser.subtool = '_selenium' - browser.caller_module = inspect.getmodule(inspect.currentframe().f_back) - - return browser - - -@deprecated(reason="Use get_drive() instead.") -def get_driver( - driver_location=config.CHROME_DRIVER, - profile='default', - porfiles_dir='.metadrive/sessions/_selenium', - headless=False, - load_images=True, - load_adblocker=True, - recreate_profile=False, - download_to='', - proxies='default', -): - - return get_drive( - driver_location=driver_location, - profile=profile, - porfiles_dir=porfiles_dir, - headless=headless, - load_images=load_images, - load_adblocker=load_adblocker, - recreate_profile=recreate_profile, - download_to=download_to, - proxies=proxies - ) - - -def save_as(element, driver): - ''' - Click "Save as ..." on element with driver. - ''' - from selenium.webdriver.common.action_chains import ActionChains - ActionChains(driver).context_click(element).perform() - - import pyautogui - import base64 - from PIL import Image - from io import BytesIO - - as__ = Image.open( # noqa - BytesIO( - base64.b64decode( - 'iVBORw0KGgoAAAANSUhEUgAAACMAAAAQCAIAAAATVVENAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4gsTAjMlHs+8UAAAAB1pVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVBkLmUHAAABrElEQVQ4y2P8//8/A10AEwO9wKhNlAAWZM7SFStXrVn78tUrXl5eTzfXvOwsFhYWiPiSZcs/ffrExsbm7elRUlhAjlX/kcCOXbsfPX7879+/e/fvewcELVm2/P///w8ePDSxsrl3//7///+/fft26cqV/2QBlNBzd3WRlZFhZGRUVFAIDw05efo0AwMDCwvL////r9+4+fnLF05OTl1tbfJCjxE5Px06cnThkiVPnjxlYGD4/uO7nKzckvlzGRgY9h04uHzVqsuXr6ioqKQlJ9nZWFMUem/fvTM0tzx4+Mjfv3////+/eOmy6PhEZO///Plz1dq1RhZWX758oSj0vn/7/u/fPw01VSYmpg8fP67buBEi/vDho+MnT/78+ZONjU1QQJCRkZGJmZmBgeHYiRPzFy2GqHnz9m3vhIlv3r6FcOcvWnzsxAmcaU9aWqogJzs5I0tQUICPl8/R3v7kqdMMDAw/f/2cOn3m/QcPGJmYpKUk+7o6OTk4GBgYLl66vH3XrsS4WAYGhg8fPixauszf10dEWJiBgWH9pk2ebm5WFhY442m0jBi0NgEAiOU/HUCPdjUAAAAASUVORK5CYII='))) # noqa - - position = pyautogui.locateOnScreen(as__) - - if position: - print(position) - pyautogui.click(x=position[0], y=position[1]) - else: - print('Could find "Save as ..." position. Try local, non-headless.') diff --git a/metadrive/_xarray.py b/metadrive/_xarray.py deleted file mode 100644 index 7ad9e51..0000000 --- a/metadrive/_xarray.py +++ /dev/null @@ -1,39 +0,0 @@ -import inspect -import os - -# import xarray -import pandas - -from metadrive import mixins - -SUBTOOL = os.path.basename(__file__).split('.py')[0] - - -class XarrayDrive: - - def __init__(self): - self.desired_capabilities = {} - self.metaname = '' - self.df = None - - def read_csv(self, *args, **kwargs): - self.df = pandas.read_csv(*args, **kwargs) - - -def get_drive( - profile='default', - porfiles_dir='.metadrive/sessions/_xarray', - recreate_profile=False, - proxies='default'): - - mixins.set_proxies(proxies) - mixins.init_profile(profile, porfiles_dir, recreate_profile) - - drive = XarrayDrive() - drive.subtool = SUBTOOL - drive.profile = profile - - # LATER MOVE TO MIXIN - drive.caller_module = inspect.getmodule(inspect.currentframe().f_back) - - return drive diff --git a/metadrive/auth.py b/metadrive/auth.py deleted file mode 100644 index 1541292..0000000 --- a/metadrive/auth.py +++ /dev/null @@ -1,62 +0,0 @@ -import random - -import requests - -from metadrive import utils -from metadrive._requests import get_session - - -class UserAgents: - - @classmethod - def random_android(self): - - if not hasattr(self, 'user_agents'): - self.user_agents = requests.get( - 'https://raw.githubusercontent.com/N0taN3rd/userAgentLists/master/json/android-browser.json').json() - - user_agent = random.choice(self.user_agents).get('ua') - - return user_agent - - -class RequestsCookieAuthentication: - - def __init__(self, raw_cookie, key_name, proxies={}): - self.key_name = key_name - self.raw_cookie = raw_cookie - self.proxies = proxies - - def authenticate(self): - session = get_session() - session.metaname = utils.get_metaname(self.key_name) - - if self.raw_cookie is None: - - session_data = utils.load_session_data(namespace=self.key_name) - - if session_data: - session.cookies.update( - requests.utils.cookiejar_from_dict( - session_data)) - return session - - else: - credential = utils.get_or_ask_credentials( - namespace=self.key_name, - variables=['cookie']) - - if credential: - session.headers.update(dict({ - 'content-type': 'text/plain', - }, **credential)) - else: - raise Warning("Credential is not provided, some data may not be retrieved.") - - else: - session.headers.update({ - 'content-type': 'text/plain', - 'cookie': self.raw_cookie - }) - - return session diff --git a/metadrive/cli.py b/metadrive/cli.py deleted file mode 100644 index 0cbd859..0000000 --- a/metadrive/cli.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -from urllib.parse import urlparse - -import click - -import metadrive -from metadrive import utils -from metadrive.config import ENSURE_SITES, SITES_DIR -from metadrive.mnt import mount - -# Cause ecryptfs supports max 143 chars. -FILENAME_LENGTH_LIMIT = 143 - - -@click.command() -@click.argument('resource', required=True, metavar='') -@click.argument('mountpoint', required=False, metavar='') -@click.option('-u', '--user', required=False, type=str, help='Reuse a drive by name.') -@click.option('-p', '--period', required=False, type=float, help='Period of resynchronization in number of seconds.') -def connect(resource, mountpoint=None, user=None, period=900): - """ - Mounts interactive data from a resource as a filesystem to OS. - $ connnect [location] - """ - if period is None: - period = 900 - shorthand = resource - - from metadrive import drivers - ENSURE_SITES() - # finding driver by default domain - - # 1. Getting default domain - from metadrive._selenium import get_drive - drive = get_drive(headless=True) - if not resource.startswith('http'): - resource = 'http://' + resource - drive.get(resource) - url = drive.current_url - drive.quit() - default_domain = urlparse(url).hostname - - # 2. Checking for driver with it. - index = drivers.index() - results = list(filter(lambda x: x['domain'] == default_domain, index)) - - if not results: - print("No drivers found for {}.".format(resource)) - print("Drivers currently available for domains:\n") - for item in index: - if item.get('domain'): - print(' - {} [{}]'.format(item.get('domain'), item.get('package'))) - - print("\n You can create a new driver by publishing a PyPI package,\n\ -that ends with _driver or -driver, and has __site_url__ variable\n\ -specified its __init__.py file. More details on creating a driver\n\ -are in https://github.com/wefindx/metadrive/blob/master/docs/DRIVER_PACKAGE_STRUCTURE.md\n\ -and https://github.com/wefindx/metadrive/blob/master/docs/RULES.md files,\n\ -as well as look at examples on https://github.com/drivernet (such as\n\ -https://github.com/drivernet/halfbakery-driver)") - return - - first_driver = results[0] - - try: - import pkg_resources - package_version = pkg_resources.require(first_driver.get('package'))[0].version - except Exception: - # print("The package not yet installed. Latest package found.") - package_version = first_driver.get('info')['version'] - - print("-================================================-\n[*] using: [PyPI:{packname}=={version}]".format( - packname=first_driver.get('package'), - version=package_version # '>'+first_driver.get('info')['version'] - ), - ) - - if mountpoint is None: - mountpoint = os.path.join(SITES_DIR, shorthand) - # print("Assuming mount point: {}".format(mountpoint)) - - package = utils.ensure_driver_installed( - '{packman}:{packname}'.format( - packman=first_driver.get('type'), - packname=first_driver.get('package') - ) - ) - - module = __import__(package) - # api = importlib.import_module('{}.api'.format(package)) - - # print('\nTop level methods:\n') - # for met in dir(module): - # if not met.startswith('__') and met not in ['api']: - # print(' - ', met) - # - - # print('\nAvailable api classes:\n') - # for cls in dir(api): - # if cls[0].isupper() and not cls.startswith('_') and cls not in ['Dict']: - # print(' - ', cls) - # - - drive_name = 'default' # input("Enter the name of drive [default]: ") or 'default' - - # drive = get_drive( - # profile=profile, - # recreate_profile=recreate_profile, - # proxies=proxies) - - if user is None: - drive = metadrive.drives.get(package.replace('_', '-'), interactive=True) - drive_name = drive.drive_id.rsplit(':', 1)[-1] - else: - drive_name = user - drive_fullname = '{package}:{drive}'.format( - package=package.replace('_', '-'), - drive=drive_name - ) - drive = metadrive.drives.get(drive_fullname, interactive=False) - - mountpoint = '{}:{}'.format(mountpoint, drive_name) - if not os.path.exists(mountpoint): - os.makedirs(mountpoint) - - drive_fullname = '{package}:{drive}'.format( - package=package.replace('_', '-'), - drive=drive_name - ) - - # drive == metadrive.drives.get(drive_fullname) - savedir = os.path.join(metadrive.config.DATA_DIR, drive_fullname) - - if user is None: - print("Pass '--user {}' next time, to reuse the session.".format(drive_name)) - - def sync(): - import inspect - if 'period' in inspect.getfullargspec(module._harvest).args: - module._harvest(drive=drive, period=period) - else: - module._harvest(drive=drive) - - from multiprocessing import Process - syncer = Process(target=sync) - syncer.daemon = True - syncer.start() - - print("[*] mount: {}\n-================================================-".format(mountpoint)) - mount(savedir, mountpoint) - syncer.terminate() diff --git a/metadrive/config.py b/metadrive/config.py deleted file mode 100644 index f347196..0000000 --- a/metadrive/config.py +++ /dev/null @@ -1,157 +0,0 @@ -import configparser -import imp -import os -from pathlib import Path - -import gpgrecord -import requests - -config = configparser.ConfigParser() - -INSTALLED = imp.find_module('metadrive')[1] - -HOME = str(Path.home()) -DEFAULT_LOCATION = os.path.join(HOME, '.metadrive') -CONFIG_LOCATION = os.path.join(DEFAULT_LOCATION, 'config') -CREDENTIALS_DIR = os.path.join(DEFAULT_LOCATION, '-/+') -SESSIONS_DIR = os.path.join(DEFAULT_LOCATION, 'sessions') -DATA_DIR = os.path.join(DEFAULT_LOCATION, 'data') -SITES_DIR = os.path.join(HOME, 'Sites') -KNOWN_DRIVERS = os.path.join(DEFAULT_LOCATION, 'known_drivers') - -SUBTOOLS = [ - fn.rsplit('.py')[0] - for fn in os.listdir(INSTALLED) - if fn.startswith('_') and fn.endswith('.py') and not fn == '__init__.py' -] - - -def ENSURE_SESSIONS(): - if not os.path.exists(SESSIONS_DIR): - os.makedirs(SESSIONS_DIR) - - for subtool in SUBTOOLS: - subtool_profiles_path = os.path.join(SESSIONS_DIR, subtool) - if not os.path.exists(subtool_profiles_path): - if subtool != '__init__': - os.makedirs(subtool_profiles_path) - - -ENSURE_SESSIONS() - - -def ENSURE_DATA(): - if not os.path.exists(DATA_DIR): - os.makedirs(DATA_DIR) - - -ENSURE_DATA() - - -def ENSURE_SITES(): - if not os.path.exists(SITES_DIR): - os.makedirs(SITES_DIR) - - -ENSURE_SITES() - - -if not os.path.exists(CONFIG_LOCATION): - username = "seva" # input("Type your GitHub username: ") - - config['GITHUB'] = {'USERNAME': username} - config['PROXIES'] = {'http': '', 'https': ''} - config['DRIVERS'] = {'auto_upgrade': False} - config['SELENIUM'] = {'headless': False} - config['DRIVER_BACKENDS'] = { - 'CHROME': '/usr/local/bin/chromedriver' # e.g., or http://0.0.0.0:4444/wd/hub, etc. - } - - with open(CONFIG_LOCATION, 'w') as configfile: - config.write(configfile) - -config.read(CONFIG_LOCATION) - -GITHUB_USER = config['GITHUB']['USERNAME'] -REPO_PATH = os.path.join(DEFAULT_LOCATION, '-') -DRIVERS_PATH = os.path.join(DEFAULT_LOCATION, 'drivers') -CHROME_DRIVER = config['DRIVER_BACKENDS']['CHROME'] -SELENIUM = config['SELENIUM'] - -if str(config['DRIVERS']['auto_upgrade']) == 'False': - AUTO_UPGRADE_DRIVERS = False -elif str(config['DRIVERS']['auto_upgrade']) == 'True': - AUTO_UPGRADE_DRIVERS = True -elif str(config['DRIVERS']['auto_upgrade']) == 'None': - AUTO_UPGRADE_DRIVERS = None -else: - AUTO_UPGRADE_DRIVERS = False - - -def ENSURE_REPO(): - - while not requests.get('https://github.com/{}/-'.format(GITHUB_USER)).ok: - input("Please, create repository named `-` on your GitHub. Type [ENTER] to continue... ") - - if os.path.exists(REPO_PATH): - # git pull # - os.system('cd {}; git pull'.format(REPO_PATH)) - else: - # git clone # - os.system('cd {}; git clone {}'.format( - DEFAULT_LOCATION, - 'git@github.com:{}/-.git'.format(GITHUB_USER))) - - if not os.path.exists(CREDENTIALS_DIR): - os.makedirs(CREDENTIALS_DIR) - os.system("cd {}; git add .; git commit -m 'credentials (+)'; git push origin master".format( - REPO_PATH - )) - - -def ENSURE_GPG(): - config.read(CONFIG_LOCATION) - if 'GPG' in config.keys(): - return config['GPG']['KEY'] - - print('Choose your GPG key for encrypting credentials:') - KEY_LIST = gpgrecord.list_recipients() - - for i, key in enumerate(KEY_LIST): - print('{id}. {uid} {fingerprint}'.format( - id=i + 1, - uid=key['uids'], - fingerprint=key['fingerprint'] - )) - - i = int(input('Type key order in the list: ')) - 1 - - GPG_KEY = KEY_LIST[i]['fingerprint'] - - config['GPG'] = {'KEY': GPG_KEY} - - with open(CONFIG_LOCATION, 'w') as configfile: - config.write(configfile) - - return GPG_KEY - - -def ENSURE_PROXIES(): - config.read(CONFIG_LOCATION) - if 'PROXIES' in config.keys(): - return {key: 'socks5h://' + config['PROXIES'][key] or None - for key in config['PROXIES'] if config['PROXIES'][key]} - - SOCKS5 = input( - 'Type-in default socks5 proxy (e.g., 127.0.0.1:9999) (leave emtpy to default to direct connections) [ENTER]: ') - - config['PROXIES'] = { - 'http': SOCKS5, - 'https': SOCKS5 - } - - with open(CONFIG_LOCATION, 'w') as configfile: - config.write(configfile) - - return {key: 'socks5h://' + config['PROXIES'][key] or None - for key in config['PROXIES'] if config['PROXIES'][key]} diff --git a/metadrive/tests/__init__.py b/metadrive/core/__init__.py similarity index 100% rename from metadrive/tests/__init__.py rename to metadrive/core/__init__.py diff --git a/metadrive/core/driver.py b/metadrive/core/driver.py new file mode 100644 index 0000000..777c9b7 --- /dev/null +++ b/metadrive/core/driver.py @@ -0,0 +1,32 @@ +import logging +import os +import time + +import aiofiles as aiof + +logger = logging.getLogger(__file__) + + +class Driver: + def __init__(self, loop, resource, root): + self.loop = loop + self.resource = resource + self.root = root + + async def sync(self): + raise NotImplementedError + + +class TestDriver(Driver): + + async def sync(self): + logger.debug('test') + filename = os.path.join(self.root, '%s.txt' % time.time()) + async with aiof.open(filename, "w", loop=self.loop) as out: + out.write('%s' % time.time()) + out.flush() + + +async def get_driver(resource): + # TODO identify driver by resource + return TestDriver diff --git a/metadrive/mnt.py b/metadrive/core/mount.py similarity index 70% rename from metadrive/mnt.py rename to metadrive/core/mount.py index 83e73b5..1fcd75d 100644 --- a/metadrive/mnt.py +++ b/metadrive/core/mount.py @@ -1,28 +1,21 @@ -from __future__ import with_statement - import errno import os -import sys from fuse import FUSE, FuseOSError, Operations +# https://github.com/skorokithakis/python-fuse-sample + class Passthrough(Operations): def __init__(self, root): self.root = root - # Helpers - # ======= - def _full_path(self, partial): if partial.startswith("/"): partial = partial[1:] path = os.path.join(self.root, partial) return path - # Filesystem methods - # ================== - def access(self, path, mode): full_path = self._full_path(path) if not os.access(full_path, mode): @@ -135,35 +128,3 @@ def fsync(self, path, fdatasync, fh): def mount(root, mountpoint): FUSE(Passthrough(root), mountpoint, nothreads=True, foreground=True) - - -if __name__ == '__main__': - mount(sys.argv[1], sys.argv[2]) - - -# Copyright (c) 2016, Stavros Korokithakis -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# https://github.com/skorokithakis/python-fuse-sample -# diff --git a/metadrive/core/runner.py b/metadrive/core/runner.py new file mode 100644 index 0000000..2e93600 --- /dev/null +++ b/metadrive/core/runner.py @@ -0,0 +1,62 @@ +import asyncio +import os +import signal +from concurrent.futures import ProcessPoolExecutor + +from metadrive import settings +from metadrive.core.driver import get_driver +from metadrive.core.mount import mount + + +class MetaDriveRunner: + def __init__(self, resource, mountpoint=None, user=None, period=900): + self.resource = resource + self.mountpoint = mountpoint + self.user = user or 'default' + self.period = period + + if self.mountpoint is None: + self.mountpoint = os.path.join( + settings.MOUNT_DIR, + resource, + self.user + ) + + self.root = os.path.join( + settings.DATA_DIR, + resource, + self.user + ) + + self.loop = asyncio.get_event_loop() + self.executor = ProcessPoolExecutor() + self.loop.add_signal_handler(signal.SIGINT, self.shutdown) + + async def mount_filesystem(self): + await self.loop.run_in_executor( + self.executor, + mount, + self.root, + self.mountpoint + ) + + async def sync_by_driver(self): + driver_cls = await get_driver(self.resource) + driver_obj = driver_cls(self.loop, self.resource, self.root) + await driver_obj.sync() + + def run(self): + self.loop.create_task(self.mount_filesystem()) + self.loop.create_task(self.sync_by_driver()) + self.loop.run_forever() + + pending = asyncio.Task.all_tasks(loop=self.loop) + group = asyncio.gather(*pending) + self.loop.run_until_complete(group) + self.loop.close() + + def shutdown(self): + self.executor.shutdown() + # for task in asyncio.Task.all_tasks(): + # task.cancel() + self.loop.stop() diff --git a/metadrive/drivers.py b/metadrive/drivers.py deleted file mode 100644 index 23d5c8d..0000000 --- a/metadrive/drivers.py +++ /dev/null @@ -1,153 +0,0 @@ -import io -import json -import os -import tarfile -import urllib - -import bs4 -import requests -import tqdm -import yaml - -from metadrive.config import KNOWN_DRIVERS - - -def auto_discover(refresh=True): - ''' - Goes over all PyPI packages in existence, which end with "-driver" in their name - and returns those which have __site_url__ in their package __init__.py file, as - a mapping enabling the discovery of drivers for Internet sites. - ''' - - if os.path.exists(KNOWN_DRIVERS) and not refresh: - site_drivers = yaml.load(open(KNOWN_DRIVERS).read(), Loader=yaml.Loader) - return site_drivers - - print("Downloading PyPI ...") - response = requests.get('https://pypi.org/simple/', stream=True) - - f = io.BytesIO() - total_length = response.headers.get('content-length') - if total_length is None: - f.write(response.content) - else: - total_length = 10000000 # int(total_length) - - with tqdm.tqdm(total=total_length) as pbar: - for data in response.iter_content(chunk_size=4096): - pbar.update(len(data)) - f.write(data) - - bindata = f.getvalue() - print("Done.") - print("Reading package URLs ...") - soup = bs4.BeautifulSoup(bindata, 'html.parser') - links = soup.find_all('a') - - # packages known to be not a metadrive drivers - ignore = [ - 'https://pypi.org/simple/aquasystems-driver/', - 'https://pypi.org/simple/arun-cassandra-driver/', - 'https://pypi.org/simple/dlogg-driver/', - 'https://pypi.org/simple/dse-driver/', - 'https://pypi.org/simple/moxel-http-driver/', - 'https://pypi.org/simple/moxel-python-driver/', - 'https://pypi.org/simple/oceandb-elasticsearch-driver/', - 'https://pypi.org/simple/oceandb-mongodb-driver/', - 'https://pypi.org/simple/openstack-vim-driver/', - 'https://pypi.org/simple/osmosis-aws-driver/', - 'https://pypi.org/simple/osmosis-azure-driver/', - 'https://pypi.org/simple/osmosis-on-premise-driver/', - 'https://pypi.org/simple/pg-driver/', - 'https://pypi.org/simple/sensirion-shdlc-driver/', - 'https://pypi.org/simple/slipstream-libcloud-driver/', - 'https://pypi.org/simple/supy-driver/', - 'https://pypi.org/simple/wolphin-driver/', - 'https://pypi.org/simple/yb-cassandra-driver/', - ] - - site_drivers = [] - - # This is for retrieving setup.py details. - from metadrive.utils import stdoutIO - import setuptools - - def setup(**kwargs): - print(json.dumps(kwargs)) - setuptools.setup = setup - - print("Looking for driver packages and reading __init__.py files ...") - for link in tqdm.tqdm(links): - if link.attrs['href'].endswith('-driver/'): - - name = link.text - link = urllib.parse.urljoin('https://pypi.org', link.attrs['href']) - - if link in ignore: - continue - response = requests.get(link) - if response.ok: - - if response.text: - soup = bs4.BeautifulSoup(response.content, 'html.parser') - versions = soup.find_all('a') - - if versions: - last_version = versions[-1] - response = requests.get(last_version.attrs['href']) - - try: - tar = tarfile.open(mode="r:gz", fileobj=io.BytesIO(response.content)) - except Exception: - tar = None - - if tar is not None: - # import pdb; pdb.set_trace() - for member in tar.getnames(): - - if member.endswith('driver/__init__.py'): - content = tar.extractfile(member).read() - if content: - text = content.decode('utf-8') - if text: - for line in text.split('\n'): - if '__site_url__' in line: - if '=' in line: - site = line.split('=', 1)[-1].strip()[1:-1] - if site: - if site.startswith('http'): - domain = urllib.parse.urlparse(site).hostname - else: - domain = None - - for member in tar.getnames(): - if member.count('/') == 1: - if member.endswith('/setup.py'): - setup = tar.extractfile(member).read() - if setup: - info = setup.decode('utf-8') - with stdoutIO() as s: - exec(info) - info = json.loads(s.getvalue()) - else: - info = None - - site_drivers.append( - {'site_url': site, - 'domain': domain, - 'package': name, - 'type': 'pypi', - 'info': info}) - break - break - - with open(KNOWN_DRIVERS, 'w') as f: - f.write(yaml.dump(site_drivers, Dumper=yaml.Dumper)) - - print("Saved to {}. Now you can use metadrive.drivers.all() to quickly access them.".format(KNOWN_DRIVERS)) - - return site_drivers - - -def index(): - return auto_discover(refresh=False) diff --git a/metadrive/drives.py b/metadrive/drives.py deleted file mode 100644 index 9d6e2b5..0000000 --- a/metadrive/drives.py +++ /dev/null @@ -1,109 +0,0 @@ -import os - -import pkg_resources - -from metadrive import utils -from metadrive.config import SESSIONS_DIR, SUBTOOLS - -# This package manages what profiles are created, and actually the sessions on disk, -# rather than active sessions on API. - -ACTIVE = {} - - -def all(): - ''' - second coordinate uniquely identifies drives - second item in tuples uniquely identifies drives, without the first. - ''' - drives_map = [] - - for subtool in SUBTOOLS: - subtool_dir = os.path.join(SESSIONS_DIR, subtool) - - for drive_dir in os.listdir(subtool_dir): - drives_map.append((subtool, drive_dir, 'ALIVE' if ACTIVE.get(drive_dir) else 'DEAD')) - - return drives_map - - -def get(driver_or_drive, interactive=False): - - if driver_or_drive in ACTIVE: - return ACTIVE[driver_or_drive] - - if ':' in driver_or_drive: - driver, drive_id = driver_or_drive.split(':', 1) - drive = driver_or_drive - else: - driver = driver_or_drive - drive = None - - ndriver = driver.replace('-', '_') - package = utils.ensure_driver_installed(driver_name='pypi:{}'.format(ndriver)) - module = __import__(package) - - d = all() - drives = list(zip(*d))[1] if d else [] - - if drive in drives: - drive_obj = module.get_drive(profile=drive) - elif drive is not None: - if os.name in ['nt']: - drive_obj = module.get_drive(profile=drive.replace(':', '__')) - else: - drive_obj = module.get_drive(profile=drive) - else: - import inspect - if interactive and 'interactive' in inspect.getfullargspec(module._login).args: - drive_obj = module._login(interactive=interactive) - drive = drive_obj.profile.rsplit(':', 1)[-1] - drive = '{}:{}'.format(driver, drive) - else: - drive = '{}:{}'.format(driver, 'default') - drive_obj = module.get_drive(profile=drive) - - ACTIVE[drive] = drive_obj - - drive_obj.drive_id = drive - driver_version = pkg_resources.require(ndriver)[0].version - - # TODO: refactor with api.py#creating-informative-drive - drive_obj.spec = '{packman}::{driver}=={version}:{profile}.{namespace}'.format( - packman='PyPI', - driver=drive_obj.drive_id.split(':', 1)[0], # .replace('-', '_'), - version=driver_version, - profile=drive_obj.drive_id.rsplit(':', 1)[-1], - namespace='api.', - # namspace not present, because it's a drive, but we prepare based on drivers package convention, the .api. - # then, in packages we only have to provide type(self).__name__, e.g.: - # item['@'] = drive.spec + type(self).__name__ - ) - - return drive_obj - - -def close(drive_obj): - found = False - for name, drive in ACTIVE.items(): - if drive == drive_obj: - found = True - break - - if found: - drive_obj.quit() - del ACTIVE[name] - - -def remove(drive_obj): - drive_id = drive_obj.drive_id - - subtool = None - for drive in all(): - if drive[1] == drive_id: - subtool = drive[0] - - if subtool is not None: - close(drive_obj) - import shutil - shutil.rmtree(os.path.join(SESSIONS_DIR, subtool, drive_id)) diff --git a/metadrive/entrypoints/__init__.py b/metadrive/entrypoints/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metadrive/entrypoints/connect.py b/metadrive/entrypoints/connect.py new file mode 100644 index 0000000..d829398 --- /dev/null +++ b/metadrive/entrypoints/connect.py @@ -0,0 +1,48 @@ +import logging +import sys + +import click + +from metadrive.core.runner import MetaDriveRunner + +logging.basicConfig( + level=logging.WARNING, + format="%(message)s", + stream=sys.stderr +) + + +@click.command() +@click.argument( + 'resource', + required=True, + metavar='' +) +@click.argument( + 'mountpoint', + required=False, + metavar='' +) +@click.option( + '-u', '--user', + required=False, + type=str, + help='Reuse a drive by name.' +) +@click.option( + '-p', '--period', + required=False, + type=float, + help='Period of resynchronization in number of seconds.' +) +def connect(resource, mountpoint=None, user=None, period=900): + runner = MetaDriveRunner( + resource, + mountpoint=mountpoint, + user=user, + period=period + ) + try: + runner.run() + except KeyboardInterrupt: + sys.exit(0) diff --git a/metadrive/helpers.py b/metadrive/helpers.py deleted file mode 100644 index 9acc662..0000000 --- a/metadrive/helpers.py +++ /dev/null @@ -1,36 +0,0 @@ -import inspect - -import yaml - - -def get_actions(cls): - ''' - Convenience function to create summaries of actions. - ''' - actions = {} - - for k, v in cls.__dict__.items(): - if not k.startswith('__'): - sig = inspect.signature(v) - actions[k] = \ - '<' + ', '.join([ - p + str(sig.parameters[p].annotation.__name__ != '_empty' and ': ' + - sig.parameters[p].annotation.__name__ or '') - for p in sig.parameters - if sig.parameters[p].name != 'self' - ]) + '>' - - if v.__doc__: - actions[k] += ' - ' + v.__doc__.strip() - - return actions - - -def print_actions(cls): - actions = get_actions(cls) - - string = yaml.dump({'_:actions': actions}, default_flow_style=False) - - string = string.replace('_:actions:', "\'_:actions\':") - - print(string, end='') diff --git a/metadrive/mixins.py b/metadrive/mixins.py deleted file mode 100644 index 8c6bec6..0000000 --- a/metadrive/mixins.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import pathlib - -from metadrive import config - - -def set_proxies(proxies): - - if proxies == 'default': - proxies = config.ENSURE_PROXIES() - - if not proxies: - proxies = { - 'httpProxy': None, - 'sslProxy': None, - 'socksProxy': None - } - - proxy = {'proxyType': 'MANUAL'} - for key in proxies: - if proxies[key] is not None: - if key == 'http_proxy': - Key = 'httpProxy' - elif key == 'ssl_proxy': - Key = 'sslProxy' - elif key == 'socks_proxy': - Key = 'socksProxy' - else: - Key = key - - if Key in ['http', 'https']: - Value = proxies[key].rsplit('//', 1)[-1] - proxy['socksProxy'] = Value - else: - proxy[Key] = proxies[key] - - if len(proxy) <= 1: - proxy = None - - return proxy - - -def init_profile(profile, porfiles_dir, recreate_profile): - - local = True - - if local: - profile_path = os.path.join( - str(pathlib.Path.home()), - os.path.join(porfiles_dir, profile)) - else: - profile_path = None - - if profile_path is not None: - - if not os.path.exists(profile_path): - os.makedirs(profile_path) - - elif recreate_profile: - import shutil - shutil.rmtree(profile_path) - - return local diff --git a/metadrive/settings.py b/metadrive/settings.py new file mode 100644 index 0000000..48c620b --- /dev/null +++ b/metadrive/settings.py @@ -0,0 +1,7 @@ +import os +from pathlib import Path + +HOME = str(Path.home()) +CONFIG_DIR = os.path.join(HOME, '.metadrive') +DATA_DIR = os.path.join(CONFIG_DIR, 'data') +MOUNT_DIR = os.path.join(HOME, 'Sites') diff --git a/metadrive/tests/config.stub b/metadrive/tests/config.stub deleted file mode 100644 index d57eae6..0000000 --- a/metadrive/tests/config.stub +++ /dev/null @@ -1,18 +0,0 @@ -[GITHUB] -username = mindey - -[DRIVERS] -auto_upgrade = False - -[DRIVER_BACKENDS] -chrome = /usr/bin/chromedriver - -[PROXIES] -http = -https = - -[GPG] -key = 5AFDB16B89805133F450688BDA580D1D5F5CC7AD - -[SELENIUM] -headless = True diff --git a/metadrive/tests/sample.csv b/metadrive/tests/sample.csv deleted file mode 100644 index aa0ba03..0000000 --- a/metadrive/tests/sample.csv +++ /dev/null @@ -1,3 +0,0 @@ -x,y,z -1,2,3 -4,5,6 diff --git a/metadrive/tests/test_wrapper.py b/metadrive/tests/test_wrapper.py deleted file mode 100644 index 008f735..0000000 --- a/metadrive/tests/test_wrapper.py +++ /dev/null @@ -1,19 +0,0 @@ -import metadrive - - -def test_read_table(): - drive = metadrive.drives.get('table-driver:default') - from table_driver.api import Row - - expect = {'x': 1, 'y': 2, 'z': 3, - '-': 'metadrive/tests/sample.csv#0', - '@': 'PyPI::table-driver==0.0.3:default.api.Row'} - - result = Row._get('metadrive/tests/sample.csv#0', drive=drive) - - assert result['@'].startswith('PyPI::table-driver') - assert result['@'].endswith('default.api.Row') - - del expect['@'] - del result['@'] - assert expect == result diff --git a/metadrive/utils.py b/metadrive/utils.py deleted file mode 100644 index 518cfc0..0000000 --- a/metadrive/utils.py +++ /dev/null @@ -1,273 +0,0 @@ -import contextlib -import importlib -import json -import os -import re -import sys -from io import StringIO - -import gpgrecord -import pkg_resources -import yaml - -from metadrive import config - -MAIN = 'default' -VENV = os.getenv('VIRTUAL_ENV') - - -def find_drivers(): - distros = pkg_resources.AvailableDistributions() - - drivers = [] - - for key in distros: - - resources = distros[key] - resource = resources[0] - egg = resource.egg_name() - folder = egg.split('-', 1)[0].lower() - path = os.path.join(resource.location, folder) - fname = os.path.join(path, '__init__.py') - - if os.path.exists(fname): - - with open(fname, 'r', encoding='utf-8') as f: - - for line in f: - if '__site_url__' in line: - - site_url = line.split('=')[-1].strip()[1:-1] - package = '{}=={}'.format(key, resource.version) - - drivers.append((site_url, package, path)) - break - - return drivers - - -def get_metaname(namespace, anchor=None): - ''' - A default place to store authentication information, like - passwords, encrypted with user's public key, in user's github. - - By default, the auth data is stored in the markdown file, under - the main anchor. - ''' - return '-:{gituser}/+/{namespace}.md#{main}'.format( - gituser=config.GITHUB_USER, - namespace=namespace, - main=anchor if anchor else MAIN - ) - - -def get_credential(namespace): - ''' - namespace: -- service name, by directory - - Example: - >>> get_credential('gmail') - ''' - - from metaform import get_schema - - try: - data = get_schema(get_metaname(namespace)) - - credential = gpgrecord.decrypt_data(data) - - return credential - - except Exception: - return None - - -def set_credential(namespace, credential): - ''' - namespace: -- service name, by directory - - Example: - >>> set_credential('gmail', {'username': '', 'password': ''}) - ''' - GPG_KEY = config.ENSURE_GPG() - - if credential: - encrypted_credential = gpgrecord.encrypt_data( - credential, - GPG_KEY - ) - - content = '''## {main} -```yaml -{cont} -```'''.format( - main=MAIN, - cont=yaml.dump(encrypted_credential) - ) - - # repo = config.ENSURE_REPO() - - with open( - os.path.join( - config.CREDENTIALS_DIR, - namespace + '.md'), 'w') as f: - f.write(content) - - os.system('cd {}; git add .; git commit -m "update"; git push origin master'.format( - config.REPO_PATH)) - - return - - -def get_or_ask_credentials(namespace, variables, ask_refresh=False): - credential = get_credential(namespace) - - refresh = False - - if ask_refresh: - if credential: - if input("Found credential, do you want to refresh? [N/y] ") in ['y', 'Y']: - refresh = True - - if not credential or refresh: - credential = {} - - print('Type credentials for your {}:'.format(namespace.title())) - for variable in variables: - credential[variable] = input('{} = '.format( - variable - )) - - if all(credential.values()): - set_credential( - namespace, - credential) - return credential - else: - raise Exception('Some of the credentials were not set.') - else: - return credential - - -def load_session_data(namespace): - session_path = os.path.join(config.SESSIONS_DIR, namespace) - if os.path.exists(session_path): - session_data = json.load(open(session_path, 'r')) - return session_data - else: - return {} - - -def save_session_data(namespace, session_data): - session_path = os.path.join(config.SESSIONS_DIR, namespace) - json.dump(session_data, open(session_path, 'w')) - - -def ensure_driver_installed(driver_name): - reader = driver_name - - SUPPORTED_PACKAGE_MANAGERS = ['pypi'] - - if reader.lower().split(':', 1)[0] not in SUPPORTED_PACKAGE_MANAGERS: - raise Exception( - "Unknown package manager. " + - "Make sure the reader you chose starts with one of these: " + - "{}. Your chosen reader is: {}".format( - ', '.join(SUPPORTED_PACKAGE_MANAGERS), - reader - ) - ) - - # SUPPORTED_PACKAGES = [ - # 'pypi:metadrive', - # 'pypi:drivers', - # 'pypi:subtools' - # ] - # - package_name = reader.split('.', 1)[0].lower() - # - # if package_name not in SUPPORTED_PACKAGES: - # raise Exception( - # "Unsupported reader package. " + - # "Make sure the reader package is one of these: " + - # "{}. Your chosen reader is: {}".format( - # ', '.join(SUPPORTED_PACKAGES), - # package_name - # ) - # - # ) - - # cause in wikis, we used only one ':', e.g., - # https://github.com/mindey/-/wiki/topic#linkedin - # TBD: unify the way we refer to package manager, use '::' in all cases - packman, package = package_name.split(':') - - # Make sure we have that package installed. - spec = importlib.util.find_spec(package) - if spec is None: - # answer = input(package +" is not installed. Install it? [Y/n] ") - # if answer in ['y', 'Y', '']: - try: - # easy_install.main( ["-U", package_name] ) - os.system('pip install --no-input -U {} --no-cache'.format(package)) - except SystemExit: - pass - # else: - # raise Exception(package_name +" is required. Install it and run again.") - else: - # Check the version installed. - import pkg_resources - importlib.reload(pkg_resources) - installed_version = pkg_resources.get_distribution(package).version - - # Check the latest version in PyPI - from yolk.pypi import CheeseShop - - def get_lastest_version_number(package_name): - pkg, all_versions = CheeseShop().query_versions_pypi(package_name) - if len(all_versions): - return all_versions[0] - return None - - latest_version = get_lastest_version_number(package) - - def cmp_version(version1, version2): - def norm(v): - return [int(x) for x in re.sub(r'(\.0+)*$', '', v).split(".")] - a, b = norm(version1), norm(version2) - return (a > b) - (a < b) - - if latest_version is not None: - if cmp_version(installed_version, latest_version) < 0: - - print('You are running {}=={}'.format(package, installed_version) + - ", but there is newer ({}) version.".format(latest_version)) - - if config.AUTO_UPGRADE_DRIVERS is None: - answer = input("Upgrade it? [y/N] ") - if answer in ['y', 'Y']: - try: - os.system('pip install --no-input -U {} --no-cache'.format(package)) - except SystemExit: - pass - - elif config.AUTO_UPGRADE_DRIVERS: - try: - os.system('pip install --no-input -U {} --no-cache'.format(package)) - except SystemExit: - pass - - else: # config.AUTO_UPGRADE_DRIVERS == False: - pass - - return package - - -@contextlib.contextmanager -def stdoutIO(stdout=None): - old = sys.stdout - if stdout is None: - stdout = StringIO() - sys.stdout = stdout - yield stdout - sys.stdout = old diff --git a/setup.py b/setup.py index de2f4c1..7cf781e 100644 --- a/setup.py +++ b/setup.py @@ -24,13 +24,14 @@ 'deprecated==1.2.5', 'click==7.0', 'fusepy==3.0.1', - 'tqdm==4.31.1', - 'yolk3k==0.9', # TODO - 'metatype', - 'metawiki', - 'metaform', - 'typology', + 'aiofiles==0.4.0', + # 'tqdm==4.31.1', + # 'yolk3k==0.9', # TODO + # 'metatype', + # 'metawiki', + # 'metaform', + # 'typology', # 'selenium==3.141.0', # for _selenuim # 'xarray==0.12.1', # for _xarray ], @@ -40,7 +41,7 @@ zip_safe=False, entry_points={ 'console_scripts': [ - 'connect=metadrive.cli:connect' + 'connect=metadrive.entrypoints.connect:connect' ], }, package_data={ From c3a20511800650abeedcf5e693876d013a62bdad Mon Sep 17 00:00:00 2001 From: Seva D Date: Sun, 30 Jun 2019 21:02:04 +0200 Subject: [PATCH 2/4] Initial changes --- metadrive/core/driver.py | 26 +------------------------- metadrive/core/runner.py | 16 ++++++++++------ metadrive/subtools/__init__.py | 0 metadrive/subtools/_example.py | 18 ++++++++++++++++++ metadrive/subtools/_requests.py | 5 +++++ metadrive/subtools/_selenium.py | 5 +++++ metadrive/subtools/_xarray.py | 5 +++++ 7 files changed, 44 insertions(+), 31 deletions(-) create mode 100644 metadrive/subtools/__init__.py create mode 100644 metadrive/subtools/_example.py create mode 100644 metadrive/subtools/_requests.py create mode 100644 metadrive/subtools/_selenium.py create mode 100644 metadrive/subtools/_xarray.py diff --git a/metadrive/core/driver.py b/metadrive/core/driver.py index 777c9b7..b49f933 100644 --- a/metadrive/core/driver.py +++ b/metadrive/core/driver.py @@ -1,13 +1,4 @@ -import logging -import os -import time - -import aiofiles as aiof - -logger = logging.getLogger(__file__) - - -class Driver: +class AbstractDriver: def __init__(self, loop, resource, root): self.loop = loop self.resource = resource @@ -15,18 +6,3 @@ def __init__(self, loop, resource, root): async def sync(self): raise NotImplementedError - - -class TestDriver(Driver): - - async def sync(self): - logger.debug('test') - filename = os.path.join(self.root, '%s.txt' % time.time()) - async with aiof.open(filename, "w", loop=self.loop) as out: - out.write('%s' % time.time()) - out.flush() - - -async def get_driver(resource): - # TODO identify driver by resource - return TestDriver diff --git a/metadrive/core/runner.py b/metadrive/core/runner.py index 2e93600..ea43dc2 100644 --- a/metadrive/core/runner.py +++ b/metadrive/core/runner.py @@ -4,7 +4,6 @@ from concurrent.futures import ProcessPoolExecutor from metadrive import settings -from metadrive.core.driver import get_driver from metadrive.core.mount import mount @@ -32,7 +31,12 @@ def __init__(self, resource, mountpoint=None, user=None, period=900): self.executor = ProcessPoolExecutor() self.loop.add_signal_handler(signal.SIGINT, self.shutdown) - async def mount_filesystem(self): + async def _get_driver(self): + # TODO identify driver by resource + from metadrive.subtools._example import ExampleDriver + return ExampleDriver + + async def _mount_filesystem(self): await self.loop.run_in_executor( self.executor, mount, @@ -40,14 +44,14 @@ async def mount_filesystem(self): self.mountpoint ) - async def sync_by_driver(self): - driver_cls = await get_driver(self.resource) + async def _sync_by_driver(self): + driver_cls = await self._get_driver() driver_obj = driver_cls(self.loop, self.resource, self.root) await driver_obj.sync() def run(self): - self.loop.create_task(self.mount_filesystem()) - self.loop.create_task(self.sync_by_driver()) + self.loop.create_task(self._mount_filesystem()) + self.loop.create_task(self._sync_by_driver()) self.loop.run_forever() pending = asyncio.Task.all_tasks(loop=self.loop) diff --git a/metadrive/subtools/__init__.py b/metadrive/subtools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metadrive/subtools/_example.py b/metadrive/subtools/_example.py new file mode 100644 index 0000000..03017ee --- /dev/null +++ b/metadrive/subtools/_example.py @@ -0,0 +1,18 @@ +import os +import time +import logging +import aiofiles as aiof + +from metadrive.core.driver import AbstractDriver + +logger = logging.getLogger(__file__) + + +class ExampleDriver(AbstractDriver): + + async def sync(self): + logger.debug('test') + filename = os.path.join(self.root, '%s.txt' % time.time()) + async with aiof.open(filename, "w", loop=self.loop) as out: + out.write('%s' % time.time()) + out.flush() diff --git a/metadrive/subtools/_requests.py b/metadrive/subtools/_requests.py new file mode 100644 index 0000000..a79958e --- /dev/null +++ b/metadrive/subtools/_requests.py @@ -0,0 +1,5 @@ +from metadrive.core.driver import AbstractDriver + + +class RequestsDriver(AbstractDriver): + pass diff --git a/metadrive/subtools/_selenium.py b/metadrive/subtools/_selenium.py new file mode 100644 index 0000000..72cec55 --- /dev/null +++ b/metadrive/subtools/_selenium.py @@ -0,0 +1,5 @@ +from metadrive.core.driver import AbstractDriver + + +class SeleniumDriver(AbstractDriver): + pass diff --git a/metadrive/subtools/_xarray.py b/metadrive/subtools/_xarray.py new file mode 100644 index 0000000..5966a89 --- /dev/null +++ b/metadrive/subtools/_xarray.py @@ -0,0 +1,5 @@ +from metadrive.core.driver import AbstractDriver + + +class XArrayDriver(AbstractDriver): + pass From 7208c4275e3a3cb05a7bbab01ea1250a448c367d Mon Sep 17 00:00:00 2001 From: Seva D Date: Mon, 1 Jul 2019 12:37:09 +0200 Subject: [PATCH 3/4] Restructed --- metadrive/core/driver.py | 8 --- metadrive/core/mount.py | 3 + metadrive/core/runner.py | 59 +++++++++++-------- metadrive/drives/__init__.py | 11 ++++ .../_example.py => drives/drive_example.py} | 9 ++- .../interfaces}/__init__.py | 0 metadrive/drives/interfaces/_generic.py | 11 ++++ metadrive/drives/interfaces/_requests.py | 5 ++ metadrive/drives/interfaces/_selenium.py | 5 ++ metadrive/drives/interfaces/_xarray.py | 5 ++ metadrive/entrypoints/connect.py | 3 +- metadrive/settings.py | 4 ++ metadrive/subtools/_requests.py | 5 -- metadrive/subtools/_selenium.py | 5 -- metadrive/subtools/_xarray.py | 5 -- 15 files changed, 87 insertions(+), 51 deletions(-) delete mode 100644 metadrive/core/driver.py create mode 100644 metadrive/drives/__init__.py rename metadrive/{subtools/_example.py => drives/drive_example.py} (53%) rename metadrive/{subtools => drives/interfaces}/__init__.py (100%) create mode 100644 metadrive/drives/interfaces/_generic.py create mode 100644 metadrive/drives/interfaces/_requests.py create mode 100644 metadrive/drives/interfaces/_selenium.py create mode 100644 metadrive/drives/interfaces/_xarray.py delete mode 100644 metadrive/subtools/_requests.py delete mode 100644 metadrive/subtools/_selenium.py delete mode 100644 metadrive/subtools/_xarray.py diff --git a/metadrive/core/driver.py b/metadrive/core/driver.py deleted file mode 100644 index b49f933..0000000 --- a/metadrive/core/driver.py +++ /dev/null @@ -1,8 +0,0 @@ -class AbstractDriver: - def __init__(self, loop, resource, root): - self.loop = loop - self.resource = resource - self.root = root - - async def sync(self): - raise NotImplementedError diff --git a/metadrive/core/mount.py b/metadrive/core/mount.py index 1fcd75d..702cf2f 100644 --- a/metadrive/core/mount.py +++ b/metadrive/core/mount.py @@ -16,6 +16,9 @@ def _full_path(self, partial): path = os.path.join(self.root, partial) return path + # Filesystem methods + # ================== + def access(self, path, mode): full_path = self._full_path(path) if not os.access(full_path, mode): diff --git a/metadrive/core/runner.py b/metadrive/core/runner.py index ea43dc2..902e553 100644 --- a/metadrive/core/runner.py +++ b/metadrive/core/runner.py @@ -1,53 +1,66 @@ import asyncio import os import signal +import importlib +import re +import logging from concurrent.futures import ProcessPoolExecutor from metadrive import settings from metadrive.core.mount import mount +logger = logging.getLogger(__file__) + class MetaDriveRunner: - def __init__(self, resource, mountpoint=None, user=None, period=900): + def __init__(self, resource, mountpoint=None, session=None): self.resource = resource - self.mountpoint = mountpoint - self.user = user or 'default' - self.period = period - - if self.mountpoint is None: - self.mountpoint = os.path.join( - settings.MOUNT_DIR, - resource, - self.user - ) - - self.root = os.path.join( + self.session = session or 'default' + self.mountpoint = mountpoint or os.path.join( + settings.MOUNT_DIR, + '{}:{}'.format(self.resource, self.session) + ) + self.rootpath = os.path.join( settings.DATA_DIR, - resource, - self.user + '{}:{}'.format(self.resource, self.session) ) self.loop = asyncio.get_event_loop() self.executor = ProcessPoolExecutor() self.loop.add_signal_handler(signal.SIGINT, self.shutdown) - async def _get_driver(self): - # TODO identify driver by resource - from metadrive.subtools._example import ExampleDriver - return ExampleDriver + try: + os.mkdir(self.rootpath) + except FileExistsError: + pass + + self.drives = [] + for drive in settings.DRIVES: + module_name, class_name = drive.split(':') + module = importlib.import_module(module_name) + drive_class = getattr(module, class_name) + self.drives.append( + drive_class(self.loop, self.resource, self.rootpath) + ) + + async def _get_drive(self): + for drive in self.drives: + if re.match(drive.get_resource_pattern(), self.resource): + return drive + logger.warning('No drive for resource %s', self.resource) async def _mount_filesystem(self): await self.loop.run_in_executor( self.executor, mount, - self.root, + self.rootpath, self.mountpoint ) async def _sync_by_driver(self): - driver_cls = await self._get_driver() - driver_obj = driver_cls(self.loop, self.resource, self.root) - await driver_obj.sync() + drive = await self._get_drive() + if drive: + await drive.sync() def run(self): self.loop.create_task(self._mount_filesystem()) diff --git a/metadrive/drives/__init__.py b/metadrive/drives/__init__.py new file mode 100644 index 0000000..d00173a --- /dev/null +++ b/metadrive/drives/__init__.py @@ -0,0 +1,11 @@ +class AbstractDrive: + # regexp pattern + resource_pattern = None + + def __init__(self, loop, resource, rootpath): + self.loop = loop + self.resource = resource + self.rootpath = rootpath + + async def sync(self): + raise NotImplementedError diff --git a/metadrive/subtools/_example.py b/metadrive/drives/drive_example.py similarity index 53% rename from metadrive/subtools/_example.py rename to metadrive/drives/drive_example.py index 03017ee..8db358b 100644 --- a/metadrive/subtools/_example.py +++ b/metadrive/drives/drive_example.py @@ -3,16 +3,19 @@ import logging import aiofiles as aiof -from metadrive.core.driver import AbstractDriver +from metadrive.drives.interfaces._generic import GenericDriveInterface logger = logging.getLogger(__file__) -class ExampleDriver(AbstractDriver): +class ExampleDrive(GenericDriveInterface): + + def get_resource_pattern(self): + return r'^test.com$' async def sync(self): logger.debug('test') - filename = os.path.join(self.root, '%s.txt' % time.time()) + filename = os.path.join(self.rootpath, '%s.txt' % time.time()) async with aiof.open(filename, "w", loop=self.loop) as out: out.write('%s' % time.time()) out.flush() diff --git a/metadrive/subtools/__init__.py b/metadrive/drives/interfaces/__init__.py similarity index 100% rename from metadrive/subtools/__init__.py rename to metadrive/drives/interfaces/__init__.py diff --git a/metadrive/drives/interfaces/_generic.py b/metadrive/drives/interfaces/_generic.py new file mode 100644 index 0000000..fe38cae --- /dev/null +++ b/metadrive/drives/interfaces/_generic.py @@ -0,0 +1,11 @@ +class GenericDriveInterface: + def __init__(self, loop, resource, rootpath): + self.loop = loop + self.resource = resource + self.rootpath = rootpath + + def get_resource_pattern(self): + raise NotImplementedError + + async def sync(self): + raise NotImplementedError diff --git a/metadrive/drives/interfaces/_requests.py b/metadrive/drives/interfaces/_requests.py new file mode 100644 index 0000000..78fd6ae --- /dev/null +++ b/metadrive/drives/interfaces/_requests.py @@ -0,0 +1,5 @@ +from metadrive.drives.interfaces._generic import GenericDriveInterface + + +class RequestsDriveInterface(GenericDriveInterface): + pass diff --git a/metadrive/drives/interfaces/_selenium.py b/metadrive/drives/interfaces/_selenium.py new file mode 100644 index 0000000..40db48d --- /dev/null +++ b/metadrive/drives/interfaces/_selenium.py @@ -0,0 +1,5 @@ +from metadrive.drives.interfaces._generic import GenericDriveInterface + + +class SeleniumDriveInterface(GenericDriveInterface): + pass diff --git a/metadrive/drives/interfaces/_xarray.py b/metadrive/drives/interfaces/_xarray.py new file mode 100644 index 0000000..a328f28 --- /dev/null +++ b/metadrive/drives/interfaces/_xarray.py @@ -0,0 +1,5 @@ +from metadrive.drives.interfaces._generic import GenericDriveInterface + + +class XArrayDriveInterface(GenericDriveInterface): + pass diff --git a/metadrive/entrypoints/connect.py b/metadrive/entrypoints/connect.py index d829398..c1bfcfe 100644 --- a/metadrive/entrypoints/connect.py +++ b/metadrive/entrypoints/connect.py @@ -39,8 +39,7 @@ def connect(resource, mountpoint=None, user=None, period=900): runner = MetaDriveRunner( resource, mountpoint=mountpoint, - user=user, - period=period + session=user, ) try: runner.run() diff --git a/metadrive/settings.py b/metadrive/settings.py index 48c620b..860584c 100644 --- a/metadrive/settings.py +++ b/metadrive/settings.py @@ -5,3 +5,7 @@ CONFIG_DIR = os.path.join(HOME, '.metadrive') DATA_DIR = os.path.join(CONFIG_DIR, 'data') MOUNT_DIR = os.path.join(HOME, 'Sites') + +DRIVES = [ + 'metadrive.drives.drive_example:ExampleDrive', +] diff --git a/metadrive/subtools/_requests.py b/metadrive/subtools/_requests.py deleted file mode 100644 index a79958e..0000000 --- a/metadrive/subtools/_requests.py +++ /dev/null @@ -1,5 +0,0 @@ -from metadrive.core.driver import AbstractDriver - - -class RequestsDriver(AbstractDriver): - pass diff --git a/metadrive/subtools/_selenium.py b/metadrive/subtools/_selenium.py deleted file mode 100644 index 72cec55..0000000 --- a/metadrive/subtools/_selenium.py +++ /dev/null @@ -1,5 +0,0 @@ -from metadrive.core.driver import AbstractDriver - - -class SeleniumDriver(AbstractDriver): - pass diff --git a/metadrive/subtools/_xarray.py b/metadrive/subtools/_xarray.py deleted file mode 100644 index 5966a89..0000000 --- a/metadrive/subtools/_xarray.py +++ /dev/null @@ -1,5 +0,0 @@ -from metadrive.core.driver import AbstractDriver - - -class XArrayDriver(AbstractDriver): - pass From 4f594e3a5981f01f0d007e4b6a97040f32eddd9e Mon Sep 17 00:00:00 2001 From: Seva D Date: Mon, 1 Jul 2019 13:37:49 +0200 Subject: [PATCH 4/4] Drives interface read/write --- metadrive/core/mount.py | 25 ++++++++++--------- metadrive/core/runner.py | 33 +++++++++++++------------ metadrive/drives/drive_example.py | 3 ++- metadrive/drives/interfaces/_generic.py | 20 ++++++++++++--- 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/metadrive/core/mount.py b/metadrive/core/mount.py index 702cf2f..dc4d25b 100644 --- a/metadrive/core/mount.py +++ b/metadrive/core/mount.py @@ -6,14 +6,14 @@ # https://github.com/skorokithakis/python-fuse-sample -class Passthrough(Operations): - def __init__(self, root): - self.root = root +class MetaFuseOperations(Operations): + def __init__(self, drive_cls, resource, rootpath): + self.drive = drive_cls(resource, rootpath) def _full_path(self, partial): if partial.startswith("/"): partial = partial[1:] - path = os.path.join(self.root, partial) + path = os.path.join(self.drive.rootpath, partial) return path # Filesystem methods @@ -55,7 +55,7 @@ def readlink(self, path): pathname = os.readlink(self._full_path(path)) if pathname.startswith("/"): # Path name is absolute, sanitize it. - return os.path.relpath(pathname, self.root) + return os.path.relpath(pathname, self.drive.rootpath) else: return pathname @@ -107,12 +107,10 @@ def create(self, path, mode, fi=None): return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode) def read(self, path, length, offset, fh): - os.lseek(fh, offset, os.SEEK_SET) - return os.read(fh, length) + return self.drive.read(path, length, offset, fh) def write(self, path, buf, offset, fh): - os.lseek(fh, offset, os.SEEK_SET) - return os.write(fh, buf) + return self.drive.write(path, buf, offset, fh) def truncate(self, path, length, fh=None): full_path = self._full_path(path) @@ -129,5 +127,10 @@ def fsync(self, path, fdatasync, fh): return self.flush(path, fh) -def mount(root, mountpoint): - FUSE(Passthrough(root), mountpoint, nothreads=True, foreground=True) +def mount(mountpoint, *args): + FUSE( + MetaFuseOperations(*args), + mountpoint, + nothreads=True, + foreground=True + ) diff --git a/metadrive/core/runner.py b/metadrive/core/runner.py index 902e553..17e4a26 100644 --- a/metadrive/core/runner.py +++ b/metadrive/core/runner.py @@ -4,6 +4,7 @@ import importlib import re import logging +import sys from concurrent.futures import ProcessPoolExecutor from metadrive import settings @@ -34,37 +35,37 @@ def __init__(self, resource, mountpoint=None, session=None): except FileExistsError: pass - self.drives = [] + self.drive = None for drive in settings.DRIVES: module_name, class_name = drive.split(':') module = importlib.import_module(module_name) drive_class = getattr(module, class_name) - self.drives.append( - drive_class(self.loop, self.resource, self.rootpath) - ) - - async def _get_drive(self): - for drive in self.drives: - if re.match(drive.get_resource_pattern(), self.resource): - return drive - logger.warning('No drive for resource %s', self.resource) + if re.match(drive_class.get_resource_pattern(), self.resource): + self.drive = drive_class( + self.resource, + self.rootpath + ) + break + if self.drive is None: + logger.warning('No drive for resource %s', self.resource) + sys.exit(0) async def _mount_filesystem(self): await self.loop.run_in_executor( self.executor, mount, + self.mountpoint, + self.drive.__class__, + self.resource, self.rootpath, - self.mountpoint ) - async def _sync_by_driver(self): - drive = await self._get_drive() - if drive: - await drive.sync() + async def _sync_drive(self): + await self.drive.sync() def run(self): self.loop.create_task(self._mount_filesystem()) - self.loop.create_task(self._sync_by_driver()) + self.loop.create_task(self._sync_drive()) self.loop.run_forever() pending = asyncio.Task.all_tasks(loop=self.loop) diff --git a/metadrive/drives/drive_example.py b/metadrive/drives/drive_example.py index 8db358b..daa8226 100644 --- a/metadrive/drives/drive_example.py +++ b/metadrive/drives/drive_example.py @@ -10,7 +10,8 @@ class ExampleDrive(GenericDriveInterface): - def get_resource_pattern(self): + @classmethod + def get_resource_pattern(cls): return r'^test.com$' async def sync(self): diff --git a/metadrive/drives/interfaces/_generic.py b/metadrive/drives/interfaces/_generic.py index fe38cae..dbffd95 100644 --- a/metadrive/drives/interfaces/_generic.py +++ b/metadrive/drives/interfaces/_generic.py @@ -1,11 +1,25 @@ + +import asyncio +import os + + class GenericDriveInterface: - def __init__(self, loop, resource, rootpath): - self.loop = loop + def __init__(self, resource, rootpath): + self.loop = asyncio.get_event_loop() self.resource = resource self.rootpath = rootpath - def get_resource_pattern(self): + @classmethod + def get_resource_pattern(cls): raise NotImplementedError async def sync(self): raise NotImplementedError + + def read(self, path, length, offset, fh): + os.lseek(fh, offset, os.SEEK_SET) + return os.read(fh, length) + + def write(self, path, buf, offset, fh): + os.lseek(fh, offset, os.SEEK_SET) + return os.write(fh, buf)