diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..0e19b31 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,102 @@ +version: 2.1 + +aliases: + - &environ + run: + name: setup virtual environment + # The below ensures the venv is activated for every subsequent step + command: | + virtualenv venv + echo "source /home/circleci/project/venv/bin/activate" >> $BASH_ENV + + - &install + run: + name: install dependencies + command: | + pip install -U pip setuptools wheel tox tox-factor codecov + + - &test-steps + steps: + - checkout + - *environ + - *install + - run: tox + - run: coverage combine + - run: coverage report + - run: codecov + +jobs: + lint: + steps: + - checkout + - *environ + - *install + - run: tox -e isort,lint + docker: + - image: circleci/python:3.8 + + dist: + steps: + - checkout + - *environ + - *install + - run: | + python setup.py bdist_wheel + tox -e dist --installpkg ./dist/jsonfield-*.whl + tox -e dist + docker: + - image: circleci/python:3.8 + + test-py38: + <<: *test-steps + docker: + - image: circleci/python:3.8 + environment: + TOXFACTOR: py38 + + test-py37: + <<: *test-steps + docker: + - image: circleci/python:3.7 + environment: + TOXFACTOR: py37 + + test-py36: + <<: *test-steps + docker: + - image: circleci/python:3.6 + environment: + TOXFACTOR: py36 + + +workflows: + version: 2 + commit: &test-workflow + jobs: + - lint + - dist: + requires: + - lint + + - test-py38: + requires: + - lint + + - test-py37: + requires: + - lint + + - test-py36: + requires: + - lint + + weekly: + <<: *test-workflow + triggers: + - schedule: + # 8/9 AM PST/PDT every Monday + cron: "0 16 * * 1" + filters: + branches: + only: + - master diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 902d25c..0000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -source = jsonfield -omit = jsonfield/tests.py -branch = 1 - -[report] -omit = jsonfield/tests.py diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 4da5a92..0000000 --- a/.dockerignore +++ /dev/null @@ -1,91 +0,0 @@ -# Git -.git -.gitignore - -# CI -.codeclimate.yml -.travis.yml - -# Docker -docker-compose.yml - -# Byte-compiled / optimized / DLL files -__pycache__/ -*/__pycache__/ -*/*/__pycache__/ -*/*/*/__pycache__/ -*.py[cod] -*/*.py[cod] -*/*/*.py[cod] -*/*/*/*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.cache -nosetests.xml -coverage.xml - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Virtual environment -.env/ -.venv/ -venv/ - -# PyCharm -.idea - -# Python mode for VIM -.ropeproject -*/.ropeproject -*/*/.ropeproject -*/*/*/.ropeproject - -# Vim swap files -*.swp -*/*.swp -*/*/*.swp -*/*/*/*.swp diff --git a/.gitignore b/.gitignore index bf5c760..0bd81c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,11 @@ -.coverage +.coverage* .tox htmlcov *.pyc +*.egg-info/ build/ dist/ -MANIFEST -.project -.pydevproject .DS_Store -.idea/* +.env +.env/ +.venv/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 20be682..0000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -language: python -sudo: false -python: - - "2.7" - - "3.3" - - "3.4" - - "3.5" - - "3.6" -env: - - DJANGO=1.8 - - DJANGO=1.9 - - DJANGO=1.10 - - DJANGO=1.11 - - DJANGO=master -matrix: - exclude: - - python: "2.7" - env: DJANGO=master - - python: "3.3" - env: DJANGO=1.9 - - python: "3.3" - env: DJANGO=1.10 - - python: "3.3" - env: DJANGO=1.11 - - python: "3.3" - env: DJANGO=master - - python: "3.4" - env: DJANGO=master - - python: "3.6" - env: DJANGO=1.8 - - python: "3.6" - env: DJANGO=1.9 - - python: "3.6" - env: DJANGO=1.10 -install: - - pip install tox coveralls -script: - - tox -e py${TRAVIS_PYTHON_VERSION//[.]/}-$DJANGO -after_success: - - coveralls -notifications: - email: - recipients: - - bjasper@gmail.com - - paltman@gmail.com - - dkoch@mm.st diff --git a/CHANGES.rst b/CHANGES.rst index 808d645..bd3a2ba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,55 @@ Changes ------- + +v3.0.0 02/14/2020 +^^^^^^^^^^^^^^^^^ + +This release is a major rewrite of ``jsonfield``, merging in changes from the +``jsonfield2`` fork. Changelog entries for ``jsonfield2`` are included below +for completeness. + +- Add source distribution to release process +- Update ``JSONEncoder`` from DRF +- Fix re-rendering of invalid field inputs +- Fix form field cleaning of string inputs +- Fix indentation for ``Textarea`` widgets +- Allow form field error message to be overridden +- Obey form ``Field.disabled`` + +jsonfield2 v3.1.0 12/06/2019 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Fix use with ``select_related`` across a foreign key +- Fix field deconstruction +- Drop Python 3.5 support +- Drop Django 2.1 (and below) support + +jsonfield2 v3.0.3 10/23/2019 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Add Python 3.8 & Django 3.0 support +- Drop Python 3.4 support + +jsonfield2 v3.0.2 12/21/2018 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Add Python 3.7 & Django 2.1 support + +jsonfield2 v3.0.1 05/21/2018 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Fix model full_clean behavior + +jsonfield2 v3.0.0 05/07/2018 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Add Django 2.0 support +- Drop Django 1.8, 1.9, and 1.10 support +- Drop Python 2.7 and 3.3 support +- Rework field serialization/deserialization +- Remove support for South +- Rename JSONFieldBase to JSONFieldMixin +- Move form fields into separate module +- Rename JSONFormFieldBase to forms.JSONFieldMixin +- Rename JSONFormField to forms.JSONField +- Remove JSONCharFormField +- Update JSONEncoder from DRF + v2.0.2, 6/18/2017 ^^^^^^^^^^^^^^^^^ - Fixed issue with GenericForeignKey field @@ -7,7 +57,7 @@ v2.0.2, 6/18/2017 v2.0.1, 3/8/2017 ^^^^^^^^^^^^^^^^ - Support upcoming Django 1.11 in test suite -- Renamed method `get_db_prep_value` to `get_prep_value` +- Renamed method ``get_db_prep_value`` to ``get_prep_value`` v2.0.0, 3/4/2017 ^^^^^^^^^^^^^^^^ @@ -27,7 +77,6 @@ v1.0.1, 2/2/2015 v1.0.0, 9/4/2014 ^^^^^^^^^^^^^^^^ - - Removed native JSON datatype support for PostgreSQL (breaking change) & added Python 3.4 to tests v0.9.23, 9/3/2014 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f5555e6..0000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM themattrix/tox - -MAINTAINER Dan Koch diff --git a/MANIFEST.in b/MANIFEST.in index 7545927..908c4f9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ -include CHANGES.rst -include LICENSE -include README.rst +include README.rst CHANGES.rst LICENSE + +include tox.ini +recursive-include tests *.py diff --git a/README.rst b/README.rst index 3ef6595..811d12c 100644 --- a/README.rst +++ b/README.rst @@ -1,25 +1,36 @@ -django-jsonfield ----------------- +jsonfield +========= -django-jsonfield is a reusable Django field that allows you to store validated JSON in your model. +.. image:: https://circleci.com/gh/rpkilby/jsonfield.svg?style=shield + :target: https://circleci.com/gh/rpkilby/jsonfield +.. image:: https://codecov.io/gh/rpkilby/jsonfield/branch/master/graph/badge.svg + :target: https://codecov.io/gh/rpkilby/jsonfield +.. image:: https://img.shields.io/pypi/v/jsonfield.svg + :target: https://pypi.org/project/jsonfield +.. image:: https://img.shields.io/pypi/l/jsonfield.svg + :target: https://pypi.org/project/jsonfield -It silently takes care of serialization. To use, simply add the field to one of your models. +**jsonfield** is a reusable model field that allows you to store validated JSON, automatically handling +serialization to and from the database. To use, add ``jsonfield.JSONField`` to one of your models. -Python 3 & Django 1.8 through 1.11 supported! +**Note:** `django.contrib.postgres`_ now supports PostgreSQL's jsonb type, which includes extended querying +capabilities. If you're an end user of PostgreSQL and want full-featured JSON support, then it is +recommended that you use the built-in JSONField. However, jsonfield is still useful when your app +needs to be database-agnostic, or when the built-in JSONField's extended querying is not being leveraged. +e.g., a configuration field. -**Use PostgreSQL?** 1.0.0 introduced a breaking change to the underlying data type, so if you were using < 1.0.0 please read https://github.com/dmkoch/django-jsonfield/issues/57 before upgrading. Also, consider switching to Django's native JSONField that was added in Django 1.9. +.. _django.contrib.postgres: https://docs.djangoproject.com/en/dev/ref/contrib/postgres/fields/#jsonfield -**Note:** There are a couple of third-party add-on JSONFields for Django. This project is django-jsonfield here on GitHub but is named `jsonfield on PyPI`_. There is another `django-jsonfield on Bitbucket`_, but that one is `django-jsonfield on PyPI`_. I realize this naming conflict is confusing and I am open to merging the two projects. -.. _jsonfield on PyPI: https://pypi.python.org/pypi/jsonfield -.. _django-jsonfield on Bitbucket: https://bitbucket.org/schinckel/django-jsonfield -.. _django-jsonfield on PyPI: https://pypi.python.org/pypi/django-jsonfield +Requirements +------------ -**Note:** Django 1.9 added native PostgreSQL JSON support in `django.contrib.postgres.fields.JSONField`_. This module is still useful if you need to support JSON in databases other than PostgreSQL or are creating a third-party module that needs to be database-agnostic. But if you're an end user using PostgreSQL and want full-featured JSON support, I recommend using the built-in JSONField from Django instead of this module. +**jsonfield** aims to support all current `versions of Django`_, however the explicity tested versions are: -.. _django.contrib.postgres.fields.JSONField: https://docs.djangoproject.com/en/dev/ref/contrib/postgres/fields/#jsonfield +* **Python:** 3.6, 3.7, 3.8 +* **Django:** 2.2, 3.0 -**Note:** Semver is followed after the 1.0 release. +.. _versions of Django: https://www.djangoproject.com/download/#supported-versions Installation @@ -39,20 +50,22 @@ Usage from jsonfield import JSONField class MyModel(models.Model): - json = JSONField() + json = JSONField() + Advanced Usage -------------- -By default python deserializes json into dict objects. This behavior differs from the standard json behavior because python dicts do not have ordered keys. - -To overcome this limitation and keep the sort order of OrderedDict keys the deserialisation can be adjusted on model initialisation: +By default python deserializes json into dict objects. This behavior differs from the standard json +behavior because python dicts do not have ordered keys. To overcome this limitation and keep the +sort order of OrderedDict keys the deserialisation can be adjusted on model initialisation: .. code-block:: python import collections + class MyModel(models.Model): - json = JSONField(load_kwargs={'object_pairs_hook': collections.OrderedDict}) + json = JSONField(load_kwargs={'object_pairs_hook': collections.OrderedDict}) Other Fields @@ -60,60 +73,54 @@ Other Fields **jsonfield.JSONCharField** -If you need to use your JSON field in an index or other constraint, you can use **JSONCharField** which subclasses **CharField** instead of **TextField**. You'll also need to specify a **max_length** parameter if you use this field. - +Subclasses **models.CharField** instead of **models.TextField**. -Compatibility --------------- -django-jsonfield aims to support the same versions of Django currently maintained by the main Django project. See `Django supported versions`_, currently: +Running the tests +----------------- - * Django 1.8 (LTS) with Python 2.7, 3.3, 3.4, or 3.5 - * Django 1.9 with Python 2.7, 3.4, or 3.5 - * Django 1.10 with Python 2.7, 3.4, or 3.5 - * Django 1.11 (LTS) with Python 2.7, 3.4, 3.5 or 3.6 +The test suite requires ``tox`` and ``tox-venv``. -.. _Django supported versions: https://www.djangoproject.com/download/#supported-versions +.. code-block:: shell + $ pip install tox tox-venv -Testing django-jsonfield Locally --------------------------------- -To test against all supported versions of Django: +To test against all supported versions of Django, install and run ``tox``: .. code-block:: shell - $ docker-compose build && docker-compose up + $ tox -Or just one version (for example Django 1.10 on Python 3.5): +Or, to test just one version (for example Django 2.0 on Python 3.6): .. code-block:: shell - $ docker-compose build && docker-compose run tox tox -e py35-1.10 - - -Travis CI ---------- + $ tox -e py36-django20 -.. image:: https://travis-ci.org/dmkoch/django-jsonfield.svg?branch=master - :target: https://travis-ci.org/dmkoch/django-jsonfield -Contact -------- -Web: http://bradjasper.com - -Twitter: `@bradjasper`_ +Release Process +--------------- -Email: `contact@bradjasper.com`_ +* Update changelog +* Update package version in setup.py +* Check supported versions in setup.py and readme +* Create git tag for version +* Upload release to PyPI test server +* Upload release to official PyPI server +.. code-block:: shell + $ pip install -U pip setuptools wheel twine + $ rm -rf dist/ build/ + $ python setup.py sdist bdist_wheel + $ twine upload -r test dist/* + $ twine upload dist/* -.. _contact@bradjasper.com: mailto:contact@bradjasper.com -.. _@bradjasper: https://twitter.com/bradjasper Changes ------- Take a look at the `changelog`_. -.. _changelog: https://github.com/dmkoch/django-jsonfield/blob/master/CHANGES.rst +.. _changelog: https://github.com/rpkilby/jsonfield/blob/master/CHANGES.rst diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index ddd84c9..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,4 +0,0 @@ -tox: - build: . - volumes: - - ".:/src:ro" diff --git a/jsonfield/__init__.py b/jsonfield/__init__.py deleted file mode 100644 index 54360e2..0000000 --- a/jsonfield/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .fields import JSONField, JSONCharField # noqa diff --git a/jsonfield/fields.py b/jsonfield/fields.py deleted file mode 100644 index 4150bbf..0000000 --- a/jsonfield/fields.py +++ /dev/null @@ -1,175 +0,0 @@ -import copy -from django.db import models -from django.utils.translation import ugettext_lazy as _ -try: - from django.utils import six -except ImportError: - import six - -try: - import json -except ImportError: - from django.utils import simplejson as json - -from django.forms import fields -try: - from django.forms.utils import ValidationError -except ImportError: - from django.forms.util import ValidationError - -from .subclassing import SubfieldBase -from .encoder import JSONEncoder - - -class JSONFormFieldBase(object): - def __init__(self, *args, **kwargs): - self.load_kwargs = kwargs.pop('load_kwargs', {}) - super(JSONFormFieldBase, self).__init__(*args, **kwargs) - - def to_python(self, value): - if isinstance(value, six.string_types) and value: - try: - return json.loads(value, **self.load_kwargs) - except ValueError: - raise ValidationError(_("Enter valid JSON")) - return value - - def clean(self, value): - - if not value and not self.required: - return None - - # Trap cleaning errors & bubble them up as JSON errors - try: - return super(JSONFormFieldBase, self).clean(value) - except TypeError: - raise ValidationError(_("Enter valid JSON")) - - -class JSONFormField(JSONFormFieldBase, fields.CharField): - pass - - -class JSONCharFormField(JSONFormFieldBase, fields.CharField): - pass - - -class JSONFieldBase(six.with_metaclass(SubfieldBase, models.Field)): - - def __init__(self, *args, **kwargs): - self.dump_kwargs = kwargs.pop('dump_kwargs', { - 'cls': JSONEncoder, - 'separators': (',', ':') - }) - self.load_kwargs = kwargs.pop('load_kwargs', {}) - - super(JSONFieldBase, self).__init__(*args, **kwargs) - - def pre_init(self, value, obj): - """Convert a string value to JSON only if it needs to be deserialized. - - SubfieldBase metaclass has been modified to call this method instead of - to_python so that we can check the obj state and determine if it needs to be - deserialized""" - - try: - if obj._state.adding: - # Make sure the primary key actually exists on the object before - # checking if it's empty. This is a special case for South datamigrations - # see: https://github.com/bradjasper/django-jsonfield/issues/52 - if getattr(obj, "pk", None) is not None: - if isinstance(value, six.string_types): - try: - return json.loads(value, **self.load_kwargs) - except ValueError: - raise ValidationError(_("Enter valid JSON")) - - except AttributeError: - # south fake meta class doesn't create proper attributes - # see this: - # https://github.com/bradjasper/django-jsonfield/issues/52 - pass - - return value - - def to_python(self, value): - """The SubfieldBase metaclass calls pre_init instead of to_python, however to_python - is still necessary for Django's deserializer""" - return value - - def get_prep_value(self, value): - """Convert JSON object to a string""" - if self.null and value is None: - return None - return json.dumps(value, **self.dump_kwargs) - - def value_to_string(self, obj): - value = self.value_from_object(obj, dump=False) - return self.get_db_prep_value(value, None) - - def value_from_object(self, obj, dump=True): - value = super(JSONFieldBase, self).value_from_object(obj) - if self.null and value is None: - return None - return self.dumps_for_display(value) if dump else value - - def dumps_for_display(self, value): - return json.dumps(value, **self.dump_kwargs) - - def formfield(self, **kwargs): - - if "form_class" not in kwargs: - kwargs["form_class"] = self.form_class - - field = super(JSONFieldBase, self).formfield(**kwargs) - - if isinstance(field, JSONFormFieldBase): - field.load_kwargs = self.load_kwargs - - if not field.help_text: - field.help_text = "Enter valid JSON" - - return field - - def get_default(self): - """ - Returns the default value for this field. - - The default implementation on models.Field calls force_unicode - on the default, which means you can't set arbitrary Python - objects as the default. To fix this, we just return the value - without calling force_unicode on it. Note that if you set a - callable as a default, the field will still call it. It will - *not* try to pickle and encode it. - - """ - if self.has_default(): - if callable(self.default): - return self.default() - return copy.deepcopy(self.default) - # If the field doesn't have a default, then we punt to models.Field. - return super(JSONFieldBase, self).get_default() - - -class JSONField(JSONFieldBase, models.TextField): - """JSONField is a generic textfield that serializes/deserializes JSON objects""" - form_class = JSONFormField - - def dumps_for_display(self, value): - kwargs = {"indent": 2} - kwargs.update(self.dump_kwargs) - return json.dumps(value, ensure_ascii=False, **kwargs) - - -class JSONCharField(JSONFieldBase, models.CharField): - """JSONCharField is a generic textfield that serializes/deserializes JSON objects, - stored in the database like a CharField, which enables it to be used - e.g. in unique keys""" - form_class = JSONCharFormField - - -try: - from south.modelsinspector import add_introspection_rules - add_introspection_rules([], ["^jsonfield\.fields\.(JSONField|JSONCharField)"]) -except ImportError: - pass diff --git a/jsonfield/models.py b/jsonfield/models.py deleted file mode 100644 index e5faf1b..0000000 --- a/jsonfield/models.py +++ /dev/null @@ -1 +0,0 @@ -# Django needs this to see it as a project diff --git a/jsonfield/subclassing.py b/jsonfield/subclassing.py deleted file mode 100644 index 49e30e1..0000000 --- a/jsonfield/subclassing.py +++ /dev/null @@ -1,62 +0,0 @@ -# This file was copied from django.db.models.fields.subclassing so that we could -# change the Creator.__set__ behavior. Read the comment below for full details. - -""" -Convenience routines for creating non-trivial Field subclasses, as well as -backwards compatibility utilities. - -Add SubfieldBase as the __metaclass__ for your Field subclass, implement -to_python() and the other necessary methods and everything will work seamlessly. -""" - - -class SubfieldBase(type): - """ - A metaclass for custom Field subclasses. This ensures the model's attribute - has the descriptor protocol attached to it. - """ - def __new__(cls, name, bases, attrs): - new_class = super(SubfieldBase, cls).__new__(cls, name, bases, attrs) - new_class.contribute_to_class = make_contrib( - new_class, attrs.get('contribute_to_class') - ) - return new_class - - -class Creator(object): - """ - A placeholder class that provides a way to set the attribute on the model. - """ - def __init__(self, field): - self.field = field - - def __get__(self, obj, type=None): - if obj is None: - return self - return obj.__dict__[self.field.name] - - def __set__(self, obj, value): - # Usually this would call to_python, but we've changed it to pre_init - # so that we can tell which state we're in. By passing an obj, - # we can definitively tell if a value has already been deserialized - # More: https://github.com/bradjasper/django-jsonfield/issues/33 - obj.__dict__[self.field.name] = self.field.pre_init(value, obj) - - -def make_contrib(superclass, func=None): - """ - Returns a suitable contribute_to_class() method for the Field subclass. - - If 'func' is passed in, it is the existing contribute_to_class() method on - the subclass and it is called before anything else. It is assumed in this - case that the existing contribute_to_class() calls all the necessary - superclass methods. - """ - def contribute_to_class(self, cls, name): - if func: - func(self, cls, name) - else: - super(superclass, self).contribute_to_class(cls, name) - setattr(cls, self.name, Creator(self)) - - return contribute_to_class diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..e9e7fdd --- /dev/null +++ b/manage.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) diff --git a/setup.cfg b/setup.cfg index 2a9acf1..ea461e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,20 @@ -[bdist_wheel] -universal = 1 +[flake8] +max_line_length = 120 +max_complexity = 10 +exclude = migrations + +[isort] +skip = .tox,migrations +atomic = true +line_length = 120 +multi_line_output = 3 +include_trailing_comma = true +known_third_party = django +known_first_party = jsonfield + +[coverage:run] +branch = true +source = jsonfield +omit = + src/jsonfield/encoder.py + tests diff --git a/setup.py b/setup.py index 9a349ae..ce2b9bc 100644 --- a/setup.py +++ b/setup.py @@ -1,55 +1,33 @@ -from distutils.core import Command -from setuptools import setup - - -class TestCommand(Command): - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - from django.conf import settings - settings.configure( - DATABASES={'default': {'NAME': ':memory:', 'ENGINE': 'django.db.backends.sqlite3'}}, - INSTALLED_APPS=('jsonfield', 'django.contrib.contenttypes') - ) - from django.core.management import call_command - import django - - if django.VERSION[:2] >= (1, 7): - django.setup() - call_command('test', 'jsonfield') +from setuptools import find_packages, setup setup( name='jsonfield', - version='2.0.2', - packages=['jsonfield'], + version='3.0.0', license='MIT', include_package_data=True, - author='Dan Koch', - author_email='dmkoch@gmail.com', - url='https://github.com/dmkoch/django-jsonfield/', + author='Brad Jasper', + author_email='contact@bradjasper.com', + maintainer='Ryan P Kilby', + maintainer_email='kilbyr@gmail.com', + url='https://github.com/rpkilby/jsonfield/', description='A reusable Django field that allows you to store validated JSON in your model.', long_description=open("README.rst").read(), - install_requires=['Django >= 1.8.0'], - tests_require=['Django >= 1.8.0'], - cmdclass={'test': TestCommand}, + packages=find_packages('src'), + package_dir={'': 'src'}, + install_requires=['Django >= 2.2'], classifiers=[ 'Environment :: Web Environment', + 'Framework :: Django', + 'Framework :: Django :: 2.2', + 'Framework :: Django :: 3.0', 'Intended Audience :: Developers', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Framework :: Django', ], ) diff --git a/src/jsonfield/__init__.py b/src/jsonfield/__init__.py new file mode 100644 index 0000000..1a0121d --- /dev/null +++ b/src/jsonfield/__init__.py @@ -0,0 +1,3 @@ +from .fields import JSONCharField, JSONField + +__all__ = ['JSONCharField', 'JSONField'] diff --git a/jsonfield/encoder.py b/src/jsonfield/encoder.py similarity index 71% rename from jsonfield/encoder.py rename to src/jsonfield/encoder.py index 4923a90..f87644a 100644 --- a/jsonfield/encoder.py +++ b/src/jsonfield/encoder.py @@ -1,29 +1,28 @@ -from django.db.models.query import QuerySet -from django.utils import six, timezone -from django.utils.encoding import force_text -from django.utils.functional import Promise import datetime import decimal import json import uuid +from django.db.models.query import QuerySet +from django.utils import timezone +from django.utils.encoding import force_str +from django.utils.functional import Promise + class JSONEncoder(json.JSONEncoder): """ JSONEncoder subclass that knows how to encode date/time/timedelta, decimal types, generators and other basic python objects. - Taken from https://github.com/tomchristie/django-rest-framework/blob/master/rest_framework/utils/encoders.py + Taken from https://github.com/tomchristie/django-rest-framework/blob/3.11.0/rest_framework/utils/encoders.py """ - def default(self, obj): # noqa + def default(self, obj): # noqa: C901 # For Date Time string spec, see ECMA 262 - # http://ecma-international.org/ecma-262/5.1/#sec-15.9.1.15 + # https://ecma-international.org/ecma-262/5.1/#sec-15.9.1.15 if isinstance(obj, Promise): - return force_text(obj) + return force_str(obj) elif isinstance(obj, datetime.datetime): representation = obj.isoformat() - if obj.microsecond: - representation = representation[:23] + representation[26:] if representation.endswith('+00:00'): representation = representation[:-6] + 'Z' return representation @@ -33,26 +32,28 @@ def default(self, obj): # noqa if timezone and timezone.is_aware(obj): raise ValueError("JSON can't represent timezone-aware times.") representation = obj.isoformat() - if obj.microsecond: - representation = representation[:12] return representation elif isinstance(obj, datetime.timedelta): - return six.text_type(obj.total_seconds()) + return str(obj.total_seconds()) elif isinstance(obj, decimal.Decimal): # Serializers will coerce decimals to strings by default. return float(obj) elif isinstance(obj, uuid.UUID): - return six.text_type(obj) + return str(obj) elif isinstance(obj, QuerySet): return tuple(obj) + elif isinstance(obj, bytes): + # Best-effort for binary blobs. See #4187. + return obj.decode() elif hasattr(obj, 'tolist'): # Numpy arrays and array scalars. return obj.tolist() elif hasattr(obj, '__getitem__'): + cls = (list if isinstance(obj, (list, tuple)) else dict) try: - return dict(obj) - except: + return cls(obj) + except Exception: pass elif hasattr(obj, '__iter__'): return tuple(item for item in obj) - return super(JSONEncoder, self).default(obj) + return super().default(obj) diff --git a/src/jsonfield/fields.py b/src/jsonfield/fields.py new file mode 100644 index 0000000..c20e400 --- /dev/null +++ b/src/jsonfield/fields.py @@ -0,0 +1,98 @@ +import copy +import json + +from django.db import models +from django.forms import ValidationError +from django.utils.translation import gettext_lazy as _ + +from . import forms +from .encoder import JSONEncoder +from .json import checked_loads + +DEFAULT_DUMP_KWARGS = { + 'cls': JSONEncoder, +} + +DEFAULT_LOAD_KWARGS = {} + + +class JSONFieldMixin(models.Field): + form_class = forms.JSONField + + def __init__(self, *args, dump_kwargs=None, load_kwargs=None, **kwargs): + self.dump_kwargs = DEFAULT_DUMP_KWARGS if dump_kwargs is None else dump_kwargs + self.load_kwargs = DEFAULT_LOAD_KWARGS if load_kwargs is None else load_kwargs + + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + + if self.dump_kwargs != DEFAULT_DUMP_KWARGS: + kwargs['dump_kwargs'] = self.dump_kwargs + if self.load_kwargs != DEFAULT_LOAD_KWARGS: + kwargs['load_kwargs'] = self.load_kwargs + + return name, path, args, kwargs + + def to_python(self, value): + try: + return checked_loads(value, **self.load_kwargs) + except ValueError: + raise ValidationError(_("Enter valid JSON.")) + + def from_db_value(self, value, expression, connection): + if value is None: + return None + return checked_loads(value, **self.load_kwargs) + + def get_prep_value(self, value): + """Convert JSON object to a string""" + if self.null and value is None: + return None + return json.dumps(value, **self.dump_kwargs) + + def value_to_string(self, obj): + value = self.value_from_object(obj) + return json.dumps(value, **self.dump_kwargs) + + def formfield(self, **kwargs): + kwargs.setdefault('form_class', self.form_class) + if issubclass(kwargs['form_class'], forms.JSONField): + kwargs.setdefault('dump_kwargs', self.dump_kwargs) + kwargs.setdefault('load_kwargs', self.load_kwargs) + + return super().formfield(**kwargs) + + def get_default(self): + """ + Returns the default value for this field. + + The default implementation on models.Field calls force_unicode + on the default, which means you can't set arbitrary Python + objects as the default. To fix this, we just return the value + without calling force_unicode on it. Note that if you set a + callable as a default, the field will still call it. It will + *not* try to pickle and encode it. + """ + if self.has_default(): + if callable(self.default): + return self.default() + return copy.deepcopy(self.default) + # If the field doesn't have a default, then we punt to models.Field. + return super().get_default() + + +class JSONField(JSONFieldMixin, models.TextField): + """JSONField is a generic textfield that serializes/deserializes JSON objects""" + + def formfield(self, **kwargs): + field = super().formfield(**kwargs) + if isinstance(field, forms.JSONField): + # Note: TextField sets the Textarea widget + field.dump_kwargs.setdefault('indent', 4) + return field + + +class JSONCharField(JSONFieldMixin, models.CharField): + """JSONCharField is a generic textfield that serializes/deserializes JSON objects""" diff --git a/src/jsonfield/forms.py b/src/jsonfield/forms.py new file mode 100644 index 0000000..9df9d58 --- /dev/null +++ b/src/jsonfield/forms.py @@ -0,0 +1,65 @@ +import json + +from django.forms import ValidationError, fields +from django.utils.translation import gettext_lazy as _ + +from .json import checked_loads + + +class InvalidJSONInput(str): + pass + + +class JSONField(fields.CharField): + default_error_messages = { + 'invalid': _('"%(value)s" value must be valid JSON.'), + } + + def __init__(self, *args, dump_kwargs=None, load_kwargs=None, **kwargs): + self.dump_kwargs = dump_kwargs if dump_kwargs else {} + self.load_kwargs = load_kwargs if load_kwargs else {} + + kwargs.setdefault('help_text', _("Enter valid JSON.")) + super().__init__(*args, **kwargs) + + def to_python(self, value): + if self.disabled: + return value + + if value in self.empty_values: + return None + + try: + return checked_loads(value, **self.load_kwargs) + except json.JSONDecodeError: + raise ValidationError( + self.error_messages['invalid'], + code='invalid', + params={'value': value}, + ) + + def bound_data(self, data, initial): + # Note: This is a bit confusing, as there are multiple things occurring. + # First, the `initial` value is the *unencoded* python object provided + # via the form instance, while `data` is the *encoded* form input. The + # outgoing value needs to be uniform, so we decode `data` here. + # + # Second, it may seem counterintuitive to encode data, just to decode + # it in `prepare_value`. Why not just decode `initial` here? This is + # due to `BoundField.value()`, which only calls `bound_data` when the + # form is bound. If unbound, the `initial` value is provided directly + # to `prepare_value`, and the value would still need to be encoded. + # + # Lastly, we don't want to run `checked_loads` here, since we *know* + # that the input `data` isn't a decoded value (e.g., via `to_python`). + if self.disabled: + return initial + try: + return json.loads(data, **self.load_kwargs) + except json.JSONDecodeError: + return InvalidJSONInput(data) + + def prepare_value(self, value): + if isinstance(value, InvalidJSONInput): + return value + return json.dumps(value, **self.dump_kwargs) diff --git a/src/jsonfield/json.py b/src/jsonfield/json.py new file mode 100644 index 0000000..86f1736 --- /dev/null +++ b/src/jsonfield/json.py @@ -0,0 +1,22 @@ +import json + + +class JSONString(str): + pass + + +def checked_loads(value, **kwargs): + """ + Ensure that values aren't loaded twice, resulting in an encoding error. + + Loaded strings are wrapped in JSONString, as it is otherwise not possible + to differentiate between a loaded and unloaded string. + """ + if isinstance(value, (list, dict, int, float, JSONString, type(None))): + return value + + value = json.loads(value, **kwargs) + if isinstance(value, str): + value = JSONString(value) + + return value diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/migrations/0001_initial.py b/tests/migrations/0001_initial.py new file mode 100644 index 0000000..bc71b85 --- /dev/null +++ b/tests/migrations/0001_initial.py @@ -0,0 +1,82 @@ +# Generated by Django 2.2rc1 on 2019-03-27 22:51 + +import collections +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.encoder +import jsonfield.fields +import tests.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='GenericForeignKeyObj', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, null=True, verbose_name='Foreign Obj')), + ], + ), + migrations.CreateModel( + name='JSONCharModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', jsonfield.fields.JSONCharField(dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={}, max_length=100)), + ('default_json', jsonfield.fields.JSONCharField(default={'check': 34}, dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={}, max_length=100)), + ], + ), + migrations.CreateModel( + name='JSONModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', jsonfield.fields.JSONField(dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={})), + ('default_json', jsonfield.fields.JSONField(default={'check': 12}, dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={})), + ('complex_default_json', jsonfield.fields.JSONField(default=[{'checkcheck': 1212}], dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={})), + ('empty_default', jsonfield.fields.JSONField(blank=True, default={}, dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={})), + ], + ), + migrations.CreateModel( + name='JSONModelCustomEncoders', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', jsonfield.fields.JSONField(dump_kwargs={'cls': tests.models.ComplexEncoder, 'indent': 4}, load_kwargs={'object_hook': tests.models.as_complex})), + ], + ), + migrations.CreateModel( + name='JSONNotRequiredModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', jsonfield.fields.JSONField(blank=True, dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={}, null=True)), + ], + ), + migrations.CreateModel( + name='OrderedJSONModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', jsonfield.fields.JSONField(dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={'object_pairs_hook': collections.OrderedDict})), + ], + ), + migrations.CreateModel( + name='JSONModelWithForeignKey', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', jsonfield.fields.JSONField(dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs={}, null=True)), + ('object_id', models.PositiveIntegerField(blank=True, db_index=True, null=True)), + ('content_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + ), + migrations.CreateModel( + name='RemoteJSONModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('foreign', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tests.JSONModel')), + ], + ), + ] diff --git a/tests/migrations/__init__.py b/tests/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..884991b --- /dev/null +++ b/tests/models.py @@ -0,0 +1,69 @@ +import json +from collections import OrderedDict + +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from jsonfield import JSONCharField, JSONField + + +class ComplexEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, complex): + return { + '__complex__': True, + 'real': obj.real, + 'imag': obj.imag, + } + + return json.JSONEncoder.default(self, obj) + + +def as_complex(dct): + if '__complex__' in dct: + return complex(dct['real'], dct['imag']) + return dct + + +class GenericForeignKeyObj(models.Model): + name = models.CharField('Foreign Obj', max_length=255, null=True) + + +class JSONCharModel(models.Model): + json = JSONCharField(max_length=100) + default_json = JSONCharField(max_length=100, default={"check": 34}) + + +class JSONModel(models.Model): + json = JSONField() + default_json = JSONField(default={"check": 12}) + complex_default_json = JSONField(default=[{"checkcheck": 1212}]) + empty_default = JSONField(default={}, blank=True) + + +class JSONModelCustomEncoders(models.Model): + # A JSON field that can store complex numbers + json = JSONField( + dump_kwargs={'cls': ComplexEncoder, "indent": 4}, + load_kwargs={'object_hook': as_complex}, + ) + + +class JSONModelWithForeignKey(models.Model): + json = JSONField(null=True) + foreign_obj = GenericForeignKey() + object_id = models.PositiveIntegerField(blank=True, null=True, db_index=True) + content_type = models.ForeignKey(ContentType, blank=True, null=True, on_delete=models.CASCADE) + + +class JSONNotRequiredModel(models.Model): + json = JSONField(blank=True, null=True) + + +class OrderedJSONModel(models.Model): + json = JSONField(load_kwargs={'object_pairs_hook': OrderedDict}) + + +class RemoteJSONModel(models.Model): + foreign = models.ForeignKey(JSONModel, blank=True, null=True, on_delete=models.CASCADE) diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..42f1cf3 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,37 @@ +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +SECRET_KEY = 'not-a-secret' + +DEBUG = True + +INSTALLED_APPS = [ + 'django.contrib.contenttypes', + 'tests', +] + +ROOT_URLCONF = [] + + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + }, +} + + +# Internationalization +# https://docs.djangoproject.com/en/dev/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True diff --git a/tests/test_fields.py b/tests/test_fields.py new file mode 100644 index 0000000..76c132d --- /dev/null +++ b/tests/test_fields.py @@ -0,0 +1,72 @@ +import json + +from django.test import TestCase + +from jsonfield.fields import JSONField +from jsonfield.json import JSONString + + +class TestFieldAPIMethods(TestCase): + + def test_get_prep_value_always_json_dumps_if_not_null(self): + json_field_instance = JSONField(null=False) + value = {'a': 1} + prepared_value = json_field_instance.get_prep_value(value) + self.assertIsInstance(prepared_value, str) + self.assertDictEqual(value, json.loads(prepared_value)) + already_json = json.dumps(value) + double_prepared_value = json_field_instance.get_prep_value( + already_json) + self.assertDictEqual(value, + json.loads(json.loads(double_prepared_value))) + self.assertEqual(json_field_instance.get_prep_value(None), 'null') + + def test_get_prep_value_can_return_none_if_null(self): + json_field_instance = JSONField(null=True) + value = {'a': 1} + prepared_value = json_field_instance.get_prep_value(value) + self.assertIsInstance(prepared_value, str) + self.assertDictEqual(value, json.loads(prepared_value)) + already_json = json.dumps(value) + double_prepared_value = json_field_instance.get_prep_value( + already_json) + self.assertDictEqual(value, + json.loads(json.loads(double_prepared_value))) + self.assertIs(json_field_instance.get_prep_value(None), None) + + def test_deconstruct_default_kwargs(self): + field = JSONField() + + _, _, _, kwargs = field.deconstruct() + + self.assertNotIn('dump_kwargs', kwargs) + self.assertNotIn('load_kwargs', kwargs) + + def test_deconstruct_non_default_kwargs(self): + field = JSONField( + dump_kwargs={'separators': (',', ':')}, + load_kwargs={'object_pairs_hook': dict}, + ) + + _, _, _, kwargs = field.deconstruct() + + self.assertEqual(kwargs['dump_kwargs'], {'separators': (',', ':')}) + self.assertEqual(kwargs['load_kwargs'], {'object_pairs_hook': dict}) + + def test_from_db_value_loaded_types(self): + values = [ + # (label, db value, loaded type) + ('object', '{"a": "b"}', dict), + ('array', '[1, 2]', list), + ('string', '"test"', JSONString), + ('float', '1.2', float), + ('int', '1234', int), + ('bool', 'true', bool), + ('null', 'null', type(None)), + ] + + for label, db_value, inst_type in values: + with self.subTest(type=label, db_value=db_value): + value = JSONField().from_db_value(db_value, None, None) + + self.assertIsInstance(value, inst_type) diff --git a/tests/test_forms.py b/tests/test_forms.py new file mode 100644 index 0000000..44b3015 --- /dev/null +++ b/tests/test_forms.py @@ -0,0 +1,103 @@ +from django.forms import ModelForm +from django.test import TestCase + +from .models import JSONNotRequiredModel + + +class JSONModelFormTest(TestCase): + def setUp(self): + class JSONNotRequiredForm(ModelForm): + class Meta: + model = JSONNotRequiredModel + fields = '__all__' + + self.form_class = JSONNotRequiredForm + + def test_blank_form(self): + form = self.form_class(data={'json': ''}) + self.assertFalse(form.has_changed()) + + def test_form_with_data(self): + form = self.form_class(data={'json': '{}'}) + self.assertTrue(form.has_changed()) + + def test_form_save(self): + form = self.form_class(data={'json': ''}) + form.save() + + def test_save_values(self): + values = [ + # (type, form input, db value) + ('object', '{"a": "b"}', {'a': 'b'}), + ('array', '[1, 2]', [1, 2]), + ('string', '"test"', 'test'), + ('float', '1.2', 1.2), + ('int', '1234', 1234), + ('bool', 'true', True), + ('null', 'null', None), + ] + + for vtype, form_input, db_value in values: + with self.subTest(type=vtype, input=form_input, db=db_value): + form = self.form_class(data={'json': form_input}) + self.assertTrue(form.is_valid(), msg=form.errors) + + instance = form.save() + self.assertEqual(instance.json, db_value) + + def test_render_initial_values(self): + values = [ + # (type, db value, form output) + ('object', {'a': 'b'}, '{\n "a": "b"\n}'), + ('array', [1, 2], "[\n 1,\n 2\n]"), + ('string', 'test', '"test"'), + ('float', 1.2, '1.2'), + ('int', 1234, '1234'), + ('bool', True, 'true'), + ('null', None, 'null'), + ] + + for vtype, db_value, form_output in values: + with self.subTest(type=vtype, db=db_value, output=form_output): + instance = JSONNotRequiredModel.objects.create(json=db_value) + + form = self.form_class(instance=instance) + self.assertEqual(form['json'].value(), form_output) + + def test_render_bound_values(self): + values = [ + # (type, db value, form input, form output) + ('object', '{"a": "b"}', '{\n "a": "b"\n}'), + ('array', '[1, 2]', "[\n 1,\n 2\n]"), + ('string', '"test"', '"test"'), + ('float', '1.2', '1.2'), + ('int', '1234', '1234'), + ('bool', 'true', 'true'), + ('null', 'null', 'null'), + ] + + for vtype, form_input, form_output in values: + with self.subTest(type=vtype, input=form_input, output=form_output): + form = self.form_class(data={'json': form_input}) + self.assertEqual(form['json'].value(), form_output) + + def test_invalid_value(self): + form = self.form_class(data={'json': 'foo'}) + + self.assertFalse(form.is_valid()) + self.assertEqual(form.errors, { + 'json': ['"foo" value must be valid JSON.'], + }) + self.assertEqual(form['json'].value(), 'foo') + + def test_disabled_field(self): + instance = JSONNotRequiredModel.objects.create(json=100) + + form = self.form_class(data={'json': '{"foo": "bar"}'}, instance=instance) + form.fields['json'].disabled = True + + self.assertTrue(form.is_valid()) + self.assertEqual(form.cleaned_data, {'json': 100}) + + # rendered value + self.assertEqual(form['json'].value(), '100') diff --git a/jsonfield/tests.py b/tests/test_jsonfield.py similarity index 51% rename from jsonfield/tests.py rename to tests/test_jsonfield.py index 2d0b980..18be627 100644 --- a/jsonfield/tests.py +++ b/tests/test_jsonfield.py @@ -1,88 +1,48 @@ +from collections import OrderedDict from decimal import Decimal + import django -from django import forms from django.core.serializers import deserialize, serialize from django.core.serializers.base import DeserializationError -from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.contenttypes.models import ContentType -from django.db import models +from django.forms import ValidationError from django.test import TestCase -try: - import json -except ImportError: - from django.utils import simplejson as json - -from .fields import JSONField, JSONCharField -try: - from django.forms.utils import ValidationError -except ImportError: - from django.forms.util import ValidationError - -from django.utils.six import string_types - -from collections import OrderedDict - - -class JsonModel(models.Model): - json = JSONField() - default_json = JSONField(default={"check": 12}) - complex_default_json = JSONField(default=[{"checkcheck": 1212}]) - empty_default = JSONField(default={}) - - -class GenericForeignKeyObj(models.Model): - name = models.CharField('Foreign Obj', max_length=255, null=True) - - -class JSONModelWithForeignKey(models.Model): - json = JSONField(null=True) - foreign_obj = GenericForeignKey() - object_id = models.PositiveIntegerField(blank=True, null=True, db_index=True) - content_type = models.ForeignKey(ContentType, blank=True, null=True, - on_delete=models.CASCADE) - - -class JsonCharModel(models.Model): - json = JSONCharField(max_length=100) - default_json = JSONCharField(max_length=100, default={"check": 34}) +from .models import ( + GenericForeignKeyObj, + JSONCharModel, + JSONModel, + JSONModelCustomEncoders, + JSONModelWithForeignKey, + JSONNotRequiredModel, + OrderedJSONModel, + RemoteJSONModel, +) -class ComplexEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, complex): - return { - '__complex__': True, - 'real': obj.real, - 'imag': obj.imag, - } - return json.JSONEncoder.default(self, obj) +class JSONModelWithForeignKeyTestCase(TestCase): + def test_object_create(self): + foreign_obj = GenericForeignKeyObj.objects.create(name='Brain') + JSONModelWithForeignKey.objects.create(foreign_obj=foreign_obj) -def as_complex(dct): - if '__complex__' in dct: - return complex(dct['real'], dct['imag']) - return dct +class RemoteJSONFieldTests(TestCase): + """Test JSON fields across a ForeignKey""" + @classmethod + def setUpTestData(cls): + RemoteJSONModel.objects.create() -class JSONModelCustomEncoders(models.Model): - # A JSON field that can store complex numbers - json = JSONField( - dump_kwargs={'cls': ComplexEncoder, "indent": 4}, - load_kwargs={'object_hook': as_complex}, - ) + def test_related_accessor(self): + RemoteJSONModel.objects.get().foreign - -class JSONModelWithForeignKeyTestCase(TestCase): - def test_object_create(self): - foreign_obj = GenericForeignKeyObj.objects.create(name='Brain') - JSONModelWithForeignKey.objects.create(foreign_obj=foreign_obj) + def test_select_related(self): + RemoteJSONModel.objects.select_related('foreign').get() class JSONFieldTest(TestCase): """JSONField Wrapper Tests""" - json_model = JsonModel + json_model = JSONModel def test_json_field_create(self): """Test saving a JSON object in our JSONField""" @@ -183,7 +143,7 @@ def test_django_serializers(self): 'dict': {'k': 'v'}}]: obj = self.json_model.objects.create(json=json_obj) new_obj = self.json_model.objects.get(id=obj.id) - self.assert_(new_obj) + self.assertTrue(new_obj) queryset = self.json_model.objects.all() ser = serialize('json', queryset) @@ -194,7 +154,7 @@ def test_django_serializers(self): def test_default_parameters(self): """Test providing a default value to the model""" - model = JsonModel() + model = JSONModel() model.json = {"check": 12} self.assertEqual(model.json, {"check": 12}) self.assertEqual(type(model.json), dict) @@ -204,17 +164,18 @@ def test_default_parameters(self): def test_invalid_json(self): # invalid json data {] in the json and default_json fields - ser = '[{"pk": 1, "model": "jsonfield.jsoncharmodel", ' \ + ser = '[{"pk": 1, "model": "tests.jsoncharmodel", ' \ '"fields": {"json": "{]", "default_json": "{]"}}]' with self.assertRaises(DeserializationError) as cm: next(deserialize('json', ser)) - # Django 2.0+ uses PEP 3134 exception chaining + # Django 2 does not reraise DeserializationError as another DeserializationError + # Changed in: https://github.com/django/django/pull/7878 if django.VERSION < (2, 0,): - inner = cm.exception.args[0] + inner = cm.exception.__context__.__context__ else: inner = cm.exception.__context__ - self.assertTrue(isinstance(inner, ValidationError)) - self.assertEqual('Enter valid JSON', inner.messages[0]) + self.assertIsInstance(inner, ValidationError) + self.assertEqual('Enter valid JSON.', inner.messages[0]) def test_integer_in_string_in_json_field(self): """Test saving the Python string '123' in our JSONField""" @@ -234,7 +195,7 @@ def test_boolean_in_string_in_json_field(self): def test_pass_by_reference_pollution(self): """Make sure the default parameter is copied rather than passed by reference""" - model = JsonModel() + model = JSONModel() model.default_json["check"] = 144 model.complex_default_json[0]["checkcheck"] = 144 self.assertEqual(model.default_json["check"], 144) @@ -242,44 +203,51 @@ def test_pass_by_reference_pollution(self): # Make sure when we create a new model, it resets to the default value # and not to what we just set it to (it would be if it were passed by reference) - model = JsonModel() + model = JSONModel() self.assertEqual(model.default_json["check"], 12) self.assertEqual(model.complex_default_json[0]["checkcheck"], 1212) def test_normal_regex_filter(self): """Make sure JSON model can filter regex""" - JsonModel.objects.create(json={"boom": "town"}) - JsonModel.objects.create(json={"move": "town"}) - JsonModel.objects.create(json={"save": "town"}) + JSONModel.objects.create(json={"boom": "town"}) + JSONModel.objects.create(json={"move": "town"}) + JSONModel.objects.create(json={"save": "town"}) - self.assertEqual(JsonModel.objects.count(), 3) + self.assertEqual(JSONModel.objects.count(), 3) - self.assertEqual(JsonModel.objects.filter(json__regex=r"boom").count(), 1) - self.assertEqual(JsonModel.objects.filter(json__regex=r"town").count(), 3) + self.assertEqual(JSONModel.objects.filter(json__regex=r"boom").count(), 1) + self.assertEqual(JSONModel.objects.filter(json__regex=r"town").count(), 3) def test_save_blank_object(self): """Test that JSON model can save a blank object as none""" - model = JsonModel() + model = JSONModel() self.assertEqual(model.empty_default, {}) model.save() self.assertEqual(model.empty_default, {}) - model1 = JsonModel(empty_default={"hey": "now"}) + model1 = JSONModel(empty_default={"hey": "now"}) self.assertEqual(model1.empty_default, {"hey": "now"}) model1.save() self.assertEqual(model1.empty_default, {"hey": "now"}) + def test_model_full_clean(self): + instances = [ + JSONNotRequiredModel(), + JSONModel(json={'a': 'b'}), + ] -class JSONCharFieldTest(JSONFieldTest): - json_model = JsonCharModel + for instance in instances: + with self.subTest(instance=instance): + instance.full_clean() + instance.save() -class OrderedJsonModel(models.Model): - json = JSONField(load_kwargs={'object_pairs_hook': OrderedDict}) +class JSONCharFieldTest(JSONFieldTest): + json_model = JSONCharModel class OrderedDictSerializationTest(TestCase): @@ -294,99 +262,12 @@ def setUp(self): ]) self.expected_key_order = ['number', 'notes', 'alpha', 'romeo', 'juliet', 'bravo'] - def test_ordered_dict_differs_from_normal_dict(self): - self.assertEqual(list(self.ordered_dict.keys()), self.expected_key_order) - self.assertNotEqual(dict(self.ordered_dict).keys(), self.expected_key_order) - - def test_default_behaviour_loses_sort_order(self): - mod = JsonModel.objects.create(json=self.ordered_dict) - self.assertEqual(list(mod.json.keys()), self.expected_key_order) - mod_from_db = JsonModel.objects.get(id=mod.id) - - # mod_from_db lost ordering information during json.loads() - self.assertNotEqual(mod_from_db.json.keys(), self.expected_key_order) - - def test_load_kwargs_hook_does_not_lose_sort_order(self): - mod = OrderedJsonModel.objects.create(json=self.ordered_dict) - self.assertEqual(list(mod.json.keys()), self.expected_key_order) - mod_from_db = OrderedJsonModel.objects.get(id=mod.id) - self.assertEqual(list(mod_from_db.json.keys()), self.expected_key_order) - - -class JsonNotRequiredModel(models.Model): - json = JSONField(blank=True, null=True) - - -class JsonNotRequiredForm(forms.ModelForm): - class Meta: - model = JsonNotRequiredModel - fields = '__all__' - - -class JsonModelFormTest(TestCase): - def test_blank_form(self): - form = JsonNotRequiredForm(data={'json': ''}) - self.assertFalse(form.has_changed()) - - def test_form_with_data(self): - form = JsonNotRequiredForm(data={'json': '{}'}) - self.assertTrue(form.has_changed()) - - -class TestFieldAPIMethods(TestCase): - def test_get_db_prep_value_method_with_null(self): - json_field_instance = JSONField(null=True) - value = {'a': 1} - prepared_value = json_field_instance.get_db_prep_value( - value, connection=None, prepared=False) - self.assertIsInstance(prepared_value, string_types) - self.assertDictEqual(value, json.loads(prepared_value)) - self.assertIs(json_field_instance.get_db_prep_value( - None, connection=None, prepared=True), None) - self.assertIs(json_field_instance.get_db_prep_value( - None, connection=None, prepared=False), None) - - def test_get_db_prep_value_method_with_not_null(self): - json_field_instance = JSONField(null=False) - value = {'a': 1} - prepared_value = json_field_instance.get_db_prep_value( - value, connection=None, prepared=False) - self.assertIsInstance(prepared_value, string_types) - self.assertDictEqual(value, json.loads(prepared_value)) - self.assertIs(json_field_instance.get_db_prep_value( - None, connection=None, prepared=True), None) - self.assertEqual(json_field_instance.get_db_prep_value( - None, connection=None, prepared=False), 'null') - - def test_get_db_prep_value_method_skips_prepared_values(self): - json_field_instance = JSONField(null=False) - value = {'a': 1} - prepared_value = json_field_instance.get_db_prep_value( - value, connection=None, prepared=True) - self.assertIs(prepared_value, value) - - def test_get_prep_value_always_json_dumps_if_not_null(self): - json_field_instance = JSONField(null=False) - value = {'a': 1} - prepared_value = json_field_instance.get_prep_value(value) - self.assertIsInstance(prepared_value, string_types) - self.assertDictEqual(value, json.loads(prepared_value)) - already_json = json.dumps(value) - double_prepared_value = json_field_instance.get_prep_value( - already_json) - self.assertDictEqual(value, - json.loads(json.loads(double_prepared_value))) - self.assertEqual(json_field_instance.get_prep_value(None), 'null') - - def test_get_prep_value_can_return_none_if_null(self): - json_field_instance = JSONField(null=True) - value = {'a': 1} - prepared_value = json_field_instance.get_prep_value(value) - self.assertIsInstance(prepared_value, string_types) - self.assertDictEqual(value, json.loads(prepared_value)) - already_json = json.dumps(value) - double_prepared_value = json_field_instance.get_prep_value( - already_json) - self.assertDictEqual(value, - json.loads(json.loads(double_prepared_value))) - self.assertIs(json_field_instance.get_prep_value(None), None) + self.instance = OrderedJSONModel.objects.create(json=self.ordered_dict) + + def test_load_kwargs_hook(self): + from_db = OrderedJSONModel.objects.get(id=self.instance.id) + + # OrderedJSONModel explicitly sets `object_pairs_hook` to `OrderedDict` + self.assertEqual(list(self.instance.json), self.expected_key_order) + self.assertEqual(list(from_db.json), self.expected_key_order) + self.assertIsInstance(from_db.json, OrderedDict) diff --git a/tox.ini b/tox.ini index b774d21..93aa33e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,33 +1,35 @@ [tox] envlist = - py27-{1.8,1.9,1.10,1.11}, - py33-{1.8}, - py34-{1.8,1.9,1.10,1.11}, - py35-{1.8,1.9,1.10,1.11,master}, - py36-{1.11,master} -skipsdist = {env:TOXBUILD:false} + py{36,37,38}-django22, + py{36,37,38}-django30, + isort,lint,dist,warnings, [testenv] -deps = - coverage==4.3.4 - flake8==3.3.0 - 1.8: Django>=1.8,<1.9 - 1.9: Django>=1.9,<1.10 - 1.10: Django>=1.10,<1.11 - 1.11: Django>=1.11,<2.0 - master: https://github.com/django/django/tarball/master +commands = coverage run --parallel-mode manage.py test {posargs} +usedevelop = True +envdir={toxworkdir}/v/{envname} setenv = - LANG=en_US.UTF-8 - LANGUAGE=en_US:en - LC_ALL=en_US.UTF-8 -whitelist_externals = - true -commands = - {env:TOXBUILD:flake8 jsonfield} - {env:TOXBUILD:coverage run setup.py test} + PYTHONDONTWRITEBYTECODE=1 +deps = + coverage + django22: Django~=2.2.0 + django30: Django~=3.0.0 + +[testenv:isort] +commands = isort --check-only --recursive jsonfield tests {posargs:--diff} +deps = + isort -[flake8] -# ignore = E265,E501 -max-line-length = 115 -max-complexity = 10 -# exclude = migrations/*,docs/* +[testenv:lint] +commands = flake8 jsonfield tests {posargs} +deps = + flake8 + +[testenv:dist] +commands = python manage.py test {posargs} +usedevelop = False + +[testenv:warnings] +commands = python -Werror manage.py test {posargs} +deps = + https://github.com/django/django/archive/master.tar.gz