diff --git a/.gitignore b/.gitignore index dc1792933..aa0bc0a60 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ playhouse/tests/peewee_test.db .idea/ MANIFEST peewee_test.db +closure.so diff --git a/CHANGELOG.md b/CHANGELOG.md index f5b0b52f1..0245bdfb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,82 @@ releases, visit GitHub: https://github.com/coleifer/peewee/releases +## 2.8.8 + +This release contains a single important bugfix for a regression in specifying +the type of lock to use when opening a SQLite transaction. + +[View commits](https://github.com/coleifer/peewee/compare/2.8.7...2.8.8) + +## 2.8.7 + +This release contains numerous cleanups. + +### Bugs fixed + +* #1087 - Fixed a misuse of the iteration protocol in the `sqliteq` extension. +* Ensure that driver exceptions are wrapped when calling `commit` and + `rollback`. +* #1096 - Fix representation of recursive foreign key relations when using the + `model_to_dict` helper. +* #1126 - Allow `pskel` to be installed into `bin` directory. +* #1105 - Added a `Tuple()` type to Peewee to enable expressing arbitrary + tuple expressions in SQL. +* #1133 - Fixed bug in the conversion of objects to `Decimal` instances in the + `DecimalField`. +* Fixed an issue renaming a unique foreign key in MySQL. +* Remove the join predicate from CROSS JOINs. +* #1148 - Ensure indexes are created when a column is added using a schema + migration. +* #1165 - Fix bug where the primary key was being overwritten in queries using + the closure-table extension. + +### New stuff + +* Added properties to the `SqliteExtDatabase` to expose common `PRAGMA` + settings. For example, to set the cache size to 4MB, `db.cache_size = 1000`. +* Clarified documentation on calling `commit()` or `rollback()` from within the + scope of an atomic block. [See docs](http://docs.peewee-orm.com/en/latest/peewee/transactions.html#transactions). +* Allow table creation dependencies to be specified using new `depends_on` meta + option. Refs #1076. +* Allow specification of the lock type used in SQLite transactions. Previously + this behavior was only present in `playhouse.sqlite_ext.SqliteExtDatabase`, + but it now exists in `peewee.SqliteDatabase`. +* Added support for `CROSS JOIN` expressions in select queries. +* Docs on how to implement [optimistic locking](http://docs.peewee-orm.com/en/latest/peewee/hacks.html#optimistic-locking). +* Documented optional dependencies. +* Generic support for specifying select queries as locking the selected rows + `FOR X`, e.g. `FOR UPDATE` or `FOR SHARE`. +* Support for specifying the frame-of-reference in window queries, e.g. + specifying `UNBOUNDED PRECEDING`, etc. [See docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Window). + +### Backwards-incompatible changes + +* As of 9e76c99, an `OperationalError` is raised if the user calls `connect()` + on an already-open Database object. Previously, the existing connection would + remain open and a new connection would overwrite it, making it impossible to + close the previous connection. If you find this is causing breakage in your + application, you can switch the `connect()` call to `get_conn()` which will + only open a connection if necessary. The error **is** indicative of a real + issue, though, so audit your code for places where you may be opening a + connection without closing it (module-scope operations, e.g.). + +[View commits](https://github.com/coleifer/peewee/compare/2.8.5...2.8.7) + +## 2.8.6 + +This release was later removed due to containing a bug. See notes on 2.8.7. + +## 2.8.5 + +This release contains two small bugfixes. + +* #1081 - fixed the use of parentheses in compound queries on MySQL. +* Fixed some grossness in a helper function used by `prefetch` that was + clearing out the `GROUP BY` and `HAVING` clauses of sub-queries. + +[View commits](https://github.com/coleifer/peewee/compare/2.8.4...2.8.5) + ## 2.8.4 This release contains bugfixes as well as a new playhouse extension module for @@ -18,6 +94,8 @@ As a miscellaneous note, I did some major refactoring and cleanup in `ExtQueryResultsWrapper` and it's corollary in the `speedups` module. The code is much easier to read than before. +[View commits](https://github.com/coleifer/peewee/compare/2.8.3...2.8.4) + ### Bugs fixed * #1061 - @akrs patched a bug in `TimestampField` which affected the accuracy diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 077e62493..3b9a2a603 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -310,30 +310,6 @@ Models last_name='Lennon', defaults={'birthday': datetime.date(1940, 10, 9)}) - .. py:classmethod:: create_or_get([**kwargs]) - - :param kwargs: Field name to value for attempting to create a new instance. - :returns: A 2-tuple containing the model instance and a boolean indicating whether the instance was created. - - This function attempts to create a model instance based on the provided kwargs. If an ``IntegrityError`` occurs indicating the violation of a constraint, then Peewee will return the model matching the filters. - - .. note:: Peewee will not attempt to match *all* the kwargs when an ``IntegrityError`` occurs. Rather, only primary key fields or fields that have a unique constraint will be used to retrieve the matching instance. - - .. note:: Use care when calling ``create_or_get`` with ``autocommit=False``, as the ``create_or_get()`` method will call :py:meth:`Database.atomic` to create either a transaction or savepoint. - - Example: - - .. code-block:: python - - # This will succeed, there is no user named 'charlie' currently. - charlie, created = User.create_or_get(username='charlie') - - # This will return the above object, since an IntegrityError occurs - # when trying to create an object using "charlie's" primary key. - user2, created = User.create_or_get(username='foo', id=charlie.id) - - assert user2.username == 'charlie' - .. py:classmethod:: alias() :rtype: :py:class:`ModelAlias` instance diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index b0fff57e0..13ffef054 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -716,7 +716,7 @@ sqlite_ext API notes .. _sqlite_closure: -.. py:function:: ClosureTable(model_class[, foreign_key=None]) +.. py:function:: ClosureTable(model_class[, foreign_key=None[, referencing_class=None, referencing_key=None]]) Factory function for creating a model class suitable for working with a `transitive closure `_ table. Closure tables are :py:class:`VirtualModel` subclasses that work with the transitive closure SQLite extension. These special tables are designed to make it easy to efficiently query heirarchical data. The SQLite extension manages an AVL tree behind-the-scenes, transparently updating the tree when your table changes and making it easy to perform common queries on heirarchical data. @@ -735,7 +735,7 @@ sqlite_ext API notes $ gcc -g -fPIC -shared closure.c -o closure.so - 3. Create a model for your heirarchical data. The only requirement here is that the model have an integer primary key and a self-referential foreign key. Any additional fields are fine. + 3. Create a model for your hierarchical data. The only requirement here is that the model has an integer primary key and a self-referential foreign key. Any additional fields are fine. .. code-block:: python @@ -747,6 +747,27 @@ sqlite_ext API notes # Generate a model for the closure virtual table. CategoryClosure = ClosureTable(Category) + The self-referentiality can also be achieved via an intermediate table (for a many-to-many relation). + + .. code-block:: python + + class User(Model): + name = CharField() + + class UserRelations(Model): + user = ForeignKeyField(User) + knows = ForeignKeyField(User, related_name='_known_by') + + class Meta: + primary_key = CompositeKey('user', 'knows') # Alternatively, a unique index on both columns. + + # Generate a model for the closure virtual table, specifying the UserRelations as the referencing table + UserClosure = ClosureTable( + User, + referencing_class=UserRelations, + foreign_key=UserRelations.knows, + referencing_key=UserRelations.user) + 4. In your application code, make sure you load the extension when you instantiate your :py:class:`Database` object. This is done by passing the path to the shared library to the :py:meth:`~SqliteExtDatabase.load_extension` method. .. code-block:: python @@ -756,6 +777,8 @@ sqlite_ext API notes :param model_class: The model class containing the nodes in the tree. :param foreign_key: The self-referential parent-node field on the model class. If not provided, peewee will introspect the model to find a suitable key. + :param referencing_class: The intermediate table for a many-to-many relationship. + :param referencing_key: For a many-to-many relationship: the originating side of the relation. :return: Returns a :py:class:`VirtualModel` for working with a closure table. .. warning:: There are two caveats you should be aware of when using the ``transitive_closure`` extension. First, it requires that your *source model* have an integer primary key. Second, it is strongly recommended that you create an index on the self-referential foreign key. diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index cd59c5562..e8e1b189a 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -277,12 +277,16 @@ For more information, see the documentation on: Create or get ------------- -Peewee has two methods for performing "get/create" type operations: +Peewee has one helper method for performing "get/create" type operations: -* :py:meth:`Model.create_or_get`, which will attempt to create a new row. If an ``IntegrityError`` occurs indicating the violation of a constraint, then Peewee will attempt to get the object instead. * :py:meth:`Model.get_or_create`, which first attempts to retrieve the matching row. Failing that, a new row will be created. -Let's say we wish to implement registering a new user account using the :ref:`example User model `. The *User* model has a *unique* constraint on the username field, so we will rely on the database's integrity guarantees to ensure we don't end up with duplicate usernames: +For "create or get" type logic, typically one would rely on a *unique* constraint +or primary key to prevent the creation of duplicate objects. As an example, let's +say we wish to implement registering a new user account using +the :ref:`example User model `. The *User* model has a *unique* +constraint on the username field, so we will rely on the database's integrity +guarantees to ensure we don't end up with duplicate usernames: .. code-block:: python @@ -294,15 +298,17 @@ Let's say we wish to implement registering a new user account using the :ref:`ex # making it safe to call .get(). return User.get(User.username == username) -Rather than writing all this code, you can instead call either :py:meth:`~Model.create_or_get`: +You can easily encapsulate this type of logic as a ``classmethod`` on your own +``Model`` classes. -.. code-block:: python - - user, created = User.create_or_get(username=username) - -The above example first attempts at creation, then falls back to retrieval, relying on the database to enforce a unique constraint. - -If you prefer to attempt to retrieve the record first, you can use :py:meth:`~Model.get_or_create`. This method is implemented along the same lines as the Django function of the same name. You can use the Django-style keyword argument filters to specify your ``WHERE`` conditions. The function returns a 2-tuple containing the instance and a boolean value indicating if the object was created. +The above example first attempts at creation, then falls back to retrieval, +relying on the database to enforce a unique constraint. If you prefer to +attempt to retrieve the record first, you can use +:py:meth:`~Model.get_or_create`. This method is implemented along the same +lines as the Django function of the same name. You can use the Django-style +keyword argument filters to specify your ``WHERE`` conditions. The function +returns a 2-tuple containing the instance and a boolean value indicating if the +object was created. Here is how you might implement user account creation using :py:meth:`~Model.get_or_create`: @@ -310,7 +316,10 @@ Here is how you might implement user account creation using :py:meth:`~Model.get user, created = User.get_or_create(username=username) -Suppose we have a different model ``Person`` and would like to get or create a person object. The only conditions we care about when retrieving the ``Person`` are their first and last names, **but** if we end up needing to create a new record, we will also specify their date-of-birth and favorite color: +Suppose we have a different model ``Person`` and would like to get or create a +person object. The only conditions we care about when retrieving the ``Person`` +are their first and last names, **but** if we end up needing to create a new +record, we will also specify their date-of-birth and favorite color: .. code-block:: python @@ -319,9 +328,11 @@ Suppose we have a different model ``Person`` and would like to get or create a p last_name=last_name, defaults={'dob': dob, 'favorite_color': 'green'}) -Any keyword argument passed to :py:meth:`~Model.get_or_create` will be used in the ``get()`` portion of the logic, except for the ``defaults`` dictionary, which will be used to populate values on newly-created instances. +Any keyword argument passed to :py:meth:`~Model.get_or_create` will be used in +the ``get()`` portion of the logic, except for the ``defaults`` dictionary, +which will be used to populate values on newly-created instances. -For more details check out the documentation for :py:meth:`Model.create_or_get` and :py:meth:`Model.get_or_create`. +For more details check out the documentation for :py:meth:`Model.get_or_create`. Selecting multiple records -------------------------- diff --git a/peewee.py b/peewee.py index 5abc780db..9e5ff13da 100644 --- a/peewee.py +++ b/peewee.py @@ -41,7 +41,7 @@ from functools import wraps from inspect import isclass -__version__ = '2.8.8' +__version__ = '2.9.0' __all__ = [ 'BareField', 'BigIntegerField', @@ -2822,11 +2822,13 @@ def convert_dict_to_node(self, qdict): if '__' in key and key.rsplit('__', 1)[1] in DJANGO_MAP: key, op = key.rsplit('__', 1) op = DJANGO_MAP[op] + elif value is None: + op = OP.IS else: op = OP.EQ for piece in key.split('__'): model_attr = getattr(curr, piece) - if isinstance(model_attr, relationship): + if value is not None and isinstance(model_attr, relationship): curr = model_attr.rel_model joins.append(model_attr) accum.append(Expression(model_attr, op, value)) @@ -4915,19 +4917,6 @@ def get_or_create(cls, **kwargs): except cls.DoesNotExist: raise exc - @classmethod - def create_or_get(cls, **kwargs): - try: - with cls._meta.database.atomic(): - return cls.create(**kwargs), True - except IntegrityError: - query = [] # TODO: multi-column unique constraints. - for field_name, value in kwargs.items(): - field = getattr(cls, field_name) - if field.unique or field.primary_key: - query.append(field == value) - return cls.get(*query), False - @classmethod def filter(cls, *dq, **query): return cls.select().filter(*dq, **query) diff --git a/playhouse/sqlite_ext.py b/playhouse/sqlite_ext.py index 0ad0735a6..2eebb4113 100644 --- a/playhouse/sqlite_ext.py +++ b/playhouse/sqlite_ext.py @@ -667,8 +667,12 @@ class Meta: return getattr(cls, attr) -def ClosureTable(model_class, foreign_key=None): +def ClosureTable(model_class, foreign_key=None, referencing_class=None, + referencing_key=None): """Model factory for the transitive closure extension.""" + if referencing_class is None: + referencing_class = model_class + if foreign_key is None: for field_obj in model_class._meta.rel.values(): if field_obj.rel_model is model_class: @@ -677,13 +681,15 @@ def ClosureTable(model_class, foreign_key=None): else: raise ValueError('Unable to find self-referential foreign key.') - primary_key = model_class._meta.primary_key + source_key = model_class._meta.primary_key + if referencing_key is None: + referencing_key = source_key class BaseClosureTable(VirtualModel): depth = VirtualIntegerField() id = VirtualIntegerField() - idcolumn = VirtualIntegerField() - parentcolumn = VirtualIntegerField() + idcolumn = VirtualCharField() + parentcolumn = VirtualCharField() root = VirtualIntegerField() tablename = VirtualCharField() @@ -694,7 +700,7 @@ class Meta: def descendants(cls, node, depth=None, include_node=False): query = (model_class .select(model_class, cls.depth.alias('depth')) - .join(cls, on=(primary_key == cls.id)) + .join(cls, on=(source_key == cls.id)) .where(cls.root == node) .naive()) if depth is not None: @@ -707,7 +713,7 @@ def descendants(cls, node, depth=None, include_node=False): def ancestors(cls, node, depth=None, include_node=False): query = (model_class .select(model_class, cls.depth.alias('depth')) - .join(cls, on=(primary_key == cls.root)) + .join(cls, on=(source_key == cls.root)) .where(cls.id == node) .naive()) if depth: @@ -718,17 +724,33 @@ def ancestors(cls, node, depth=None, include_node=False): @classmethod def siblings(cls, node, include_node=False): - fk_value = node._data.get(foreign_key.name) - query = model_class.select().where(foreign_key == fk_value) + if referencing_class is model_class: + # self-join + fk_value = node._data.get(foreign_key.name) + query = model_class.select().where(foreign_key == fk_value) + else: + # siblings as given in reference_class + siblings = (referencing_class + .select(referencing_key) + .join(cls, on=(foreign_key == cls.root)) + .where((cls.id == node) & (cls.depth == 1))) + + # the according models + query = (model_class + .select() + .where(source_key << siblings) + .naive()) + if not include_node: - query = query.where(primary_key != node) + query = query.where(source_key != node) + return query class Meta: - database = model_class._meta.database + database = referencing_class._meta.database extension_options = { - 'tablename': model_class._meta.db_table, - 'idcolumn': model_class._meta.primary_key.db_column, + 'tablename': referencing_class._meta.db_table, + 'idcolumn': referencing_key.db_column, 'parentcolumn': foreign_key.db_column} primary_key = False diff --git a/playhouse/tests/test_models.py b/playhouse/tests/test_models.py index c43097b9a..b65012e41 100644 --- a/playhouse/tests/test_models.py +++ b/playhouse/tests/test_models.py @@ -853,68 +853,6 @@ def integrity_error(): self.assertEqual(GCModel.select().count(), 2) - def test_create_or_get(self): - assertQC = partial(self.assertQueryCount, ignore_txn=True) - - with assertQC(1): - user, new = User.create_or_get(username='charlie') - - self.assertTrue(user.id is not None) - self.assertTrue(new) - - with assertQC(2): - user_get, new = User.create_or_get(username='peewee', id=user.id) - - self.assertFalse(new) - self.assertEqual(user_get.id, user.id) - self.assertEqual(user_get.username, 'charlie') - self.assertEqual(User.select().count(), 1) - - # Test with a unique model. - with assertQC(1): - um, new = UniqueMultiField.create_or_get( - name='baby huey', - field_a='fielda', - field_b=1) - - self.assertTrue(new) - self.assertEqual(um.name, 'baby huey') - self.assertEqual(um.field_a, 'fielda') - self.assertEqual(um.field_b, 1) - - with assertQC(2): - um_get, new = UniqueMultiField.create_or_get( - name='baby huey', - field_a='fielda-modified', - field_b=2) - - self.assertFalse(new) - self.assertEqual(um_get.id, um.id) - self.assertEqual(um_get.name, um.name) - self.assertEqual(um_get.field_a, um.field_a) - self.assertEqual(um_get.field_b, um.field_b) - self.assertEqual(UniqueMultiField.select().count(), 1) - - # Test with a non-integer primary key model. - with assertQC(1): - nm, new = NonIntModel.create_or_get( - pk='1337', - data='sweet mickey') - - self.assertTrue(new) - self.assertEqual(nm.pk, '1337') - self.assertEqual(nm.data, 'sweet mickey') - - with assertQC(2): - nm_get, new = NonIntModel.create_or_get( - pk='1337', - data='michael-nuggie') - - self.assertFalse(new) - self.assertEqual(nm_get.pk, nm.pk) - self.assertEqual(nm_get.data, nm.data) - self.assertEqual(NonIntModel.select().count(), 1) - def test_peek(self): users = User.create_users(3) diff --git a/playhouse/tests/test_sqlite_ext.py b/playhouse/tests/test_sqlite_ext.py index 8e5803623..cc0f3d202 100644 --- a/playhouse/tests/test_sqlite_ext.py +++ b/playhouse/tests/test_sqlite_ext.py @@ -26,6 +26,8 @@ CLOSURE_EXTENSION = os.environ.get('CLOSURE_EXTENSION') +if not CLOSURE_EXTENSION and os.path.exists('closure.so'): + CLOSURE_EXTENSION = 'closure.so' FTS5_EXTENSION = FTS5Model.fts5_installed() @@ -1028,6 +1030,61 @@ def test_clean_query(self): self.assertEqual(FTS5Model.clean_query(inval, '_'), outval) +@skip_if(lambda: not CLOSURE_EXTENSION) +class TestTransitiveClosureManyToMany(PeeweeTestCase): + def setUp(self): + super(TestTransitiveClosureManyToMany, self).setUp() + ext_db.load_extension(CLOSURE_EXTENSION.rstrip('.so')) + ext_db.close() + + def tearDown(self): + super(TestTransitiveClosureManyToMany, self).tearDown() + ext_db.unload_extension(CLOSURE_EXTENSION.rstrip('.so')) + ext_db.close() + + def test_manytomany(self): + class Person(BaseExtModel): + name = CharField() + + class Relationship(BaseExtModel): + person = ForeignKeyField(Person) + relation = ForeignKeyField(Person, related_name='related_to') + + PersonClosure = ClosureTable( + Person, + referencing_class=Relationship, + foreign_key=Relationship.relation, + referencing_key=Relationship.person) + + ext_db.drop_tables([Person, Relationship, PersonClosure], safe=True) + ext_db.create_tables([Person, Relationship, PersonClosure]) + + c = Person.create(name='charlie') + m = Person.create(name='mickey') + h = Person.create(name='huey') + z = Person.create(name='zaizee') + Relationship.create(person=c, relation=h) + Relationship.create(person=c, relation=m) + Relationship.create(person=h, relation=z) + Relationship.create(person=h, relation=m) + + def assertPeople(query, expected): + self.assertEqual(sorted([p.name for p in query]), expected) + + PC = PersonClosure + assertPeople(PC.descendants(c), []) + assertPeople(PC.ancestors(c), ['huey', 'mickey', 'zaizee']) + assertPeople(PC.siblings(c), ['huey']) + + assertPeople(PC.descendants(h), ['charlie']) + assertPeople(PC.ancestors(h), ['mickey', 'zaizee']) + assertPeople(PC.siblings(h), ['charlie']) + + assertPeople(PC.descendants(z), ['charlie', 'huey']) + assertPeople(PC.ancestors(z), []) + assertPeople(PC.siblings(z), []) + + @skip_if(lambda: not CLOSURE_EXTENSION) class TestTransitiveClosureIntegration(PeeweeTestCase): tree = {