From 2c5a825776ceb65a9a07ac0f984a9200477d86a1 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 29 May 2018 15:49:22 -0500 Subject: [PATCH 01/58] More examples of sqlite JSONField.update(). --- docs/peewee/sqlite_ext.rst | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index d40843115..a054f9c66 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -332,11 +332,23 @@ APIs >>> KV.get(KV.value['k1'] == 'v1').key 'a' - It's possible to update a JSON value in-place using the :py:meth:`~JSONField.update` method: + It's possible to update a JSON value in-place using the :py:meth:`~JSONField.update` + method. Note that "k1=v1" is preserved: .. code-block:: pycon - >>> KV.update(value=KV.value.update({'k1': 'v1-x', 'k2': 'v2'})).execute() + >>> KV.update(value=KV.value.update({'k2': 'v2', 'k3': 'v3'})).execute() + 1 + >>> KV.get(KV.key == 'a').value + {'k1': 'v1', 'k2': 'v2', 'k3': 'v3'} + + We can also update existing data atomically, or remove keys by setting + their value to ``None``. In the following example, we'll update the value + of "k1" and remove "k3" ("k2" will not be modified): + + .. code-block:: pycon + + >>> KV.update(value=KV.value.update({'k1': 'v1-x', 'k3': None})).execute() 1 >>> KV.get(KV.key == 'a').value {'k1': 'v1-x', 'k2': 'v2'} @@ -413,7 +425,7 @@ APIs 'b', {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': [1, 2]} - API documentation for :py:class:`JSONField` follows. + For more information, refer to the `sqlite json1 documentation `_. .. py:method:: __getitem__(item) From a63c0be520713ecebda4381442f3b480b521a130 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 29 May 2018 15:55:30 -0500 Subject: [PATCH 02/58] Remove 3.0 release notice from docs. --- docs/index.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 807613d04..200218bf1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,10 +8,6 @@ peewee .. image:: peewee3-logo.png -.. attention:: - Peewee 3.0 has been released (you are looking at the 3.0 documentation). To - see a list of backwards-incompatible changes, see :ref:`the list of changes `. - Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. From 095136b4349b7c444932e77b1310ba44502599c6 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 30 May 2018 10:47:04 -0500 Subject: [PATCH 03/58] bugfix for user-defined aggregate with apsw. --- playhouse/apsw_ext.py | 9 ++++++--- tests/apsw_ext.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/playhouse/apsw_ext.py b/playhouse/apsw_ext.py index b757a5f53..d8fe15af5 100644 --- a/playhouse/apsw_ext.py +++ b/playhouse/apsw_ext.py @@ -24,6 +24,7 @@ from peewee import DateTimeField as _DateTimeField from peewee import DecimalField as _DecimalField from peewee import TimeField as _TimeField +from peewee import logger from playhouse.sqlite_ext import SqliteExtDatabase @@ -35,6 +36,8 @@ def __init__(self, database, **kwargs): def register_module(self, mod_name, mod_inst): self._modules[mod_name] = mod_inst + if not self.is_closed(): + self.connection().createmodule(mod_name, mod_inst) def unregister_module(self, mod_name): del(self._modules[mod_name]) @@ -62,8 +65,7 @@ def _load_modules(self, conn): def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): def make_aggregate(): - instance = klass() - return (instance, instance.step, instance.finalize) + return (klass(), klass.step, klass.finalize) conn.createaggregatefunction(name, make_aggregate) def _load_collations(self, conn): @@ -82,7 +84,7 @@ def _load_extensions(self, conn): def load_extension(self, extension): self._extensions.add(extension) if not self.is_closed(): - conn = self.get_conn() + conn = self.connection() conn.enableloadextension(True) conn.loadextension(extension) @@ -102,6 +104,7 @@ def rollback(self): self.cursor().execute('rollback;') def execute_sql(self, sql, params=None, commit=True): + logger.debug((sql, params)) with __exception_wrapper__: cursor = self.cursor() cursor.execute(sql, params or ()) diff --git a/tests/apsw_ext.py b/tests/apsw_ext.py index a7d56a9ee..d860fc7b7 100644 --- a/tests/apsw_ext.py +++ b/tests/apsw_ext.py @@ -32,6 +32,43 @@ def title(s): curs = self.database.execute_sql('SELECT title(?)', ('heLLo',)) self.assertEqual(curs.fetchone()[0], 'Hello') + def test_db_register_aggregate(self): + @database.aggregate() + class First(object): + def __init__(self): + self._value = None + + def step(self, value): + if self._value is None: + self._value = value + + def finalize(self): + return self._value + + with database.atomic(): + for i in range(10): + User.create(username='u%s' % i) + + query = User.select(fn.First(User.username)).order_by(User.username) + self.assertEqual(query.scalar(), 'u0') + + def test_db_register_collation(self): + @database.collation() + def reverse(lhs, rhs): + lhs, rhs = lhs.lower(), rhs.lower() + if lhs < rhs: + return 1 + return -1 if rhs > lhs else 0 + + with database.atomic(): + for i in range(3): + User.create(username='u%s' % i) + + query = (User + .select(User.username) + .order_by(User.username.collate('reverse'))) + self.assertEqual([u.username for u in query], ['u2', 'u1', 'u0']) + def test_db_pragmas(self): test_db = APSWDatabase(':memory:', pragmas=( ('cache_size', '1337'), From 8ceb030a85a267a97aab9dae1b69030719a0ac24 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 30 May 2018 15:12:21 -0500 Subject: [PATCH 04/58] Add test to ensure page size is preserved by backup. --- tests/cysqlite.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/tests/cysqlite.py b/tests/cysqlite.py index 360f61fe2..3f1e8643d 100644 --- a/tests/cysqlite.py +++ b/tests/cysqlite.py @@ -147,19 +147,22 @@ def test_md5(self): class TestBackup(CyDatabaseTestCase): - backup_filename = 'test_backup.db' + backup_filenames = set(('test_backup.db', 'test_backup1.db', + 'test_backup2.db')) def tearDown(self): super(TestBackup, self).tearDown() - if os.path.exists(self.backup_filename): - os.unlink(self.backup_filename) - - def _populate_test_data(self, nrows=100): - self.execute('CREATE TABLE register (id INTEGER NOT NULL PRIMARY KEY, ' - 'value INTEGER NOT NULL)') - with self.database.atomic(): + for backup_filename in self.backup_filenames: + if os.path.exists(backup_filename): + os.unlink(backup_filename) + + def _populate_test_data(self, nrows=100, db=None): + db = self.database if db is None else db + db.execute_sql('CREATE TABLE register (id INTEGER NOT NULL PRIMARY KEY' + ', value INTEGER NOT NULL)') + with db.atomic(): for i in range(nrows): - self.execute('INSERT INTO register (value) VALUES (?)', i) + db.execute_sql('INSERT INTO register (value) VALUES (?)', (i,)) def test_backup(self): self._populate_test_data() @@ -172,11 +175,26 @@ def test_backup(self): self.assertEqual([val for val, in cursor.fetchall()], list(range(100))) other_db.close() + def test_backup_preserve_pagesize(self): + db1 = CSqliteExtDatabase('test_backup1.db') + with db1.connection_context(): + db1.page_size = 8192 + self._populate_test_data(db=db1) + + db1.connect() + self.assertEqual(db1.page_size, 8192) + + db2 = CSqliteExtDatabase('test_backup2.db') + db1.backup(db2) + self.assertEqual(db2.page_size, 8192) + nrows, = db2.execute_sql('select count(*) from register;').fetchone() + self.assertEqual(nrows, 100) + def test_backup_to_file(self): self._populate_test_data() - self.database.backup_to_file(self.backup_filename) - backup_db = CSqliteExtDatabase(self.backup_filename) + self.database.backup_to_file('test_backup.db') + backup_db = CSqliteExtDatabase('test_backup.db') cursor = backup_db.execute_sql('SELECT value FROM register ORDER BY ' 'value;') self.assertEqual([val for val, in cursor.fetchall()], list(range(100))) From 295130689e8db707566df34d48b708d55b5f2008 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 30 May 2018 15:52:37 -0500 Subject: [PATCH 05/58] Add new, inheritable meta option for declaring temp tables ("temporary"). --- docs/peewee/models.rst | 1 + peewee.py | 15 ++++++++++----- tests/models.py | 16 ++++++++++++++++ tests/schema.py | 27 +++++++++++++++++++++++++-- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/peewee/models.rst b/docs/peewee/models.rst index 5e38295b9..25a5f9809 100644 --- a/docs/peewee/models.rst +++ b/docs/peewee/models.rst @@ -720,6 +720,7 @@ Option Meaning I ``schema`` the database schema for the model yes ``only_save_dirty`` when calling model.save(), only save dirty fields yes ``options`` dictionary of options for create table extensions yes +``temporary`` indicate temporary table yes ``table_alias`` an alias to use for the table in queries no ``depends_on`` indicate this table depends on another for creation no ``without_rowid`` indicate table should not have rowid (SQLite only) no diff --git a/peewee.py b/peewee.py index 19350a395..98c42d354 100644 --- a/peewee.py +++ b/peewee.py @@ -4686,13 +4686,13 @@ def create_table(self, safe=True, **options): self.database.execute(self._create_table(safe=safe, **options)) def _drop_table(self, safe=True, **options): - is_temp = options.pop('temporary', False) ctx = (self._create_context() - .literal('DROP TEMPORARY ' if is_temp else 'DROP ') - .literal('TABLE IF EXISTS ' if safe else 'TABLE ') + .literal('DROP TABLE IF EXISTS ' if safe else 'DROP TABLE ') .sql(self.model)) if options.get('cascade'): ctx = ctx.literal(' CASCADE') + elif options.get('restrict'): + ctx = ctx.literal(' RESTRICT') return ctx def drop_table(self, safe=True, **options): @@ -4807,7 +4807,7 @@ def __init__(self, model, database=None, table_name=None, indexes=None, primary_key=None, constraints=None, schema=None, only_save_dirty=False, table_alias=None, depends_on=None, options=None, db_table=None, table_function=None, - without_rowid=False, **kwargs): + without_rowid=False, temporary=False, **kwargs): if db_table is not None: __deprecated__('"db_table" has been deprecated in favor of ' '"table_name" for Models.') @@ -4847,6 +4847,7 @@ def __init__(self, model, database=None, table_name=None, indexes=None, self.table_alias = table_alias self.depends_on = depends_on self.without_rowid = without_rowid + self.temporary = temporary self.refs = {} self.backrefs = {} @@ -5078,7 +5079,7 @@ class DoesNotExist(Exception): pass class ModelBase(type): inheritable = set(['constraints', 'database', 'indexes', 'primary_key', - 'options', 'schema', 'table_function', + 'options', 'schema', 'table_function', 'temporary', 'only_save_dirty']) def __new__(cls, name, bases, attrs): @@ -5526,6 +5527,8 @@ def create_table(cls, safe=True, **options): if safe and not cls._meta.database.safe_create_index \ and cls.table_exists(): return + if cls._meta.temporary: + options.setdefault('temporary', cls._meta.temporary) cls._schema.create_all(safe, **options) @classmethod @@ -5533,6 +5536,8 @@ def drop_table(cls, safe=True, drop_sequences=True, **options): if safe and not cls._meta.database.safe_drop_index \ and not cls.table_exists(): return + if cls._meta.temporary: + options.setdefault('temporary', cls._meta.temporary) cls._schema.drop_all(safe, drop_sequences, **options) @classmethod diff --git a/tests/models.py b/tests/models.py index ad46c9c4f..d7d06ea73 100644 --- a/tests/models.py +++ b/tests/models.py @@ -2403,6 +2403,22 @@ class Meta: self.assertEqual(Overrides._meta.options, {'foo': 'bar'}) self.assertTrue(Overrides._meta.schema is None) + def test_temporary_inheritance(self): + class T0(TestModel): pass + class T1(TestModel): + class Meta: + temporary = True + + class T2(T1): pass + class T3(T1): + class Meta: + temporary = False + + self.assertFalse(T0._meta.temporary) + self.assertTrue(T1._meta.temporary) + self.assertTrue(T2._meta.temporary) + self.assertFalse(T3._meta.temporary) + class TestModelSetDatabase(BaseTestCase): def test_set_database(self): diff --git a/tests/schema.py b/tests/schema.py index 357eede41..3d0287e7d 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -112,6 +112,14 @@ def test_without_rowid(self): '"key" TEXT NOT NULL PRIMARY KEY, ' '"value" TEXT NOT NULL) WITHOUT ROWID')]) + # Subclasses do not inherit "without_rowid" setting. + class SubNoRowid(NoRowid): pass + + self.assertCreateTable(SubNoRowid, [ + ('CREATE TABLE "subnorowid" (' + '"key" TEXT NOT NULL PRIMARY KEY, ' + '"value" TEXT NOT NULL)')]) + NoRowid._meta.database = None def test_db_table(self): @@ -140,8 +148,20 @@ def test_temporary_table(self): '"id" INTEGER NOT NULL PRIMARY KEY, ' '"username" VARCHAR(255) NOT NULL)')) - sql, params = User._schema._drop_table(temporary=True).query() - self.assertEqual(sql, 'DROP TEMPORARY TABLE IF EXISTS "users"') + def test_model_temporary_table(self): + class TempUser(User): + class Meta: + temporary = True + + self.reset_sql_history() + TempUser.create_table() + TempUser.drop_table() + queries = [x.msg for x in self.history] + self.assertEqual(queries, [ + ('CREATE TEMPORARY TABLE IF NOT EXISTS "tempuser" (' + '"id" INTEGER NOT NULL PRIMARY KEY, ' + '"username" VARCHAR(255) NOT NULL)', []), + ('DROP TABLE IF EXISTS "tempuser"', [])]) def test_drop_table(self): sql, params = User._schema._drop_table().query() @@ -150,6 +170,9 @@ def test_drop_table(self): sql, params = User._schema._drop_table(cascade=True).query() self.assertEqual(sql, 'DROP TABLE IF EXISTS "users" CASCADE') + sql, params = User._schema._drop_table(restrict=True).query() + self.assertEqual(sql, 'DROP TABLE IF EXISTS "users" RESTRICT') + def test_table_and_index_creation(self): self.assertCreateTable(Person, [ ('CREATE TABLE "person" (' From 0cc62234dad0f7e6494e7464bdbbc845ff42a89f Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sun, 3 Jun 2018 13:38:16 +0100 Subject: [PATCH 06/58] Correct spelling mistakes. --- docs/peewee/api.rst | 2 +- docs/peewee/example.rst | 2 +- docs/peewee/models.rst | 2 +- docs/peewee/playhouse.rst | 6 +++--- docs/peewee/quickstart.rst | 2 +- docs/peewee/sqlite_ext.rst | 6 +++--- peewee.py | 2 +- playhouse/sqlcipher_ext.py | 4 ++-- tests/libs/mock.py | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 28674f062..f04a7212e 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -2227,7 +2227,7 @@ Fields :param str help_text: Help-text for field, metadata purposes only. :param str verbose_name: Verbose name for field, metadata purposes only. - Fields on a :py:class:`Model` are analagous to columns on a table. + Fields on a :py:class:`Model` are analogous to columns on a table. .. py:attribute:: field_type = '' diff --git a/docs/peewee/example.rst b/docs/peewee/example.rst index b2140a78f..22180ac75 100644 --- a/docs/peewee/example.rst +++ b/docs/peewee/example.rst @@ -61,7 +61,7 @@ clone, there are just three models: the *User* model and stores which users follow one another. *Message*: - Analagous to a tweet. The Message model stores the text content of + Analogous to a tweet. The Message model stores the text content of the tweet, when it was created, and who posted it (foreign key to User). If you like UML, these are the tables and relationships: diff --git a/docs/peewee/models.rst b/docs/peewee/models.rst index 25a5f9809..6c728d4a2 100644 --- a/docs/peewee/models.rst +++ b/docs/peewee/models.rst @@ -1069,7 +1069,7 @@ This will yield the following DDL: Self-referential foreign keys ----------------------------- -When creating a heirarchical structure it is necessary to create a +When creating a hierarchical structure it is necessary to create a self-referential foreign key which links a child object to its parent. Because the model class is not defined at the time you instantiate the self-referential foreign key, use the special string ``'self'`` to indicate a self-referential diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index 1cca74193..892045662 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -608,11 +608,11 @@ Notes: * [Hopefully] there's no way to tell whether the passphrase is wrong or the file is corrupt. - In both cases -- *the first time we try to acces the database* -- a + In both cases -- *the first time we try to access the database* -- a :py:class:`DatabaseError` error is raised, with the *exact* message: ``"file is encrypted or is not a database"``. - As mentioned above, this only happens when you *access* the databse, + As mentioned above, this only happens when you *access* the database, so if you need to know *right away* whether the passphrase was correct, you can trigger this check by calling [e.g.] :py:meth:`~Database.get_tables()` (see example below). @@ -1287,7 +1287,7 @@ postgres_ext API notes :param keys: One or more keys to search for. - Query rows for the existince of *any* key. + Query rows for the existence of *any* key. .. py:class:: JSONField(dumps=None, *args, **kwargs) diff --git a/docs/peewee/quickstart.rst b/docs/peewee/quickstart.rst index 6b8561590..8245b15ce 100644 --- a/docs/peewee/quickstart.rst +++ b/docs/peewee/quickstart.rst @@ -395,7 +395,7 @@ It would look like this: # Grandma L. no pets # Herb Mittens Jr -Usually this type of duplication is undesirable. To accomodate the more common +Usually this type of duplication is undesirable. To accommodate the more common (and intuitive) workflow of listing a person and attaching **a list** of that person's pets, we can use a special method called :py:meth:`~ModelSelect.prefetch`: diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index a054f9c66..e89974631 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -1295,10 +1295,10 @@ APIs `transitive closure `_ table. Closure tables are :py:class:`VirtualModel` subclasses that work with the transitive closure SQLite extension. These special tables are - designed to make it easy to efficiently query heirarchical data. The SQLite + designed to make it easy to efficiently query hierarchical data. The SQLite extension manages an AVL tree behind-the-scenes, transparently updating the tree when your table changes and making it easy to perform common queries - on heirarchical data. + on hierarchical data. To use the closure table extension in your project, you need: @@ -1536,7 +1536,7 @@ APIs ts = get_timestamp() EventLog[ts] = ('pageview', 'search', '/blog/some-post/') - # Retreive row from event log. + # Retrieve row from event log. log = EventLog[ts] print(log.action, log.sender, log.target) # Prints ("pageview", "search", "/blog/some-post/") diff --git a/peewee.py b/peewee.py index 98c42d354..e4ab19b76 100644 --- a/peewee.py +++ b/peewee.py @@ -3581,7 +3581,7 @@ def _initialize_columns(self): def _row_to_dict(self, row): result = {} for i in range(self.ncols): - result.setdefault(self.columns[i], row[i]) # Do not overwite. + result.setdefault(self.columns[i], row[i]) # Do not overwrite. return result process_row = _row_to_dict diff --git a/playhouse/sqlcipher_ext.py b/playhouse/sqlcipher_ext.py index c69d5f16e..750022fc6 100644 --- a/playhouse/sqlcipher_ext.py +++ b/playhouse/sqlcipher_ext.py @@ -5,7 +5,7 @@ **WARNING!!! EXPERIMENTAL!!!** -* Although this extention's code is short, it has not been propery +* Although this extention's code is short, it has not been properly peer-reviewed yet and may have introduced vulnerabilities. * The code contains minimum values for `passphrase` length and `kdf_iter`, as well as a default value for the later. @@ -25,7 +25,7 @@ * `passphrase`: should be "long enough". Note that *length beats vocabulary* (much exponential), and even a lowercase-only passphrase like easytorememberyethardforotherstoguess - packs more noise than 8 random printable chatacters and *can* be memorized. + packs more noise than 8 random printable characters and *can* be memorized. * `kdf_iter`: Should be "as much as the weakest target machine can afford". When opening an existing database, passphrase and kdf_iter should be identical diff --git a/tests/libs/mock.py b/tests/libs/mock.py index c8fc5d1d2..7e689caf5 100644 --- a/tests/libs/mock.py +++ b/tests/libs/mock.py @@ -1180,7 +1180,7 @@ def decorate_callable(self, func): @wraps(func) def patched(*args, **keywargs): - # don't use a with here (backwards compatability with Python 2.4) + # don't use a with here (backwards compatibility with Python 2.4) extra_args = [] entered_patchers = [] From e6cbe97240949787015a46654caba15c47f42c47 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 7 Jun 2018 08:19:50 -0500 Subject: [PATCH 07/58] Allow overriding model "__repr__", fixes #1622. --- peewee.py | 2 +- tests/models.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/peewee.py b/peewee.py index e4ab19b76..d3afd69e0 100644 --- a/peewee.py +++ b/peewee.py @@ -5157,7 +5157,7 @@ def __new__(cls, name, bases, attrs): cls._meta.add_field(name, field) # Create a repr and error class before finalizing. - if hasattr(cls, '__str__'): + if hasattr(cls, '__str__') and '__repr__' not in attrs: setattr(cls, '__repr__', lambda self: '<%s: %s>' % ( cls.__name__, self.__str__())) diff --git a/tests/models.py b/tests/models.py index d7d06ea73..3f364bd33 100644 --- a/tests/models.py +++ b/tests/models.py @@ -3414,3 +3414,20 @@ class Meta: self.assertEqual(repr(EAV.entity), '') self.assertEqual(repr(TextField()), '') + + def test_custom_reprs(self): + class User(Model): + username = TextField(primary_key=True) + + def __str__(self): + return self.username.title() + + class Foo(Model): + def __repr__(self): + return 'TWEET: %s' % self.id + + u = User(username='charlie') + self.assertEqual(repr(u), '') + + f = Foo(id=1337) + self.assertEqual(repr(f), 'TWEET: 1337') From ecdc2ffde3b0c03262790c38292a33f4029fe868 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sun, 10 Jun 2018 11:32:26 -0500 Subject: [PATCH 08/58] Clarify that db name can be None in param doc. --- docs/peewee/api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index f04a7212e..b3de65bba 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -10,7 +10,9 @@ Database .. py:class:: Database(database[, thread_safe=True[, autorollback=False[, field_types=None[, operations=None[, **kwargs]]]]]) - :param str database: Database name or filename for SQLite. + :param str database: Database name or filename for SQLite (or None to + defer initialization, in which case you must call + :py:meth:`Database.init`, specifying the database name). :param bool thread_safe: Whether to store connection state in a thread-local. :param bool autorollback: Automatically rollback queries that fail when From da782b6d7e19348c4b115490aa8ca9865a9be0ff Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sun, 10 Jun 2018 15:40:49 -0500 Subject: [PATCH 09/58] Add note to manytomany field docs about saving first. Refs #1632 --- docs/peewee/querying.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 3d30caf4b..d4472b574 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1467,6 +1467,12 @@ Modeling students and courses using :py:class:`ManyToManyField`: # Calling .clear() will remove all associated objects: cs_150.students.clear() +.. attention:: + Before many-to-many relationships can be added, the objects being + referenced will need to be saved first. In order to create relationships in + the many-to-many through table, Peewee needs to know the primary keys of + the models being referenced. + For more examples, see: * :py:meth:`ManyToManyField.add` From 2d2426fef5f96b3be210c9687f5e7375b8c41d24 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Sun, 10 Jun 2018 15:47:43 -0500 Subject: [PATCH 10/58] Ensure that we use BigInt for BigAutoField foreign-keys. Fixes #1630. --- peewee.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/peewee.py b/peewee.py index d3afd69e0..00f3ea4b8 100644 --- a/peewee.py +++ b/peewee.py @@ -4303,6 +4303,8 @@ def __init__(self, model, field=None, backref=None, on_delete=None, def field_type(self): if not isinstance(self.rel_field, AutoField): return self.rel_field.field_type + elif isinstance(self.rel_field, BigAutoField): + return BigIntegerField.field_type return IntegerField.field_type def get_modifiers(self): From 692db228c9dd4f2dc8361096467a04d31c9360b9 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 11:08:50 -0500 Subject: [PATCH 11/58] Allow aliasing fields to their underlying column name if different. Also adds tests for various edge-cases that may occur when mixing joins and custom aliases. An error is now raised when you attempt to alias a joined model to it's object_id_name. Fixes #1625 --- peewee.py | 17 +++++++++++- tests/__init__.py | 1 + tests/regressions.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/regressions.py diff --git a/peewee.py b/peewee.py index 00f3ea4b8..3851637b3 100644 --- a/peewee.py +++ b/peewee.py @@ -606,6 +606,9 @@ def coerce(self, _coerce=True): return clone return self + def is_alias(self): + return False + def unwrap(self): return self @@ -1085,6 +1088,9 @@ def __init__(self, node): self.node = node self._coerce = getattr(node, '_coerce', True) + def is_alias(self): + return self.node.is_alias() + def unwrap(self): return self.node.unwrap() @@ -1121,6 +1127,9 @@ def alias(self, alias=None): def unalias(self): return self.node + def is_alias(self): + return True + def __sql__(self, ctx): if ctx.scope == SCOPE_SOURCE: return (ctx @@ -5857,6 +5866,11 @@ def _normalize_join(self, src, dest, on, attr): attr = fk_field.name else: attr = dest_model._meta.name + elif on_alias and fk_field is not None and \ + attr == fk_field.object_id_name and not is_backref: + raise ValueError('Cannot assign join alias to "%s", as this ' + 'attribute is the object_id_name for the ' + 'foreign-key field "%s"' % (attr, fk_field)) elif isinstance(dest, Source): constructor = dict @@ -6163,7 +6177,8 @@ def _initialize_columns(self): if raw_node._coerce: converters[idx] = node.python_value fields[idx] = node - if column == node.name or column == node.column_name: + if (column == node.name or column == node.column_name) and \ + not raw_node.is_alias(): self.columns[idx] = node.name elif column in combined: if raw_node._coerce: diff --git a/tests/__init__.py b/tests/__init__.py index 6e447669c..ed51171cc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,6 +13,7 @@ from .model_sql import * from .prefetch import * from .queries import * +from .regressions import * from .results import * from .schema import * from .sql import * diff --git a/tests/regressions.py b/tests/regressions.py new file mode 100644 index 000000000..0077b307c --- /dev/null +++ b/tests/regressions.py @@ -0,0 +1,62 @@ +from peewee import * + +from .base import ModelTestCase +from .base import TestModel + + +class ColAlias(TestModel): + name = TextField(column_name='pname') + + +class CARef(TestModel): + colalias = ForeignKeyField(ColAlias, backref='carefs', column_name='ca', + object_id_name='colalias_id') + + +class TestQueryAliasToColumnName(ModelTestCase): + requires = [ColAlias, CARef] + + def setUp(self): + super(TestQueryAliasToColumnName, self).setUp() + with self.database.atomic(): + for name in ('huey', 'mickey'): + col_alias = ColAlias.create(name=name) + CARef.create(colalias=col_alias) + + def test_alias_to_column_name(self): + # The issue here occurs when we take a field whose name differs from + # it's underlying column name, then alias that field to it's column + # name. In this case, peewee was *not* respecting the alias and using + # the field name instead. + query = (ColAlias + .select(ColAlias.name.alias('pname')) + .order_by(ColAlias.name)) + self.assertEqual([c.pname for c in query], ['huey', 'mickey']) + + # Ensure that when using dicts the logic is preserved. + query = query.dicts() + self.assertEqual([r['pname'] for r in query], ['huey', 'mickey']) + + def test_alias_overlap_with_join(self): + query = (CARef + .select(CARef, ColAlias.name.alias('pname')) + .join(ColAlias) + .order_by(ColAlias.name)) + with self.assertQueryCount(1): + self.assertEqual([r.colalias.pname for r in query], + ['huey', 'mickey']) + + # Note: we cannot alias the join to "ca", as this is the object-id + # descriptor name. + query = (CARef + .select(CARef, ColAlias.name.alias('pname')) + .join(ColAlias, + on=(CARef.colalias == ColAlias.id).alias('ca')) + .order_by(ColAlias.name)) + with self.assertQueryCount(1): + self.assertEqual([r.ca.pname for r in query], ['huey', 'mickey']) + + def test_cannot_alias_join_to_object_id_name(self): + query = CARef.select(CARef, ColAlias.name.alias('pname')) + expr = (CARef.colalias == ColAlias.id).alias('colalias_id') + self.assertRaises(ValueError, query.join, ColAlias, on=expr) From 45bc720baca1e7bb01c6c8f15f540895f0a68596 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 11:20:58 -0500 Subject: [PATCH 12/58] Add changelog entries for the latest changes to master. Test re-org. --- CHANGELOG.md | 13 +++++++++++++ tests/models.py | 9 +-------- tests/regressions.py | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c13fee734..af633456f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ https://github.com/coleifer/peewee/releases ## master +* Add new model meta option for indicating that a model uses a temporary table. +* Fixed edge-case where attempting to alias a field to it's underlying + column-name (when different), Peewee would not respect the alias and use the + field name instead. See #1625 for details and discussion. +* Raise a `ValueError` when joining and aliasing the join to a foreign-key's + `object_id_name` descriptor. Should prevent accidentally introducing O(n) + queries or silently ignoring data from a joined-instance. +* Fixed bug for MySQL when creating a foreign-key to a model which used the + `BigAutoField` for it's primary-key. +* Fixed bugs in the implementation of user-defined aggregates and extensions + with the APSW SQLite driver. +* Fixed regression introduced in 3.5.0 which ignored custom Model `__repr__()`. + [View commits](https://github.com/coleifer/peewee/compare/3.5.0...HEAD) ## 3.5.0 diff --git a/tests/models.py b/tests/models.py index 3f364bd33..333ba5b2d 100644 --- a/tests/models.py +++ b/tests/models.py @@ -3415,19 +3415,12 @@ class Meta: self.assertEqual(repr(TextField()), '') - def test_custom_reprs(self): + def test_model_str_method(self): class User(Model): username = TextField(primary_key=True) def __str__(self): return self.username.title() - class Foo(Model): - def __repr__(self): - return 'TWEET: %s' % self.id - u = User(username='charlie') self.assertEqual(repr(u), '') - - f = Foo(id=1337) - self.assertEqual(repr(f), 'TWEET: 1337') diff --git a/tests/regressions.py b/tests/regressions.py index 0077b307c..bb24a04c2 100644 --- a/tests/regressions.py +++ b/tests/regressions.py @@ -1,5 +1,6 @@ from peewee import * +from .base import BaseTestCase from .base import ModelTestCase from .base import TestModel @@ -60,3 +61,18 @@ def test_cannot_alias_join_to_object_id_name(self): query = CARef.select(CARef, ColAlias.name.alias('pname')) expr = (CARef.colalias == ColAlias.id).alias('colalias_id') self.assertRaises(ValueError, query.join, ColAlias, on=expr) + + +class TestOverrideModelRepr(BaseTestCase): + def test_custom_reprs(self): + # In 3.5.0, Peewee included a new implementation and semantics for + # customizing model reprs. This introduced a regression where model + # classes that defined a __repr__() method had this override ignored + # silently. This test ensures that it is possible to completely + # override the model repr. + class Foo(Model): + def __repr__(self): + return 'FOO: %s' % self.id + + f = Foo(id=1337) + self.assertEqual(repr(f), 'FOO: 1337') From a77a79f7347dd9f786c2de591c3573279f2e5f00 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 11:32:49 -0500 Subject: [PATCH 13/58] Allow database parameter to be specified on ModelSelect.get(). Fixes #1620. --- peewee.py | 4 ++-- tests/models.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/peewee.py b/peewee.py index 3851637b3..874018c7a 100644 --- a/peewee.py +++ b/peewee.py @@ -5732,11 +5732,11 @@ def objects(self, constructor=None): def prefetch(self, *subqueries): return prefetch(self, *subqueries) - def get(self): + def get(self, database=None): clone = self.paginate(1, 1) clone._cursor_wrapper = None try: - return clone.execute()[0] + return clone.execute(database)[0] except IndexError: sql, params = clone.sql() raise self.model.DoesNotExist('%s instance matching query does ' diff --git a/tests/models.py b/tests/models.py index 333ba5b2d..721612605 100644 --- a/tests/models.py +++ b/tests/models.py @@ -3424,3 +3424,25 @@ def __str__(self): u = User(username='charlie') self.assertEqual(repr(u), '') + + +class TestGetWithSecondDatabase(ModelTestCase): + database = get_in_memory_db() + requires = [User] + + def test_get_with_second_database(self): + User.create(username='huey') + query = User.select().where(User.username == 'huey') + self.assertEqual(query.get().username, 'huey') + + alt_db = get_in_memory_db() + with User.bind_ctx(alt_db): + User.create_table() + + self.assertRaises(User.DoesNotExist, query.get, alt_db) + with User.bind_ctx(alt_db): + User.create(username='zaizee') + + query = User.select().where(User.username == 'zaizee') + self.assertRaises(User.DoesNotExist, query.get) + self.assertEqual(query.get(alt_db).username, 'zaizee') From 46c993a966b252026d1f50acf06c05cbaaa6d60c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 11:33:45 -0500 Subject: [PATCH 14/58] Add changelog note for #1620. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af633456f..41f2f998b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ https://github.com/coleifer/peewee/releases ## master * Add new model meta option for indicating that a model uses a temporary table. +* Allow `database` parameter to be specified with `ModelSelect.get()` method. + For discussion, see #1620. * Fixed edge-case where attempting to alias a field to it's underlying column-name (when different), Peewee would not respect the alias and use the field name instead. See #1625 for details and discussion. From f7535b6d2c1d4e3ef2f40482669f7170033b0636 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 22:48:17 -0500 Subject: [PATCH 15/58] Small tweaks to docs. --- docs/peewee/api.rst | 73 ++++++++++++++++++++++++++++++++------------- tests/models.py | 3 +- 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index b3de65bba..89e1b9385 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -10,9 +10,9 @@ Database .. py:class:: Database(database[, thread_safe=True[, autorollback=False[, field_types=None[, operations=None[, **kwargs]]]]]) - :param str database: Database name or filename for SQLite (or None to - defer initialization, in which case you must call - :py:meth:`Database.init`, specifying the database name). + :param str database: Database name or filename for SQLite (or ``None`` to + :ref:`defer initialization `, in which case + you must call :py:meth:`Database.init`, specifying the database name). :param bool thread_safe: Whether to store connection state in a thread-local. :param bool autorollback: Automatically rollback queries that fail when @@ -31,14 +31,40 @@ Database * Introspection .. note:: - The database can be instantiated with ``None`` as the database name if the database is not known until run-time. In this way you can create a database instance and then configure it elsewhere when the settings are - known. This is called *deferred* initialization. + known. This is called :ref:`deferred* initialization `. + + Examples: + + .. code-block:: python + + # Sqlite database using WAL-mode and 32MB page-cache. + db = SqliteDatabase('app.db', pragmas={ + 'journal_mode': 'wal', + 'cache_size': -32 * 1000}) + + # Postgresql database on remote host. + db = PostgresqlDatabase('my_app', user='postgres', host='10.1.0.3', + password='secret') + + Deferred initialization example: + + .. code-block:: python + + db = PostgresqlDatabase(None) + + class BaseModel(Model): + class Meta: + database = db - To initialize a database that has been *deferred*, use the - :py:meth:`~Database.init` method. + # Read database connection info from env, for example: + db_name = os.environ['DATABASE'] + db_host = os.environ['PGHOST'] + + # Initialize database. + db.init(db_name, host=db_host, user='postgres') .. py:attribute:: param = '?' @@ -56,7 +82,8 @@ Database database driver when a connection is created, for example ``password``, ``host``, etc. - Initialize a *deferred* database. + Initialize a *deferred* database. See :ref:`deferring_initialization` + for more info. .. py:method:: __enter__() @@ -811,6 +838,11 @@ Query-builder API for recursively unwrapping "wrapped" nodes. Base case is to return self. + .. py:method:: is_alias() + + API for determining if a node, at any point, has been explicitly + aliased by the user. + .. py:class:: Source([alias=None]) @@ -1197,7 +1229,9 @@ Query-builder .. py:function:: AsIs(value) Represents a :py:class:`Value` that is treated as-is, and passed directly - back to the database driver. + back to the database driver. This may be useful if you are using database + extensions that accept native Python data-types and you do not wish Peewee + to impose any handling of the values. .. py:class:: Cast(node, cast) @@ -2482,15 +2516,18 @@ Fields the datetime can be encoded with (for databases that do not have support for a native datetime data-type). The default supported formats are: - .. note:: - If the incoming value does not match a format, it is returned as-is. - .. code-block:: python '%Y-%m-%d %H:%M:%S.%f' # year-month-day hour-minute-second.microsecond '%Y-%m-%d %H:%M:%S' # year-month-day hour-minute-second '%Y-%m-%d' # year-month-day + .. note:: + SQLite does not have a native datetime data-type, so datetimes are + stored as strings. This is handled transparently by Peewee, but if you + have pre-existing data you should ensure it is stored as + ``YYYY-mm-dd HH:MM:SS`` or one of the other supported formats. + .. py:attribute:: year Reference the year of the value stored in the column in a query. @@ -2681,10 +2718,10 @@ Fields Take care with foreign keys in SQLite. By default, ON DELETE has no effect, which can have surprising (and usually unwanted) effects on your database integrity. This can affect you even if you don't specify - on_delete, since the default ON DELETE behaviour (to fail without + ``on_delete``, since the default ON DELETE behaviour (to fail without modifying your data) does not happen, and your data can be silently relinked. The safest thing to do is to specify - ``pragmas=(('foreign_keys', 'on'),)`` when you instantiate + ``pragmas={'foreign_keys': 1}`` when you instantiate :py:class:`SqliteDatabase`. .. py:class:: DeferredForeignKey(rel_model_name[, **kwargs]) @@ -3187,7 +3224,7 @@ Model Example: - .. code-block:: pycon + .. code-block:: python Parent = Category.alias() sq = (Category @@ -3195,10 +3232,6 @@ Model .join(Parent, on=(Category.parent == Parent.id)) .where(Parent.name == 'parent category')) - .. note:: - When using a :py:class:`ModelAlias` in a join, you must explicitly - specify the join condition. - .. py:classmethod:: select(*fields) :param fields: A list of model classes, field instances, functions or @@ -3439,7 +3472,7 @@ Model q = User.raw('select id, username from users') for user in q: - print user.id, user.username + print(user.id, user.username) .. note:: Generally the use of ``raw`` is reserved for those cases where you diff --git a/tests/models.py b/tests/models.py index 721612605..801b41f36 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1355,8 +1355,7 @@ def test_join(self): def test_join_on(self): UA = User.alias('ua') - query = self._test_query(lambda: UA) - query = query.join(UA, on=(Tweet.user == UA.id)) + query = self._test_query(lambda: UA).join(UA, on=(Tweet.user == UA.id)) self.assertTweets(query) def test_join_on_field(self): From 0548a6b39ce7517784da4fc606fe4932fb195a5c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 23:00:50 -0500 Subject: [PATCH 16/58] A few more docs tweaks. --- docs/peewee/database.rst | 3 +-- docs/peewee/example.rst | 2 +- docs/peewee/installation.rst | 4 ++-- docs/peewee/quickstart.rst | 11 ++--------- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/docs/peewee/database.rst b/docs/peewee/database.rst index a956ed3b1..9df7dc16f 100644 --- a/docs/peewee/database.rst +++ b/docs/peewee/database.rst @@ -792,8 +792,7 @@ Example test-case setup: def setUp(self): # Bind model classes to test db. Since we have a complete list of # all models, we do not need to recursively bind dependencies. - for model in MODELS: - model.bind(test_db, bind_refs=False, bind_backrefs=False) + test_db.bind(MODELS, bind_refs=False, bind_backrefs=False) test_db.connect() test_db.create_tables(MODELS) diff --git a/docs/peewee/example.rst b/docs/peewee/example.rst index 22180ac75..5908dc5f8 100644 --- a/docs/peewee/example.rst +++ b/docs/peewee/example.rst @@ -321,7 +321,7 @@ subquery: user = get_current_user() messages = (Message .select() - .where(Message.user << user.following()) + .where(Message.user.in_(user.following())) .order_by(Message.pub_date.desc())) This code corresponds to the following SQL query: diff --git a/docs/peewee/installation.rst b/docs/peewee/installation.rst index 8e9007b59..5fbc2c600 100644 --- a/docs/peewee/installation.rst +++ b/docs/peewee/installation.rst @@ -83,8 +83,8 @@ Optional dependencies * `Cython `_: used for various speedups. Can give a big boost to certain operations, particularly if you use SQLite. * `apsw `_: an optional 3rd-party SQLite - binding offering greater performance and much, much saner semantics than the - standard library ``pysqlite``. Use with :py:class:`APSWDatabase`. + binding offering greater performance and comprehensive support for SQLite's C + APIs. Use with :py:class:`APSWDatabase`. * `gevent `_ is an optional dependency for :py:class:`SqliteQueueDatabase` (though it works with ``threading`` just fine). diff --git a/docs/peewee/quickstart.rst b/docs/peewee/quickstart.rst index 8245b15ce..1024eb60f 100644 --- a/docs/peewee/quickstart.rst +++ b/docs/peewee/quickstart.rst @@ -13,7 +13,8 @@ features. This guide will cover: .. note:: If you'd like something a bit more meaty, there is a thorough tutorial on :ref:`creating a "twitter"-style web app ` using peewee and the - Flask framework. + Flask framework. In the projects ``examples/`` folder you can find more + self-contained Peewee examples, like a `blog app `_. I **strongly** recommend opening an interactive shell session and running the code. That way you can get a feel for typing in queries. @@ -162,7 +163,6 @@ adopts Fido: herb_fido.owner = uncle_bob herb_fido.save() - bob_fido = herb_fido # rename our variable for clarity .. _retrieving-data: @@ -441,13 +441,6 @@ We're done with our database, let's close the connection: db.close() This is just the basics! You can make your queries as complex as you like. - -All the other SQL clauses are available as well, such as: - -* :py:meth:`~SelectQuery.group_by` -* :py:meth:`~SelectQuery.having` -* :py:meth:`~SelectQuery.limit` and :py:meth:`~SelectQuery.offset` - Check the documentation on :ref:`querying` for more info. Working with existing databases From dada74d72306e37f81865b00efbff0c4503c6fa1 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 23:01:27 -0500 Subject: [PATCH 17/58] Fix gevent link. --- docs/peewee/database.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/peewee/database.rst b/docs/peewee/database.rst index 9df7dc16f..159443375 100644 --- a/docs/peewee/database.rst +++ b/docs/peewee/database.rst @@ -819,7 +819,7 @@ out Peewee's own `test-suite ` is recommended for doing asynchronous i/o +`gevent `_ is recommended for doing asynchronous I/O with Postgresql or MySQL. Reasons I prefer gevent: * No need for special-purpose "loop-aware" re-implementations of *everything*. From a8952eb3eb25237d530967cc4d575184537c32d4 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 23:11:23 -0500 Subject: [PATCH 18/58] Ensure that model joins with both on+alias and attr prefer attr. --- peewee.py | 2 +- tests/models.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/peewee.py b/peewee.py index 874018c7a..a6a8e76c0 100644 --- a/peewee.py +++ b/peewee.py @@ -5818,7 +5818,7 @@ def _normalize_join(self, src, dest, on, attr): # destination attribute for the joined data. on_alias = isinstance(on, Alias) if on_alias: - attr = on._alias + attr = attr or on._alias on = on.alias() # Obtain references to the source and destination models being joined. diff --git a/tests/models.py b/tests/models.py index 801b41f36..c363016c5 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1375,6 +1375,12 @@ def test_join_attr(self): query = self._test_query(lambda: UA).join(UA, attr='baz') self.assertTweets(query, 'baz') + def test_join_on_alias_attr(self): + UA = User.alias('ua') + q = self._test_query(lambda: UA) + q = q.join(UA, on=(Tweet.user == UA.id).alias('foo'), attr='bar') + self.assertTweets(q, 'bar') + def _test_query_backref(self, alias_expr): TA = alias_expr() return (User From 6c02d868b6bc6962de60887820fbf6de54073960 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 12 Jun 2018 23:15:09 -0500 Subject: [PATCH 19/58] Clarify note about join alias/attr parameter. --- docs/peewee/querying.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index d4472b574..bc33e215c 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1172,8 +1172,8 @@ which has it's username attribute set: When doing complicated joins, joins where no foreign-key exists (for example joining on a subquery), etc., it is necessary to tell Peewee where to place the -joined attributes. This is done by putting an *alias* on the join predicate -expression. +joined attributes. This is done by specifying an ``attr`` parameter in the join +method. For example, let's say that in the above query we want to put the joined user data in the *Tweet.foo* attribute: @@ -1182,13 +1182,23 @@ data in the *Tweet.foo* attribute: query = (Tweet .select(Tweet.content, Tweet.timestamp, User.username) - .join(User, on=(Tweet.user == User.id).alias('foo')) + .join(User, attr='foo') .order_by(Tweet.timestamp.desc())) for tweet in query: # Joined user data is stored in "tweet.foo": print(tweet.content, tweet.timestamp, tweet.foo.username) +Alternatively, we can also specify the attribute name by putting an *alias* on +the join predicate expression: + +.. code-block:: python + + query = (Tweet + .select(Tweet.content, Tweet.timestamp, User.username) + .join(User, on=(Tweet.user == User.id).alias('foo')) + .order_by(Tweet.timestamp.desc())) + For queries with complex joins and selections from several models, constructing this graph can be expensive. If you wish, instead, to have *all* columns as attributes on a single model, you can use :py:meth:`~ModelSelect.objects` From 7d649bedc39197af94620d03e1b8c855aba87673 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 13 Jun 2018 08:38:03 -0500 Subject: [PATCH 20/58] Small cleanups. --- peewee.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/peewee.py b/peewee.py index a6a8e76c0..8b97ef89d 100644 --- a/peewee.py +++ b/peewee.py @@ -3638,10 +3638,8 @@ def next(self): __next__ = next - # FIELDS - class FieldAccessor(object): def __init__(self, model, field, name): self.model = model @@ -3720,7 +3718,6 @@ def __set__(self, instance, value): setattr(instance, self.field.name, value) - class Field(ColumnBase): _field_counter = 0 _order = 0 From ded6d2b541efead1a94bbb07f6bece89461930c5 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 13 Jun 2018 08:52:11 -0500 Subject: [PATCH 21/58] Add test for user-defined function registration across conns. --- docs/peewee/sqlite_ext.rst | 12 ++++++++++-- tests/sqlite.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index e89974631..18c1aa2c7 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -700,6 +700,14 @@ APIs constructor. * ``primary_key`` - defaults to ``False``, indicating no primary key. + These all are combined in the following way: + + .. code-block:: sql + + CREATE VIRTUAL TABLE + USING + ([prefix_arguments, ...] fields, ... [arguments, ...], [options...]) + .. _sqlite-fts: .. py:class:: FTSModel() @@ -790,8 +798,8 @@ APIs .order_by(DocumentIndex.bm25())) .. warning:: - All SQL queries on ``FTSModel`` classes will be slow **except** - full-text searches and ``rowid`` lookups. + All SQL queries on ``FTSModel`` classes will be full-table scans + **except** full-text searches and ``rowid`` lookups. If the primary source of the content you are indexing exists in a separate table, you can save some disk space by instructing SQLite to not store an diff --git a/tests/sqlite.py b/tests/sqlite.py index 117255b5b..21fd42a37 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -989,6 +989,16 @@ def test_function_decorator(self): self.assertEqual([x[0] for x in pq.tuples()], [ 'testing', 'chatting', ' foo']) + def test_use_across_connections(self): + db = get_in_memory_db() + @db.func() + def rev(s): + return s[::-1] + + db.connect(); db.close(); db.connect() + curs = db.execute_sql('select rev(?)', ('hello',)) + self.assertEqual(curs.fetchone(), ('olleh',)) + class TestRowIDField(ModelTestCase): database = database From 4eb7824cb4209119bef6077b1704d0908b353144 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 13 Jun 2018 08:59:21 -0500 Subject: [PATCH 22/58] Test inter-op of tables/columns and models/fields. --- tests/models.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/models.py b/tests/models.py index c363016c5..b217fc32c 100644 --- a/tests/models.py +++ b/tests/models.py @@ -3451,3 +3451,24 @@ def test_get_with_second_database(self): query = User.select().where(User.username == 'zaizee') self.assertRaises(User.DoesNotExist, query.get) self.assertEqual(query.get(alt_db).username, 'zaizee') + + +class TestMixModelsTables(ModelTestCase): + database = get_in_memory_db() + requires = [User] + + def test_mix_models_tables(self): + Tbl = User._meta.table + self.assertEqual(Tbl.insert({Tbl.username: 'huey'}).execute(), 1) + + huey = Tbl.select(User.username).get() + self.assertEqual(huey, {'username': 'huey'}) + + huey = User.select(Tbl.username).get() + self.assertEqual(huey.username, 'huey') + + Tbl.update(username='huey-x').where(Tbl.username == 'huey').execute() + self.assertEqual(User.select().get().username, 'huey-x') + + Tbl.delete().where(User.username == 'huey-x').execute() + self.assertEqual(Tbl.select().count(), 0) From 809b331dd139320b831868225b8a5eab3881fa67 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 13 Jun 2018 15:59:14 -0500 Subject: [PATCH 23/58] Simplify readme. --- README.rst | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index adc856bb2..e2702d6f4 100644 --- a/README.rst +++ b/README.rst @@ -5,30 +5,20 @@ peewee Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. -* A small, expressive ORM -* Written in python with support for versions 2.7+ and 3.4+ (developed with 3.6) -* built-in support for sqlite, mysql and postgresql -* tons of extensions available in the `playhouse `_ - - * `Postgresql HStore, JSON, arrays and more `_ - * `SQLite full-text search, user-defined functions, virtual tables and more `_ - * `Schema migrations `_ and `model code generator `_ - * `Connection pool `_ - * `Encryption `_ - * `and much, much more... `_ +* a small, expressive ORM +* python 2.7+ and 3.4+ (developed with 3.6) +* supports sqlite, mysql and postgresql +* tons of `extensions `_ .. image:: https://travis-ci.org/coleifer/peewee.svg?branch=master :target: https://travis-ci.org/coleifer/peewee -New to peewee? Here is a list of documents you might find most helpful when getting -started: +New to peewee? These may help: -* `Quickstart guide `_ -- this guide covers all the essentials. It will take you between 5 and 10 minutes to go through it. -* `Example queries `_ taken from the `PostgreSQL Exercises website `_. -* `Guide to the various query operators `_ describes how to construct queries and combine expressions. -* `Field types table `_ lists the various field types peewee supports and the parameters they accept. - -For flask helpers, check out the `flask_utils extension module `_. You can also use peewee with the popular extension `flask-admin `_ to provide a Django-like admin interface for managing peewee models. +* `Quickstart `_ +* `Example twitter app `_ +* `Query operators `_ +* `Field types `_ Examples -------- @@ -115,7 +105,7 @@ Queries are expressive and composable: # Do an atomic update Counter.update(count=Counter.count + 1).where(Counter.url == request.url) -Check out the `example app `_ for a working Twitter-clone website written with Flask. +Check out the `example twitter app `_. Learning more ------------- From 9acabbeaf84ae5ad8fee0e7c063ecb64ef04f99c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 13 Jun 2018 21:54:58 -0500 Subject: [PATCH 24/58] Use noop() helper. --- examples/blog/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/blog/app.py b/examples/blog/app.py index 9f0a17eeb..2ce4ec2ef 100644 --- a/examples/blog/app.py +++ b/examples/blog/app.py @@ -121,7 +121,7 @@ def search(cls, query): words = [word.strip() for word in query.split() if word.strip()] if not words: # Return an empty query. - return Entry.select().where(Entry.id == 0) + return Entry.noop() else: search = ' '.join(words) From 058a4aefb468efeb56525c10d1a6b9fa1982d5ba Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 15 Jun 2018 10:01:22 -0500 Subject: [PATCH 25/58] Improve upsert tests, add info on MySQL upsert. --- docs/peewee/querying.rst | 47 ++++++++++++++++---- tests/models.py | 92 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 123 insertions(+), 16 deletions(-) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index bc33e215c..659be6eeb 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -108,8 +108,8 @@ The above approach is slow for a couple of reasons: 4. We are retrieving the *last insert id*, which causes an additional query to be executed in some cases. -You can get a **very significant speedup** by simply wrapping this in a -:py:meth:`~Database.atomic`. +You can get a **significant speedup** by simply wrapping this in a transaction +with :py:meth:`~Database.atomic`. .. code-block:: python @@ -127,7 +127,7 @@ tuples or dictionaries to insert. # Fastest. MyModel.insert_many(data_source).execute() - # Fastest using tuples and specifying the fields being inserted. + # We can also use tuples and specify the fields being inserted. fields = [MyModel.field1, MyModel.field2] data = [('val1-1', 'val1-2'), ('val2-1', 'val2-2'), @@ -298,9 +298,36 @@ Example of using :py:meth:`~Model.replace` and :py:meth:`~Insert.on_conflict_rep action (see: :py:meth:`~Insert.on_conflict_ignore`) if you simply wish to insert and ignore any potential constraint violation. -Postgresql and SQLite (3.24.0 and newer) provide a different syntax that allows -for more granular control over which constraint violation should trigger the -conflict resolution, and what values should be updated or preserved. +**MySQL** supports upsert via the *ON DUPLICATE KEY UPDATE* clause. For +example: + +.. code-block:: python + + class User(Model): + username = TextField(unique=True) + last_login = DateTimeField(null=True) + login_count = IntegerField() + + # Insert a new user. + User.create(username='huey', login_count=0) + + # Simulate the user logging in. The login count and timestamp will be + # either created or updated correctly. + now = datetime.now() + rowid = (User + .insert(username='huey', last_login=now, login_count=1) + .on_conflict( + preserve=[User.last_login], # Use the value we would have inserted. + update={User.login_count: User.login_count + 1}) + .execute()) + +In the above example, we could safely invoke the upsert query as many times as +we wanted. The login count will be incremented atomically, the last login +column will be updated, and no duplicate rows will be created. + +**Postgresql and SQLite** (3.24.0 and newer) provide a different syntax that +allows for more granular control over which constraint violation should trigger +the conflict resolution, and what values should be updated or preserved. Example of using :py:meth:`~Insert.on_conflict` to perform a Postgresql-style upsert (or SQLite 3.24+): @@ -321,8 +348,8 @@ upsert (or SQLite 3.24+): rowid = (User .insert(username='huey', last_login=now, login_count=1) .on_conflict( - conflict_target=(User.username,), # Which constraint? - preserve=(User.last_login,), # Use the value we would have inserted. + conflict_target=[User.username], # Which constraint? + preserve=[User.last_login], # Use the value we would have inserted. update={User.login_count: User.login_count + 1}) .execute()) @@ -330,6 +357,10 @@ In the above example, we could safely invoke the upsert query as many times as we wanted. The login count will be incremented atomically, the last login column will be updated, and no duplicate rows will be created. +.. note:: + The main difference between MySQL and Postgresql/SQLite is that Postgresql + and SQLite require that you specify a ``conflict_target``. + For more information, see :py:meth:`Insert.on_conflict` and :py:class:`OnConflict`. diff --git a/tests/models.py b/tests/models.py index b217fc32c..457c1ac2e 100644 --- a/tests/models.py +++ b/tests/models.py @@ -2554,6 +2554,7 @@ class Meta: class OCTest(TestModel): a = CharField(unique=True) b = IntegerField(default=0) + c = IntegerField(default=0) class OnConflictTestCase(ModelTestCase): @@ -2618,6 +2619,59 @@ def test_replace(self): ('nuggie', 'dog', '123'), ('beanie', 'cat', '126')]) + @requires_models(OCTest) + def test_update(self): + pk = (OCTest + .insert(a='a', b=3) + .on_conflict(update={OCTest.b: 1337}) + .execute()) + oc = OCTest.get(OCTest.a == 'a') + self.assertEqual(oc.b, 3) + + pk2 = (OCTest + .insert(a='a', b=4) + .on_conflict(update={OCTest.b: OCTest.b + 10}) + .execute()) + self.assertEqual(pk, pk2) + self.assertEqual(OCTest.select().count(), 1) + + oc = OCTest.get(OCTest.a == 'a') + self.assertEqual(oc.b, 13) + + pk3 = (OCTest + .insert(a='a2', b=5) + .on_conflict(update={OCTest.b: 1337}) + .execute()) + self.assertTrue(pk3 != pk2) + self.assertEqual(OCTest.select().count(), 2) + + oc = OCTest.get(OCTest.a == 'a2') + self.assertEqual(oc.b, 5) + + @requires_models(OCTest) + def test_update_preserve(self): + OCTest.create(a='a', b=3) + + pk = (OCTest + .insert(a='a', b=4) + .on_conflict(preserve=[OCTest.b]) + .execute()) + oc = OCTest.get(OCTest.a == 'a') + self.assertEqual(oc.b, 4) + + pk2 = (OCTest + .insert(a='a', b=5, c=6) + .on_conflict( + preserve=[OCTest.c], + update={OCTest.b: OCTest.b + 100}) + .execute()) + self.assertEqual(pk, pk2) + self.assertEqual(OCTest.select().count(), 1) + + oc = OCTest.get(OCTest.a == 'a') + self.assertEqual(oc.b, 104) + self.assertEqual(oc.c, 6) + class TestUpsertSqlite(OnConflictTestCase): database = get_in_memory_db() @@ -2746,9 +2800,9 @@ def test_update_atomic(self): # Sanity-check the SQL. self.assertSQL(query, ( - 'INSERT INTO "octest" ("a", "b") VALUES (?, ?) ' + 'INSERT INTO "octest" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") ' - 'DO UPDATE SET "b" = ("octest"."b" + ?)'), ['foo', 1, 2]) + 'DO UPDATE SET "b" = ("octest"."b" + ?)'), ['foo', 1, 0, 2]) # First execution returns rowid=1. Second execution hits the conflict- # resolution, and will update the value in "b" from 1 -> 3. @@ -2760,6 +2814,17 @@ def test_update_atomic(self): self.assertEqual(obj.a, 'foo') self.assertEqual(obj.b, 3) + query = OCTest.insert(a='foo', b=4, c=5).on_conflict( + conflict_target=[OCTest.a], + preserve=[OCTest.c], + update={OCTest.b: OCTest.b + 100}) + self.assertEqual(query.execute(), rowid2) + + obj = OCTest.get() + self.assertEqual(obj.a, 'foo') + self.assertEqual(obj.b, 103) + self.assertEqual(obj.c, 5) + @skip_unless(IS_SQLITE_24, 'requires sqlite >= 3.24') @requires_models(OCTest) def test_update_where_clause(self): @@ -2770,9 +2835,9 @@ def test_update_where_clause(self): update={OCTest.b: OCTest.b + 2}, where=(OCTest.b < 3)) self.assertSQL(query, ( - 'INSERT INTO "octest" ("a", "b") VALUES (?, ?) ' + 'INSERT INTO "octest" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") DO UPDATE SET "b" = ("octest"."b" + ?) ' - 'WHERE ("octest"."b" < ?)'), ['foo', 1, 2, 3]) + 'WHERE ("octest"."b" < ?)'), ['foo', 1, 0, 2, 3]) # First execution returns rowid=1. Second execution hits the conflict- # resolution, and will update the value in "b" from 1 -> 3. @@ -2847,9 +2912,9 @@ def test_update_atomic(self): conflict_target=(OCTest.a,), update={OCTest.b: OCTest.b + 2}) self.assertSQL(query, ( - 'INSERT INTO "octest" ("a", "b") VALUES (?, ?) ' + 'INSERT INTO "octest" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") DO UPDATE SET "b" = ("octest"."b" + ?) ' - 'RETURNING "id"'), ['foo', 1, 2]) + 'RETURNING "id"'), ['foo', 1, 0, 2]) # First execution returns rowid=1. Second execution hits the conflict- # resolution, and will update the value in "b" from 1 -> 3. @@ -2861,6 +2926,17 @@ def test_update_atomic(self): self.assertEqual(obj.a, 'foo') self.assertEqual(obj.b, 3) + query = OCTest.insert(a='foo', b=4, c=5).on_conflict( + conflict_target=[OCTest.a], + preserve=[OCTest.c], + update={OCTest.b: OCTest.b + 100}) + self.assertEqual(query.execute(), rowid2) + + obj = OCTest.get() + self.assertEqual(obj.a, 'foo') + self.assertEqual(obj.b, 103) + self.assertEqual(obj.c, 5) + @requires_models(OCTest) def test_update_where_clause(self): # Add a new row with the given "a" value. If a conflict occurs, @@ -2870,10 +2946,10 @@ def test_update_where_clause(self): update={OCTest.b: OCTest.b + 2}, where=(OCTest.b < 3)) self.assertSQL(query, ( - 'INSERT INTO "octest" ("a", "b") VALUES (?, ?) ' + 'INSERT INTO "octest" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") DO UPDATE SET "b" = ("octest"."b" + ?) ' 'WHERE ("octest"."b" < ?) ' - 'RETURNING "id"'), ['foo', 1, 2, 3]) + 'RETURNING "id"'), ['foo', 1, 0, 2, 3]) # First execution returns rowid=1. Second execution hits the conflict- # resolution, and will update the value in "b" from 1 -> 3. From 53731e6de666547aa9895267da711ab0c0b855e1 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 10:46:29 -0500 Subject: [PATCH 26/58] Remove dead code. --- peewee.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/peewee.py b/peewee.py index 8b97ef89d..0e337ae88 100644 --- a/peewee.py +++ b/peewee.py @@ -4331,9 +4331,6 @@ def python_value(self, value): return value return self.rel_field.python_value(value) - def expression(self): - return self.column == self.rel_field.column - def bind(self, model, name, set_attribute=True): if not self.column_name: self.column_name = name if name.endswith('_id') else name + '_id' From 75096bc1922c25201a77045cf7c9e8e2e6353359 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 16:27:32 -0500 Subject: [PATCH 27/58] Slight re-org to improve coverage of UPDATE...FROM tests. --- peewee.py | 1 + tests/model_sql.py | 34 +++++++++++++++++++++++++++++ tests/models.py | 54 ++++++++++++++++++---------------------------- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/peewee.py b/peewee.py index 0e337ae88..3abc17179 100644 --- a/peewee.py +++ b/peewee.py @@ -113,6 +113,7 @@ 'prefetch', 'ProgrammingError', 'Proxy', + 'QualifiedNames', 'SchemaManager', 'SmallIntegerField', 'Select', diff --git a/tests/model_sql.py b/tests/model_sql.py index 09f3fc138..6c8942c44 100644 --- a/tests/model_sql.py +++ b/tests/model_sql.py @@ -371,6 +371,40 @@ class Account(TestModel): 'FROM "salesperson" AS "t1" ' 'WHERE ("sales_id" = "id")'), []) + query = (User + .update({User.username: QualifiedNames(Tweet.content)}) + .from_(Tweet) + .where(Tweet.content == 'tx')) + self.assertSQL(query, ( + 'UPDATE "users" SET "username" = "t1"."content" ' + 'FROM "tweet" AS "t1" WHERE ("content" = ?)'), ['tx']) + + def test_update_from_qualnames(self): + data = [(1, 'u1-x'), (2, 'u2-x')] + vl = ValuesList(data, columns=('id', 'username'), alias='tmp') + query = (User + .update({User.username: QualifiedNames(vl.c.username)}) + .from_(vl) + .where(QualifiedNames(User.id == vl.c.id))) + self.assertSQL(query, ( + 'UPDATE "users" SET "username" = "tmp"."username" ' + 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username") ' + 'WHERE ("users"."id" = "tmp"."id")'), [1, 'u1-x', 2, 'u2-x']) + + def test_update_from_subselect(self): + data = [(1, 'u1-x'), (2, 'u2-x')] + vl = ValuesList(data, columns=('id', 'username'), alias='tmp') + subq = vl.select(vl.c.id, vl.c.username) + query = (User + .update({User.username: QualifiedNames(subq.c.username)}) + .from_(subq) + .where(QualifiedNames(User.id == subq.c.id))) + self.assertSQL(query, ( + 'UPDATE "users" SET "username" = "t1"."username" FROM (' + 'SELECT "tmp"."id", "tmp"."username" ' + 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username")) AS "t1" ' + 'WHERE ("users"."id" = "t1"."id")'), [1, 'u1-x', 2, 'u2-x']) + def test_delete(self): query = (Note .delete() diff --git a/tests/models.py b/tests/models.py index 457c1ac2e..0a8cb3793 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1211,12 +1211,15 @@ class TestDefaultValues(ModelTestCase): database = get_in_memory_db() requires = [Sample, SampleMeta] - def test_default_absent_on_insert(self): + def test_default_present_on_insert(self): + # Although value is not specified, it has a default, which is included + # in the INSERT. query = Sample.insert(counter=0) self.assertSQL(query, ( 'INSERT INTO "sample" ("counter", "value") ' 'VALUES (?, ?)'), [0, 1.0]) + # Default values are also included when doing bulk inserts. query = Sample.insert_many([ {'counter': '0'}, {'counter': 1, 'value': 2}, @@ -1231,7 +1234,6 @@ def test_default_absent_on_insert(self): 'INSERT INTO "sample" ("counter", "value") ' 'VALUES (?, ?), (?, ?)'), [0, 1.0, 1, 2.0]) - def test_default_present_on_create(self): s = Sample.create(counter=3) s_db = Sample.get(Sample.counter == 3) @@ -1245,6 +1247,7 @@ def test_defaults_from_cursor(self): # Simple query. query = SampleMeta.select(SampleMeta.sample).order_by(SampleMeta.value) + # Defaults are not present when doing a read query. with self.assertQueryCount(1): sm1_db, sm2_db = list(query) self.assertIsNone(sm1_db.value) @@ -3250,24 +3253,19 @@ def test_sequence(self): @requires_postgresql -class TestUpdateFrom(ModelTestCase): +class TestUpdateFromIntegration(ModelTestCase): requires = [User] def test_update_from(self): u1, u2 = [User.create(username=username) for username in ('u1', 'u2')] data = [(u1.id, 'u1-x'), (u2.id, 'u2-x')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') - query = (User - .update({User.username: QualifiedNames(vl.c.username)}) - .from_(vl) - .where(QualifiedNames(User.id == vl.c.id))) - self.assertSQL(query, ( - 'UPDATE "users" SET "username" = "tmp"."username" ' - 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username") ' - 'WHERE ("users"."id" = "tmp"."id")'), - [u1.id, 'u1-x', u2.id, 'u2-x']) + (User + .update({User.username: QualifiedNames(vl.c.username)}) + .from_(vl) + .where(QualifiedNames(User.id == vl.c.id)) + .execute()) - query.execute() usernames = [u.username for u in User.select().order_by(User.username)] self.assertEqual(usernames, ['u1-x', 'u2-x']) @@ -3276,18 +3274,12 @@ def test_update_from_subselect(self): data = [(u1.id, 'u1-y'), (u2.id, 'u2-y')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') subq = vl.select(vl.c.id, vl.c.username) - query = (User - .update({User.username: QualifiedNames(subq.c.username)}) - .from_(subq) - .where(QualifiedNames(User.id == subq.c.id))) - self.assertSQL(query, ( - 'UPDATE "users" SET "username" = "t1"."username" FROM (' - 'SELECT "tmp"."id", "tmp"."username" ' - 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username")) AS "t1" ' - 'WHERE ("users"."id" = "t1"."id")'), - [u1.id, 'u1-y', u2.id, 'u2-y']) + (User + .update({User.username: QualifiedNames(subq.c.username)}) + .from_(subq) + .where(QualifiedNames(User.id == subq.c.id)) + .execute()) - query.execute() usernames = [u.username for u in User.select().order_by(User.username)] self.assertEqual(usernames, ['u1-y', 'u2-y']) @@ -3297,15 +3289,11 @@ def test_update_from_simple(self): t1 = Tweet.create(user=u, content='t1') t2 = Tweet.create(user=u, content='t2') - query = (User - .update({User.username: QualifiedNames(Tweet.content)}) - .from_(Tweet) - .where(Tweet.content == 't2')) - self.assertSQL(query, ( - 'UPDATE "users" SET "username" = "t1"."content" ' - 'FROM "tweet" AS "t1" ' - 'WHERE ("content" = ?)'), ['t2']) - query.execute() + (User + .update({User.username: QualifiedNames(Tweet.content)}) + .from_(Tweet) + .where(Tweet.content == 't2') + .execute()) self.assertEqual(User.get(User.id == u.id).username, 't2') From 530d2f72009fd2183985499aeb98823209e29cb0 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 16:35:07 -0500 Subject: [PATCH 28/58] Updates to changelog w/current changes. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41f2f998b..2aa62d8ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,10 @@ https://github.com/coleifer/peewee/releases ## master -* Add new model meta option for indicating that a model uses a temporary table. * Allow `database` parameter to be specified with `ModelSelect.get()` method. For discussion, see #1620. +* Add `QualifiedNames` helper to peewee module exports. +* Add `temporary=` meta option to support temporary tables. * Fixed edge-case where attempting to alias a field to it's underlying column-name (when different), Peewee would not respect the alias and use the field name instead. See #1625 for details and discussion. From 5408f18e938fdf674382101e7500d098a698d4a1 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 16:35:15 -0500 Subject: [PATCH 29/58] Move older regression tests into "regressions" test module. --- tests/models.py | 120 ----------------------------------------- tests/regressions.py | 123 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 120 deletions(-) diff --git a/tests/models.py b/tests/models.py index 0a8cb3793..bad225995 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1007,76 +1007,6 @@ def test_raw_iterator(self): self.assertEqual([u.username for u in query], []) -class DiA(TestModel): - a = TextField(unique=True) -class DiB(TestModel): - a = ForeignKeyField(DiA) - b = TextField() -class DiC(TestModel): - b = ForeignKeyField(DiB) - c = TextField() -class DiD(TestModel): - c = ForeignKeyField(DiC) - d = TextField() -class DiBA(TestModel): - a = ForeignKeyField(DiA, to_field=DiA.a) - b = TextField() - - -class TestDeleteInstanceRegression(ModelTestCase): - database = get_in_memory_db() - requires = [DiA, DiB, DiC, DiD, DiBA] - - def test_delete_instance_regression(self): - with self.database.atomic(): - a1, a2, a3 = [DiA.create(a=a) for a in ('a1', 'a2', 'a3')] - for a in (a1, a2, a3): - for j in (1, 2): - b = DiB.create(a=a, b='%s-b%s' % (a.a, j)) - c = DiC.create(b=b, c='%s-c' % (b.b)) - d = DiD.create(c=c, d='%s-d' % (c.c)) - - DiBA.create(a=a, b='%s-b%s' % (a.a, j)) - - # (a1 (b1 (c (d))), (b2 (c (d)))), (a2 ...), (a3 ...) - with self.assertQueryCount(5): - a2.delete_instance(recursive=True) - - queries = [logrecord.msg for logrecord in self._qh.queries[-5:]] - self.assertEqual(sorted(queries, reverse=True), [ - ('DELETE FROM "did" WHERE ("c_id" IN (' - 'SELECT "t1"."id" FROM "dic" AS "t1" WHERE ("t1"."b_id" IN (' - 'SELECT "t2"."id" FROM "dib" AS "t2" WHERE ("t2"."a_id" = ?)' - '))))', [2]), - ('DELETE FROM "dic" WHERE ("b_id" IN (' - 'SELECT "t1"."id" FROM "dib" AS "t1" WHERE ("t1"."a_id" = ?)' - '))', [2]), - ('DELETE FROM "diba" WHERE ("a_id" = ?)', ['a2']), - ('DELETE FROM "dib" WHERE ("a_id" = ?)', [2]), - ('DELETE FROM "dia" WHERE ("id" = ?)', [2]) - ]) - - # a1 & a3 exist, plus their relations. - self.assertTrue(DiA.select().count(), 2) - for rel in (DiB, DiBA, DiC, DiD): - self.assertTrue(rel.select().count(), 4) # 2x2 - - with self.assertQueryCount(5): - a1.delete_instance(recursive=True) - - # Only the objects related to a3 exist still. - self.assertTrue(DiA.select().count(), 1) - self.assertEqual(DiA.get(DiA.a == 'a3').id, a3.id) - self.assertEqual([d.d for d in DiD.select().order_by(DiD.d)], - ['a3-b1-c-d', 'a3-b2-c-d']) - self.assertEqual([c.c for c in DiC.select().order_by(DiC.c)], - ['a3-b1-c', 'a3-b2-c']) - self.assertEqual([b.b for b in DiB.select().order_by(DiB.b)], - ['a3-b1', 'a3-b2']) - self.assertEqual([ba.b for ba in DiBA.select().order_by(DiBA.b)], - ['a3-b1', 'a3-b2']) - - class TestDeleteInstance(ModelTestCase): database = get_in_memory_db() requires = [User, Account, Tweet, Favorite] @@ -3032,56 +2962,6 @@ def get_person(name): self.assertEqual(db_data, list(data)) -class User2(TestModel): - username = TextField() - - -class Category2(TestModel): - name = TextField() - parent = ForeignKeyField('self', backref='children', null=True) - user = ForeignKeyField(User2) - - -class TestGithub1354(ModelTestCase): - @requires_models(Category2, User2) - def test_get_or_create_self_referential_fk2(self): - huey = User2.create(username='huey') - parent = Category2.create(name='parent', user=huey) - child, created = Category2.get_or_create(parent=parent, name='child', - user=huey) - child_db = Category2.get(Category2.parent == parent) - self.assertEqual(child_db.user.username, 'huey') - self.assertEqual(child_db.parent.name, 'parent') - self.assertEqual(child_db.name, 'child') - - -class TestCountUnionRegression(ModelTestCase): - @requires_mysql - @requires_models(User) - def test_count_union(self): - with self.database.atomic(): - for i in range(5): - User.create(username='user-%d' % i) - - lhs = User.select() - rhs = User.select() - query = (lhs | rhs) - self.assertSQL(query, ( - 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' - 'UNION ' - 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2"'), []) - - self.assertEqual(query.count(), 5) - - query = query.limit(3) - self.assertSQL(query, ( - 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' - 'UNION ' - 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2" ' - 'LIMIT ?'), [3]) - self.assertEqual(query.count(), 3) - - class TestSumCase(ModelTestCase): @requires_models(User) def test_sum_case(self): diff --git a/tests/regressions.py b/tests/regressions.py index bb24a04c2..92df37119 100644 --- a/tests/regressions.py +++ b/tests/regressions.py @@ -3,6 +3,10 @@ from .base import BaseTestCase from .base import ModelTestCase from .base import TestModel +from .base import get_in_memory_db +from .base import requires_models +from .base import requires_mysql +from .base_models import User class ColAlias(TestModel): @@ -76,3 +80,122 @@ def __repr__(self): f = Foo(id=1337) self.assertEqual(repr(f), 'FOO: 1337') + + +class DiA(TestModel): + a = TextField(unique=True) +class DiB(TestModel): + a = ForeignKeyField(DiA) + b = TextField() +class DiC(TestModel): + b = ForeignKeyField(DiB) + c = TextField() +class DiD(TestModel): + c = ForeignKeyField(DiC) + d = TextField() +class DiBA(TestModel): + a = ForeignKeyField(DiA, to_field=DiA.a) + b = TextField() + + +class TestDeleteInstanceRegression(ModelTestCase): + database = get_in_memory_db() + requires = [DiA, DiB, DiC, DiD, DiBA] + + def test_delete_instance_regression(self): + with self.database.atomic(): + a1, a2, a3 = [DiA.create(a=a) for a in ('a1', 'a2', 'a3')] + for a in (a1, a2, a3): + for j in (1, 2): + b = DiB.create(a=a, b='%s-b%s' % (a.a, j)) + c = DiC.create(b=b, c='%s-c' % (b.b)) + d = DiD.create(c=c, d='%s-d' % (c.c)) + + DiBA.create(a=a, b='%s-b%s' % (a.a, j)) + + # (a1 (b1 (c (d))), (b2 (c (d)))), (a2 ...), (a3 ...) + with self.assertQueryCount(5): + a2.delete_instance(recursive=True) + + queries = [logrecord.msg for logrecord in self._qh.queries[-5:]] + self.assertEqual(sorted(queries, reverse=True), [ + ('DELETE FROM "did" WHERE ("c_id" IN (' + 'SELECT "t1"."id" FROM "dic" AS "t1" WHERE ("t1"."b_id" IN (' + 'SELECT "t2"."id" FROM "dib" AS "t2" WHERE ("t2"."a_id" = ?)' + '))))', [2]), + ('DELETE FROM "dic" WHERE ("b_id" IN (' + 'SELECT "t1"."id" FROM "dib" AS "t1" WHERE ("t1"."a_id" = ?)' + '))', [2]), + ('DELETE FROM "diba" WHERE ("a_id" = ?)', ['a2']), + ('DELETE FROM "dib" WHERE ("a_id" = ?)', [2]), + ('DELETE FROM "dia" WHERE ("id" = ?)', [2]) + ]) + + # a1 & a3 exist, plus their relations. + self.assertTrue(DiA.select().count(), 2) + for rel in (DiB, DiBA, DiC, DiD): + self.assertTrue(rel.select().count(), 4) # 2x2 + + with self.assertQueryCount(5): + a1.delete_instance(recursive=True) + + # Only the objects related to a3 exist still. + self.assertTrue(DiA.select().count(), 1) + self.assertEqual(DiA.get(DiA.a == 'a3').id, a3.id) + self.assertEqual([d.d for d in DiD.select().order_by(DiD.d)], + ['a3-b1-c-d', 'a3-b2-c-d']) + self.assertEqual([c.c for c in DiC.select().order_by(DiC.c)], + ['a3-b1-c', 'a3-b2-c']) + self.assertEqual([b.b for b in DiB.select().order_by(DiB.b)], + ['a3-b1', 'a3-b2']) + self.assertEqual([ba.b for ba in DiBA.select().order_by(DiBA.b)], + ['a3-b1', 'a3-b2']) + + +class TestCountUnionRegression(ModelTestCase): + @requires_mysql + @requires_models(User) + def test_count_union(self): + with self.database.atomic(): + for i in range(5): + User.create(username='user-%d' % i) + + lhs = User.select() + rhs = User.select() + query = (lhs | rhs) + self.assertSQL(query, ( + 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' + 'UNION ' + 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2"'), []) + + self.assertEqual(query.count(), 5) + + query = query.limit(3) + self.assertSQL(query, ( + 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' + 'UNION ' + 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2" ' + 'LIMIT ?'), [3]) + self.assertEqual(query.count(), 3) + + +class User2(TestModel): + username = TextField() + +class Category2(TestModel): + name = TextField() + parent = ForeignKeyField('self', backref='children', null=True) + user = ForeignKeyField(User2) + + +class TestGithub1354(ModelTestCase): + @requires_models(Category2, User2) + def test_get_or_create_self_referential_fk2(self): + huey = User2.create(username='huey') + parent = Category2.create(name='parent', user=huey) + child, created = Category2.get_or_create(parent=parent, name='child', + user=huey) + child_db = Category2.get(Category2.parent == parent) + self.assertEqual(child_db.user.username, 'huey') + self.assertEqual(child_db.parent.name, 'parent') + self.assertEqual(child_db.name, 'child') From d00ffe9d06ea23aa660e377e8f8616c19e074fab Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 16:43:44 -0500 Subject: [PATCH 30/58] More test cleanups. --- tests/database.py | 15 ++++++++------- tests/expressions.py | 7 +++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/database.py b/tests/database.py index 2fc1a9345..077b3d12b 100644 --- a/tests/database.py +++ b/tests/database.py @@ -251,22 +251,23 @@ def assertBatches(n_objs, batch_size, n_commits): class TestThreadSafety(ModelTestCase): + nthreads = 4 + nrows = 10 requires = [User] def test_multiple_writers(self): def create_users(idx): - n = 10 - for i in range(idx * n, (idx + 1) * n): + for i in range(idx * self.nrows, (idx + 1) * self.nrows): User.create(username='u%d' % i) threads = [] - for i in range(4): + for i in range(self.nthreads): threads.append(threading.Thread(target=create_users, args=(i,))) for t in threads: t.start() for t in threads: t.join() - self.assertEqual(User.select().count(), 40) + self.assertEqual(User.select().count(), self.nrows * self.nthreads) def test_multiple_readers(self): data = Queue() @@ -275,13 +276,13 @@ def read_user_count(n): data.put(User.select().count()) threads = [] - for i in range(4): + for i in range(self.nthreads): threads.append(threading.Thread(target=read_user_count, - args=(10,))) + args=(self.nrows,))) for t in threads: t.start() for t in threads: t.join() - self.assertEqual(data.qsize(), 40) + self.assertEqual(data.qsize(), self.nrows * self.nthreads) class TestDeferredDatabase(BaseTestCase): diff --git a/tests/expressions.py b/tests/expressions.py index 4f1a38fdb..bd4c696ca 100644 --- a/tests/expressions.py +++ b/tests/expressions.py @@ -55,6 +55,13 @@ class UpperModel(TestModel): class TestValueConversion(ModelTestCase): + """ + Test the conversion of field values using a field's db_value() function. + + It is possible that a field's `db_value()` function may returns a Node + subclass (e.g. a SQL function). These tests verify and document how such + conversions are applied in various parts of the query. + """ database = get_in_memory_db() requires = [UpperModel] From f4b9f70881b54484e7f2ec5f2d84b46a3512a342 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 19:19:11 -0500 Subject: [PATCH 31/58] Simplify quickstart model. --- docs/peewee/quickstart.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/peewee/quickstart.rst b/docs/peewee/quickstart.rst index 1024eb60f..7b7a71418 100644 --- a/docs/peewee/quickstart.rst +++ b/docs/peewee/quickstart.rst @@ -46,7 +46,6 @@ data model, by defining one or more :py:class:`Model` classes: class Person(Model): name = CharField() birthday = DateField() - is_relative = BooleanField() class Meta: database = db # This model uses the "people.db" database. @@ -107,7 +106,7 @@ people's records. .. code-block:: python from datetime import date - uncle_bob = Person(name='Bob', birthday=date(1960, 1, 15), is_relative=True) + uncle_bob = Person(name='Bob', birthday=date(1960, 1, 15)) uncle_bob.save() # bob is now stored in the database # Returns: 1 @@ -120,8 +119,8 @@ returns a model instance: .. code-block:: python - grandma = Person.create(name='Grandma', birthday=date(1935, 3, 1), is_relative=True) - herb = Person.create(name='Herb', birthday=date(1950, 5, 5), is_relative=False) + grandma = Person.create(name='Grandma', birthday=date(1935, 3, 1)) + herb = Person.create(name='Herb', birthday=date(1950, 5, 5)) To update a row, modify the model instance and call :py:meth:`~Model.save` to persist the changes. Here we will change Grandma's name and then save the @@ -197,12 +196,12 @@ Let's list all the people in the database: .. code-block:: python for person in Person.select(): - print(person.name, person.is_relative) + print(person.name) # prints: - # Bob True - # Grandma L. True - # Herb False + # Bob + # Grandma L. + # Herb Let's list all the cats and their owner's name: From 112833cc0f70a21d2efc998a84db35f359a1c0e7 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 18 Jun 2018 20:38:45 -0500 Subject: [PATCH 32/58] Docs index cleanup. --- README.rst | 4 ++-- docs/index.rst | 19 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index e2702d6f4..0fa5e6c18 100644 --- a/README.rst +++ b/README.rst @@ -17,8 +17,8 @@ New to peewee? These may help: * `Quickstart `_ * `Example twitter app `_ -* `Query operators `_ -* `Field types `_ +* `Models and fields `_ +* `Querying `_ Examples -------- diff --git a/docs/index.rst b/docs/index.rst index 200218bf1..043611841 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,10 +11,10 @@ peewee Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. -* A small, expressive ORM -* Written in python with support for versions 2.7+ and 3.4+ (developed with 3.6) -* Built-in support for SQLite, MySQL and Postgresql. -* :ref:`numerous extensions available ` (:ref:`postgres hstore/json/arrays `, :ref:`sqlite full-text-search `, :ref:`schema migrations `, and much more). +* a small, expressive ORM +* python 2.7+ and 3.4+ (developed with 3.6) +* supports sqlite, mysql and postgresql +* :ref:`tons of extensions ` .. image:: postgresql.png :target: peewee/database.html#using-postgresql @@ -30,13 +30,12 @@ it easy to learn and intuitive to use. Peewee's source code hosted on `GitHub `_. -New to peewee? Here is a list of documents you might find most helpful when getting -started: +New to peewee? These may help: -* :ref:`Quickstart guide ` -- this guide covers all the bare essentials. It will take you between 5 and 10 minutes to go through it. -* :ref:`Example Twitter app ` written using Flask framework. -* :ref:`Guide to the various query operators ` describes how to construct queries and combine expressions. -* :ref:`Field types table ` lists the various field types peewee supports and the parameters they accept. There is also an :ref:`extension module ` that contains :ref:`special/custom field types `. +* :ref:`Quickstart ` +* :ref:`Example twitter app ` +* :ref:`Models and fields ` +* :ref:`Querying ` Contents: --------- From 50231d9f9b9f4a74e5955da1d787d4887d40d31d Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 19 Jun 2018 09:44:37 -0500 Subject: [PATCH 33/58] Small fixes for query-builder and tests. --- peewee.py | 7 +++---- tests/queries.py | 19 +++++++++++++++++++ tests/sql.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/peewee.py b/peewee.py index 3abc17179..42eaa50da 100644 --- a/peewee.py +++ b/peewee.py @@ -831,7 +831,7 @@ def update(self, update=None, **kwargs): update = {} if update is None else update for key, value in kwargs.items(): src = self if self._columns else self.c - update[getattr(self, key)] = value + update[getattr(src, key)] = value return Update(self, update=update) @__bind_database__ @@ -910,14 +910,13 @@ def __sql__(self, ctx): class CTE(_HashableSource, Source): def __init__(self, name, query, recursive=False, columns=None): self._alias = name - self._nested_cte_list = query._cte_list - query._cte_list = () self._query = query self._recursive = recursive if columns is not None: columns = [Entity(c) if isinstance(c, basestring) else c for c in columns] self._columns = columns + query._cte_list = () super(CTE, self).__init__(alias=name) def select_from(self, *columns): @@ -1629,7 +1628,7 @@ def __getitem__(self, value): index = value if index is not None and index >= 0: index += 1 - self._cursor_wrapper.fill_cache(index) + self._cursor_wrapper.fill_cache(index if index > 0 else 0) return self._cursor_wrapper.row_cache[value] def __len__(self): diff --git a/tests/queries.py b/tests/queries.py index 3608e157e..8f991efd9 100644 --- a/tests/queries.py +++ b/tests/queries.py @@ -116,6 +116,25 @@ def test_scalar(self): query = query.where(Register.value >= 2) self.assertEqual(query.scalar(as_tuple=True), (15, 3, 5)) + def test_slicing_select(self): + values = [1., 1., 2., 3., 5., 8.] + (Register + .insert([(v,) for v in values], columns=(Register.value,)) + .execute()) + + query = (Register + .select(Register.value) + .order_by(Register.value) + .tuples()) + with self.assertQueryCount(1): + self.assertEqual(query[0], (1.,)) + self.assertEqual(query[:2], [(1.,), (1.,)]) + self.assertEqual(query[1:4], [(1.,), (2.,), (3.,)]) + self.assertEqual(query[-1], (8.,)) + self.assertEqual(query[-2], (5.,)) + self.assertEqual(query[-2:], [(5.,), (8.,)]) + self.assertEqual(query[2:-2], [(2.,), (3.,)]) + class TestQueryCloning(BaseTestCase): def test_clone_tables(self): diff --git a/tests/sql.py b/tests/sql.py index f062f5e22..450124db7 100644 --- a/tests/sql.py +++ b/tests/sql.py @@ -220,6 +220,29 @@ def test_two_ctes(self): 'WHERE (("user_ids"."id" = "t2"."id") AND ' '("user_names"."username" = "t2"."username"))'), []) + def test_select_from_cte(self): + # Use the "select_from()" helper on the CTE object. + cte = User.select(User.c.username).cte('user_cte') + query = cte.select_from(cte.c.username).order_by(cte.c.username) + self.assertSQL(query, ( + 'WITH "user_cte" AS (SELECT "t1"."username" FROM "users" AS "t1") ' + 'SELECT "user_cte"."username" FROM "user_cte" ' + 'ORDER BY "user_cte"."username"'), []) + + # Test selecting from multiple CTEs, which is done manually. + c1 = User.select(User.c.username).where(User.c.is_admin == 1).cte('c1') + c2 = User.select(User.c.username).where(User.c.is_staff == 1).cte('c2') + query = (Select((c1, c2), (c1.c.username, c2.c.username)) + .with_cte(c1, c2)) + self.assertSQL(query, ( + 'WITH "c1" AS (' + 'SELECT "t1"."username" FROM "users" AS "t1" ' + 'WHERE ("t1"."is_admin" = ?)), ' + '"c2" AS (' + 'SELECT "t1"."username" FROM "users" AS "t1" ' + 'WHERE ("t1"."is_staff" = ?)) ' + 'SELECT "c1"."username", "c2"."username" FROM "c1", "c2"'), [1, 1]) + def test_fibonacci_cte(self): q1 = Select(columns=( Value(1).alias('n'), @@ -559,6 +582,29 @@ def test_update_subquery(self): 'GROUP BY "users"."id" ' 'HAVING ("ct" > ?)))'), [0, True, 100]) + def test_update_from(self): + data = [(1, 'u1-x'), (2, 'u2-x')] + vl = ValuesList(data, columns=('id', 'username'), alias='tmp') + query = (User + .update(username=QualifiedNames(vl.c.username)) + .from_(vl) + .where(QualifiedNames(User.c.id == vl.c.id))) + self.assertSQL(query, ( + 'UPDATE "users" SET "username" = "tmp"."username" ' + 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username") ' + 'WHERE ("users"."id" = "tmp"."id")'), [1, 'u1-x', 2, 'u2-x']) + + subq = vl.select(vl.c.id, vl.c.username) + query = (User + .update({User.c.username: QualifiedNames(subq.c.username)}) + .from_(subq) + .where(QualifiedNames(User.c.id == subq.c.id))) + self.assertSQL(query, ( + 'UPDATE "users" SET "username" = "t1"."username" FROM (' + 'SELECT "tmp"."id", "tmp"."username" ' + 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username")) AS "t1" ' + 'WHERE ("users"."id" = "t1"."id")'), [1, 'u1-x', 2, 'u2-x']) + def test_update_returning(self): query = (User .update({User.c.is_admin: True}) From 0ebdc48594674be932aa7e6e3da71d8c6d29b181 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 19 Jun 2018 11:57:35 -0500 Subject: [PATCH 34/58] Fix failing test. --- peewee.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/peewee.py b/peewee.py index 42eaa50da..d71aa23c6 100644 --- a/peewee.py +++ b/peewee.py @@ -1626,9 +1626,9 @@ def __getitem__(self, value): index = value.stop else: index = value - if index is not None and index >= 0: - index += 1 - self._cursor_wrapper.fill_cache(index if index > 0 else 0) + if index is not None: + index = index + 1 if index >= 0 else 0 + self._cursor_wrapper.fill_cache(index) return self._cursor_wrapper.row_cache[value] def __len__(self): From 9f09e46a8ca0ede482cdc81ca824fc73c52b0ffa Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 21 Jun 2018 11:57:34 -0500 Subject: [PATCH 35/58] Add note about register_hstore. Fixes #1639. --- docs/peewee/changes.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/peewee/changes.rst b/docs/peewee/changes.rst index 472190557..4df42d579 100644 --- a/docs/peewee/changes.rst +++ b/docs/peewee/changes.rst @@ -132,6 +132,13 @@ like this: arguments, options) +Postgresql Extension +^^^^^^^^^^^^^^^^^^^^ + +The `PostgresqlExtDatabase` no longer registers the `hstore` extension by +default. To use the `hstore` extension in 3.0 and onwards, pass +`register_hstore=True` when initializing the database object. + Signals Extension ^^^^^^^^^^^^^^^^^ From ee4189f0747d2399369a81ef37034e06f6bfe807 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 21 Jun 2018 15:40:29 -0500 Subject: [PATCH 36/58] Add help request for documentation. --- docs/conf.py | 2 +- docs/index.rst | 2 ++ docs/peewee/api.rst | 4 ++++ docs/peewee/help-request.rst | 5 +++++ docs/peewee/models.rst | 4 +++- docs/peewee/querying.rst | 2 ++ docs/peewee/sqlite_ext.rst | 4 ++-- 7 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 docs/peewee/help-request.rst diff --git a/docs/conf.py b/docs/conf.py index ca684a643..97699e8d5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -86,7 +86,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'pastie' +#pygments_style = 'pastie' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] diff --git a/docs/index.rst b/docs/index.rst index 043611841..5080b4279 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,6 +37,8 @@ New to peewee? These may help: * :ref:`Models and fields ` * :ref:`Querying ` +.. include:: help-request.rst + Contents: --------- diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 89e1b9385..788ee431e 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -1,5 +1,7 @@ .. _api: +.. include:: help-request.rst + API Documentation ================= @@ -2241,6 +2243,8 @@ Query-builder # index will be created. Article.add_index(idx) +.. _fields-api: + Fields ------ diff --git a/docs/peewee/help-request.rst b/docs/peewee/help-request.rst new file mode 100644 index 000000000..541ade08d --- /dev/null +++ b/docs/peewee/help-request.rst @@ -0,0 +1,5 @@ +.. attention:: + Peewee needs your help! Do you have suggestions on how the documentation + could be improved? If so, please leave a comment on this GitHub issue: + + https://github.com/coleifer/peewee/issues/1640 diff --git a/docs/peewee/models.rst b/docs/peewee/models.rst index 6c728d4a2..536216e6b 100644 --- a/docs/peewee/models.rst +++ b/docs/peewee/models.rst @@ -1,5 +1,7 @@ .. _models: +.. include:: help-request.rst + Models and Fields ================= @@ -986,7 +988,7 @@ key, you must set the ``primary_key`` attribute of the model options to a .. warning:: Peewee does not support foreign-keys to models that define a - :ref:`CompositeKey` primary key. If you wish to add a foreign-key to a + :py:class:`CompositeKey` primary key. If you wish to add a foreign-key to a model that has a composite primary key, replicate the columns on the related model and add a custom accessor (e.g. a property). diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 659be6eeb..884b131a0 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1,5 +1,7 @@ .. _querying: +.. include:: help-request.rst + Querying ======== diff --git a/docs/peewee/sqlite_ext.rst b/docs/peewee/sqlite_ext.rst index 18c1aa2c7..f4e1bba3d 100644 --- a/docs/peewee/sqlite_ext.rst +++ b/docs/peewee/sqlite_ext.rst @@ -55,7 +55,7 @@ APIs :param bool rank_functions: Make search result ranking functions available. :param bool hash_functions: Make hashing functions available (md5, sha1, etc). :param bool regexp_function: Make the REGEXP function available. - :param bool bloomfilter: Make the :ref:`sqlite-bloomfilter` available. + :param bool bloomfilter: Make the :ref:`bloom filter ` available. Extends :py:class:`SqliteDatabase` and inherits methods for declaring user-defined functions, pragmas, etc. @@ -71,7 +71,7 @@ APIs :param bool rank_functions: Make search result ranking functions available. :param bool hash_functions: Make hashing functions available (md5, sha1, etc). :param bool regexp_function: Make the REGEXP function available. - :param bool bloomfilter: Make the :ref:`sqlite-bloomfilter` available. + :param bool bloomfilter: Make the :ref:`bloom filter ` available. :param bool replace_busy_handler: Use a smarter busy-handler implementation. Extends :py:class:`SqliteExtDatabase` and requires that the From 911e1dee5f24bd840a67d303076a529634bdf0a0 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 21 Jun 2018 15:42:51 -0500 Subject: [PATCH 37/58] Small tweaks. --- docs/index.rst | 2 -- docs/peewee/query_operators.rst | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5080b4279..043611841 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,8 +37,6 @@ New to peewee? These may help: * :ref:`Models and fields ` * :ref:`Querying ` -.. include:: help-request.rst - Contents: --------- diff --git a/docs/peewee/query_operators.rst b/docs/peewee/query_operators.rst index 13e163bce..d5e3cc269 100644 --- a/docs/peewee/query_operators.rst +++ b/docs/peewee/query_operators.rst @@ -1,5 +1,7 @@ .. _query-operators: +.. include:: help-request.rst + Query operators =============== From 8232b24afebd185bcc4fe5f65f6813dd110d1514 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 21 Jun 2018 15:49:11 -0500 Subject: [PATCH 38/58] Add section linking to sqlite ext. --- docs/peewee/playhouse.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index 892045662..fbaa122fc 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -40,6 +40,10 @@ make up the ``playhouse``. * :ref:`test_utils` * :ref:`flask_utils` +Sqlite Extensions +----------------- + +The Sqlite extensions have been moved to :ref:`their own page `. .. _sqliteq: From 46c2d8beae643d9159d78f332e85d4db99506d3c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Thu, 21 Jun 2018 16:57:35 -0500 Subject: [PATCH 39/58] Improve CTE docs and add test. --- docs/peewee/api.rst | 45 +++++++++++++++++++++++++++++++++- docs/peewee/querying.rst | 53 ++++++++++++++++++++++++++++++++++++++++ tests/models.py | 30 +++++++++++++++++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 788ee431e..5a97f1ac0 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -1675,7 +1675,7 @@ Query-builder .. py:method:: with_cte(*cte_list) - :param cte_list: zero or more CTE objects. + :param cte_list: zero or more :py:class:`CTE` objects. Include the given common-table-expressions in the query. Any previously specified CTEs will be overwritten. @@ -1757,6 +1757,49 @@ Query-builder Select query helper-class that implements operator-overloads for creating compound queries. + .. py:method:: cte(name[, recursive=False[, columns=None]]) + + :param str name: Alias for common table expression. + :param bool recursive: Will this be a recursive CTE? + :param list columns: List of column names (as strings). + + Indicate that a query will be used as a common table expression. For + example, if we are modelling a category tree and are using a + parent-link foreign key, we can retrieve all categories and their + absolute depths using a recursive CTE: + + .. code-block:: python + + class Category(Model): + name = TextField() + parent = ForeignKeyField('self', backref='children', null=True) + + # The base case of our recursive CTE will be categories that are at + # the root level -- in other words, categories without parents. + roots = (Category + .select(Category.name, Value(0).alias('level')) + .where(Category.parent.is_null()) + .cte(name='roots', recursive=True)) + + # The recursive term will select the category name and increment + # the depth, joining on the base term so that the recursive term + # consists of all children of the base category. + RTerm = Category.alias() + recursive = (RTerm + .select(RTerm.name, (roots.c.level + 1).alias('level')) + .join(roots, on=(RTerm.parent == roots.c.id))) + + # Express UNION ALL . + cte = roots.union_all(recursive) + + # Select name and level from the recursive CTE. + query = (cte + .select_from(cte.c.name, cte.c.level) + .order_by(cte.c.name)) + + for category in query: + print(category.name, category.level) + .. py:method:: union_all(dest) Create a UNION ALL query with ``dest``. diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 884b131a0..d03fc525c 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1077,6 +1077,59 @@ columns from tables listed in the query's ``FROM`` clause. To select all columns from a particular table, you can simply pass in the :py:class:`Model` class. +Common Table Expressions +------------------------ + +Peewee supports the inclusion of common table expressions (CTEs) in all types +of queries. To declare a :py:class:`Select` query for use as a CTE, use +:py:meth:`~SelectQuery.cte` method, which wraps the query in a :py:class:`CTE` +object. To indicate that a :py:class:`CTE` should be included as part of a +query, use the :py:meth:`Query.with_cte` method, passing a list of CTE objects. + +For an example, let's say we have some data points that consist of a key and a +floating-point value. We wish to, for each distinct key, find the values that +were above-average for that key. + +.. code-block:: python + + class Sample(Model): + key = TextField() + value = FloatField() + + data = ( + ('a', (1.25, 1.5, 1.75)), + ('b', (2.1, 2.3, 2.5, 2.7, 2.9)), + ('c', (3.5, 3.5))) + + # Populate data. + for key, values in data: + Sample.insert_many([(key, value) for value in values], + fields=[Sample.key, Sample.value]).execute() + + # First we'll declare the query that will be used as a CTE. This query + # simply determines the average value for each key. + cte = (Sample + .select(Sample.key, fn.AVG(Sample.value).alias('avg_value')) + .group_by(Sample.key) + .cte('key_avgs', columns=('key', 'avg_value'))) + + # Now we'll query the sample table, using our CTE to find rows whose value + # exceeds the average for the given key. We'll calculate how far above the + # average the given sample's value is, as well. + query = (Sample + .select(Sample.key, (Sample.value - cte.c.avg_value).alias('diff')) + .join(cte, on=(Sample.key == cte.c.key)) + .where(Sample.value > cte.c.avg_value) + .order_by(Sample.value) + .with_cte(cte)) + + for sample in query: + print(sample.key, sample.diff) + + # 'a', .25 -- for (a, 1.75) + # 'b', .2 -- for (b, 2.7) + # 'b', .4 -- for (b, 2.9) + Foreign Keys and Joins ---------------------- diff --git a/tests/models.py b/tests/models.py index bad225995..8e9276627 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1977,6 +1977,36 @@ def test_recursive_cte2(self): ('p3', 1), ('root', 0)]) + @requires_models(Sample) + def test_cte_reuse_aggregate(self): + data = ( + (1, (1.25, 1.5, 1.75)), + (2, (2.1, 2.3, 2.5, 2.7, 2.9)), + (3, (3.5, 3.5))) + with self.database.atomic(): + for counter, values in data: + (Sample + .insert_many([(counter, value) for value in values], + fields=[Sample.counter, Sample.value]) + .execute()) + + cte = (Sample + .select(Sample.counter, fn.AVG(Sample.value).alias('avg_value')) + .group_by(Sample.counter) + .cte('count_to_avg', columns=('counter', 'avg_value'))) + + query = (Sample + .select(Sample.counter, + (Sample.value - cte.c.avg_value).alias('diff')) + .join(cte, on=(Sample.counter == cte.c.counter)) + .where(Sample.value > cte.c.avg_value) + .order_by(Sample.value) + .with_cte(cte)) + self.assertEqual([(a, round(b, 2)) for a, b in query.tuples()], [ + (1, .25), + (2, .2), + (2, .4)]) + @skip_unless(IS_POSTGRESQL or IS_SQLITE_15, 'requires row-value support') class TestTupleComparison(ModelTestCase): From 0995d888b39d66b2c08cf0b3ee46f50a540b99c6 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 22 Jun 2018 09:14:14 -0500 Subject: [PATCH 40/58] More CTE docs and test fix. --- docs/peewee/api.rst | 9 ++++-- docs/peewee/querying.rst | 60 ++++++++++++++++++++++++++++++++++++++++ tests/models.py | 40 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index 5a97f1ac0..b479453db 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -1099,7 +1099,7 @@ Query-builder .. py:class:: CTE(name, query[, recursive=False[, columns=None]]) - Represent a common-table-expression. + Represent a common-table-expression. For example queries, see :ref:`cte`. :param name: Name for the CTE. :param query: :py:class:`Select` query describing CTE. @@ -1677,8 +1677,9 @@ Query-builder :param cte_list: zero or more :py:class:`CTE` objects. - Include the given common-table-expressions in the query. Any previously - specified CTEs will be overwritten. + Include the given common-table expressions in the query. Any previously + specified CTEs will be overwritten. For examples of common-table + expressions, see :ref:`cte`. .. py:method:: where(*expressions) @@ -1800,6 +1801,8 @@ Query-builder for category in query: print(category.name, category.level) + For more examples of CTEs, see :ref:`cte`. + .. py:method:: union_all(dest) Create a UNION ALL query with ``dest``. diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index d03fc525c..6454e4d8c 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1077,6 +1077,8 @@ columns from tables listed in the query's ``FROM`` clause. To select all columns from a particular table, you can simply pass in the :py:class:`Model` class. +.. _cte: + Common Table Expressions ------------------------ @@ -1130,6 +1132,64 @@ were above-average for that key. # 'b', .2 -- for (b, 2.7) # 'b', .4 -- for (b, 2.9) +Recursive CTEs +^^^^^^^^^^^^^^ + +Peewee supports recursive CTEs. Recursive CTEs can be useful when, for example, +you have a tree data-structure represented by a parent-link foreign key. +Suppose, for example, that we have a hierarchy of categories for an online +bookstore. We wish to generate a table showing all categories and their +absolute depths, along with the path from the root to the category. + +For this, we can use a recursive CTE: + +.. code-block:: python + + class Category(Model): + name = TextField() + parent = ForeignKeyField('self', backref='children', null=True) + + # Define the base case of our recursive CTE. This will be categories that + # have a null parent foreign-key. + Base = Category.alias() + level = Value(1).alias('level') + path = Base.name.alias('path') + base_case = (Base + .select(Base.name, Base.parent, level, path) + .where(Base.parent.is_null()) + .cte('base', recursive=True)) + + # Define the recursive terms. + RTerm = Category.alias() + rlevel = (base_case.c.level + 1).alias('level') + rpath = base_case.c.path.concat('->').concat(RTerm.name).alias('path') + recursive = (RTerm + .select(RTerm.name, RTerm.parent, rlevel, rpath) + .join(base_case, on=(RTerm.parent == base_case.c.id))) + + # The recursive CTE is created by taking the base case and UNION ALL with + # the recursive term. + cte = base_case.union_all(recursive) + + # We will now query from the CTE to get the categories, their levels, and + # their paths. + query = (cte + .select_from(cte.c.name, cte.c.level, cte.c.path) + .order_by(cte.c.path)) + + # We can now iterate over a list of all categories and print their names, + # absolute levels, and path from root -> category. + for category in query: + print(category.name, category.level, category.path) + + # Example output: + # root, 1, root + # p1, 2, root->p1 + # c1-1, 3, root->p1->c1-1 + # c1-2, 3, root->p1->c1-2 + # p2, 2, root->p2 + # c2-1, 3, root->p2->c2-1 + Foreign Keys and Joins ---------------------- diff --git a/tests/models.py b/tests/models.py index 8e9276627..f61fd0361 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1977,7 +1977,47 @@ def test_recursive_cte2(self): ('p3', 1), ('root', 0)]) + @skip_if(IS_SQLITE_OLD or IS_MYSQL, 'requires recursive cte support') + def test_recursive_cte_docs_example(self): + # Define the base case of our recursive CTE. This will be categories that + # have a null parent foreign-key. + Base = Category.alias() + level = Value(1).alias('level') + path = Base.name.alias('path') + base_case = (Base + .select(Base.name, Base.parent, level, path) + .where(Base.parent.is_null()) + .cte('base', recursive=True)) + + # Define the recursive terms. + RTerm = Category.alias() + rlevel = (base_case.c.level + 1).alias('level') + rpath = base_case.c.path.concat('->').concat(RTerm.name).alias('path') + recursive = (RTerm + .select(RTerm.name, RTerm.parent, rlevel, rpath) + .join(base_case, on=(RTerm.parent == base_case.c.name))) + + # The recursive CTE is created by taking the base case and UNION ALL with + # the recursive term. + cte = base_case.union_all(recursive) + + # We will now query from the CTE to get the categories, their levels, and + # their paths. + query = (cte + .select_from(cte.c.name, cte.c.level, cte.c.path) + .order_by(cte.c.path)) + data = [(obj.name, obj.level, obj.path) for obj in query] + self.assertEqual(data, [ + ('root', 1, 'root'), + ('p1', 2, 'root->p1'), + ('c11', 3, 'root->p1->c11'), + ('c12', 3, 'root->p1->c12'), + ('p2', 2, 'root->p2'), + ('p3', 2, 'root->p3'), + ('c31', 3, 'root->p3->c31')]) + @requires_models(Sample) + @skip_if(IS_SQLITE_OLD, 'sqlite too old for ctes') def test_cte_reuse_aggregate(self): data = ( (1, (1.25, 1.5, 1.75)), From f290011233c3babe064b3ddaf8296e5b3d5ab91b Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 22 Jun 2018 09:16:51 -0500 Subject: [PATCH 41/58] Fix test that was failing due to missing CASTs with Postgres. --- tests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models.py b/tests/models.py index f61fd0361..cccc51b0d 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1982,8 +1982,8 @@ def test_recursive_cte_docs_example(self): # Define the base case of our recursive CTE. This will be categories that # have a null parent foreign-key. Base = Category.alias() - level = Value(1).alias('level') - path = Base.name.alias('path') + level = Value(1).cast('integer').alias('level') + path = Base.name.cast('text').alias('path') base_case = (Base .select(Base.name, Base.parent, level, path) .where(Base.parent.is_null()) From ccf8ac09e6cd571a3becea0b31af9418fcb05a94 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 22 Jun 2018 15:42:33 -0500 Subject: [PATCH 42/58] More documentation for pwiz. Fixes #1642. --- docs/peewee/playhouse.rst | 85 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index fbaa122fc..dba6bd0c1 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -2641,6 +2641,10 @@ This will print a bunch of models to standard output. So you can do this: >>> from mymodels import Blog, Entry, Tag, Whatever >>> print [blog.name for blog in Blog.select()] +Command-line options +^^^^^^^^^^^^^^^^^^^^ + +pwiz accepts the following command-line options: ====== ========================= ============================================ Option Meaning Example @@ -2660,6 +2664,83 @@ The following are valid parameters for the engine: * mysql * postgresql +pwiz examples +^^^^^^^^^^^^^ + +Examples of introspecting various databases: + +.. code-block:: console + + # Introspect a Sqlite database. + python -m pwiz -e sqlite path/to/sqlite_database.db + + # Introspect a MySQL database, logging in as root:secret. + python -m pwiz -e mysql -u root -P secret mysql_db_name + + # Introspect a Postgresql database on a remote server. + python -m pwiz -e postgres -u postgres -H 10.1.0.3 pg_db_name + +Full example: + +.. code-block:: console + + $ sqlite3 example.db << EOM + CREATE TABLE "user" ("id" INTEGER NOT NULL PRIMARY KEY, "username" TEXT NOT NULL); + CREATE TABLE "tweet" ( + "id" INTEGER NOT NULL PRIMARY KEY, + "content" TEXT NOT NULL, + "timestamp" DATETIME NOT NULL, + "user_id" INTEGER NOT NULL, + FOREIGN KEY ("user_id") REFERENCES "user" ("id")); + CREATE UNIQUE INDEX "user_username" ON "user" ("username"); + EOM + + $ python -m pwiz -e sqlite example.db + +Produces the following output: + +.. code-block:: python + + from peewee import * + + database = SqliteDatabase('example.db', **{}) + + class UnknownField(object): + def __init__(self, *_, **__): pass + + class BaseModel(Model): + class Meta: + database = database + + class User(BaseModel): + username = TextField(unique=True) + + class Meta: + table_name = 'user' + + class Tweet(BaseModel): + content = TextField() + timestamp = DateTimeField() + user = ForeignKeyField(column_name='user_id', field='id', model=User) + + class Meta: + table_name = 'tweet' + +Observations: + +* The foreign-key ``Tweet.user_id`` is detected and mapped correctly. +* The ``User.username`` UNIQUE constraint is detected. +* Each model explicitly declares its table name, even in cases where it is not + necessary (as Peewee would automatically translate the class name into the + appropriate table name). +* All the parameters of the :py:class:`ForeignKeyField` are explicitly + declared, even though they follow the conventions Peewee uses by default. + +.. note:: + The ``UnknownField`` is a placeholder that is used in the event your schema + contains a column declaration that Peewee doesn't know how to map to a + field class. + .. _migrate: Schema Migrations @@ -2717,11 +2798,11 @@ Use :py:func:`migrate` to execute one or more operations: .. warning:: Migrations are not run inside a transaction. If you wish the migration to run in a transaction you will need to wrap the call to `migrate` in a - transaction block, e.g. + :py:meth:`~Database.atomic` context-manager, e.g. .. code-block:: python - with my_db.transaction(): + with my_db.atomic(): migrate(...) Supported Operations From dd34ac8a4537e2ba72be55d3cf2cad8b78637e64 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 25 Jun 2018 16:35:03 -0500 Subject: [PATCH 43/58] Add new relationships document. --- docs/index.rst | 1 + docs/peewee/relationships.rst | 389 ++++++++++++++++++++++++++++++++++ 2 files changed, 390 insertions(+) create mode 100644 docs/peewee/relationships.rst diff --git a/docs/index.rst b/docs/index.rst index 043611841..8fb2031a3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -52,6 +52,7 @@ Contents: peewee/models peewee/querying peewee/query_operators + peewee/relationships peewee/api peewee/sqlite_ext peewee/playhouse diff --git a/docs/peewee/relationships.rst b/docs/peewee/relationships.rst new file mode 100644 index 000000000..d8cec4863 --- /dev/null +++ b/docs/peewee/relationships.rst @@ -0,0 +1,389 @@ +.. _relationships: + +Relationships and Joins +======================= + +In this document we'll cover how Peewee handles relationships between models. + +Model definitions +----------------- + +We'll use the following model definitions for our examples: + +.. code-block:: python + + import datetime + from peewee import * + + + db = SqliteDatabase(':memory:') + + class BaseModel(Model): + class Meta: + database = db + + class User(BaseModel): + username = TextField() + + class Tweet(BaseModel): + content = TextField() + timestamp = DateTimeField(default=datetime.datetime.now) + user = ForeignKeyField(User, backref='tweets') + + class Favorite(BaseModel): + user = ForeignKeyField(User, backref='favorites') + tweet = ForeignKeyField(Tweet, backref='favorites') + + +Peewee uses :py:class:`ForeignKeyField` to define foreign-key relationships +between models. Every foreign-key field has an implied back-reference, which is +exposed as a pre-filtered :py:class:`Select` query using the provided +``backref`` attribute. + +Creating test data +^^^^^^^^^^^^^^^^^^ + +To follow along with the examples, let's populate this database with some test +data: + +.. code-block:: python + + def populate_test_data(): + data = ( + ('huey', ('meow', 'hiss', 'purr')), + ('mickey', ('woof', 'whine')), + ('zaizee', ())) + for username, tweets in data: + user = User.create(username=username) + for tweet in tweets: + Tweet.create(user=user, content=tweet) + + # Populate a few favorites for our users, such that: + favorite_data = ( + ('huey', ['whine']), + ('mickey', ['purr']), + ('zaizee', ['meow', 'purr'])) + for username, favorites in favorite_data: + user = User.get(User.username == username) + for content in favorites: + tweet = Tweet.get(Tweet.content == content) + Favorite.create(user=user, tweet=tweet) + +This gives us the following: + +========= ========== =========================== +User Tweet Favorited by +========= ========== =========================== +huey meow zaizee +huey hiss +huey purr mickey, zaizee +mickey woof +mickey whine huey +========= ========== =========================== + +Performing simple joins +----------------------- + +As an exercise in learning how to perform joins with Peewee, let's write a +query to print out all the tweets by "huey". To do this we'll select from the +``Tweet`` model and join on the ``User`` model, so we can then filter on the +``User.username`` field: + +.. code-block:: pycon + + >>> query = Tweet.select().join(User).where(User.username == 'huey') + >>> for tweet in query: + ... print(tweet.content) + ... + meow + hiss + purr + +.. note:: + We did not have to explicitly specify the join predicate (the "ON" clause), + because Peewee inferred from the models that when we joined from Tweet to + User, we were joining on the ``Tweet.user`` foreign-key. + + The following code is equivalent, but more explicit: + + .. code-block:: python + + query = (Tweet + .select() + .join(User, on=(Tweet.user == User.id)) + .where(User.username == 'huey')) + +If we already had a reference to the ``User`` object for "huey", we could use +the ``User.tweets`` back-reference to list all of huey's tweets: + +.. code-block:: pycon + + >>> huey = User.get(User.username == 'huey') + >>> for tweet in huey.tweets: + ... print(tweet.content) + ... + meow + hiss + purr + +Taking a closer look at ``huey.tweets``, we can see that it is just a simple +pre-filtered ``SELECT`` query: + +.. code-block:: pycon + + >>> huey.tweets + + + >>> huey.tweets.sql() + ('SELECT "t1"."id", "t1"."content", "t1"."timestamp", "t1"."user_id" + FROM "tweet" AS "t1" WHERE ("t1"."user_id" = ?)', [1]) + +Joining multiple tables +----------------------- + +Let's take another look at joins by querying the list of users and getting the +count of how many tweet's they've authored that were favorited. This will +require us to join twice: from user to tweet, and from tweet to favorite. We'll +add the additional requirement that users should be included who have not +created any tweets, as well as users whose tweets have not been favorited. The +query, expressed in SQL, would be: + +.. code-block:: sql + + SELECT user.username, COUNT(favorite.id) + FROM user + LEFT OUTER JOIN tweet ON tweet.user_id = user.id + LEFT OUTER JOIN favorite ON favorite.tweet_id = tweet.id + GROUP BY user.username + +.. note:: + In the above query both joins are LEFT OUTER, since a user may not have any + tweets or, if they have tweets, none of them may have been favorited. + +Peewee has a concept of a *join context*, meaning that whenever we call the +:py:meth:`~ModelSelect.join` method, we are implicitly joining on the +previously-joined model (or if this is the first call, the model we are +selecting from). Since we are joining straight through, from user to tweet, +then from tweet to favorite, we can simply write: + +.. code-block:: python + + query = (User + .select(User.username, fn.COUNT(Favorite.id).alias('count')) + .join(Tweet, JOIN.LEFT_OUTER) # Joins user -> tweet. + .join(Favorite, JOIN.LEFT_OUTER) # Joins tweet -> favorite. + .group_by(User.username)) + +Iterating over the results: + +.. code-block:: pycon + + >>> for user in query: + ... print(user.username, user.count) + ... + huey 3 + mickey 1 + zaizee 0 + +For a more complicated example involving multiple joins and switching join +contexts, let's find all the tweets by Huey and the number of times they've +been favorited. To do this we'll need to perform two joins and we'll also use +an aggregate function to calculate the favorite count. + +Here is how we would write this query in SQL: + +.. code-block:: sql + + SELECT tweet.content, COUNT(favorite.id) + FROM tweet + INNER JOIN user ON tweet.user_id = user.id + LEFT OUTER JOIN favorite ON favorite.tweet_id = tweet.id + WHERE user.username = 'huey' + GROUP BY tweet.content; + +.. note:: + We use a LEFT OUTER join from tweet to favorite since a tweet may not have + any favorites, yet we still wish to display it's content (along with a + count of zero) in the result set. + +With Peewee, the resulting Python code looks very similar to what we would +write in SQL: + +.. code-block:: python + + query = (Tweet + .select(Tweet.content, fn.COUNT(Favorite.id).alias('count')) + .join(User) # Join from tweet -> user. + .switch(Tweet) # Move "join context" back to tweet. + .join(Favorite, JOIN.LEFT_OUTER) # Join from tweet -> favorite. + .where(User.username == 'huey') + .group_by(Tweet.content)) + +Note the call to :py:meth:`~ModelSelect.switch` - that instructs Peewee to set +the *join context* back to ``Tweet``. If we had omitted the explicit call to +switch, Peewee would have used ``User`` (the last model we joined) as the join +context and constructed the join from User to Favorite using the +``Favorite.user`` foreign-key, which would have given us incorrect results. + +We can iterate over the results of the above query to print the tweet's content +and the favorite count: + +.. code-block:: pycon + + >>> for tweet in query: + ... print('%s favorited %d times' % (tweet.content, tweet.count)) + ... + meow favorited 1 times + hiss favorited 0 times + purr favorited 2 times + +Selecting from multiple sources +------------------------------- + +If we wished to list all the tweets in the database, along with the username of +their author, you might try writing this: + +.. code-block:: pycon + + >>> for tweet in Tweet.select(): + ... print(tweet.user.username, '->', tweet.content) + ... + huey -> meow + huey -> hiss + huey -> purr + mickey -> woof + mickey -> whine + +There is a big problem with the above loop: it executes an additional query for +every tweet to look up the ``tweet.user`` foreign-key. For our small table the +performance penalty isn't obvious, but we would find the delays grew as the +number of rows increased. + +If you're familiar with SQL, you might remember that it's possible to SELECT +from multiple tables, allowing us to get the tweet content *and* the username +in a single query: + +.. code-block:: sql + + SELECT tweet.content, user.username + FROM tweet + INNER JOIN user ON tweet.user_id = user.id; + +Peewee makes this quite easy. In fact, we only need to modify our query a +little bit. We tell Peewee we wish to select ``Tweet.content`` as well as +the ``User.username`` field, then we include a join from tweet to user. +To make it a bit more obvious that it's doing the correct thing, we can ask +Peewee to return the rows as dictionaries. + +.. code-block:: pycon + + >>> for row in Tweet.select(Tweet.content, User.username).join(User).dicts(): + ... print(row) + ... + {'content': 'meow', 'username': 'huey'} + {'content': 'hiss', 'username': 'huey'} + {'content': 'purr', 'username': 'huey'} + {'content': 'woof', 'username': 'mickey'} + {'content': 'whine', 'username': 'mickey'} + +Now we'll leave off the call to ".dicts()" and return the rows as ``Tweet`` +objects. Notice that Peewee assigns the ``username`` value to +``tweet.user.username`` -- NOT ``tweet.username``! Because there is a +foreign-key from tweet to user, and we have selected fields from both models, +Peewee will reconstruct the model-graph for us: + +.. code-block:: pycon + + >>> for tweet in Tweet.select(Tweet.content, User.username).join(User): + ... print(tweet.user.username, '->', tweet.content) + ... + huey -> meow + huey -> hiss + huey -> purr + mickey -> woof + mickey -> whine + +If we wish to, we can control where Peewee puts the joined ``User`` instance in +the above query, by specifying an ``attr`` in the ``join()`` method: + +.. code-block:: pycon + + >>> query = Tweet.select(Tweet.content, User.username).join(User, attr='author') + >>> for tweet in query: + ... print(tweet.author.username, '->', tweet.content) + ... + huey -> meow + huey -> hiss + huey -> purr + mickey -> woof + mickey -> whine + +Conversely, if we simply wish *all* attributes we select to me attributes of +the ``Tweet`` instance, we can add a call to :py:meth:`~ModelSelect.objects` at +the end of our query (similar to how we called ``dicts()``): + +.. code-block:: pycon + + >>> for tweet in query.objects(): + ... print(tweet.username, '->', tweet.content) + ... + huey -> meow + (etc) + +More complex example +^^^^^^^^^^^^^^^^^^^^ + +As a more complex example, in this query, we will write a single query that +selects all the favorites, along with the user who created the favorite, the +tweet that was favorited, and that tweet's author. + +In SQL we would write: + +.. code-block:: sql + + SELECT owner.username, tweet.content, author.username AS author + FROM favorite + INNER JOIN user AS owner ON (favorite.user_id = owner.id) + INNER JOIN tweet ON (favorite.tweet_id = tweet.id) + INNER JOIN user AS author ON (tweet.user_id = author.id); + +Note that we are selecting from the user table twice - once in the context of +the user who created the favorite, and again as the author of the tweet. + +With Peewee, we use :py:meth:`Model.alias` to alias a model class so it can be +referenced twice in a single query: + +.. code-block:: python + + Owner = User.alias() + query = (Favorite + .select(Favorite, Tweet.content, User.username, Owner.username) + .join(Owner) # Join favorite -> user (owner of favorite). + .switch(Favorite) + .join(Tweet) # Join favorite -> tweet + .join(User)) # Join tweet -> user + +We can iterate over the results and access the joined values in the following +way. Note how Peewee has resolved the fields from the various models we +selected and reconstructed the model graph: + +.. code-block:: pycon + + >>> for fav in query: + ... print(fav.user.username, 'liked', fav.tweet.content, 'by', fav.tweet.user.username) + ... + huey liked whine by mickey + mickey liked purr by huey + zaizee liked meow by huey + zaizee liked purr by huey + +.. attention:: + If you are unsure how many queries are being executed, you can add the + following code, which will log all queries to the console: + + .. code-block:: python + + import logging + logger = logging.getLogger('peewee') + logger.addHandler(logging.StreamHandler()) + logger.setLevel(logging.DEBUG) From 7f8c5cabfd0ce5a39323bcab2285c5a480c8e885 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 25 Jun 2018 19:11:00 -0500 Subject: [PATCH 44/58] Subqueries and CTE join docs. --- docs/peewee/relationships.rst | 130 ++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/docs/peewee/relationships.rst b/docs/peewee/relationships.rst index d8cec4863..2fe16891b 100644 --- a/docs/peewee/relationships.rst +++ b/docs/peewee/relationships.rst @@ -49,6 +49,8 @@ data: .. code-block:: python def populate_test_data(): + db.create_tables([User, Tweet, Favorite]) + data = ( ('huey', ('meow', 'hiss', 'purr')), ('mickey', ('woof', 'whine')), @@ -387,3 +389,131 @@ selected and reconstructed the model graph: logger = logging.getLogger('peewee') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) + +Subqueries +---------- + +Peewee allows you to join on any table-like object, including subqueries or +common table expressions (CTEs). To demonstrate joining on a subquery, let's +query for all users and their latest tweet. + +Here is the SQL: + +.. code-block:: sql + + SELECT tweet.*, user.* + FROM tweet + INNER JOIN ( + SELECT latest.user_id, MAX(latest.timestamp) AS max_ts + FROM tweet AS latest + GROUP BY latest.user_id) AS latest_query + ON ((tweet.user_id = latest_query.user_id) AND (tweet.timestamp = latest_query.max_ts)) + INNER JOIN user ON (tweet.user_id = user.id) + +We'll do this by creating a subquery which selects each user and the timestamp +of their latest tweet. Then we can query the tweets table in the outer query +and join on the user and timestamp combination from the subquery. + +.. code-block:: python + + # Define our subquery first. We'll use an alias of the Tweet model, since + # we will be querying from the Tweet model directly in the outer query. + Latest = Tweet.alias() + latest_query = (Latest + .select(Latest.user, fn.MAX(Latest.timestamp).alias('max_ts')) + .group_by(Latest.user) + .alias('latest_query')) + + # Our join predicate will ensure that we match tweets based on their + # timestamp *and* user_id. + predicate = ((Tweet.user == latest_query.c.user_id) & + (Tweet.timestamp == latest_query.c.max_ts)) + + # We put it all together, querying from tweet and joining on the subquery + # using the above predicate. + query = (Tweet + .select(Tweet, User) # Select all columns from tweet and user. + .join(latest_query, on=predicate) # Join tweet -> subquery. + .join_from(Tweet, User)) # Join from tweet -> user. + +Iterating over the query, we can see each user and their latest tweet. + +.. code-block:: pycon + + >>> for tweet in query: + ... print(tweet.user.username, '->', tweet.content) + ... + huey -> purr + mickey -> whine + +There are a couple things you may not have seen before in the code we used to +create the query in this section: + +* We used :py:meth:`~ModelSelect.join_from` to explicitly specify the join + context. We wrote ``.join_from(Tweet, User)``, which is equivalent to + ``.switch(Tweet).join(User)``. +* We referenced columns in the subquery using the magic ``.c`` attribute, + for example ``latest_query.c.max_ts``. The ``.c`` attribute is used to + dynamically create column references. +* Instead of passing individual fields to ``Tweet.select()``, we passed the + ``Tweet`` and ``User`` models. This is shorthand for selecting all fields on + the given model. + +Common-table Expressions +^^^^^^^^^^^^^^^^^^^^^^^^ + +In the previous section we joined on a subquery, but we could just as easily +have used a common-table expression (CTE). We will repeat the same query as +before, listing users and their latest tweets, but this time we will do it +using a CTE. + +Here is the SQL: + +.. code-block:: sql + + WITH latest AS ( + SELECT user_id, MAX(timestamp) AS max_ts + FROM tweet + GROUP BY user_id) + SELECT tweet.*, user.* + FROM tweet + INNER JOIN latest + ON ((latest.user_id = tweet.user_id) AND (latest.max_ts = tweet.timestamp)) + INNER JOIN user + ON (tweet.user_id = user.id) + +This example looks very similar to the previous example with the subquery: + +.. code-block:: python + + # Define our CTE first. We'll use an alias of the Tweet model, since + # we will be querying from the Tweet model directly in the main query. + Latest = Tweet.alias() + cte = (Latest + .select(Latest.user, fn.MAX(Latest.timestamp).alias('max_ts')) + .group_by(Latest.user) + .cte('latest')) + + # Our join predicate will ensure that we match tweets based on their + # timestamp *and* user_id. + predicate = ((Tweet.user == cte.c.user_id) & + (Tweet.timestamp == cte.c.max_ts)) + + # We put it all together, querying from tweet and joining on the CTE + # using the above predicate. + query = (Tweet + .select(Tweet, User) # Select all columns from tweet and user. + .join(cte, on=predicate) # Join tweet -> CTE. + .join_from(Tweet, User) # Join from tweet -> user. + .with_cte(cte)) + +We can iterate over the result-set, which consists of the latest tweets for +each user: + +.. code-block:: pycon + + >>> for tweet in query: + ... print(tweet.user.username, '->', tweet.content) + ... + huey -> purr + mickey -> whine From 0e029560c23174e24d351e1947a81f92dda1147c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 08:55:53 -0500 Subject: [PATCH 45/58] Allow SQL() objects as insert/from source. Fixes #1645 --- peewee.py | 2 +- tests/regressions.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/peewee.py b/peewee.py index d71aa23c6..0fd1f87af 100644 --- a/peewee.py +++ b/peewee.py @@ -2224,7 +2224,7 @@ def __sql__(self, ctx): except self.DefaultValuesException: self._default_values(ctx) self._query_type = Insert.SIMPLE - elif isinstance(self._insert, SelectQuery): + elif isinstance(self._insert, (SelectQuery, SQL)): self._query_insert(ctx) self._query_type = Insert.QUERY else: diff --git a/tests/regressions.py b/tests/regressions.py index 92df37119..e1cae3a02 100644 --- a/tests/regressions.py +++ b/tests/regressions.py @@ -199,3 +199,23 @@ def test_get_or_create_self_referential_fk2(self): self.assertEqual(child_db.user.username, 'huey') self.assertEqual(child_db.parent.name, 'parent') self.assertEqual(child_db.name, 'child') + + +class TestInsertFromSQL(ModelTestCase): + def setUp(self): + super(TestInsertFromSQL, self).setUp() + + self.database.execute_sql('create table if not exists user_src ' + '(name TEXT);') + tbl = Table('user_src').bind(self.database) + tbl.insert(name='foo').execute() + + def tearDown(self): + super(TestInsertFromSQL, self).tearDown() + self.database.execute_sql('drop table if exists user_src') + + @requires_models(User) + def test_insert_from_sql(self): + query_src = SQL('SELECT name FROM user_src') + User.insert_from(query=query_src, fields=[User.username]).execute() + self.assertEqual([u.username for u in User.select()], ['foo']) From 0e2063c7a1f286fb358f273612fd65135e653a45 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 09:26:39 -0500 Subject: [PATCH 46/58] Slight reorg to relationships docs. --- docs/peewee/relationships.rst | 36 ++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/peewee/relationships.rst b/docs/peewee/relationships.rst index 2fe16891b..b092766e7 100644 --- a/docs/peewee/relationships.rst +++ b/docs/peewee/relationships.rst @@ -83,6 +83,18 @@ mickey woof mickey whine huey ========= ========== =========================== +.. attention:: + In the following examples we will be executing a number of queries. If you + are unsure how many queries are being executed, you can add the following + code, which will log all queries to the console: + + .. code-block:: python + + import logging + logger = logging.getLogger('peewee') + logger.addHandler(logging.StreamHandler()) + logger.setLevel(logging.DEBUG) + Performing simple joins ----------------------- @@ -227,6 +239,19 @@ switch, Peewee would have used ``User`` (the last model we joined) as the join context and constructed the join from User to Favorite using the ``Favorite.user`` foreign-key, which would have given us incorrect results. +If we wanted to omit the join-context switching we could instead use the +:py:meth:`~ModelSelect.join_from` method. The following query is equivalent to +the previous one: + +.. code-block:: python + + query = (Tweet + .select(Tweet.content, fn.COUNT(Favorite.id).alias('count')) + .join_from(Tweet, User) # Join tweet -> user. + .join_from(Tweet, Favorite, JOIN.LEFT_OUTER) # Join tweet -> favorite. + .where(User.username == 'huey') + .group_by(Tweet.content)) + We can iterate over the results of the above query to print the tweet's content and the favorite count: @@ -379,17 +404,6 @@ selected and reconstructed the model graph: zaizee liked meow by huey zaizee liked purr by huey -.. attention:: - If you are unsure how many queries are being executed, you can add the - following code, which will log all queries to the console: - - .. code-block:: python - - import logging - logger = logging.getLogger('peewee') - logger.addHandler(logging.StreamHandler()) - logger.setLevel(logging.DEBUG) - Subqueries ---------- From 98f75f528623eeb4ccd90648d810d1493204f942 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 09:34:48 -0500 Subject: [PATCH 47/58] Move relevant sections into relationships document. --- docs/peewee/querying.rst | 514 +--------------------------------- docs/peewee/relationships.rst | 331 ++++++++++++++++++++++ 2 files changed, 332 insertions(+), 513 deletions(-) diff --git a/docs/peewee/querying.rst b/docs/peewee/querying.rst index 6454e4d8c..9343dd3a8 100644 --- a/docs/peewee/querying.rst +++ b/docs/peewee/querying.rst @@ -1193,519 +1193,7 @@ For this, we can use a recursive CTE: Foreign Keys and Joins ---------------------- -Foreign keys are created using a special field class -:py:class:`ForeignKeyField`. Each foreign key also creates a back-reference on -the related model using the specified *backref*. - -.. note:: - In SQLite, foreign keys are not enabled by default. Most things, including - the Peewee foreign-key API, will work fine, but ON DELETE behaviour will be - ignored, even if you explicitly specify on_delete to your ForeignKeyField. - In conjunction with the default PrimaryKeyField behaviour (where deleted - record IDs can be reused), this can lead to surprising (and almost - certainly unwanted) behaviour where if you delete a record in table A - referenced by a foreign key in table B, and then create a new, unrelated, - record in table A, the new record will end up mis-attached to the undeleted - record in table B. To avoid the mis-attachment, you can use - :py:class:`AutoIncrementField`, but it may be better overall to - ensure that foreign keys are enabled with - ``pragmas=(('foreign_keys', 'on'),)`` when you - instantiate :py:class:`SqliteDatabase`. - - -Traversing foreign keys -^^^^^^^^^^^^^^^^^^^^^^^ - -Referring back to the :ref:`User and Tweet models `, note that -there is a :py:class:`ForeignKeyField` from *Tweet* to *User*. The foreign key -can be traversed, allowing you access to the associated user instance: - -.. code-block:: pycon - - >>> tweet.user.username - 'charlie' - -.. note:: - Unless the *User* model was explicitly selected when retrieving the - *Tweet*, an additional query will be required to load the *User* data. To - learn how to avoid the extra query, see the :ref:`N+1 query documentation - `. - -The reverse is also true, and we can iterate over the tweets associated with a -given *User* instance: - -.. code-block:: python - - >>> for tweet in user.tweets: - ... print(tweet.message) - ... - http://www.youtube.com/watch?v=xdhLQCYQ-nQ - -Under the hood, the *tweets* attribute is just a :py:class:`Select` with the -*WHERE* clause pre-populated to point to the given *User* instance: - -.. code-block:: python - - >>> user.tweets - - - >>> user.tweets.sql() - ('SELECT "t1"."id", "t1"."user_id", "t1"."content", "t1"."timestamp" FROM "tweet" AS "t1" WHERE ("t1"."user_id" = ?)', - [1]) - -Joining tables -^^^^^^^^^^^^^^ - -Use the :py:meth:`~ModelSelect.join` method to *JOIN* additional tables. When a -foreign key exists between the source model and the join model, you do not need -to specify any additional parameters: - -.. code-block:: pycon - - >>> my_tweets = Tweet.select().join(User).where(User.username == 'charlie') - -By default peewee will use an *INNER* join, but you can use *LEFT OUTER*, -*RIGHT OUTER*, *FULL*, or *CROSS* joins as well: - -.. code-block:: python - - users = (User - .select(User, fn.Count(Tweet.id).alias('num_tweets')) - .join(Tweet, JOIN.LEFT_OUTER) - .group_by(User) - .order_by(fn.Count(Tweet.id).desc())) - for user in users: - print(user.username, 'has created', user.num_tweets, 'tweet(s).') - -Selecting from multiple models -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -SQL makes it easy to select columns from multiple tables and return it all at -once. Peewee makes this possible, too, but since Peewee models form a graph -(via foreign-keys), the selected data is returned as a graph of model -instances. To see what I mean, consider this query: - -.. code-block:: sql - - SELECT tweet.content, tweet.timestamp, user.username - FROM tweet - INNER JOIN user ON tweet.user_id = user.id - ORDER BY tweet.timestamp DESC; - - -- Returns rows like - -- "Meow I'm a tweet" | 2017-01-17 13:37:00 | huey - -- "Woof woof" | 2017-01-17 11:59:00 | mickey - -- "Purr" | 2017-01-17 10:00:00 | huey - -With Peewee we would write this query: - -.. code-block:: python - - query = (Tweet - .select(Tweet.content, Tweet.timestamp, User.username) - .join(User) - .order_by(Tweet.timestamp.desc())) - -The question is: where is the "username" attribute to be found? The answer is -that Peewee, because there is a foreign-key relationship between Tweet and -User, will return each row as a Tweet model *with* the associated User model, -which has it's username attribute set: - -.. code-block:: python - - for tweet in query: - print(tweet.content, tweet.timestamp, tweet.user.username) - -When doing complicated joins, joins where no foreign-key exists (for example -joining on a subquery), etc., it is necessary to tell Peewee where to place the -joined attributes. This is done by specifying an ``attr`` parameter in the join -method. - -For example, let's say that in the above query we want to put the joined user -data in the *Tweet.foo* attribute: - -.. code-block:: python - - query = (Tweet - .select(Tweet.content, Tweet.timestamp, User.username) - .join(User, attr='foo') - .order_by(Tweet.timestamp.desc())) - - for tweet in query: - # Joined user data is stored in "tweet.foo": - print(tweet.content, tweet.timestamp, tweet.foo.username) - -Alternatively, we can also specify the attribute name by putting an *alias* on -the join predicate expression: - -.. code-block:: python - - query = (Tweet - .select(Tweet.content, Tweet.timestamp, User.username) - .join(User, on=(Tweet.user == User.id).alias('foo')) - .order_by(Tweet.timestamp.desc())) - -For queries with complex joins and selections from several models, constructing -this graph can be expensive. If you wish, instead, to have *all* columns as -attributes on a single model, you can use :py:meth:`~ModelSelect.objects` -method: - -.. code-block:: python - - for tweet in query.objects(): - # Now "username" is on the Tweet model itself: - print(tweet.content, tweet.timestamp, tweet.username) - -For additional performance gains, consider using :py:meth:`~BaseQuery.dicts`, -:py:meth:`~BaseQuery.tuples` or :py:meth:`~BaseQuery.namedtuples` when -iterating large and/or complex result-sets. - -Multiple Foreign Keys to the Same Model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When there are multiple foreign keys to the same model, it is good practice to -explicitly specify which field you are joining on. - -Referring back to the :ref:`example app's models `, -consider the *Relationship* model, which is used to denote when one user -follows another. Here is the model definition: - -.. code-block:: python - - class Relationship(BaseModel): - from_user = ForeignKeyField(User, backref='relationships') - to_user = ForeignKeyField(User, backref='related_to') - - class Meta: - indexes = ( - # Specify a unique multi-column index on from/to-user. - (('from_user', 'to_user'), True), - ) - -Since there are two foreign keys to *User*, we should always specify which -field we are using in a join. - -For example, to determine which users I am following, I would write: - -.. code-block:: python - - (User - .select() - .join(Relationship, on=Relationship.to_user) - .where(Relationship.from_user == charlie)) - -On the other hand, if I wanted to determine which users are following me, I -would instead join on the *from_user* column and filter on the relationship's -*to_user*: - -.. code-block:: python - - (User - .select() - .join(Relationship, on=Relationship.from_user) - .where(Relationship.to_user == charlie)) - -Joining on arbitrary fields -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If a foreign key does not exist between two tables you can still perform a -join, but you must manually specify the join predicate. - -In the following example, there is no explicit foreign-key between *User* and -*ActivityLog*, but there is an implied relationship between the -*ActivityLog.object_id* field and *User.id*. Rather than joining on a specific -:py:class:`Field`, we will join using an :py:class:`Expression`. - -.. code-block:: python - - user_log = (User - .select(User, ActivityLog) - .join( - ActivityLog, - on=(User.id == ActivityLog.object_id).alias('log')) - .where( - (ActivityLog.activity_type == 'user_activity') & - (User.username == 'charlie'))) - - for user in user_log: - print(user.username, user.log.description) - - #### Print something like #### - charlie logged in - charlie posted a tweet - charlie retweeted - charlie posted a tweet - charlie logged out - -.. note:: - By specifying an alias on the join condition, you can control the attribute - peewee will assign the joined instance to. In the previous example, we used - the following *join*: - - .. code-block:: python - - (User.id == ActivityLog.object_id).alias('log') - - Then when iterating over the query, we were able to directly access the - joined *ActivityLog* without incurring an additional query: - - .. code-block:: python - - for user in user_log: - print(user.username, user.log.description) - -Joining on Multiple Tables -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When calling :py:meth:`~ModelSelect.join`, peewee will use the *last joined table* -as the source table. For example: - -.. code-block:: python - - User.select().join(Tweet).join(Comment) - -This query will result in a join from *User* to *Tweet*, and another join from -*Tweet* to *Comment*. - -If you would like to join the same table twice, use the :py:meth:`~ModelSelect.switch` method: - -.. code-block:: python - - # Join the Artist table on both `Album` and `Genre`. - Artist.select().join(Album).switch(Artist).join(Genre) - -Alternatively, you can use :py:meth:`~ModelSelect.join_from`: - -.. code-block:: python - - query = (Artist - .select() - .join(Album) - .join_from(Artist, Genre)) - -.. _manytomany: - -Implementing Many to Many -------------------------- - -Peewee provides a field for representing many-to-many relationships, much like -Django does. This feature was added due to many requests from users, but I -strongly advocate against using it, since it conflates the idea of a field with -a junction table and hidden joins. It's just a nasty hack to provide convenient -accessors. - -To implement many-to-many **correctly** with peewee, you will therefore create -the intermediary table yourself and query through it: - -.. code-block:: python - - class Student(Model): - name = CharField() - - class Course(Model): - name = CharField() - - class StudentCourse(Model): - student = ForeignKeyField(Student) - course = ForeignKeyField(Course) - -To query, let's say we want to find students who are enrolled in math class: - -.. code-block:: python - - query = (Student - .select() - .join(StudentCourse) - .join(Course) - .where(Course.name == 'math')) - for student in query: - print(student.name) - -To query what classes a given student is enrolled in: - -.. code-block:: python - - courses = (Course - .select() - .join(StudentCourse) - .join(Student) - .where(Student.name == 'da vinci')) - - for course in courses: - print(course.name) - -To efficiently iterate over a many-to-many relation, i.e., list all students -and their respective courses, we will query the *through* model -``StudentCourse`` and *precompute* the Student and Course: - -.. code-block:: python - - query = (StudentCourse - .select(StudentCourse, Student, Course) - .join(Course) - .switch(StudentCourse) - .join(Student) - .order_by(Student.name)) - -To print a list of students and their courses you might do the following: - -.. code-block:: python - - for student_course in query: - print(student_course.student.name, '->', student_course.course.name) - -Since we selected all fields from ``Student`` and ``Course`` in the *select* -clause of the query, these foreign key traversals are "free" and we've done the -whole iteration with just 1 query. - -ManyToManyField -^^^^^^^^^^^^^^^ - -The :py:class:`ManyToManyField` provides a *field-like* API over many-to-many -fields. For all but the simplest many-to-many situations, you're better off -using the standard peewee APIs. But, if your models are very simple and your -querying needs are not very complex, you can get a big boost by using -:py:class:`ManyToManyField`. Check out the :ref:`extra-fields` extension module -for details. - -Modeling students and courses using :py:class:`ManyToManyField`: - -.. code-block:: python - - from peewee import * - from playhouse.fields import ManyToManyField - - db = SqliteDatabase('school.db') - - class BaseModel(Model): - class Meta: - database = db - - class Student(BaseModel): - name = CharField() - - class Course(BaseModel): - name = CharField() - students = ManyToManyField(Student, backref='courses') - - StudentCourse = Course.students.get_through_model() - - db.create_tables([ - Student, - Course, - StudentCourse]) - - # Get all classes that "huey" is enrolled in: - huey = Student.get(Student.name == 'Huey') - for course in huey.courses.order_by(Course.name): - print(course.name) - - # Get all students in "English 101": - engl_101 = Course.get(Course.name == 'English 101') - for student in engl_101.students: - print(student.name) - - # When adding objects to a many-to-many relationship, we can pass - # in either a single model instance, a list of models, or even a - # query of models: - huey.courses.add(Course.select().where(Course.name.contains('English'))) - - engl_101.students.add(Student.get(Student.name == 'Mickey')) - engl_101.students.add([ - Student.get(Student.name == 'Charlie'), - Student.get(Student.name == 'Zaizee')]) - - # The same rules apply for removing items from a many-to-many: - huey.courses.remove(Course.select().where(Course.name.startswith('CS'))) - - engl_101.students.remove(huey) - - # Calling .clear() will remove all associated objects: - cs_150.students.clear() - -.. attention:: - Before many-to-many relationships can be added, the objects being - referenced will need to be saved first. In order to create relationships in - the many-to-many through table, Peewee needs to know the primary keys of - the models being referenced. - -For more examples, see: - -* :py:meth:`ManyToManyField.add` -* :py:meth:`ManyToManyField.remove` -* :py:meth:`ManyToManyField.clear` -* :py:meth:`ManyToManyField.get_through_model` - -Self-joins ----------- - -Peewee supports constructing queries containing a self-join. - -Using model aliases -^^^^^^^^^^^^^^^^^^^ - -To join on the same model (table) twice, it is necessary to create a model -alias to represent the second instance of the table in a query. Consider the -following model: - -.. code-block:: python - - class Category(Model): - name = CharField() - parent = ForeignKeyField('self', backref='children') - -What if we wanted to query all categories whose parent category is -*Electronics*. One way would be to perform a self-join: - -.. code-block:: python - - Parent = Category.alias() - query = (Category - .select() - .join(Parent, on=(Category.parent == Parent.id)) - .where(Parent.name == 'Electronics')) - -When performing a join that uses a :py:class:`ModelAlias`, it is necessary to -specify the join condition using the ``on`` keyword argument. In this case we -are joining the category with its parent category. - -Using subqueries -^^^^^^^^^^^^^^^^ - -Another less common approach involves the use of subqueries. Here is another -way we might construct a query to get all the categories whose parent category -is *Electronics* using a subquery: - -.. code-block:: python - - Parent = Category.alias() - join_query = Parent.select().where(Parent.name == 'Electronics') - - # Subqueries used as JOINs need to have an alias. - join_query = join_query.alias('jq') - - query = (Category - .select() - .join(join_query, on=(Category.parent == join_query.c.id))) - -This will generate the following SQL query: - -.. code-block:: sql - - SELECT t1."id", t1."name", t1."parent_id" - FROM "category" AS t1 - INNER JOIN ( - SELECT t2."id" - FROM "category" AS t2 - WHERE (t2."name" = ?)) AS jq ON (t1."parent_id" = "jq"."id") - -To access the ``id`` value from the subquery, we use the ``.c`` magic lookup -which will generate the appropriate SQL expression: - -.. code-block:: python - - Category.parent == join_query.c.id - # Becomes: (t1."parent_id" = "jq"."id") +This section have been moved into its own document: :ref:`relationships`. Performance Techniques ---------------------- diff --git a/docs/peewee/relationships.rst b/docs/peewee/relationships.rst index b092766e7..e59b3e920 100644 --- a/docs/peewee/relationships.rst +++ b/docs/peewee/relationships.rst @@ -95,6 +95,21 @@ mickey whine huey logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) +.. note:: + In SQLite, foreign keys are not enabled by default. Most things, including + the Peewee foreign-key API, will work fine, but ON DELETE behaviour will be + ignored, even if you explicitly specify on_delete to your ForeignKeyField. + In conjunction with the default PrimaryKeyField behaviour (where deleted + record IDs can be reused), this can lead to surprising (and almost + certainly unwanted) behaviour where if you delete a record in table A + referenced by a foreign key in table B, and then create a new, unrelated, + record in table A, the new record will end up mis-attached to the undeleted + record in table B. To avoid the mis-attachment, you can use + :py:class:`AutoIncrementField`, but it may be better overall to + ensure that foreign keys are enabled with + ``pragmas=(('foreign_keys', 'on'),)`` when you + instantiate :py:class:`SqliteDatabase`. + Performing simple joins ----------------------- @@ -531,3 +546,319 @@ each user: ... huey -> purr mickey -> whine + +Multiple foreign-keys to the same Model +--------------------------------------- + +When there are multiple foreign keys to the same model, it is good practice to +explicitly specify which field you are joining on. + +Referring back to the :ref:`example app's models `, +consider the *Relationship* model, which is used to denote when one user +follows another. Here is the model definition: + +.. code-block:: python + + class Relationship(BaseModel): + from_user = ForeignKeyField(User, backref='relationships') + to_user = ForeignKeyField(User, backref='related_to') + + class Meta: + indexes = ( + # Specify a unique multi-column index on from/to-user. + (('from_user', 'to_user'), True), + ) + +Since there are two foreign keys to *User*, we should always specify which +field we are using in a join. + +For example, to determine which users I am following, I would write: + +.. code-block:: python + + (User + .select() + .join(Relationship, on=Relationship.to_user) + .where(Relationship.from_user == charlie)) + +On the other hand, if I wanted to determine which users are following me, I +would instead join on the *from_user* column and filter on the relationship's +*to_user*: + +.. code-block:: python + + (User + .select() + .join(Relationship, on=Relationship.from_user) + .where(Relationship.to_user == charlie)) + +Joining on arbitrary fields +--------------------------- + +If a foreign key does not exist between two tables you can still perform a +join, but you must manually specify the join predicate. + +In the following example, there is no explicit foreign-key between *User* and +*ActivityLog*, but there is an implied relationship between the +*ActivityLog.object_id* field and *User.id*. Rather than joining on a specific +:py:class:`Field`, we will join using an :py:class:`Expression`. + +.. code-block:: python + + user_log = (User + .select(User, ActivityLog) + .join(ActivityLog, on=(User.id == ActivityLog.object_id), attr='log') + .where( + (ActivityLog.activity_type == 'user_activity') & + (User.username == 'charlie'))) + + for user in user_log: + print(user.username, user.log.description) + + #### Print something like #### + charlie logged in + charlie posted a tweet + charlie retweeted + charlie posted a tweet + charlie logged out + +.. note:: + Recall that we can control the attribute Peewee will assign the joined + instance to by specifying the ``attr`` parameter in the ``join()`` method. + In the previous example, we used the following *join*: + + .. code-block:: python + + join(ActivityLog, on=(User.id == ActivityLog.object_id), attr='log') + + Then when iterating over the query, we were able to directly access the + joined *ActivityLog* without incurring an additional query: + + .. code-block:: python + + for user in user_log: + print(user.username, user.log.description) + +.. _manytomany: + +Implementing Many to Many +------------------------- + +Peewee provides a field for representing many-to-many relationships, much like +Django does. This feature was added due to many requests from users, but I +strongly advocate against using it, since it conflates the idea of a field with +a junction table and hidden joins. It's just a nasty hack to provide convenient +accessors. + +To implement many-to-many **correctly** with peewee, you will therefore create +the intermediary table yourself and query through it: + +.. code-block:: python + + class Student(Model): + name = CharField() + + class Course(Model): + name = CharField() + + class StudentCourse(Model): + student = ForeignKeyField(Student) + course = ForeignKeyField(Course) + +To query, let's say we want to find students who are enrolled in math class: + +.. code-block:: python + + query = (Student + .select() + .join(StudentCourse) + .join(Course) + .where(Course.name == 'math')) + for student in query: + print(student.name) + +To query what classes a given student is enrolled in: + +.. code-block:: python + + courses = (Course + .select() + .join(StudentCourse) + .join(Student) + .where(Student.name == 'da vinci')) + + for course in courses: + print(course.name) + +To efficiently iterate over a many-to-many relation, i.e., list all students +and their respective courses, we will query the *through* model +``StudentCourse`` and *precompute* the Student and Course: + +.. code-block:: python + + query = (StudentCourse + .select(StudentCourse, Student, Course) + .join(Course) + .switch(StudentCourse) + .join(Student) + .order_by(Student.name)) + +To print a list of students and their courses you might do the following: + +.. code-block:: python + + for student_course in query: + print(student_course.student.name, '->', student_course.course.name) + +Since we selected all fields from ``Student`` and ``Course`` in the *select* +clause of the query, these foreign key traversals are "free" and we've done the +whole iteration with just 1 query. + +ManyToManyField +^^^^^^^^^^^^^^^ + +The :py:class:`ManyToManyField` provides a *field-like* API over many-to-many +fields. For all but the simplest many-to-many situations, you're better off +using the standard peewee APIs. But, if your models are very simple and your +querying needs are not very complex, you can get a big boost by using +:py:class:`ManyToManyField`. Check out the :ref:`extra-fields` extension module +for details. + +Modeling students and courses using :py:class:`ManyToManyField`: + +.. code-block:: python + + from peewee import * + from playhouse.fields import ManyToManyField + + db = SqliteDatabase('school.db') + + class BaseModel(Model): + class Meta: + database = db + + class Student(BaseModel): + name = CharField() + + class Course(BaseModel): + name = CharField() + students = ManyToManyField(Student, backref='courses') + + StudentCourse = Course.students.get_through_model() + + db.create_tables([ + Student, + Course, + StudentCourse]) + + # Get all classes that "huey" is enrolled in: + huey = Student.get(Student.name == 'Huey') + for course in huey.courses.order_by(Course.name): + print(course.name) + + # Get all students in "English 101": + engl_101 = Course.get(Course.name == 'English 101') + for student in engl_101.students: + print(student.name) + + # When adding objects to a many-to-many relationship, we can pass + # in either a single model instance, a list of models, or even a + # query of models: + huey.courses.add(Course.select().where(Course.name.contains('English'))) + + engl_101.students.add(Student.get(Student.name == 'Mickey')) + engl_101.students.add([ + Student.get(Student.name == 'Charlie'), + Student.get(Student.name == 'Zaizee')]) + + # The same rules apply for removing items from a many-to-many: + huey.courses.remove(Course.select().where(Course.name.startswith('CS'))) + + engl_101.students.remove(huey) + + # Calling .clear() will remove all associated objects: + cs_150.students.clear() + +.. attention:: + Before many-to-many relationships can be added, the objects being + referenced will need to be saved first. In order to create relationships in + the many-to-many through table, Peewee needs to know the primary keys of + the models being referenced. + +For more examples, see: + +* :py:meth:`ManyToManyField.add` +* :py:meth:`ManyToManyField.remove` +* :py:meth:`ManyToManyField.clear` +* :py:meth:`ManyToManyField.get_through_model` + +Self-joins +---------- + +Peewee supports constructing queries containing a self-join. + +Using model aliases +^^^^^^^^^^^^^^^^^^^ + +To join on the same model (table) twice, it is necessary to create a model +alias to represent the second instance of the table in a query. Consider the +following model: + +.. code-block:: python + + class Category(Model): + name = CharField() + parent = ForeignKeyField('self', backref='children') + +What if we wanted to query all categories whose parent category is +*Electronics*. One way would be to perform a self-join: + +.. code-block:: python + + Parent = Category.alias() + query = (Category + .select() + .join(Parent, on=(Category.parent == Parent.id)) + .where(Parent.name == 'Electronics')) + +When performing a join that uses a :py:class:`ModelAlias`, it is necessary to +specify the join condition using the ``on`` keyword argument. In this case we +are joining the category with its parent category. + +Using subqueries +^^^^^^^^^^^^^^^^ + +Another less common approach involves the use of subqueries. Here is another +way we might construct a query to get all the categories whose parent category +is *Electronics* using a subquery: + +.. code-block:: python + + Parent = Category.alias() + join_query = Parent.select().where(Parent.name == 'Electronics') + + # Subqueries used as JOINs need to have an alias. + join_query = join_query.alias('jq') + + query = (Category + .select() + .join(join_query, on=(Category.parent == join_query.c.id))) + +This will generate the following SQL query: + +.. code-block:: sql + + SELECT t1."id", t1."name", t1."parent_id" + FROM "category" AS t1 + INNER JOIN ( + SELECT t2."id" + FROM "category" AS t2 + WHERE (t2."name" = ?)) AS jq ON (t1."parent_id" = "jq"."id") + +To access the ``id`` value from the subquery, we use the ``.c`` magic lookup +which will generate the appropriate SQL expression: + +.. code-block:: python + + Category.parent == join_query.c.id + # Becomes: (t1."parent_id" = "jq"."id") From ff5624cf899011b28a2d83ce16ba83a1d5689c1c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 11:33:04 -0500 Subject: [PATCH 48/58] Remove outdated reference. --- docs/peewee/relationships.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/peewee/relationships.rst b/docs/peewee/relationships.rst index e59b3e920..6d69706f2 100644 --- a/docs/peewee/relationships.rst +++ b/docs/peewee/relationships.rst @@ -720,9 +720,7 @@ ManyToManyField The :py:class:`ManyToManyField` provides a *field-like* API over many-to-many fields. For all but the simplest many-to-many situations, you're better off using the standard peewee APIs. But, if your models are very simple and your -querying needs are not very complex, you can get a big boost by using -:py:class:`ManyToManyField`. Check out the :ref:`extra-fields` extension module -for details. +querying needs are not very complex, :py:class:`ManyToManyField` may work. Modeling students and courses using :py:class:`ManyToManyField`: From 82b43874a14f7e19c4bb40cf1f5206ccf00a83bf Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 14:13:00 -0500 Subject: [PATCH 49/58] Add some additional test-cases for emulating window functions. --- tests/models.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/models.py b/tests/models.py index cccc51b0d..8fb755e06 100644 --- a/tests/models.py +++ b/tests/models.py @@ -654,6 +654,52 @@ def test_join_subquery_cte(self): self.assertEqual([t.content for t in query], ['meow', 'purr', 'hiss']) + @requires_models(User) + def test_subquery_emulate_window(self): + # We have duplicated users. Select a maximum of 2 instances of the + # username. + name2count = { + 'beanie': 6, + 'huey': 5, + 'mickey': 3, + 'pipey': 1, + 'zaizee': 4} + names = [] + for name, count in sorted(name2count.items()): + names += [name] * count + User.insert_many([(n,) for n in names], [User.username]).execute() + + # The results we are trying to obtain. + expected = [ + ('beanie', 1), ('beanie', 2), + ('huey', 7), ('huey', 8), + ('mickey', 12), ('mickey', 13), + ('pipey', 15), + ('zaizee', 16), ('zaizee', 17)] + + # Using a self-join. + UA = User.alias() + query = (User + .select(User.username, UA.id) + .join(UA, on=((UA.username == User.username) & + (UA.id >= User.id))) + .group_by(User.username, UA.id) + .having(fn.COUNT(UA.id) < 3) + .order_by(User.username, UA.id)) + self.assertEqual(query.tuples()[:], expected) + + # Using a correlated subquery. + subq = (UA + .select(UA.id) + .where(User.username == UA.username) + .order_by(UA.id) + .limit(2)) + query = (User + .select(User.username, User.id) + .where(User.id.in_(subq.alias('subq'))) + .order_by(User.username, User.id)) + self.assertEqual(query.tuples()[:], expected) + @requires_models(User, Tweet) def test_insert_query_value(self): huey = self.add_user('huey') From 61d980efe9d437faa92254724af014f5c3d3fe61 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 14:24:37 -0500 Subject: [PATCH 50/58] Fix failing test. --- tests/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models.py b/tests/models.py index 8fb755e06..f9e9fa08f 100644 --- a/tests/models.py +++ b/tests/models.py @@ -654,6 +654,7 @@ def test_join_subquery_cte(self): self.assertEqual([t.content for t in query], ['meow', 'purr', 'hiss']) + @skip_if(IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES) @requires_models(User) def test_subquery_emulate_window(self): # We have duplicated users. Select a maximum of 2 instances of the From 904fe0aa30e73292bf6e7af6d54bc47c297227f9 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 14:51:53 -0500 Subject: [PATCH 51/58] Fix failing test...for real. --- tests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models.py b/tests/models.py index f9e9fa08f..f13ee43c2 100644 --- a/tests/models.py +++ b/tests/models.py @@ -654,7 +654,7 @@ def test_join_subquery_cte(self): self.assertEqual([t.content for t in query], ['meow', 'purr', 'hiss']) - @skip_if(IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES) + @skip_if(IS_MYSQL) # MariaDB does not support LIMIT in subqueries! @requires_models(User) def test_subquery_emulate_window(self): # We have duplicated users. Select a maximum of 2 instances of the From cd8d0580180417f25de217fac2b236654c73843b Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 16:15:25 -0500 Subject: [PATCH 52/58] More references to the relationships document. --- docs/peewee/api.rst | 6 ++++++ docs/peewee/models.rst | 8 ++++++++ docs/peewee/quickstart.rst | 17 ++++++++++------- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/docs/peewee/api.rst b/docs/peewee/api.rst index b479453db..ca0058e0c 100644 --- a/docs/peewee/api.rst +++ b/docs/peewee/api.rst @@ -2755,6 +2755,9 @@ Fields Another tweet Yet another tweet + For an in-depth discussion of foreign-keys, joins and relationships between + models, refer to :ref:`relationships`. + .. note:: Foreign keys do not have a particular ``field_type`` as they will take their field type depending on the type of primary key on the model they @@ -4012,6 +4015,9 @@ Model sq = User.select().join(Relationship, on=Relationship.to_user) + For an in-depth discussion of foreign-keys, joins and relationships + between models, refer to :ref:`relationships`. + .. py:method:: join_from(src, dest[, join_type='INNER'[, on=None[, attr=None]]]) :param src: Source for join. diff --git a/docs/peewee/models.rst b/docs/peewee/models.rst index 536216e6b..ff1b01763 100644 --- a/docs/peewee/models.rst +++ b/docs/peewee/models.rst @@ -136,6 +136,10 @@ This allows you to write code like the following: another message yet another message +.. note:: + Refer to the :ref:`relationships` document for an in-depth discussion of + foreign-keys, joins and relationships between models. + For full documentation on fields, see the :ref:`Fields API notes ` .. _field_types_table: @@ -304,6 +308,10 @@ means that all the users are stored in their own table, as are the tweets, and the foreign key from tweet to user allows each tweet to *point* to a particular user object. +.. note:: + Refer to the :ref:`relationships` document for an in-depth discussion of + foreign keys, joins and relationships between models. + In peewee, accessing the value of a :py:class:`ForeignKeyField` will return the entire related object, e.g.: diff --git a/docs/peewee/quickstart.rst b/docs/peewee/quickstart.rst index 7b7a71418..0e25cc218 100644 --- a/docs/peewee/quickstart.rst +++ b/docs/peewee/quickstart.rst @@ -61,8 +61,7 @@ by the database, so you can use Python types in your code without having to worry. Things get interesting when we set up relationships between models using -`foreign keys (wikipedia) `_. This is -easy to do with peewee: +:ref:`foreign key relationships `. This is simple with peewee: .. code-block:: python @@ -215,11 +214,15 @@ Let's list all the cats and their owner's name: # Kitty Bob # Mittens Jr Herb -There is a big problem with the previous query: because we are accessing -``pet.owner.name`` and we did not select this relation in our original query, -peewee will have to perform an additional query to retrieve the pet's owner. -This behavior is referred to as :ref:`N+1 ` and it should generally -be avoided. +.. attention:: + There is a big problem with the previous query: because we are accessing + ``pet.owner.name`` and we did not select this relation in our original + query, peewee will have to perform an additional query to retrieve the + pet's owner. This behavior is referred to as :ref:`N+1 ` and it + should generally be avoided. + + For an in-depth guide to working with relationships and joins, refer to the + :ref:`relationships` documentation. We can avoid the extra queries by selecting both *Pet* and *Person*, and adding a *join*. From 520b97a2fde3732faff8eccc2026cbbe292b360c Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Tue, 26 Jun 2018 16:44:39 -0500 Subject: [PATCH 53/58] Links in index/readme to relationships doc. --- README.rst | 1 + docs/index.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 0fa5e6c18..e22a915fb 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,7 @@ New to peewee? These may help: * `Example twitter app `_ * `Models and fields `_ * `Querying `_ +* `Relationships and joins `_ Examples -------- diff --git a/docs/index.rst b/docs/index.rst index 8fb2031a3..aa42c03cf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,6 +36,7 @@ New to peewee? These may help: * :ref:`Example twitter app ` * :ref:`Models and fields ` * :ref:`Querying ` +* :ref:`Relationships and joins ` Contents: --------- From 95dcea8f7218b71451a8fb98756ca76c32f14748 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 27 Jun 2018 08:58:39 -0500 Subject: [PATCH 54/58] Add test for read-only support with sqlite. --- tests/sqlite.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/sqlite.py b/tests/sqlite.py index 21fd42a37..ae0c67cac 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -14,6 +14,7 @@ from .base import requires_models from .base import skip_if from .base import skip_unless +from .base_models import User from .sqlite_helpers import json_installed @@ -1726,3 +1727,22 @@ def assertC(query, expected): # Sorting of column c is performed using the NOCASE collating sequence. assertC(base.order_by(Datum.c.collate('NOCASE'), Datum.id), [2, 4, 3, 1]) + + +class TestReadOnly(ModelTestCase): + @skip_if(sys.version_info < (3, 4, 0), 'requres python >= 3.4.0') + @requires_models(User) + def test_read_only(self): + User.create(username='foo') + + db_filename = self.database.database + db = SqliteDatabase('file:%s?mode=ro' % db_filename, uri=True) + cursor = db.execute_sql('select username from users') + self.assertEqual(cursor.fetchone(), ('foo',)) + + self.assertRaises(OperationalError, db.execute_sql, + 'insert into users (username) values (?)', ('huey',)) + + # We cannot create a database if in read-only mode. + db = SqliteDatabase('file:xx_not_exists.db?mode=ro', uri=True) + self.assertRaises(OperationalError, db.connect) From 2d24303c7b9ec28681a0ddf6ba047323e364936e Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 27 Jun 2018 10:06:19 -0500 Subject: [PATCH 55/58] Ensure sqlite is used for sqlite readonly test. --- tests/sqlite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/sqlite.py b/tests/sqlite.py index ae0c67cac..9ea851c7f 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -10,6 +10,7 @@ from .base import IS_SQLITE_9 from .base import ModelTestCase from .base import TestModel +from .base import db_loader from .base import get_in_memory_db from .base import requires_models from .base import skip_if @@ -1730,6 +1731,8 @@ def assertC(query, expected): class TestReadOnly(ModelTestCase): + database = db_loader('sqlite3') + @skip_if(sys.version_info < (3, 4, 0), 'requres python >= 3.4.0') @requires_models(User) def test_read_only(self): From d1f42fedde8ac3345dc7cf17e03056cca8cb661e Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 27 Jun 2018 10:25:14 -0500 Subject: [PATCH 56/58] Allow database object to be passed to DataSet constructor. --- playhouse/dataset.py | 16 +++++++++++----- tests/dataset.py | 9 +++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/playhouse/dataset.py b/playhouse/dataset.py index 0a456d756..8351455c5 100644 --- a/playhouse/dataset.py +++ b/playhouse/dataset.py @@ -22,12 +22,18 @@ class DataSet(object): def __init__(self, url, bare_fields=False): - self._url = url - parse_result = urlparse(url) - self._database_path = parse_result.path[1:] + if isinstance(url, Database): + self._url = None + self._database = url + self._database_path = self._database.database + else: + self._url = url + parse_result = urlparse(url) + self._database_path = parse_result.path[1:] + + # Connect to the database. + self._database = connect(url) - # Connect to the database. - self._database = connect(url) self._database.connect() # Introspect the database and generate models. diff --git a/tests/dataset.py b/tests/dataset.py index 9bbdc79bd..c322e8fff 100644 --- a/tests/dataset.py +++ b/tests/dataset.py @@ -49,6 +49,15 @@ def tearDown(self): self.dataset.close() super(TestDataSet, self).tearDown() + def test_pass_database(self): + db = SqliteDatabase(':memory:') + dataset = DataSet(db) + self.assertEqual(dataset._database_path, ':memory:') + + users = dataset['users'] + users.insert(username='charlie') + self.assertEqual(list(users), [{'id': 1, 'username': 'charlie'}]) + def create_users(self, n=2): user = self.dataset['user'] for i in range(min(n, len(self.names))): From e189e3dc1def2b1ad0f8ef96f19c40b8e424ecca Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 27 Jun 2018 10:26:10 -0500 Subject: [PATCH 57/58] Update docs according to dataset url parameter change. --- docs/peewee/playhouse.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/peewee/playhouse.rst b/docs/peewee/playhouse.rst index dba6bd0c1..ebafb4250 100644 --- a/docs/peewee/playhouse.rst +++ b/docs/peewee/playhouse.rst @@ -1762,7 +1762,8 @@ API .. py:class:: DataSet(url) - :param str url: A database URL. See :ref:`db_url` for examples. + :param url: A database URL or a :py:class:`Database` instance. For + details on using a URL, see :ref:`db_url` for examples. The *DataSet* class provides a high-level API for working with relational databases. From 8d3faa9947033d9e986afae5380a9a9100ecaefb Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Wed, 27 Jun 2018 10:31:26 -0500 Subject: [PATCH 58/58] 3.5.1 --- CHANGELOG.md | 17 ++++++++++++++++- peewee.py | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aa62d8ff..5c7b00617 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,23 @@ https://github.com/coleifer/peewee/releases ## master +[View commits](https://github.com/coleifer/peewee/compare/3.5.1...master) + +## 3.5.1 + +**New features** + +* New documentation for working with [relationships](http://docs.peewee-orm.com/en/latest/peewee/relationships.html) + in Peewee. +* Improved tests and documentation for MySQL upsert functionality. * Allow `database` parameter to be specified with `ModelSelect.get()` method. For discussion, see #1620. * Add `QualifiedNames` helper to peewee module exports. * Add `temporary=` meta option to support temporary tables. +* Allow a `Database` object to be passed to constructor of `DataSet` helper. + +**Bug fixes** + * Fixed edge-case where attempting to alias a field to it's underlying column-name (when different), Peewee would not respect the alias and use the field name instead. See #1625 for details and discussion. @@ -22,8 +35,10 @@ https://github.com/coleifer/peewee/releases * Fixed bugs in the implementation of user-defined aggregates and extensions with the APSW SQLite driver. * Fixed regression introduced in 3.5.0 which ignored custom Model `__repr__()`. +* Fixed regression from 2.x in which inserting from a query using a `SQL()` was + no longer working. Refs #1645. -[View commits](https://github.com/coleifer/peewee/compare/3.5.0...HEAD) +[View commits](https://github.com/coleifer/peewee/compare/3.5.0...3.5.1) ## 3.5.0 diff --git a/peewee.py b/peewee.py index 0fd1f87af..f3f591f6c 100644 --- a/peewee.py +++ b/peewee.py @@ -57,7 +57,7 @@ mysql = None -__version__ = '3.5.0' +__version__ = '3.5.1' __all__ = [ 'AsIs', 'AutoField',