From 3646bfdd7154e2db221d1c67f4e17e1820cafe5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Tue, 30 Apr 2024 14:28:08 +0200 Subject: [PATCH 1/8] Add a way not to have transaction partionning applying --- .gitignore | 1 + psqlextra/backend/schema.py | 14 ++++++++++---- psqlextra/partitioning/config.py | 2 ++ psqlextra/partitioning/plan.py | 15 ++++++++++++--- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 97ebaa67..80fbf68f 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ dist/ # Ignore PyCharm / IntelliJ files .idea/ +*.iml \ No newline at end of file diff --git a/psqlextra/backend/schema.py b/psqlextra/backend/schema.py index 28e9211a..9c0a6ef6 100644 --- a/psqlextra/backend/schema.py +++ b/psqlextra/backend/schema.py @@ -75,6 +75,7 @@ class PostgresSchemaEditor(SchemaEditor): sql_add_range_partition = ( "CREATE TABLE %s PARTITION OF %s FOR VALUES FROM (%s) TO (%s)" ) + sql_detach_partition = "ALTER TABLE %s DETACH PARTITION %s" sql_add_list_partition = ( "CREATE TABLE %s PARTITION OF %s FOR VALUES IN (%s)" ) @@ -807,11 +808,16 @@ def add_default_partition( def delete_partition(self, model: Type[Model], name: str) -> None: """Deletes the partition with the specified name.""" - - sql = self.sql_delete_partition % self.quote_name( - self.create_partition_table_name(model, name) + partition_table_name = self.create_partition_table_name(model, name) + detach_sql = self.sql_detach_partition % ( + self.quote_name(model._meta.db_table), + self.quote_name(partition_table_name), ) - self.execute(sql) + delete_sql = self.sql_delete_partition % self.quote_name( + partition_table_name + ) + self.execute(detach_sql) + self.execute(delete_sql) def alter_db_table( self, model: Type[Model], old_db_table: str, new_db_table: str diff --git a/psqlextra/partitioning/config.py b/psqlextra/partitioning/config.py index 976bf1ae..07cc729b 100644 --- a/psqlextra/partitioning/config.py +++ b/psqlextra/partitioning/config.py @@ -13,9 +13,11 @@ def __init__( self, model: Type[PostgresPartitionedModel], strategy: PostgresPartitioningStrategy, + atomic: bool = True, ) -> None: self.model = model self.strategy = strategy + self.atomic = atomic __all__ = ["PostgresPartitioningConfig"] diff --git a/psqlextra/partitioning/plan.py b/psqlextra/partitioning/plan.py index 3fcac44d..fd2e6153 100644 --- a/psqlextra/partitioning/plan.py +++ b/psqlextra/partitioning/plan.py @@ -1,5 +1,7 @@ +import contextlib + from dataclasses import dataclass, field -from typing import TYPE_CHECKING, List, Optional, cast +from typing import TYPE_CHECKING, ContextManager, List, Optional, Union, cast from django.db import connections, transaction @@ -36,8 +38,15 @@ def apply(self, using: Optional[str]) -> None: connection = connections[using or "default"] - with transaction.atomic(): - with connection.schema_editor() as schema_editor: + cm: Union[transaction.Atomic, ContextManager[None]] = ( + transaction.atomic() + if self.config.atomic + else contextlib.nullcontext() + ) + with cm: + with connection.schema_editor( + atomic=self.config.atomic + ) as schema_editor: for partition in self.creations: partition.create( self.config.model, From 363071d93c0817efb333903bbcd68a33a369fd04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 10:40:53 +0200 Subject: [PATCH 2/8] Add atomic in shorthand --- psqlextra/partitioning/shorthands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psqlextra/partitioning/shorthands.py b/psqlextra/partitioning/shorthands.py index 30175273..e3f0dcdc 100644 --- a/psqlextra/partitioning/shorthands.py +++ b/psqlextra/partitioning/shorthands.py @@ -18,6 +18,7 @@ def partition_by_current_time( days: Optional[int] = None, max_age: Optional[relativedelta] = None, name_format: Optional[str] = None, + atomic: bool = True, ) -> PostgresPartitioningConfig: """Short-hand for generating a partitioning config that partitions the specified model by time. @@ -53,6 +54,9 @@ def partition_by_current_time( name_format: The datetime format which is being passed to datetime.strftime to generate the partition name. + + atomic: + If set to True, the partitioning operations will be run inside a transaction. """ size = PostgresTimePartitionSize( @@ -61,6 +65,7 @@ def partition_by_current_time( return PostgresPartitioningConfig( model=model, + atomic=atomic, strategy=PostgresCurrentTimePartitioningStrategy( size=size, count=count, From 78a01b95b2bec8961b69d75870b3b1cffc76beae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 10:52:25 +0200 Subject: [PATCH 3/8] Add some documentation regarding atomic parameter --- docs/source/table_partitioning.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/table_partitioning.rst b/docs/source/table_partitioning.rst index 1bb5ba6f..588b1df2 100644 --- a/docs/source/table_partitioning.rst +++ b/docs/source/table_partitioning.rst @@ -177,6 +177,16 @@ Time-based partitioning ]) +Running management operations in a non atomic way +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Partitions creating and deletion can be done in a non-atomic way. +This can be useful to reduce lock contention when performing partition operations on a table while it is under heavy load. +Note that obviously this can lead to partially created/deleted partitions if something goes wrong during the operations. +By default all operations are done in an atomic way. + +You can disable atomic operations by setting the `atomic` parameter to `False` in the `PostgresPartitioningConfig` constructor. + Changing a time partitioning strategy ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 776b28102ba94a882bd78a5c056ea6242b65e5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 13:56:32 +0200 Subject: [PATCH 4/8] Add atomic false unit test --- tests/test_partitioning_time.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_partitioning_time.py b/tests/test_partitioning_time.py index 9f6b5bf1..b2c80ae3 100644 --- a/tests/test_partitioning_time.py +++ b/tests/test_partitioning_time.py @@ -458,6 +458,28 @@ def test_partitioning_time_delete(kwargs, timepoints): assert len(table.partitions) == partition_count +def test_partitioning_time_when_non_atomic(): + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} + ) + + schema_editor = connection.schema_editor() + schema_editor.create_partitioned_model(model) + + manager = PostgresPartitioningManager( + [partition_by_current_time(model=model, count=6, days=7, atomic=False)] + ) + + with freezegun.freeze_time("2019-1-1"): + manager.plan().apply() + + with freezegun.freeze_time("2019-1-15"): + manager.plan(skip_create=True).apply() + + table = _get_partitioned_table(model) + assert len(table.partitions) == 4 + + @pytest.mark.postgres_version(lt=110000) def test_partitioning_time_delete_ignore_manual(): """Tests whether partitions that were created manually are ignored. From 97df68baaab8116b3d2dc18fdc77950c36b9e48c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 14:04:00 +0200 Subject: [PATCH 5/8] Missing max age param in test --- tests/test_partitioning_time.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_partitioning_time.py b/tests/test_partitioning_time.py index b2c80ae3..a2cb5742 100644 --- a/tests/test_partitioning_time.py +++ b/tests/test_partitioning_time.py @@ -467,7 +467,15 @@ def test_partitioning_time_when_non_atomic(): schema_editor.create_partitioned_model(model) manager = PostgresPartitioningManager( - [partition_by_current_time(model=model, count=6, days=7, atomic=False)] + [ + partition_by_current_time( + model=model, + count=6, + days=7, + max_age=relativedelta(weeks=1), + atomic=False, + ) + ] ) with freezegun.freeze_time("2019-1-1"): From 9b67c13b03458dacd5c278319e48c15867437a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 14:24:25 +0200 Subject: [PATCH 6/8] Write implem for python < 3.7 --- psqlextra/partitioning/plan.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/psqlextra/partitioning/plan.py b/psqlextra/partitioning/plan.py index fd2e6153..a5882d0a 100644 --- a/psqlextra/partitioning/plan.py +++ b/psqlextra/partitioning/plan.py @@ -1,4 +1,5 @@ import contextlib +import sys from dataclasses import dataclass, field from typing import TYPE_CHECKING, ContextManager, List, Optional, Union, cast @@ -38,12 +39,7 @@ def apply(self, using: Optional[str]) -> None: connection = connections[using or "default"] - cm: Union[transaction.Atomic, ContextManager[None]] = ( - transaction.atomic() - if self.config.atomic - else contextlib.nullcontext() - ) - with cm: + with self._migration_context_manager(): with connection.schema_editor( atomic=self.config.atomic ) as schema_editor: @@ -60,6 +56,22 @@ def apply(self, using: Optional[str]) -> None: cast("PostgresSchemaEditor", schema_editor), ) + def _migration_context_manager( + self, + ) -> Union[transaction.Atomic, ContextManager[None]]: + if sys.version_info >= (3, 7): + return ( + transaction.atomic() + if self.config.atomic + else contextlib.nullcontext() + ) + else: + return ( + transaction.atomic() + if self.config.atomic + else contextlib.suppress() + ) + def print(self) -> None: """Prints this model plan to the terminal in a readable format.""" From b5bfb70e7ee1606d0fb10de5c8683d5844b668d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 14:49:09 +0200 Subject: [PATCH 7/8] Add missing postgres_version test tag --- tests/test_partitioning_time.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_partitioning_time.py b/tests/test_partitioning_time.py index a2cb5742..f1952f99 100644 --- a/tests/test_partitioning_time.py +++ b/tests/test_partitioning_time.py @@ -458,6 +458,7 @@ def test_partitioning_time_delete(kwargs, timepoints): assert len(table.partitions) == partition_count +@pytest.mark.postgres_version(lt=110000) def test_partitioning_time_when_non_atomic(): model = define_fake_partitioned_model( {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} From c20633ba0be3b189976692482a4552836e8b9ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pradal?= Date: Mon, 13 May 2024 15:28:59 +0200 Subject: [PATCH 8/8] Fix typo --- docs/source/table_partitioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/table_partitioning.rst b/docs/source/table_partitioning.rst index 588b1df2..eba33c10 100644 --- a/docs/source/table_partitioning.rst +++ b/docs/source/table_partitioning.rst @@ -180,7 +180,7 @@ Time-based partitioning Running management operations in a non atomic way ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Partitions creating and deletion can be done in a non-atomic way. +Partitions creation and deletion can be done in a non-atomic way. This can be useful to reduce lock contention when performing partition operations on a table while it is under heavy load. Note that obviously this can lead to partially created/deleted partitions if something goes wrong during the operations. By default all operations are done in an atomic way.