From c6c4ba212704434df5a7da872e8d798866df60e3 Mon Sep 17 00:00:00 2001 From: lamco-office Date: Sat, 20 Dec 2025 00:33:42 +0200 Subject: [PATCH] Update PostgreSQLEnhanced to v1.7.3 Major update with dual backend registration and ConfigManager integration. Changes: - Dual backend registration: Monolithic and Separate modes - ConfigManager integration for per-tree settings.ini - Gramps 6.0.6 API compatibility (json_extract_expression, composite index) - Import guard for psycopg dependency checking - Bug fixes for config loading and database creation - Migration scripts for existing users - Documentation updates for Windows AIO and Flatpak Breaking changes: - Backend IDs changed to postgresqlenhanced-monolithic/separate - Migration scripts provided in scripts/ directory - Users must run migrate_to_dual_backends.py and migrate_to_settings_ini.py Fixes GitHub issues #1 and #2 --- PostgreSQLEnhanced/MANIFEST | 2 + PostgreSQLEnhanced/README.md | 66 +- PostgreSQLEnhanced/concurrency.py | 74 +- PostgreSQLEnhanced/connection.py | 30 +- PostgreSQLEnhanced/postgresqlenhanced.gpr.py | 90 ++- PostgreSQLEnhanced/postgresqlenhanced.py | 743 +++++++++++-------- PostgreSQLEnhanced/schema.py | 16 +- PostgreSQLEnhanced/schema_migrations.py | 76 +- PostgreSQLEnhanced/undo_postgresql.py | 126 ++-- 9 files changed, 691 insertions(+), 532 deletions(-) diff --git a/PostgreSQLEnhanced/MANIFEST b/PostgreSQLEnhanced/MANIFEST index 505870cef..b641610de 100644 --- a/PostgreSQLEnhanced/MANIFEST +++ b/PostgreSQLEnhanced/MANIFEST @@ -10,3 +10,5 @@ schema_columns.py schema_migrations.py undo_postgresql.py README.md +INSTALL.md +CHANGELOG.md diff --git a/PostgreSQLEnhanced/README.md b/PostgreSQLEnhanced/README.md index 10d597ed9..ffd624a98 100644 --- a/PostgreSQLEnhanced/README.md +++ b/PostgreSQLEnhanced/README.md @@ -2,11 +2,17 @@ A high-performance PostgreSQL database backend for Gramps genealogy software that provides advanced database capabilities and superior performance for genealogical data management while maintaining full compatibility with the Gramps data model. -**Version:** 1.5.1 +**Version:** 1.6.0 **Project Status:** Experimental - Rigorous testing completed | [GitHub Repository](https://github.com/glamberson/gramps-postgresql-enhanced) | [Submit Issues](https://github.com/glamberson/gramps-postgresql-enhanced/issues) ## Recent Updates +### Version 1.6.0 (2025-12-19) +- **Gramps 6.0.6 compatibility** - Added required json_extract_expression() method +- **Import guard** - Prevents registration when psycopg not available +- **Performance index** - Added person_name_composite index for faster name searches +- **Better error messages** - Clear guidance when dependencies missing + ### Version 1.5.1 (2025-08-11) - **Fixed VARCHAR(255) truncation issue** - All string fields now use TEXT type to match SQLite behavior - **Automatic migration** - Existing databases are automatically upgraded when opened @@ -22,9 +28,9 @@ The PostgreSQL Enhanced addon provides a professional-grade database backend for - **Modern psycopg3** - Uses the latest PostgreSQL adapter (psycopg 3, not psycopg2) - **Dual Storage Format** - Maintains both pickle blobs (for Gramps compatibility) and JSONB (for advanced queries) -- **Two Database Modes** - Both fully tested and working: - - **Monolithic Mode** - All family trees in one database with table prefixes - - **Separate Mode** - Each family tree gets its own PostgreSQL database +- **Two Backend Options** - Select mode at tree creation: + - **Monolithic** - All family trees in one database with table prefixes (recommended) + - **Separate** - Each family tree gets its own PostgreSQL database - **Full Gramps Compatibility** - Works with all existing Gramps tools and reports - **Transaction Safety** - Proper savepoint handling and rollback capabilities - **Data Preservation** - Intelligent design that preserves data when trees are removed from Gramps @@ -197,76 +203,83 @@ Create or edit `connection_info.txt` with the following format: ```ini # PostgreSQL Enhanced Connection Configuration -# This file controls how the addon connects to PostgreSQL # Connection details -host = 192.168.10.90 # PostgreSQL server address +host = localhost # PostgreSQL server address port = 5432 # PostgreSQL port user = genealogy_user # Database username password = YourPassword # Database password -# Database mode: 'separate' or 'monolithic' -database_mode = monolithic - -# For monolithic mode only: name of the shared database -shared_database_name = gramps_monolithic +# For Monolithic mode: name of the shared database +shared_database_name = gramps_shared -# Optional settings (uncomment to use) +# Optional settings # pool_size = 10 # Connection pool size # sslmode = prefer # SSL connection mode -# connect_timeout = 10 # Connection timeout in seconds ``` -### Database Modes - Both Fully Tested and Working +**Note**: Mode selection (Monolithic vs Separate) is now done by choosing the backend in the tree creation dialog, not in this file. + +### Database Modes + +**Mode selection is now part of backend choice** - you pick the mode when creating a tree. -#### Monolithic Mode +#### Monolithic Mode (Recommended) + +**Select**: "PostgreSQL Enhanced (Monolithic)" in backend dropdown - **How it works**: All family trees share one PostgreSQL database - **Table naming**: Each tree's tables are prefixed with `tree__` - **Example**: Tree "68932301" creates tables like `tree_68932301_person` -- **Configuration**: Uses central `connection_info.txt` in plugin directory - **Advantages**: - Single database to manage and backup - Can query across multiple trees - Works without CREATEDB privilege - Simplified administration -- **Best for**: Organizations managing multiple related trees +- **Best for**: Most users, organizations managing multiple related trees + +#### Separate Mode (Advanced) -#### Separate Mode +**Select**: "PostgreSQL Enhanced (Separate)" in backend dropdown - **How it works**: Each family tree gets its own PostgreSQL database - **Database naming**: Creates database named after the tree ID - **Table naming**: Direct names without prefixes -- **Configuration**: Uses central `connection_info.txt` in plugin directory - **Advantages**: - Complete isolation between trees - Simpler table structure - Per-tree backup/restore - Better for multi-user scenarios - Independent database tuning per tree - - Lightning fast - **Best for**: Large independent trees or multi-tenant environments +- **Requires**: PostgreSQL user with CREATEDB privilege ## Creating a Family Tree ### Step 1: Configure the Connection -Before creating a tree, ensure your `connection_info.txt` is properly configured: +Ensure your `connection_info.txt` is configured in the plugin directory: ```bash # Edit the configuration file nano ~/.local/share/gramps/gramps60/plugins/PostgreSQLEnhanced/connection_info.txt ``` +Set at minimum: host, port, user, password, and shared_database_name (for Monolithic mode) + ### Step 2: Create Tree in Gramps 1. Open Gramps 2. Go to **Family Trees → Manage Family Trees** 3. Click **New** 4. Enter a name for your tree -5. For **Database backend**, select "PostgreSQL Enhanced" +5. For **Database backend**, select: + - **PostgreSQL Enhanced (Monolithic)** - Recommended: shared database + - **PostgreSQL Enhanced (Separate)** - Advanced: individual databases 6. Click **Load Family Tree** +**The mode you select determines how your tree is stored.** No additional configuration needed. + ### What Happens Behind the Scenes When you create a new tree: @@ -302,12 +315,13 @@ After registration, restart Gramps and the tree will appear in the Family Tree M ### Switching Between Modes -To switch from monolithic to separate mode (or vice versa): +To switch a tree from one mode to another: 1. Export your tree as GEDCOM or Gramps XML -2. Edit `connection_info.txt` to change `database_mode` -3. Create a new tree in Gramps -4. Import your exported data +2. Create a new tree selecting the desired backend: + - PostgreSQL Enhanced (Monolithic), or + - PostgreSQL Enhanced (Separate) +3. Import your exported data ## Design Features diff --git a/PostgreSQLEnhanced/concurrency.py b/PostgreSQLEnhanced/concurrency.py index a8dbcc70e..684d15552 100644 --- a/PostgreSQLEnhanced/concurrency.py +++ b/PostgreSQLEnhanced/concurrency.py @@ -65,7 +65,7 @@ class ConcurrencyError(Exception): class PostgreSQLConcurrency: """ PostgreSQL concurrency control features with graceful fallbacks. - + Provides advanced concurrency features that are safe to use regardless of PostgreSQL version or capabilities. """ @@ -79,18 +79,18 @@ def __init__(self, connection): """ self.conn = connection self.log = logging.getLogger(".PostgreSQLEnhanced.Concurrency") - + # Track capabilities self._listen_notify_available = None self._advisory_locks_available = None self._isolation_levels_available = None - + # Track active listeners and locks self._active_listeners: Dict[str, Callable] = {} self._active_locks: Dict[str, int] = {} self._notification_thread: Optional[threading.Thread] = None self._stop_notifications = threading.Event() - + # Initialize capabilities self._check_capabilities() @@ -100,19 +100,19 @@ def _check_capabilities(self): # Check PostgreSQL version self.conn.execute("SELECT version()") version_str = self.conn.fetchone()[0] - + # LISTEN/NOTIFY available in all modern PostgreSQL versions self._listen_notify_available = True - + # Advisory locks available in PostgreSQL 8.2+ self._advisory_locks_available = True - + # Isolation levels available in all PostgreSQL versions self._isolation_levels_available = True - + self.log.debug("Concurrency capabilities: LISTEN/NOTIFY=%s, Advisory Locks=%s, Isolation Levels=%s", self._listen_notify_available, self._advisory_locks_available, self._isolation_levels_available) - + except Exception as e: self.log.warning(f"Could not check concurrency capabilities: {e}") self._listen_notify_available = False @@ -143,11 +143,11 @@ def setup_listen_notify(self, channels: List[str] = None): for channel in channels: self.conn.execute(f"LISTEN {channel}") self.log.debug(f"Listening on channel: {channel}") - + # Start notification processing thread if not already running if not self._notification_thread or not self._notification_thread.is_alive(): self._start_notification_thread() - + return True except Exception as e: self.log.warning(f"Failed to set up LISTEN/NOTIFY: {e}") @@ -172,7 +172,7 @@ def notify_change(self, obj_type: str, handle: str, change_type: str, payload: D return False channel = f"{obj_type}_changes" - + # Create notification message message_data = { 'change_type': change_type, @@ -185,7 +185,7 @@ def notify_change(self, obj_type: str, handle: str, change_type: str, payload: D try: # PostgreSQL NOTIFY payload is limited to 8000 bytes message = str(message_data)[:7900] # Leave room for safety - + self.conn.execute("NOTIFY %s, %s", (channel, message)) self.log.debug(f"Sent notification: {channel} - {change_type}:{handle}") return True @@ -243,18 +243,18 @@ def _process_notifications(self): # Check for notifications (non-blocking) if hasattr(self.conn.connection, 'notifies'): self.conn.connection.poll() - + while self.conn.connection.notifies: notify = self.conn.connection.notifies.popleft() self._handle_notification(notify) - + # Sleep briefly to avoid busy waiting time.sleep(0.1) - + except Exception as e: self.log.warning(f"Error processing notifications: {e}") time.sleep(1) # Wait longer after error - + except Exception as e: self.log.error(f"Notification thread crashed: {e}") @@ -263,10 +263,10 @@ def _handle_notification(self, notify): try: channel = notify.channel payload = notify.payload - + if channel in self._active_listeners: callback = self._active_listeners[channel] - + # Parse payload back to dict import ast try: @@ -274,13 +274,13 @@ def _handle_notification(self, notify): except (ValueError, SyntaxError): # Fallback to string payload data = {'message': payload} - + # Call listener in thread-safe way try: callback(data) except Exception as e: self.log.error(f"Error in change listener callback: {e}") - + except Exception as e: self.log.warning(f"Error handling notification: {e}") @@ -316,29 +316,29 @@ def acquire_object_lock(self, obj_type: str, handle: str, exclusive: bool = True # Generate consistent lock ID from object type and handle lock_key = f"{obj_type}:{handle}" lock_id = abs(hash(lock_key)) % (2**31) # 32-bit signed int - + try: if exclusive: query = "SELECT pg_try_advisory_lock(%s)" else: query = "SELECT pg_try_advisory_lock_shared(%s)" - + start_time = time.time() while time.time() - start_time < timeout: self.conn.execute(query, (lock_id,)) result = self.conn.fetchone() - + if result and result[0]: self._active_locks[lock_key] = lock_id self.log.debug(f"Acquired {'exclusive' if exclusive else 'shared'} lock on {lock_key}") return True - + # Wait briefly before retrying time.sleep(0.1) - + self.log.warning(f"Failed to acquire lock on {lock_key} within {timeout}s") return False - + except Exception as e: self.log.warning(f"Error acquiring lock on {lock_key}: {e}") return False # Fail safe - allow access @@ -358,7 +358,7 @@ def release_object_lock(self, obj_type: str, handle: str): return True lock_key = f"{obj_type}:{handle}" - + if lock_key not in self._active_locks: self.log.debug(f"No active lock found for {lock_key}") return True @@ -366,11 +366,11 @@ def release_object_lock(self, obj_type: str, handle: str): try: lock_id = self._active_locks[lock_key] self.conn.execute("SELECT pg_advisory_unlock(%s)", (lock_id,)) - + del self._active_locks[lock_key] self.log.debug(f"Released lock on {lock_key}") return True - + except Exception as e: self.log.warning(f"Error releasing lock on {lock_key}: {e}") return False @@ -445,13 +445,13 @@ def get_object_version(self, obj_type: str, handle: str) -> Optional[float]: try: # Try to get change_time from object data table_name = obj_type # Assuming table name matches object type - + query = f"SELECT change_time FROM {table_name} WHERE handle = %s" self.conn.execute(query, (handle,)) result = self.conn.fetchone() - + return float(result[0]) if result else None - + except Exception as e: self.log.debug(f"Could not get version for {obj_type}:{handle}: {e}") return None @@ -469,11 +469,11 @@ def check_object_version(self, obj_type: str, handle: str, expected_version: flo :raises ConcurrencyError: If object was modified by another user """ current_version = self.get_object_version(obj_type, handle) - + if current_version is None: # Object doesn't exist or version check not available return - + if abs(current_version - expected_version) > 0.001: # Allow small floating point differences raise ConcurrencyError( _("Object {obj_type}:{handle} was modified by another user") @@ -486,7 +486,7 @@ def check_object_version(self, obj_type: str, handle: str, expected_version: flo class ObjectLock: """Context manager for object locks.""" - + def __init__(self, concurrency, obj_type: str, handle: str, exclusive: bool = True, timeout: float = 5.0): self.concurrency = concurrency self.obj_type = obj_type @@ -512,7 +512,7 @@ def object_lock(self, obj_type: str, handle: str, exclusive: bool = True, timeou Create a context manager for object locking. :param obj_type: Type of object - :param handle: Object handle + :param handle: Object handle :param exclusive: Whether to acquire exclusive lock :param timeout: Lock acquisition timeout :returns: Context manager for the lock diff --git a/PostgreSQLEnhanced/connection.py b/PostgreSQLEnhanced/connection.py index cc76b3ed5..285bbffd0 100644 --- a/PostgreSQLEnhanced/connection.py +++ b/PostgreSQLEnhanced/connection.py @@ -78,6 +78,7 @@ def __init__(self, connection_info, username=None, password=None): self._pool = None self._connection = None self._savepoints = [] + self._in_transaction = False self._persistent_cursor = None self._persistent_conn = None self._last_cursor = None @@ -100,8 +101,16 @@ def __init__(self, connection_info, username=None, password=None): else: self._create_connection(conninfo) - # Set up the connection - self._setup_connection() + # Set up the connection (create helper functions) + # This may fail if user lacks CREATE FUNCTION privilege + try: + self._setup_connection() + except Exception as e: + self.log.warning( + "Could not create helper functions (regexp): %s. " + "Some search features may be limited.", e + ) + # Continue without helper functions - core functionality still works # Configure JSONB handling for Gramps compatibility self._setup_jsonb_handling() @@ -495,6 +504,8 @@ def commit(self): pass else: self._connection.commit() + self._in_transaction = False + self._savepoints = [] def _commit(self): """Internal commit method.""" @@ -508,19 +519,20 @@ def rollback(self): pass else: self._connection.rollback() - self._savepoints.clear() + self._in_transaction = False + self._savepoints = [] def begin(self): """ Begin a transaction. - PostgreSQL starts transactions automatically, but we - track this for savepoint support. + With autocommit=False, PostgreSQL requires explicit BEGIN + to start a transaction. Without it, each statement auto-commits. """ - # Create a savepoint for nested transaction support - savepoint_name = "sp_%s" % len(self._savepoints) - self.execute("SAVEPOINT %s" % savepoint_name) - self._savepoints.append(savepoint_name) + if not self._in_transaction: + self.execute("BEGIN") + self._in_transaction = True + self._savepoints = [] def begin_savepoint(self, name=None): """Create a named savepoint.""" diff --git a/PostgreSQLEnhanced/postgresqlenhanced.gpr.py b/PostgreSQLEnhanced/postgresqlenhanced.gpr.py index 120951b14..603e92fa9 100644 --- a/PostgreSQLEnhanced/postgresqlenhanced.gpr.py +++ b/PostgreSQLEnhanced/postgresqlenhanced.gpr.py @@ -22,31 +22,65 @@ PostgreSQL Enhanced Database Backend Registration """ -register( - DATABASE, - id="postgresqlenhanced", - name=_("PostgreSQL Enhanced"), - name_accell=_("PostgreSQL _Enhanced Database"), - description=_( - "Advanced PostgreSQL backend with JSONB storage, " - "graph database support (Apache AGE), vector similarity (pgvector), " - "and AI/ML capabilities. For advanced users. " - "Requires PostgreSQL 15+ with extensions. Gramps Web compatible." - ), - version = '1.5.2', - gramps_target_version="6.0", - status=STABLE, - audience=EXPERT, # For advanced users who can configure PostgreSQL - fname="postgresqlenhanced.py", - databaseclass="PostgreSQLEnhanced", - authors=["Greg Lamberson"], - authors_email=["lamberson@yahoo.com"], - maintainers=["Greg Lamberson"], - maintainers_email=["lamberson@yahoo.com"], - requires_mod=['psycopg'], # psycopg3 (checks before installation) - requires_exe=[], # No external executables required - depends_on=[], # No dependencies on other Gramps plugins - help_url="https://github.com/gramps-project/addons-source/wiki/PostgreSQLEnhanced", - # Note: features attribute may not be supported in all Gramps versions - # Capabilities: monolithic-mode, separate-mode, grampsweb-compatible, jsonb-storage -) +import importlib + +from gramps.gen.plug._pluginreg import register, STABLE, DATABASE, DEVELOPER +from gramps.gen.const import GRAMPS_LOCALE as glocale + +_ = glocale.translation.gettext + +# Check for psycopg3 availability before registering +try: + import importlib.util + + PSYCOPG_AVAILABLE = importlib.util.find_spec("psycopg") is not None +except (ImportError, ValueError, AttributeError): + PSYCOPG_AVAILABLE = False + +# Only register if dependency available or building addon +if PSYCOPG_AVAILABLE or locals().get("build_script"): + # Register Monolithic mode - all trees share one database + register( + DATABASE, + id="postgresqlenhanced-monolithic", + name=_("PostgreSQL Enhanced (Monolithic)"), + name_accell=_("PostgreSQL Enhanced (_Monolithic)"), + description=_( + "PostgreSQL backend with all trees in shared database. " + "Recommended for most users. Uses table prefixes for multi-tree support. " + "Requires PostgreSQL 15+ and psycopg 3+." + ), + version = '1.7.3', + gramps_target_version="6.0", + status=STABLE, + fname="postgresqlenhanced.py", + databaseclass="PostgreSQLEnhancedMonolithic", + authors=["Greg Lamberson"], + authors_email=["lamberson@yahoo.com"], + maintainers=["Greg Lamberson"], + maintainers_email=["lamberson@yahoo.com"], + help_url="https://github.com/glamberson/gramps-postgresql-enhanced", + ) + + # Register Separate mode - one database per tree + register( + DATABASE, + id="postgresqlenhanced-separate", + name=_("PostgreSQL Enhanced (Separate)"), + name_accell=_("PostgreSQL Enhanced (_Separate)"), + description=_( + "PostgreSQL backend with individual database per tree. " + "For advanced users requiring complete tree isolation. " + "Requires PostgreSQL 15+ with CREATEDB privilege and psycopg 3+." + ), + version = '1.7.3', + gramps_target_version="6.0", + status=STABLE, + fname="postgresqlenhanced.py", + databaseclass="PostgreSQLEnhancedSeparate", + authors=["Greg Lamberson"], + authors_email=["lamberson@yahoo.com"], + maintainers=["Greg Lamberson"], + maintainers_email=["lamberson@yahoo.com"], + help_url="https://github.com/glamberson/gramps-postgresql-enhanced", + ) diff --git a/PostgreSQLEnhanced/postgresqlenhanced.py b/PostgreSQLEnhanced/postgresqlenhanced.py index a275d6fad..749ef9b69 100644 --- a/PostgreSQLEnhanced/postgresqlenhanced.py +++ b/PostgreSQLEnhanced/postgresqlenhanced.py @@ -131,20 +131,25 @@ # ------------------------------------------------------------ # -# PostgreSQLEnhanced +# PostgreSQLEnhancedBase - Shared implementation # # ------------------------------------------------------------ -class PostgreSQLEnhanced(DBAPI): +class PostgreSQLEnhancedBase(DBAPI): """ - PostgreSQL Enhanced interface for Gramps. + PostgreSQL Enhanced base implementation for Gramps. Provides advanced PostgreSQL features while maintaining full compatibility with the standard Gramps DBAPI interface. + + This is the base class - use PostgreSQLEnhancedMonolithic or + PostgreSQLEnhancedSeparate subclasses. """ - def __init__(self): + def __init__(self, force_mode=None): """Initialize the PostgreSQL Enhanced backend.""" super().__init__() + # Store forced mode (set by subclass) + self.force_mode = force_mode # Initialize logger self.log = logging.getLogger(__name__) # Check psycopg3 availability @@ -193,12 +198,21 @@ def __init__(self): if DEBUG_ENABLED and DEBUG_AVAILABLE: self._debug_context = DebugContext(LOG) LOG.debug("Debug context initialized") - + # Detect Gramps Web environment self.grampsweb_active = self._detect_grampsweb_environment() if self.grampsweb_active: LOG.info("PostgreSQL Enhanced: Gramps Web environment detected") + def requires_login(self): + """ + Returns True for backends that require a login dialog, else False. + + PostgreSQL requires username/password authentication. + Gramps will prompt for credentials before calling load(). + """ + return True + def get_summary(self): """ Return a dictionary of information about this database backend. @@ -288,210 +302,246 @@ def get_summary(self): def _initialize(self, directory, username, password): """ - Initialize the PostgreSQL Enhanced database connection. - - The 'directory' parameter contains connection information: - - * postgresql://user:pass@host:port/dbname - * host:port:dbname:schema - * dbname (for local connection) - - Special features can be enabled via query parameters: + Initialize PostgreSQL Enhanced using Gramps standard ConfigManager. - * ?use_jsonb=false (disable JSONB, use blob only) - * ?pool_size=10 (connection pool size) - - :param directory: Path to database directory or connection string + :param directory: Path to database directory :type directory: str - :param username: Database username (may be overridden by config) + :param username: Database username (overrides config) :type username: str - :param password: Database password (may be overridden by config) + :param password: Database password (overrides config) :type password: str :raises DbConnectionError: If configuration cannot be loaded or connection fails """ - LOG.info(f"_initialize called with directory='{directory}', POSTGRESQL_ENHANCED_MODE='{os.environ.get('POSTGRESQL_ENHANCED_MODE')}'") - - # Check for monolithic mode first - # Also detect if directory looks like a tree ID (8 hex chars) - # OR if it's a filesystem path ending with a tree ID - actual_tree_id = None - - # Extract tree ID from filesystem path if present - if directory and '/' in directory: - # Path like /root/.gramps/grampsdb/6894f36d - last_part = directory.rstrip('/').split('/')[-1] - # Check if it looks like a tree ID (8 chars, hex or similar format) - if last_part and (len(last_part) == 8 or '-' not in last_part): - # For now, accept any 8-char string as potential tree ID - # This handles both hex IDs and other formats - if len(last_part) <= 20: # Reasonable limit for tree ID - actual_tree_id = last_part - LOG.info(f"Extracted tree ID '{actual_tree_id}' from path '{directory}'") - - # Check if directory itself is a tree ID - is_tree_id = ( - directory and - len(directory) == 8 and - all(c in '0123456789abcdef' for c in directory.lower()) - ) - - if os.environ.get('POSTGRESQL_ENHANCED_MODE') == 'monolithic' or is_tree_id or actual_tree_id: - # In monolithic mode, build connection from environment variables - config = self._build_config_from_env() - self.directory = directory # Store original path - # Use extracted tree ID if available, otherwise use directory - tree_id_to_use = actual_tree_id or directory - self.tree_id = tree_id_to_use # Tree ID - self.table_prefix = f"tree_{tree_id_to_use}_" - self.shared_db_mode = True - - # Build connection string - connection_string = ( - "postgresql://{}:{}@{}:{}/{}".format( - config['user'], - config['password'], - config['host'], - config['port'], - config['database'] - ) - ) - - LOG.info( - "Monolithic mode - Tree ID: '%s', Table prefix: '%s', Database: '%s'", - self.tree_id, - self.table_prefix, - config['database'] - ) - # Check if this is a Gramps file-based path - # (like /home/user/.local/share/gramps/grampsdb/xxx) - # or a test directory with connection_info.txt - elif ( - directory - and os.path.isabs(directory) - and ( - "/grampsdb/" in directory - or (os.path.exists(os.path.join(directory, "connection_info.txt"))) - ) - ): - # Extract tree name from path - path_parts = directory.rstrip("/").split("/") - tree_name = path_parts[-1] if path_parts else "gramps_default" + from gramps.gen.utils.configmanager import ConfigManager + from gramps.gen.config import config as global_config + + # Extract tree ID from directory + tree_id = os.path.basename(directory.rstrip('/')) + self.directory = directory + self.tree_id = tree_id + self.path = directory + + LOG.info("Initializing tree '%s' with force_mode='%s'", tree_id, self.force_mode) + + # Check for GrampsWeb environment variable mode + explicit_env_mode = os.environ.get('POSTGRESQL_ENHANCED_MODE') == 'monolithic' + + if explicit_env_mode: + # GrampsWeb mode - use environment variables + config_dict = self._build_config_from_env() + host = config_dict['host'] + port = config_dict['port'] + db_user = config_dict['user'] + actual_password = config_dict['password'] + db_name = config_dict['database'] - # Store directory for config file lookup - self.directory = directory + self.table_prefix = f"tree_{tree_id}_" + self.shared_db_mode = True - # Load connection configuration - config = self._load_connection_config(directory) + LOG.info("Environment variable mode - database=%s, prefix=%s", db_name, self.table_prefix) - if config["database_mode"] == "separate": - # Separate database per tree - db_name = tree_name + else: + # Standard Gramps mode - use ConfigManager + config_file = os.path.join(directory, 'settings.ini') + config_mgr = ConfigManager(config_file) + + # Register configuration keys + config_mgr.register('database.host', 'localhost') + config_mgr.register('database.port', 5432) + config_mgr.register('database.user', 'gramps_user') + config_mgr.register('database.shared-database', 'gramps_shared') + config_mgr.register('database.pool-size', 5) + + # Load or create configuration + if not os.path.exists(config_file): + LOG.info("Creating settings.ini for tree %s", tree_id) + + # Check for connection_info.txt migration + plugin_dir = os.path.dirname(os.path.abspath(__file__)) + old_config_file = os.path.join(plugin_dir, 'connection_info.txt') + + if os.path.exists(old_config_file): + # Migrate from connection_info.txt + LOG.info("Migrating from connection_info.txt") + old_config = self._read_config_file(old_config_file) + + config_mgr.set('database.host', old_config.get('host', 'localhost')) + config_mgr.set('database.port', int(old_config.get('port', '5432'))) + config_mgr.set('database.user', old_config.get('user', 'gramps_user')) + config_mgr.set('database.shared-database', + old_config.get('shared_database_name', 'gramps_shared')) + config_mgr.set('database.pool-size', int(old_config.get('pool_size', '5'))) + else: + # Use defaults from global Gramps config + LOG.info("Using defaults from global Gramps preferences") + config_mgr.set('database.host', + global_config.get('database.host') or 'localhost') + port_str = global_config.get('database.port') or '5432' + config_mgr.set('database.port', + int(port_str) if port_str else 5432) + config_mgr.set('database.user', 'gramps_user') + config_mgr.set('database.shared-database', 'gramps_shared') + config_mgr.set('database.pool-size', 5) + + config_mgr.save() + LOG.info("Created settings.ini at %s", config_file) + + # Load configuration + config_mgr.load() + + # Get connection parameters from settings.ini + host = config_mgr.get('database.host') + port = config_mgr.get('database.port') + db_user = config_mgr.get('database.user') + + # Determine database name based on force_mode + if self.force_mode == 'separate': + db_name = tree_id self.table_prefix = "" self.shared_db_mode = False + LOG.info("Separate mode: database=%s", db_name) # Try to create database if it doesn't exist - if config.get("user") and config.get("password"): - self._ensure_database_exists(db_name, config) + config_dict = { + 'host': host, + 'port': port, + 'user': db_user, + 'password': password or '' + } + self._ensure_database_exists(db_name, config_dict) else: - # Shared database with table prefixes - db_name = config.get("shared_database_name", "gramps_shared") - # Sanitize tree name for use as table prefix - # Ensure prefix starts with 'tree_' to avoid PostgreSQL identifier issues - # (identifiers can't start with numbers) - safe_tree_name = re.sub(r"[^a-zA-Z0-9_]", "_", tree_name) - self.table_prefix = "tree_%s_" % safe_tree_name + db_name = config_mgr.get('database.shared-database') or 'gramps_shared' + safe_tree_id = re.sub(r"[^a-zA-Z0-9_]", "_", tree_id) + self.table_prefix = f"tree_{safe_tree_id}_" self.shared_db_mode = True - LOG.info( - "Using shared database mode with prefix: %s", self.table_prefix - ) + LOG.info("Monolithic mode: database=%s, prefix=%s", db_name, self.table_prefix) - # Build connection string - connection_string = ( - "postgresql://%s:%s@%s:%s/%s" % ( - config['user'], - config['password'], - config['host'], - config['port'], - db_name - ) - ) + # Username/password from parameters override config + # (password never stored in settings.ini for security) + actual_password = password or '' - LOG.info( - "Tree name: '%s', Database: '%s', Mode: '%s'", - tree_name, - db_name, - config["database_mode"], - ) - else: - # Direct connection string - connection_string = directory - self.table_prefix = "" - self.shared_db_mode = False + # Override username if provided as parameter + actual_user = username or db_user or 'gramps_user' + + # Build connection string + connection_string = f"postgresql://{actual_user}:{actual_password}@{host}:{port}/{db_name}" # Parse connection options self._parse_connection_options(connection_string) - # Store path for compatibility - self.path = directory - # Create connection try: - self.dbapi = PostgreSQLConnection(connection_string, username, password) + self.dbapi = PostgreSQLConnection(connection_string, actual_user, actual_password) # In monolithic mode, wrap the connection to add table prefixes - if hasattr(self, "table_prefix") and self.table_prefix: + if self.table_prefix: self.dbapi = TablePrefixWrapper(self.dbapi, self.table_prefix) except Exception as e: raise DbConnectionError(str(e), connection_string) from e - # Set serializer - DBAPI expects JSONSerializer - # JSONSerializer has object_to_data method that DBAPI needs - self.serializer = JSONSerializer() + # Initialize components - wrapped in try/finally to ensure connection cleanup + try: + # Set serializer + self.serializer = JSONSerializer() + + # Initialize schema + schema = PostgreSQLSchema( + self.dbapi, + use_jsonb=self._use_jsonb, + table_prefix=self.table_prefix, + ) + schema.check_and_init_schema() - # Initialize schema with table prefix if in shared mode - schema = PostgreSQLSchema( - self.dbapi, - use_jsonb=self._use_jsonb, - table_prefix=getattr(self, "table_prefix", ""), - ) - schema.check_and_init_schema() + # Initialize migration manager + self.migration_manager = MigrationManager(self.dbapi) - # Initialize migration manager - self.migration_manager = MigrationManager(self.dbapi) + # Initialize enhanced queries if JSONB is enabled + if self._use_jsonb: + self.enhanced_queries = EnhancedQueries(self.dbapi) - # Initialize enhanced queries if JSONB is enabled - if self._use_jsonb: - self.enhanced_queries = EnhancedQueries(self.dbapi) - - # Initialize search capabilities - try: - # TODO: Re-enable when search_capabilities module is available - # self.search_capabilities = SearchCapabilities(self.dbapi) - # self.search_api = SearchAPI(self, self.search_capabilities) - # mode = 'monolithic' if self.table_prefix else 'separate' - # self.search_capabilities.setup_search_infrastructure(mode) - self.search_api = None # Temporarily disabled - - LOG.info("Search capabilities temporarily disabled") - except Exception as e: - LOG.warning(f"Could not initialize search capabilities: {e}") - # Continue without advanced search features + # Initialize concurrency features (graceful degradation if fails) + try: + self.concurrency = PostgreSQLConcurrency(self.dbapi) + LOG.debug("Concurrency features initialized") + except Exception as e: + LOG.warning("Could not initialize concurrency features: %s", e) + self.concurrency = None + + # Log success + LOG.info("PostgreSQL Enhanced initialized successfully") + + # Set database as writable + self.readonly = False + self._is_open = True - # Initialize concurrency features - try: - self.concurrency = PostgreSQLConcurrency(self.dbapi) - LOG.info("Concurrency features initialized successfully") except Exception as e: - LOG.warning(f"Could not initialize concurrency features: {e}") - # Continue without concurrency features + # Clean up connection on initialization failure + LOG.error("Initialization failed, closing connection: %s", e) + if hasattr(self, 'dbapi') and self.dbapi: + try: + self.dbapi.close() + except Exception: + pass # Ignore errors during cleanup + raise + + def json_extract_expression(self, json_column, json_path): + """ + Generate PostgreSQL-specific JSON extraction expression. - # Log successful initialization - LOG.info("PostgreSQL Enhanced initialized successfully") + Converts JSONPath notation to PostgreSQL JSONB operators. - # Set database as writable - self.readonly = False - self._is_open = True + :param json_column: Name of the JSON column + :type json_column: str + :param json_path: JSONPath expression (e.g., '$.type' or '$.date.sort') + :type json_path: str + :returns: PostgreSQL JSONB extraction expression + :rtype: str + """ + path = json_path.strip("$").strip(".") + + if not path: + return json_column + + # Parse path into components + parts = [] + current = "" + in_bracket = False + + for char in path: + if char == "[": + if current: + parts.append(("key", current)) + current = "" + in_bracket = True + elif char == "]": + parts.append(("index", current)) + current = "" + in_bracket = False + elif char == "." and not in_bracket: + if current: + parts.append(("key", current)) + current = "" + else: + current += char + + if current: + parts.append(("key", current)) + + # Build PostgreSQL JSONB expression + expr = json_column + for i, (ptype, value) in enumerate(parts): + is_last = i == len(parts) - 1 + + if ptype == "key": + if is_last: + expr = "(%s->>%s)" % (expr, repr(value)) + else: + expr = "(%s->%s)" % (expr, repr(value)) + elif ptype == "index": + expr = "(%s->%s)" % (expr, value) + + return expr def is_open(self): """ @@ -561,80 +611,80 @@ def load( :param kwargs: Additional keyword arguments (unused) :returns: Always returns True :rtype: bool - + .. versionchanged:: 1.4 Added full DBAPI compatibility attributes for Gramps Web. """ # Handle both 'user' and 'username' parameters actual_username = username or user or None actual_password = password or None - + # Store original directory for later use self._original_directory = directory # Call our initialize method self._initialize(directory, actual_username, actual_password) - + # ===== BLOCK 1: TRIVIAL FLAGS AND ATTRIBUTES ===== # CRITICAL: This flag is required for Gramps Web to work self.db_is_open = True - + # Support read-only mode from gramps.gen.db.dbconst import DBMODE_R self.readonly = mode == DBMODE_R if mode else False - + # Initialize change tracking counter self.has_changed = 0 - + # Set minimal directory attributes for compatibility self._directory = directory self.path = directory # Some Gramps code expects this - + # Set serializer (always JSON for PostgreSQL Enhanced) self.set_serializer("json") - + # ===== END BLOCK 1 ===== - + # ===== BLOCK 2: BOOKMARKS AND BASIC METADATA ===== # Initialize all bookmark collections (required for Gramps Web) self._initialize_bookmarks() - + # Load name formats and researcher info from gramps.gen.lib import Researcher self.name_formats = self._get_metadata("name_formats", []) self.owner = self._get_metadata("researcher", default=Researcher()) - + # Initialize all custom type attributes self._initialize_custom_types() - + # Load gender statistics from gramps.gen.lib import GenderStats gstats = self._get_metadata("gender_stats", {}) self.genderStats = GenderStats(gstats) - + # ===== END BLOCK 2 ===== - + # ===== BLOCK 3: MODE-AWARE METADATA (ID COUNTERS & SURNAME LIST) ===== # Initialize ID counters with mode awareness (critical for monolithic mode) self._initialize_id_counters() - + # Load surname list with mode awareness (must be isolated per tree) self.surname_list = self._get_mode_aware_surname_list() - + # ===== END BLOCK 3 ===== - + # ===== BLOCK 4: RECENT FILES TRACKING ===== # Update recent files to fix "Last Accessed: NEVER" issue self._update_recent_files() - + # ===== END BLOCK 4 ===== # Set up the undo manager without calling parent's full load # which tries to run upgrades on non-existent files - + # Detect if we're running under GrampsWeb grampsweb_mode = self._detect_grampsweb_mode() - + if grampsweb_mode: # Use our PostgreSQL undo that has get_transactions try: @@ -651,18 +701,18 @@ def load( from gramps.gen.db.generic import DbGenericUndo self.undolog = None self.undodb = DbGenericUndo(self, self.undolog) - + self.undodb.open() # Set proper version to avoid upgrade prompts self._set_metadata("version", "21") - + return True - + def _initialize_bookmarks(self): """ Initialize all bookmark collections. - + .. versionadded:: 1.4 Required for Gramps Web compatibility. """ @@ -678,7 +728,7 @@ def _initialize_bookmarks(self): self.media_bookmarks = Bookmarks() self.place_bookmarks = Bookmarks() self.note_bookmarks = Bookmarks() - + # Load bookmark data from metadata self.bookmarks.load(self._get_metadata("bookmarks", [])) self.family_bookmarks.load(self._get_metadata("family_bookmarks", [])) @@ -689,13 +739,13 @@ def _initialize_bookmarks(self): self.media_bookmarks.load(self._get_metadata("media_bookmarks", [])) self.place_bookmarks.load(self._get_metadata("place_bookmarks", [])) self.note_bookmarks.load(self._get_metadata("note_bookmarks", [])) - + def _initialize_custom_types(self): """ Initialize all custom type attributes. - + These populate UI dropdowns in Gramps. - + .. versionadded:: 1.4 Required for Gramps Web UI functionality. """ @@ -717,14 +767,14 @@ def _initialize_custom_types(self): self.media_attributes = self._get_metadata("mattr_names", set()) self.event_attributes = self._get_metadata("eattr_names", set()) self.place_types = self._get_metadata("place_types", set()) - + def _initialize_id_counters(self): """ Initialize ID generation counters with mode awareness. - + In monolithic mode, each tree must have its own counters to prevent ID collisions between trees. - + .. versionadded:: 1.4 Mode-aware ID counter initialization. """ @@ -751,14 +801,14 @@ def _initialize_id_counters(self): self.omap_index = self._get_metadata("omap_index", 0) self.rmap_index = self._get_metadata("rmap_index", 0) self.nmap_index = self._get_metadata("nmap_index", 0) - + def _get_mode_aware_surname_list(self): """ Get surname list with mode awareness. - + In monolithic mode, returns surnames only from the current tree's table. In separate mode, returns surnames from the single person table. - + .. versionadded:: 1.4 Mode-aware surname list retrieval. """ @@ -769,60 +819,60 @@ def _get_mode_aware_surname_list(self): else: # Separate mode: standard person table table_name = "person" - + query = f""" - SELECT DISTINCT + SELECT DISTINCT json_data->'primary_name'->>'surname' as surname FROM {table_name} - WHERE json_data IS NOT NULL + WHERE json_data IS NOT NULL AND json_data->'primary_name'->>'surname' IS NOT NULL AND json_data->'primary_name'->>'surname' != '' ORDER BY surname """ - + result = self.dbapi.execute(query) return [row[0] for row in result.fetchall()] except Exception as e: LOG.warning(f"Failed to load surname list: {e}") return [] - + def _detect_grampsweb_mode(self): """ Detect if running under GrampsWeb without importing it. - + :returns: True if running under GrampsWeb, False otherwise :rtype: bool """ # Check for GrampsWeb-specific environment variables if os.environ.get('GRAMPSWEB_TREE'): return True - + # Check if gramps_webapi is in sys.modules (already imported) import sys if 'gramps_webapi' in sys.modules: return True - + # Check call stack for GrampsWeb import inspect for frame_info in inspect.stack(): if 'gramps_webapi' in frame_info.filename: return True - + return False - + def _update_recent_files(self): """ Update Gramps' recent files tracking with current timestamp. - + Creates meaningful virtual PostgreSQL paths instead of filesystem paths. Fixes the "Last Accessed: NEVER" issue. - + .. versionadded:: 1.4 Recent files tracking for PostgreSQL databases. """ try: from gramps.gen.recentfiles import recent_files - + # Create a meaningful virtual path for PostgreSQL if hasattr(self, 'table_prefix') and self.table_prefix: # Monolithic mode: use tree identifier @@ -840,15 +890,15 @@ def _update_recent_files(self): db_name = "postgresql_db" virtual_path = f"postgresql://separate/{db_name}" display_name = f"PostgreSQL: {db_name}" - + # Get the tree name from metadata if available tree_name = self._get_metadata("name", display_name) - + # Update recent files with current timestamp recent_files(virtual_path, tree_name) - + LOG.debug(f"Updated recent files: {virtual_path} -> {tree_name}") - + except Exception as e: # Don't fail the entire load if recent files update fails LOG.warning(f"Could not update recent files tracking: {e}") @@ -869,7 +919,9 @@ def _read_config_file(self, config_path): line = line.strip() if line and not line.startswith("#") and "=" in line: key, value = line.split("=", 1) - config[key.strip()] = value.strip() + # Strip inline comments before storing value + value = value.split('#')[0].strip() + config[key.strip()] = value return config def _load_connection_config(self, directory): @@ -999,19 +1051,22 @@ def _ensure_database_exists(self, db_name, config): def _build_config_from_env(self): """ Build configuration from environment variables for monolithic mode. - + + Used by GrampsWeb when POSTGRESQL_ENHANCED_MODE=monolithic is set. + All GRAMPSWEB_POSTGRES_* environment variables should be configured. + :return: Configuration dictionary :rtype: dict """ return { - 'host': os.environ.get('GRAMPSWEB_POSTGRES_HOST', '192.168.10.90'), + 'host': os.environ.get('GRAMPSWEB_POSTGRES_HOST', 'localhost'), 'port': os.environ.get('GRAMPSWEB_POSTGRES_PORT', '5432'), - 'database': os.environ.get('GRAMPSWEB_POSTGRES_DB', 'gramps_monolithic_v13_test'), - 'user': os.environ.get('GRAMPSWEB_POSTGRES_USER', 'genealogy_user'), - 'password': os.environ.get('GRAMPSWEB_POSTGRES_PASSWORD', 'GenealogyData2025'), + 'database': os.environ.get('GRAMPSWEB_POSTGRES_DB', 'gramps'), + 'user': os.environ.get('GRAMPSWEB_POSTGRES_USER', 'gramps'), + 'password': os.environ.get('GRAMPSWEB_POSTGRES_PASSWORD', ''), 'database_mode': 'monolithic' } - + def _parse_connection_options(self, connection_string): """ Parse connection options from the connection string. @@ -1193,15 +1248,18 @@ def search_all_text(self, search_term, limit=100): # Use modern SearchAPI with tsvector if available if self.search_api: return self.search_api.fulltext_search( - search_term, + search_term, limit=limit, - obj_types=['person', 'family', 'event', 'place', 'source', 'note', 'citation', 'media', 'repository', 'tag'] + obj_types=[ + 'person', 'family', 'event', 'place', 'source', + 'note', 'citation', 'media', 'repository', 'tag' + ] ) - + # Fallback to enhanced queries ILIKE search if JSONB enabled elif self.enhanced_queries: return self.enhanced_queries.search_all_text(search_term, limit) - + else: raise RuntimeError(_("Full-text search requires JSONB support or search capabilities")) @@ -1498,7 +1556,7 @@ def get_dbid(self): """ Return unique database identifier. Required by Gramps Web for tree identification. - + :returns: Unique database identifier (UUID) :rtype: str """ @@ -1509,7 +1567,7 @@ def get_dbid(self): return dbid except Exception: pass - + # Generate new UUID and store it import uuid dbid = str(uuid.uuid4()) @@ -1517,13 +1575,13 @@ def get_dbid(self): self.set_metadata('dbid', dbid) except Exception as e: LOG.warning("Could not store database ID in metadata: %s", e) - + return dbid - + def _detect_grampsweb_environment(self): """ Detect if running under Gramps Web. - + :returns: True if Gramps Web environment variables are present :rtype: bool """ @@ -1533,29 +1591,29 @@ def _detect_grampsweb_environment(self): 'GRAMPSWEB_NEW_DB_BACKEND', # Backend specification 'GRAMPSWEB_POSTGRES_HOST', # PostgreSQL configuration ] - + for indicator in grampsweb_indicators: if os.environ.get(indicator): LOG.debug("Gramps Web detected via %s", indicator) return True - + return False - + def is_read_only(self): """ Check if database is read-only. Used by Gramps Web for UI permission handling. - + :returns: True if database is read-only :rtype: bool """ return getattr(self, 'readonly', False) - + def get_mediapath(self): """ Get media directory path. Used by Gramps Web for media file handling. - + :returns: Path to media directory or None :rtype: str or None """ @@ -1566,17 +1624,17 @@ def get_mediapath(self): return path except Exception: pass - + # Default to subdirectory of tree directory if hasattr(self, 'directory') and self.directory: return os.path.join(self.directory, 'media') - + return None - + def set_mediapath(self, path): """ Set media directory path. - + :param path: Path to media directory :type path: str """ @@ -1584,15 +1642,15 @@ def set_mediapath(self, path): self.set_metadata('mediapath', path) except Exception as e: LOG.warning("Could not store media path in metadata: %s", e) - + # ======================================================================== # Search API for Gramps Web and other consumers # ======================================================================== - + def search(self, query, search_type='auto', limit=100, **kwargs): """ Unified search interface for Gramps Web compatibility. - + :param query: Search query string :type query: str :param search_type: Type of search ('auto', 'exact', 'fuzzy', 'phonetic', 'semantic') @@ -1605,73 +1663,73 @@ def search(self, query, search_type='auto', limit=100, **kwargs): if not self.search_api: # Fallback to basic search if capabilities not initialized return self._basic_search_fallback(query, limit, **kwargs) - + return self.search_api.search(query, search_type, limit=limit, **kwargs) - + def setup_fulltext_search(self): """ Set up PostgreSQL native full-text search. This replaces the need for sifts entirely. """ LOG.info("Setting up PostgreSQL native full-text search") - + # Add search vectors to all object tables - for obj_type in ['person', 'family', 'event', 'place', 'source', 'citation', + for obj_type in ['person', 'family', 'event', 'place', 'source', 'citation', 'repository', 'media', 'note', 'tag']: table = self.schema._table_name(obj_type) - + try: # Add search column if not exists with self.dbapi.execute(f""" - ALTER TABLE {table} + ALTER TABLE {table} ADD COLUMN IF NOT EXISTS search_vector tsvector """): pass - + # Create GIN index for fast searching with self.dbapi.execute(f""" - CREATE INDEX IF NOT EXISTS idx_{table}_search + CREATE INDEX IF NOT EXISTS idx_{table}_search ON {table} USING GIN(search_vector) """): pass - + # Create trigger to auto-update search vector with self.dbapi.execute(f""" - CREATE OR REPLACE FUNCTION {table}_search_trigger() + CREATE OR REPLACE FUNCTION {table}_search_trigger() RETURNS trigger AS $$ BEGIN - NEW.search_vector := to_tsvector('simple', + NEW.search_vector := to_tsvector('simple', coalesce(NEW.json_data::text, '')); RETURN NEW; END; $$ LANGUAGE plpgsql; - + DROP TRIGGER IF EXISTS {table}_search_update ON {table}; - - CREATE TRIGGER {table}_search_update + + CREATE TRIGGER {table}_search_update BEFORE INSERT OR UPDATE ON {table} - FOR EACH ROW + FOR EACH ROW EXECUTE FUNCTION {table}_search_trigger(); """): pass - + # Update existing rows with self.dbapi.execute(f""" - UPDATE {table} + UPDATE {table} SET search_vector = to_tsvector('simple', coalesce(json_data::text, '')) WHERE search_vector IS NULL """): pass - + LOG.debug(f"Full-text search enabled for {table}") - + except Exception as e: LOG.warning(f"Could not setup full-text search for {table}: {e}") - + def get_search_capabilities(self): """ Get available search capabilities for Gramps Web. - + :returns: Dictionary of available search features :rtype: dict """ @@ -1681,17 +1739,17 @@ def get_search_capabilities(self): 'features': ['basic_search'], 'extensions': {} } - + return { 'level': self.search_capabilities.search_level, 'features': self.search_capabilities.get_available_features(), 'extensions': self.search_capabilities.capabilities } - + def enable_search_extension(self, extension): """ Try to enable a search extension if available. - + :param extension: Extension name (pg_trgm, fuzzystrmatch, etc.) :type extension: str :returns: True if successful @@ -1699,9 +1757,9 @@ def enable_search_extension(self, extension): """ if not self.search_capabilities: return False - + return self.search_capabilities.enable_extension(extension) - + def _basic_search_fallback(self, query, limit=100, **kwargs): """ Basic search fallback when advanced features unavailable. @@ -1709,7 +1767,7 @@ def _basic_search_fallback(self, query, limit=100, **kwargs): """ results = [] query_pattern = f"%{query}%" - + # Search persons table = self.schema._table_name('person') with self.dbapi.execute(f""" @@ -1726,7 +1784,7 @@ def _basic_search_fallback(self, query, limit=100, **kwargs): 'data': row[2], 'relevance': 1.0 }) - + return results def get_event_from_handle(self, handle): @@ -1963,28 +2021,28 @@ def _set_metadata(self, key, value, use_txn=True): continue else: raise - + # ======================================================================== # Public Metadata Methods for GrampsWeb Compatibility # ======================================================================== - + def set_metadata(self, key, value): """ Public wrapper for _set_metadata. Required by GrampsWeb for metadata storage. - + :param key: Metadata key :type key: str :param value: Metadata value :type value: Any """ return self._set_metadata(key, value) - + def get_metadata(self, key, default=None): """ Public wrapper for _get_metadata. Required by GrampsWeb for metadata retrieval. - + :param key: Metadata key :type key: str :param default: Default value if key not found @@ -1994,17 +2052,17 @@ def get_metadata(self, key, default=None): """ result = self._get_metadata(key, "_") return default if result == "_" else result - + # ======================================================================== # Transaction History Support for GrampsWeb # ======================================================================== - + def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, ascending=False, before=None, after=None): """ Get transaction history with pagination. Required by GrampsWeb API for /api/transactions/history/ - + :param page: Page number (1-based) :type page: int :param pagesize: Number of transactions per page @@ -2026,22 +2084,22 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, # TODO: Implement actual transaction history from undo table transactions = [] total_count = 0 - + # If we have undo data, we could query it here # This would require querying the undo table with proper filtering - + return transactions, total_count - + # ======================================================================== # Gramps Web Multi-Tree Support (Class Methods) # ======================================================================== - + @classmethod def create_tree(cls, tree_id=None, name=None): """ Create a new family tree. Required by Gramps Web API for POST /api/trees/ - + :param tree_id: Optional tree identifier (UUID if not provided) :type tree_id: str or None :param name: Optional human-readable tree name @@ -2051,97 +2109,97 @@ def create_tree(cls, tree_id=None, name=None): """ import uuid import tempfile - + # Generate tree ID if not provided if not tree_id: tree_id = str(uuid.uuid4()) - + # Determine tree directory gramps_home = os.environ.get('GRAMPS_HOME', tempfile.gettempdir()) tree_dir = os.path.join(gramps_home, 'gramps_tree_%s' % tree_id) os.makedirs(tree_dir, exist_ok=True) - + # Determine database mode from environment database_mode = os.environ.get('POSTGRESQL_ENHANCED_MODE', 'separate') - + # Create connection_info.txt config_path = os.path.join(tree_dir, 'connection_info.txt') with open(config_path, 'w') as f: f.write("# PostgreSQL Enhanced Configuration\n") f.write("# Auto-generated for Gramps Web\n\n") f.write("# Connection details\n") - f.write("host = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_HOST', '192.168.10.90')) + f.write("host = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_HOST', 'localhost')) f.write("port = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_PORT', '5432')) - f.write("user = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_USER', 'genealogy_user')) - f.write("password = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_PASSWORD', 'GenealogyData2025')) + f.write("user = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_USER', 'gramps')) + f.write("password = %s\n" % os.environ.get('GRAMPSWEB_POSTGRES_PASSWORD', '')) f.write("\n# Database mode\n") f.write("database_mode = %s\n" % database_mode) - + if database_mode == 'monolithic': f.write("\n# Monolithic mode configuration\n") - f.write("monolithic_database = %s\n" % - os.environ.get('GRAMPSWEB_POSTGRES_DB', 'henderson_unified')) + f.write("monolithic_database = %s\n" % + os.environ.get('GRAMPSWEB_POSTGRES_DB', 'gramps')) f.write("tree_prefix = tree_%s_\n" % tree_id[:8]) - + # Write database.txt with open(os.path.join(tree_dir, 'database.txt'), 'w') as f: f.write('postgresqlenhanced') - + # Write name.txt with open(os.path.join(tree_dir, 'name.txt'), 'w') as f: f.write(name or 'Tree %s' % tree_id[:8]) - + # Initialize the database try: db = cls() db._initialize(tree_dir, None, None) - + # Set initial metadata db.set_metadata('dbid', tree_id) db.set_metadata('name', name or 'Tree %s' % tree_id[:8]) db.set_metadata('created', str(time.time())) - + LOG.info("Created tree %s in %s mode", tree_id, database_mode) - + except Exception as e: LOG.error("Failed to create tree %s: %s", tree_id, e) raise - + return tree_id - + @classmethod def list_trees(cls): """ List available family trees. Optional for Gramps Web multi-tree mode. - + :returns: List of tree dictionaries with id, name, and path :rtype: list """ trees = [] - + # For monolithic mode, query the database if os.environ.get('POSTGRESQL_ENHANCED_MODE') == 'monolithic': try: # Connect to monolithic database conn_params = { - 'host': os.environ.get('GRAMPSWEB_POSTGRES_HOST', '192.168.10.90'), + 'host': os.environ.get('GRAMPSWEB_POSTGRES_HOST', 'localhost'), 'port': int(os.environ.get('GRAMPSWEB_POSTGRES_PORT', 5432)), - 'dbname': os.environ.get('GRAMPSWEB_POSTGRES_DB', 'gramps_monolithic'), - 'user': os.environ.get('GRAMPSWEB_POSTGRES_USER', 'genealogy_user'), - 'password': os.environ.get('GRAMPSWEB_POSTGRES_PASSWORD', 'GenealogyData2025'), + 'dbname': os.environ.get('GRAMPSWEB_POSTGRES_DB', 'gramps'), + 'user': os.environ.get('GRAMPSWEB_POSTGRES_USER', 'gramps'), + 'password': os.environ.get('GRAMPSWEB_POSTGRES_PASSWORD', ''), } - + with psycopg.connect(**conn_params) as conn: with conn.cursor() as cur: # Query for all tree prefixes cur.execute(""" - SELECT DISTINCT + SELECT DISTINCT substring(tablename from 'tree_(.*)_metadata') as tree_id - FROM pg_tables + FROM pg_tables WHERE tablename LIKE 'tree_%_metadata' """) - + for row in cur.fetchall(): tree_id = row[0] trees.append({ @@ -2149,10 +2207,10 @@ def list_trees(cls): 'name': 'Tree %s' % tree_id, 'mode': 'monolithic' }) - + except Exception as e: LOG.error("Failed to list trees from database: %s", e) - + # Also check file system gramps_home = os.environ.get('GRAMPS_HOME', '/tmp') if os.path.exists(gramps_home): @@ -2160,7 +2218,7 @@ def list_trees(cls): if entry.startswith('gramps_tree_'): tree_dir = os.path.join(gramps_home, entry) tree_id = entry.replace('gramps_tree_', '') - + # Read name if available name_file = os.path.join(tree_dir, 'name.txt') name = 'Tree %s' % tree_id[:8] @@ -2170,7 +2228,7 @@ def list_trees(cls): name = f.read().strip() except: pass - + # Check database mode mode = 'separate' config_file = os.path.join(tree_dir, 'connection_info.txt') @@ -2178,14 +2236,14 @@ def list_trees(cls): with open(config_file) as f: if 'database_mode = monolithic' in f.read(): mode = 'monolithic' - + trees.append({ 'id': tree_id, 'name': name, 'path': tree_dir, 'mode': mode }) - + return trees @@ -2482,3 +2540,34 @@ def __getattr__(self, name): :rtype: object """ return getattr(self._cursor, name) + + +# ------------------------------------------------------------ +# +# Mode-specific wrapper classes +# +# ------------------------------------------------------------ +class PostgreSQLEnhancedMonolithic(PostgreSQLEnhancedBase): + """ + PostgreSQL Enhanced with Monolithic mode. + + All family trees share one PostgreSQL database with table prefixes. + Recommended for most users. + """ + + def __init__(self): + """Initialize with monolithic mode forced.""" + super().__init__(force_mode="monolithic") + + +class PostgreSQLEnhancedSeparate(PostgreSQLEnhancedBase): + """ + PostgreSQL Enhanced with Separate mode. + + Each family tree gets its own PostgreSQL database. + For advanced users requiring complete tree isolation. + """ + + def __init__(self): + """Initialize with separate mode forced.""" + super().__init__(force_mode="separate") diff --git a/PostgreSQLEnhanced/schema.py b/PostgreSQLEnhanced/schema.py index d04726c41..b39cecfb1 100644 --- a/PostgreSQLEnhanced/schema.py +++ b/PostgreSQLEnhanced/schema.py @@ -127,7 +127,7 @@ def check_and_init_schema(self): "Upgrading schema from v%s to v%s", current_version, SCHEMA_VERSION ) self._upgrade_schema(current_version) - + # Also check for internal schema migrations (VARCHAR to TEXT, etc.) migrator = SchemaMigrator(self.conn, self.table_prefix) migrator.check_and_migrate() @@ -235,7 +235,7 @@ def _create_schema(self): # Set initial schema version self._set_schema_version(SCHEMA_VERSION) - + # Set initial internal schema version (for our own migrations) migrator = SchemaMigrator(self.conn, self.table_prefix) migrator.set_internal_version((1, 1, 0)) # Current version with TEXT fields @@ -253,7 +253,7 @@ def _create_object_table(self, obj_type): # Determine column type # Default to TEXT for all strings (matches SQLite behavior) col_type = "TEXT" - + # Special cases for specific types if "INTEGER" in json_path: col_type = "INTEGER" @@ -329,7 +329,15 @@ def _create_object_specific_indexes(self, obj_type): # Then add our enhanced indexes for better performance if obj_type == "person": - # Name searches + # Composite index for name searches (required by Gramps 6.0.6+) + self.conn.execute( + f""" + CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}person_name_composite + ON {self._table_name('person')} (surname, given_name) + """ + ) + + # Name searches (JSONB) self.conn.execute( f""" CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}person_names diff --git a/PostgreSQLEnhanced/schema_migrations.py b/PostgreSQLEnhanced/schema_migrations.py index 9449feda6..e48f02396 100644 --- a/PostgreSQLEnhanced/schema_migrations.py +++ b/PostgreSQLEnhanced/schema_migrations.py @@ -51,38 +51,38 @@ def migrate_1_0_to_1_1(conn, table_prefix=""): """ logger = logging.getLogger(".PostgreSQLEnhanced.Migration") logger.info("Migrating schema from 1.0 to 1.1: VARCHAR(255) to TEXT") - + migrations = [ # Metadata table (f"{table_prefix}metadata", "setting", "TEXT"), - - # Gender stats table + + # Gender stats table (f"{table_prefix}gender_stats", "given_name", "TEXT"), - + # Surname table ("surname", "surname", "TEXT"), # Note: no prefix for surname table - + # Name group table ("name_group", "name", "TEXT"), # Note: no prefix for name_group table ("name_group", "grouping", "TEXT"), - + # Reference table - class columns (f"{table_prefix}reference", "obj_class", "TEXT"), (f"{table_prefix}reference", "ref_class", "TEXT"), ] - + # Also need to migrate any dynamic columns in object tables # These would have been created as VARCHAR(255) by default object_tables = [ - "person", "family", "event", "place", "source", + "person", "family", "event", "place", "source", "citation", "media", "repository", "note", "tag" ] - + with conn.cursor() as cur: # First, get list of all VARCHAR(255) columns in object tables for obj_type in object_tables: table_name = f"{table_prefix}{obj_type}" - + cur.execute(""" SELECT column_name FROM information_schema.columns @@ -91,10 +91,10 @@ def migrate_1_0_to_1_1(conn, table_prefix=""): AND data_type = 'character varying' AND character_maximum_length = 255 """, [table_name]) - + for (column_name,) in cur.fetchall(): migrations.append((table_name, column_name, "TEXT")) - + # Now perform all migrations success_count = 0 for table, column, new_type in migrations: @@ -107,7 +107,7 @@ def migrate_1_0_to_1_1(conn, table_prefix=""): AND column_name = %s AND table_schema = 'public' """, [table, column]) - + if cur.fetchone()[0] > 0: # Column exists, migrate it alter_sql = sql.SQL( @@ -117,15 +117,15 @@ def migrate_1_0_to_1_1(conn, table_prefix=""): sql.Identifier(column), sql.SQL(new_type) ) - + cur.execute(alter_sql) logger.debug(f"Migrated {table}.{column} to {new_type}") success_count += 1 - + except Exception as e: logger.warning(f"Could not migrate {table}.{column}: {e}") # Continue with other migrations - + logger.info(f"Migration complete: {success_count} columns converted to TEXT") return True @@ -140,29 +140,29 @@ class SchemaMigrator: """ Handles automatic schema migrations for PostgreSQL Enhanced. """ - + def __init__(self, connection, table_prefix=""): """ Initialize the migrator. - + :param connection: PostgreSQLConnection instance :param table_prefix: Table prefix for shared database mode """ self.conn = connection self.table_prefix = table_prefix self.log = logging.getLogger(".PostgreSQLEnhanced.SchemaMigrator") - + def get_internal_version(self): """ Get the internal schema version from the database. - + Returns tuple (major, minor, patch) or (1, 0, 0) if not set. """ try: self.conn.execute( f""" - SELECT json_data - FROM {self.table_prefix}metadata + SELECT json_data + FROM {self.table_prefix}metadata WHERE setting = 'internal_schema_version' """ ) @@ -177,17 +177,17 @@ def get_internal_version(self): except Exception as e: self.log.debug(f"Could not get internal version: {e}") return (1, 0, 0) - + def set_internal_version(self, version): """ Set the internal schema version in the database. - + :param version: Tuple (major, minor, patch) """ from psycopg.types.json import Jsonb - + version_data = {'version': list(version)} - + self.conn.execute( f""" INSERT INTO {self.table_prefix}metadata (setting, json_data) @@ -197,53 +197,53 @@ def set_internal_version(self, version): """, [Jsonb(version_data)] ) - + def check_and_migrate(self): """ Check if migrations are needed and apply them. - + This is called automatically when opening a database. """ current_version = self.get_internal_version() target_version = INTERNAL_SCHEMA_VERSION - + if current_version >= target_version: # No migration needed return True - + self.log.info( f"Database schema migration needed: {current_version} -> {target_version}" ) - + # Apply migrations in order if current_version < (1, 1, 0) and target_version >= (1, 1, 0): # Apply VARCHAR to TEXT migration if not migrate_1_0_to_1_1(self.conn, self.table_prefix): return False - + # Update version self.set_internal_version(target_version) self.conn.commit() - + self.log.info(f"Schema migration complete: now at version {target_version}") return True - + def get_migration_status(self): """ Get information about migration status. - + Returns dict with current version and available migrations. """ current = self.get_internal_version() target = INTERNAL_SCHEMA_VERSION - + status = { 'current_version': f"{current[0]}.{current[1]}.{current[2]}", 'target_version': f"{target[0]}.{target[1]}.{target[2]}", 'up_to_date': current >= target, 'migrations_available': [] } - + # List available migrations for version, description in MIGRATIONS.items(): if version > current and version <= target: @@ -251,5 +251,5 @@ def get_migration_status(self): 'version': f"{version[0]}.{version[1]}.{version[2]}", 'description': description }) - + return status \ No newline at end of file diff --git a/PostgreSQLEnhanced/undo_postgresql.py b/PostgreSQLEnhanced/undo_postgresql.py index e3a939050..9f51c4e38 100644 --- a/PostgreSQLEnhanced/undo_postgresql.py +++ b/PostgreSQLEnhanced/undo_postgresql.py @@ -52,7 +52,7 @@ class DbUndoPostgreSQL(DbUndo): """ PostgreSQL-native undo implementation. - + Stores undo data in PostgreSQL tables instead of files, providing: - Full transaction history with timestamps - User tracking capability @@ -60,52 +60,52 @@ class DbUndoPostgreSQL(DbUndo): - Compatible with GrampsWeb's get_transactions() method - No external dependencies beyond psycopg3 """ - + def __init__(self, grampsdb, connection): """ Initialize PostgreSQL undo system. - + :param grampsdb: The Gramps database object :param connection: PostgreSQL connection object """ super().__init__(grampsdb) self.connection = connection - + # IMPORTANT: Handle both monolithic and separate modes # In monolithic mode, table_prefix is like "tree_6894f36d_" # In separate mode, table_prefix is empty "" self.table_prefix = getattr(grampsdb, 'table_prefix', '') - + # If we have a wrapped connection (TablePrefixWrapper), get the real connection if hasattr(connection, '_connection'): self.connection = connection._connection else: self.connection = connection - + self.log = logging.getLogger(".PostgreSQLUndo") - + # Create undo tables if they don't exist self._create_undo_tables() - + # Track current transaction self.current_trans_id = None self.current_trans_changes = [] - + # Get current user if available self.user_id = os.environ.get('USER', 'unknown') if 'GRAMPSWEB_USER' in os.environ: self.user_id = os.environ['GRAMPSWEB_USER'] - + def open(self): """Open the undo system (tables already created in __init__).""" self.log.debug("PostgreSQL undo system opened for %s", self.table_prefix) - + def close(self): """Close the undo system (commit any pending changes).""" if self.current_trans_id: self._finalize_transaction() self.log.debug("PostgreSQL undo system closed") - + def _create_undo_tables(self): """Create undo tables if they don't exist.""" with self.connection.cursor() as cur: @@ -121,7 +121,7 @@ def _create_undo_tables(self): is_undo BOOLEAN DEFAULT FALSE ) """).format(sql.Identifier(f"{self.table_prefix}transactions"))) - + # Individual changes table cur.execute(sql.SQL(""" CREATE TABLE IF NOT EXISTS {} ( @@ -139,7 +139,7 @@ def _create_undo_tables(self): sql.Identifier(f"{self.table_prefix}changes"), sql.Identifier(f"{self.table_prefix}transactions") )) - + # Create indexes for performance cur.execute(sql.SQL(""" CREATE INDEX IF NOT EXISTS {} ON {}(trans_id) @@ -147,36 +147,36 @@ def _create_undo_tables(self): sql.Identifier(f"{self.table_prefix}changes_trans_idx"), sql.Identifier(f"{self.table_prefix}changes") )) - + cur.execute(sql.SQL(""" CREATE INDEX IF NOT EXISTS {} ON {}(obj_handle) """).format( sql.Identifier(f"{self.table_prefix}changes_handle_idx"), sql.Identifier(f"{self.table_prefix}changes") )) - + cur.execute(sql.SQL(""" CREATE INDEX IF NOT EXISTS {} ON {}(timestamp DESC) """).format( sql.Identifier(f"{self.table_prefix}transactions_ts_idx"), sql.Identifier(f"{self.table_prefix}transactions") )) - + self.connection.commit() self.log.debug("Undo tables created/verified for %s", self.table_prefix) - + def commit(self, msg=""): """ Commit current transaction with optional message. - + :param msg: Description of the transaction """ if self.current_trans_changes: self._finalize_transaction(msg) - + # Start new transaction self._start_transaction(msg) - + def _start_transaction(self, description=""): """Start a new undo transaction.""" with self.connection.cursor() as cur: @@ -186,16 +186,16 @@ def _start_transaction(self, description=""): RETURNING trans_id """).format(sql.Identifier(f"{self.table_prefix}transactions")), (self.user_id, description)) - + self.current_trans_id = cur.fetchone()[0] self.current_trans_changes = [] self.connection.commit() - + def _finalize_transaction(self, description=""): """Finalize the current transaction.""" if not self.current_trans_id: return - + # Update transaction with change IDs if self.current_trans_changes: with self.connection.cursor() as cur: @@ -213,21 +213,21 @@ def _finalize_transaction(self, description=""): self.current_trans_id )) self.connection.commit() - + self.current_trans_id = None self.current_trans_changes = [] - + def append(self, value): """ Add a change to current transaction. - + :param value: Pickled tuple of (obj_type, trans_type, handle, old_data, new_data) """ try: # Ensure we have a current transaction if not self.current_trans_id: self._start_transaction() - + # Unpack the change data data = pickle.loads(value) if len(data) == 5: @@ -235,17 +235,17 @@ def append(self, value): else: self.log.warning("Unexpected undo data format: %s items", len(data)) return - + # Handle tuple handles (for references) if isinstance(handle, tuple): obj_handle, ref_handle = handle else: obj_handle, ref_handle = handle, None - + # Store change in database with self.connection.cursor() as cur: cur.execute(sql.SQL(""" - INSERT INTO {} + INSERT INTO {} (trans_id, obj_class, trans_type, obj_handle, ref_handle, old_data, new_data) VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING change_id @@ -259,49 +259,49 @@ def append(self, value): json.dumps(self._serialize_for_json(old_data)) if old_data else None, json.dumps(self._serialize_for_json(new_data)) if new_data else None )) - + change_id = cur.fetchone()[0] self.current_trans_changes.append(change_id) self.connection.commit() - + except Exception as e: self.log.error("Error appending undo data: %s", e) self.connection.rollback() - + def _serialize_for_json(self, data): """ Convert Gramps objects to JSON-serializable format. - + :param data: Data to serialize :returns: JSON-serializable version of the data """ if data is None: return None - + # If it's already a dict/list, return as-is if isinstance(data, (dict, list, str, int, float, bool)): return data - + # If it has a serialize method (Gramps objects), use it if hasattr(data, 'serialize'): return data.serialize() - + # For bytes, convert to base64 if isinstance(data, bytes): import base64 return {'__bytes__': base64.b64encode(data).decode('ascii')} - + # For other types, try to convert to string return str(data) - + def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, ascending=False, before=None, after=None): """ Get transaction history with pagination. - + This method is compatible with GrampsWeb's expectations but doesn't require any GrampsWeb imports. - + :param page: Page number (1-based) :param pagesize: Number of results per page :param old_data: Include old values in results @@ -312,23 +312,23 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, :returns: (transactions, total_count) tuple """ offset = (page - 1) * pagesize - + try: with self.connection.cursor() as cur: # Build WHERE clause where_clauses = [] params = [] - + if before: where_clauses.append("timestamp < %s") params.append(datetime.fromtimestamp(before)) - + if after: where_clauses.append("timestamp > %s") params.append(datetime.fromtimestamp(after)) - + where_sql = " AND ".join(where_clauses) if where_clauses else "TRUE" - + # Get total count cur.execute(sql.SQL(""" SELECT COUNT(*) FROM {} @@ -338,7 +338,7 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, sql.SQL(where_sql) ), params) total_count = cur.fetchone()[0] - + # Get paginated results order_dir = "ASC" if ascending else "DESC" cur.execute(sql.SQL(""" @@ -352,7 +352,7 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, sql.SQL(where_sql), sql.SQL(order_dir) ), params + [pagesize, offset]) - + transactions = [] for row in cur.fetchall(): trans = { @@ -362,7 +362,7 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, 'description': row[3] or '', 'is_undo': row[4] or False } - + # Get changes for this transaction if data is requested if old_data or new_data: cur.execute(sql.SQL(""" @@ -372,7 +372,7 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, ORDER BY change_id """).format(sql.Identifier(f"{self.table_prefix}changes")), (row[0],)) - + changes = [] for change_row in cur.fetchall(): change = { @@ -380,30 +380,30 @@ def get_transactions(self, page=1, pagesize=20, old_data=False, new_data=False, 'trans_type': change_row[1], 'obj_handle': change_row[2] } - + if old_data and change_row[3]: change['old_data'] = change_row[3] - + if new_data and change_row[4]: change['new_data'] = change_row[4] - + changes.append(change) - + if changes: trans['changes'] = changes - + transactions.append(trans) - + return transactions, total_count - + except Exception as e: self.log.error("Error getting transactions: %s", e) return [], 0 - + def undo(self, update_history=True): """ Undo the last committed transaction. - + :param update_history: Whether to update history :returns: True if successful """ @@ -411,18 +411,18 @@ def undo(self, update_history=True): # This would need to replay changes in reverse self.log.info("Undo requested but not yet implemented in PostgreSQL backend") return False - + def redo(self, update_history=True): """ Redo the last undone transaction. - + :param update_history: Whether to update history :returns: True if successful """ # For now, return False as full redo is complex self.log.info("Redo requested but not yet implemented in PostgreSQL backend") return False - + def clean(self): """Clean up old transaction data.""" # Could implement cleanup of old transactions here