fix: enable automatic cache refresh for small datasets and optimize configuration

Implement dynamic cache invalidation for datasets under COUNT_LIMIT threshold
to ensure entity counts remain accurate without manual intervention.

Fix SPARQL BIND clause syntax in display rules to prevent query errors.

Author: arcangelo7

config.example.py

@@ -36,6 +36,11 @@ class Config(object):
     REDIS_URL = os.environ.get("REDIS_URL")
 
     # Query Configuration
+    # COUNT_LIMIT serves dual purpose:
+    # 1. Maximum entity count to display (shows "10000+" if exceeded)
+    # 2. Threshold for automatic cache refresh after entity modifications
+    #    - Datasets below this limit: auto-refresh enabled (always accurate counts)
+    #    - Datasets above this limit: cache remains static (manual refresh via admin endpoint)
     COUNT_LIMIT = int(os.environ.get("COUNT_LIMIT", "10000"))
 
     # Database configuration

dataset_database/virtuoso.ini

@@ -1,247 +1,120 @@
-;
-;  virtuoso.ini
-;
-;  Configuration file for the OpenLink Virtuoso VDBMS Server
-;
-;  To learn more about this product, or any other product in our
-;  portfolio, please check out our web site at:
-;
-;      http://virtuoso.openlinksw.com/
-;
-;  or contact us at:
-;
-;      [email protected]
-;
-;  If you have any technical questions, please contact our support
-;  staff at:
-;
-;      [email protected]
-;
-;
-;  Database setup
-;
 [Database]
-DatabaseFile       = /database/virtuoso.db
-ErrorLogFile       = /database/virtuoso.log
-LockFile           = /database/virtuoso.lck
-TransactionFile    = /database/virtuoso.trx
-xa_persistent_file = /database/virtuoso.pxa
-ErrorLogLevel      = 7
-FileExtend         = 200
+DatabaseFile = ../database/virtuoso.db
+ErrorLogFile = ../database/virtuoso.log
+LockFile = ../database/virtuoso.lck
+TransactionFile = ../database/virtuoso.trx
+xa_persistent_file = ../database/virtuoso.pxa
+ErrorLogLevel = 7
+FileExtend = 200
 MaxCheckpointRemap = 2000
-Striping           = 0
-TempStorage        = TempDatabase
+Striping = 0
+TempStorage = TempDatabase
 
 [TempDatabase]
-DatabaseFile       = /database/virtuoso-temp.db
-TransactionFile    = /database/virtuoso-temp.trx
+DatabaseFile = ../database/virtuoso-temp.db
+TransactionFile = ../database/virtuoso-temp.trx
 MaxCheckpointRemap = 2000
-Striping           = 0
-;MaxTempDBPages                 = 50G        ; Limit size of the TempDatabase file to 50GB on disk
+Striping = 0
 
-;
-;  Server parameters
-;
 [Parameters]
-ServerPort               = 1111
-LiteMode                 = 0
-DisableUnixSocket        = 1
-DisableTcpSocket         = 0
-;SSLServerPort			= 2111
-;SSLCertificate			= cert.pem
-;SSLPrivateKey			= pk.pem
-;X509ClientVerify		= 0
-;X509ClientVerifyDepth		= 0
-;X509ClientVerifyCAFile		= ca.pem
-MaxClientConnections     = 10
-CheckpointInterval       = 60
-O_DIRECT                 = 0
-CaseMode                 = 2
-MaxStaticCursorRows      = 5000
-CheckpointAuditTrail     = 0
-AllowOSCalls             = 0
-SchedulerInterval        = 10
-DirsAllowed              = ., ../vad, /usr/share/proj, /data
-ThreadCleanupInterval    = 1
-ThreadThreshold          = 10
+ServerPort = 1111
+LiteMode = 0
+DisableUnixSocket = 1
+DisableTcpSocket = 0
+MaxClientConnections = 10
+CheckpointInterval = 60
+O_DIRECT = 0
+CaseMode = 2
+MaxStaticCursorRows = 5000
+CheckpointAuditTrail = 0
+AllowOSCalls = 0
+SchedulerInterval = 10
+DirsAllowed = ../virtuoso_input,/usr/share/proj,../vad,.,/opt/virtuoso-opensource/database
+ThreadCleanupInterval = 1
+ThreadThreshold = 10
 ResourcesCleanupInterval = 1
-FreeTextBatchSize        = 100000
-SingleCPU                = 0
-VADInstallDir            = ../vad/
-PrefixResultNames        = 0
-RdfFreeTextRulesSize     = 100
-IndexTreeMaps            = 64
-MaxMemPoolSize           = 200000000
-PrefixResultNames        = 0
-MacSpotlight             = 0
-MaxQueryMem              = 2G	; memory allocated to query processor
-VectorSize               = 1000	; initial parallel query vector (array of query operations) size
-MaxVectorSize            = 1000000	; query vector size threshold.
-AdjustVectorSize         = 0
-ThreadsPerQuery          = 4
-AsyncQueueMaxThreads     = 10
-;;
-;; When running with large data sets, one should configure the Virtuoso
-;; process to use between 2/3 to 3/5 of free system memory and to stripe
-;; storage on all available disks.
-;;
-;; Uncomment next two lines if there is 2 GB system memory free
-;NumberOfBuffers          = 170000
-;MaxDirtyBuffers          = 130000
-;; Uncomment next two lines if there is 4 GB system memory free
-;NumberOfBuffers          = 340000
-; MaxDirtyBuffers          = 250000
-;; Uncomment next two lines if there is 8 GB system memory free
-;NumberOfBuffers          = 680000
-;MaxDirtyBuffers          = 500000
-;; Uncomment next two lines if there is 16 GB system memory free
-;NumberOfBuffers          = 1360000
-;MaxDirtyBuffers          = 1000000
-;; Uncomment next two lines if there is 32 GB system memory free
-;NumberOfBuffers          = 2720000
-;MaxDirtyBuffers          = 2000000
-;; Uncomment next two lines if there is 48 GB system memory free
-;NumberOfBuffers          = 4000000
-;MaxDirtyBuffers          = 3000000
-;; Uncomment next two lines if there is 64 GB system memory free
-;NumberOfBuffers          = 5450000
-;MaxDirtyBuffers          = 4000000
-;;
-;; Note the default settings will take very little memory
-;; but will not result in very good performance
-;;
-NumberOfBuffers          = 10000
-MaxDirtyBuffers          = 6000
-SSLServerPort            = 1112
-SSLCertificate           = virtuoso.crt
-SSLPrivateKey            = virtuoso.key
+FreeTextBatchSize = 100000
+SingleCPU = 0
+VADInstallDir = ../vad/
+PrefixResultNames = 0
+RdfFreeTextRulesSize = 100
+IndexTreeMaps = 64
+MaxMemPoolSize = 200000000
+MacSpotlight = 0
+MaxQueryMem = 2G	; memory allocated to query processor
+VectorSize = 1000	; initial parallel query vector (array of query operations) size
+MaxVectorSize = 1000000	; query vector size threshold.
+AdjustVectorSize = 0
+ThreadsPerQuery = 4
+AsyncQueueMaxThreads = 10
+NumberOfBuffers = 88583
+MaxDirtyBuffers = 66437
 
 [HTTPServer]
-ServerPort                  = 8890
-ServerRoot                  = ../vsp
-MaxClientConnections        = 10
-DavRoot                     = DAV
-EnabledDavVSP               = 0
-HTTPProxyEnabled            = 0
-TempASPXDir                 = 0
-DefaultMailServer           = localhost:25
-MaxKeepAlives               = 10
-KeepAliveTimeout            = 10
-MaxCachedProxyConnections   = 10
+ServerPort = 8890
+ServerRoot = ../vsp
+MaxClientConnections = 10
+DavRoot = DAV
+EnabledDavVSP = 0
+HTTPProxyEnabled = 0
+TempASPXDir = 0
+DefaultMailServer = localhost:25
+MaxKeepAlives = 10
+KeepAliveTimeout = 10
+MaxCachedProxyConnections = 10
 ProxyConnectionCacheTimeout = 15
-HTTPThreadSize              = 280000
-HttpPrintWarningsInOutput   = 0
-Charset                     = UTF-8
-;HTTPLogFile		        = logs/http.log
-MaintenancePage             = atomic.html
-EnabledGzipContent          = 1
-SSLPort                     = 8891
-SSLCertificate              = virtuoso.crt
-SSLPrivateKey               = virtuoso.key
+HTTPThreadSize = 280000
+HttpPrintWarningsInOutput = 0
+Charset = UTF-8
+MaintenancePage = atomic.html
+EnabledGzipContent = 1
 
 [AutoRepair]
 BadParentLinks = 0
 
 [Client]
-SQL_PREFETCH_ROWS  = 100
+SQL_PREFETCH_ROWS = 100
 SQL_PREFETCH_BYTES = 16000
-SQL_QUERY_TIMEOUT  = 0
-SQL_TXN_TIMEOUT    = 0
-;SQL_NO_CHAR_C_ESCAPE		= 1
-;SQL_UTF8_EXECS			= 0
-;SQL_NO_SYSTEM_TABLES		= 0
-;SQL_BINARY_TIMESTAMP		= 1
-;SQL_ENCRYPTION_ON_PASSWORD	= -1
+SQL_QUERY_TIMEOUT = 0
+SQL_TXN_TIMEOUT = 0
 
 [VDB]
-ArrayOptimization           = 0
-NumArrayParameters          = 10
-VDBDisconnectTimeout        = 1000
+ArrayOptimization = 0
+NumArrayParameters = 10
+VDBDisconnectTimeout = 1000
 KeepConnectionOnFixedThread = 0
 
 [Replication]
-ServerName   = db-ALPINE-PORT
+ServerName = db-BIONIC-PORT
 ServerEnable = 1
-QueueMax     = 50000
+QueueMax = 50000
 
-;
-;  Striping setup
-;
-;  These parameters have only effect when Striping is set to 1 in the
-;  [Database] section, in which case the DatabaseFile parameter is ignored.
-;
-;  With striping, the database is spawned across multiple segments
-;  where each segment can have multiple stripes.
-;
-;  Format of the lines below:
-;    Segment<number> = <size>, <stripe file name> [, <stripe file name> .. ]
-;
-;  <number> must be ordered from 1 up.
-;
-;  The <size> is the total size of the segment which is equally divided
-;  across all stripes forming  the segment. Its specification can be in
-;  gigabytes (g), megabytes (m), kilobytes (k) or in database blocks
-;  (b, the default)
-;
-;  Note that the segment size must be a multiple of the database page size
-;  which is currently 8k. Also, the segment size must be divisible by the
-;  number of stripe files forming  the segment.
-;
-;  The example below creates a 200 meg database striped on two segments
-;  with two stripes of 50 meg and one of 100 meg.
-;
-;  You can always add more segments to the configuration, but once
-;  added, do not change the setup.
-;
 [Striping]
 Segment1 = 100M, db-seg1-1.db, db-seg1-2.db
 Segment2 = 100M, db-seg2-1.db
-;...
-;[TempStriping]
-;Segment1			= 100M, db-seg1-1.db, db-seg1-2.db
-;Segment2			= 100M, db-seg2-1.db
-;...
-;[Ucms]
-;UcmPath			= <path>
-;Ucm1				= <file>
-;Ucm2				= <file>
-;...
 
 [Zero Config]
-ServerName = virtuoso (ALPINE-PORT)
-;ServerDSN			= ZDSN
-;SSLServerName			=
-;SSLServerDSN			=
+ServerName = virtuoso (BIONIC-PORT)
 
 [Mono]
-;MONO_TRACE			= Off
-;MONO_PATH			= <path_here>
-;MONO_ROOT			= <path_here>
-;MONO_CFG_DIR			= <path_here>
-;virtclr.dll			=
 
 [URIQA]
 DynamicLocal = 0
-DefaultHost  = localhost:8890
+DefaultHost = localhost:8890
 
 [SPARQL]
-;ExternalQuerySource		= 1
-;ExternalXsltSource 		= 1
-;DefaultGraph      		= http://localhost:8890/dataspace
-;ImmutableGraphs    		= http://localhost:8890/dataspace
-ResultSetMaxRows           = 100000
-MaxConstructTriples        = 100000
+ResultSetMaxRows = 10000
+MaxConstructTriples = 10000
 MaxQueryCostEstimationTime = 400	; in seconds
-MaxQueryExecutionTime      = 60	; in seconds
-DefaultQuery               = select distinct ?Concept where {[] a ?Concept} LIMIT 100
-DeferInferenceRulesInit    = 0	; controls inference rules loading
-MaxMemInUse                = 0	; limits the amount of memory for construct dict (0=unlimited)
-;LabelInferenceName		= facets	; Only needed when using the Faceted Browser
-;PingService       		= http://rpc.pingthesemanticweb.com/
+MaxQueryExecutionTime = 60	; in seconds
+DefaultQuery = SELECT (COUNT(*) AS ?triples) WHERE {?s ?p ?o}
+DeferInferenceRulesInit = 0	; controls inference rules loading
+MaxMemInUse = 0	; limits the amount of memory for construct dict (0=unlimited)
 
 [Plugins]
 LoadPath = ../hosting
-Load1    = plain, geos
-Load2    = plain, graphql
-Load3    = plain, proj4
-Load4    = plain, shapefileio
+Load1 = plain, geos
+Load2 = plain, graphql
+Load3 = plain, proj4
+Load4 = plain, shapefileio
+

display_rules.yaml

@@ -2722,9 +2722,8 @@ rules:
           GROUP BY ?title ?pubDate ?publisher ?edition
         }
         
-        BIND(
-          IF(BOUND(?editorList) && STRLEN(?editorList) > 0, CONCAT(?editorList, " (Ed.)"),
-            COALESCE(?authorList, "")),
+        BIND(CONCAT(
+          IF(BOUND(?editorList) && STRLEN(?editorList) > 0, CONCAT(?editorList, " (Ed.)"), COALESCE(?authorList, "")),
           IF((BOUND(?editorList) && STRLEN(?editorList) > 0 || BOUND(?authorList)) && BOUND(?pubDate), " ", ""),
           IF(BOUND(?pubDate), CONCAT("(", ?pubDate, ")"), ""),
           IF(BOUND(?title) && STRLEN(?title) > 0, CONCAT(

heritrace/__init__.py

@@ -10,7 +10,6 @@
 from heritrace.cli import register_cli_commands
 from heritrace.utils.sparql_utils import precompute_available_classes_cache
 
-
 def create_app(config_object=None):
     app = Flask(__name__)
 
@@ -42,12 +41,8 @@ def create_app(config_object=None):
 
         with app.app_context():
             app.logger.info("[STARTUP] Pre-computing available classes cache...")
-            try:
-                precompute_available_classes_cache()
-                app.logger.info("[STARTUP] Available classes cache computed successfully")
-            except Exception as e:
-                app.logger.warning(f"[STARTUP] Failed to pre-compute classes cache: {e}")
-                app.logger.warning("[STARTUP] Classes will be computed on first request")
+            precompute_available_classes_cache()
+            app.logger.info("[STARTUP] Available classes cache computed successfully")
 
         register_blueprints(app)
 

heritrace/utils/sparql_utils.py

@@ -199,9 +199,15 @@ def get_available_classes():
     """
     Fetch and format all available entity classes.
     Returns cached result if available (computed at startup).
+    For small datasets (< COUNT_LIMIT), cache is invalidated to keep counts accurate.
     """
     global _AVAILABLE_CLASSES_CACHE
 
+    if _AVAILABLE_CLASSES_CACHE is not None:
+        total_count = sum(cls.get('count_numeric', 0) for cls in _AVAILABLE_CLASSES_CACHE)
+        if total_count < COUNT_LIMIT:
+            _AVAILABLE_CLASSES_CACHE = None
+
     if _AVAILABLE_CLASSES_CACHE is not None:
         return _AVAILABLE_CLASSES_CACHE