From d0d1d734b4095dcb79ea1ee7cab35034b223e3c2 Mon Sep 17 00:00:00 2001 From: KaterynaLove Date: Wed, 25 Jan 2023 21:21:45 +0000 Subject: [PATCH] Adding support for delayed deletion of orphaned external files. --- classes/local/manager.php | 7 +++ .../task/delete_orphaned_object_metadata.php | 12 ++++- db/install.xml | 3 +- db/tasks.php | 1 - db/upgrade.php | 16 ++++++ lang/en/tool_objectfs.php | 2 + settings.php | 4 ++ tests/local/tasks_test.php | 54 +++++++++++++++++++ version.php | 2 +- 9 files changed, 96 insertions(+), 5 deletions(-) diff --git a/classes/local/manager.php b/classes/local/manager.php index c746b9c2..0e4099eb 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -60,6 +60,7 @@ public static function get_objectfs_config() { $config->batchsize = 10000; $config->useproxy = 0; $config->deleteexternal = 0; + $config->delaydeleteexternalobject = 0; $config->filesystem = ''; $config->enablepresignedurls = 0; @@ -185,6 +186,12 @@ public static function update_object(stdClass $object, $newlocation) { $object->timeduplicated = time(); } + // If location change is 'orphaned' we update timeorphaned. + // Set time orphaned clock is ticking now for delay deletion comparison... + if ($newlocation === OBJECT_LOCATION_ORPHANED) { + $object->timeorphaned = time(); + } + $object->location = $newlocation; $DB->update_record('tool_objectfs_objects', $object); diff --git a/classes/task/delete_orphaned_object_metadata.php b/classes/task/delete_orphaned_object_metadata.php index c631bafd..2268ed26 100644 --- a/classes/task/delete_orphaned_object_metadata.php +++ b/classes/task/delete_orphaned_object_metadata.php @@ -52,15 +52,23 @@ public function execute() { 'ageforremoval' => time() - $ageforremoval ]; + // Check for delay deletion if enabled. + $delayquery = ''; + if (!empty($this->config->delaydeleteexternalobject)) { + $params['deletetime'] = time() - $this->config->delaydeleteexternalobject; + $delayquery = 'AND o.timeorphaned < :deletetime'; + } + if (!empty($this->config->deleteexternal) && $this->config->deleteexternal == TOOL_OBJECTFS_DELETE_EXTERNAL_TRASH) { // We need to delete the external files as well as the orphaned data. $filesystem = new $this->config->filesystem(); // Join with files table to make extra sure we aren't deleting something that already exists. - $sql = 'SELECT o.* + $sql = "SELECT o.* FROM {tool_objectfs_objects} o LEFT JOIN {files} f ON o.contenthash = f.contenthash - WHERE f.id is null AND o.location = :location AND timeduplicated < :ageforremoval'; + WHERE f.id is null AND o.location = :location AND o.timeduplicated < :ageforremoval + $delayquery"; $objects = $DB->get_recordset_sql($sql, $params); $count = 0; diff --git a/db/install.xml b/db/install.xml index 855c9786..d91e79ae 100644 --- a/db/install.xml +++ b/db/install.xml @@ -1,5 +1,5 @@ - @@ -11,6 +11,7 @@ + diff --git a/db/tasks.php b/db/tasks.php index 5125d734..39bf803e 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -108,4 +108,3 @@ 'month' => '*' ), ); - diff --git a/db/upgrade.php b/db/upgrade.php index 5515dc43..1867dd73 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -154,5 +154,21 @@ function xmldb_tool_objectfs_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2022070401, 'tool', 'objectfs'); } + + if ($oldversion < 2022120806) { + + // Define field timeorphaned to be added to tool_objectfs_objects. + $table = new xmldb_table('tool_objectfs_objects'); + $field = new xmldb_field('timeorphaned', XMLDB_TYPE_INTEGER, '20', null, null, null, null, 'filesize'); + + // Conditionally launch add field timeorphaned. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + // Objectfs savepoint reached. + upgrade_plugin_savepoint(true, 2022120806, 'tool', 'objectfs'); + } + return true; } diff --git a/lang/en/tool_objectfs.php b/lang/en/tool_objectfs.php index 168c3221..da45005e 100644 --- a/lang/en/tool_objectfs.php +++ b/lang/en/tool_objectfs.php @@ -169,6 +169,8 @@ $string['settings:batchsize_help'] = 'Number of files to be transferred in one cron run'; $string['settings:maxorphanedage'] = 'Max orphaned object age'; $string['settings:maxorphanedage_help'] = 'If set to zero, this will not delete old orphaned metadata for objects. Otherwise, it will remove these records as they are no longer relevant. An orphaned object is one where the metadata exists on the {tool_objectfs_objects} table but referenced file no longer exists.'; +$string['settings:delaydeleteexternalobject'] = 'Delay delete external object'; +$string['settings:delaydeleteexternalobject_help'] = 'Delay delete of external orphaned objects. They will be deleted once this time has passed after they became orphaned.'; $string['settings:minimumage'] = 'Minimum age'; $string['settings:minimumage_help'] = 'Minimum age that a object must exist on the local filedir before it will be considered for transfer.'; $string['settings:deleteexternal'] = 'Delete external objects'; diff --git a/settings.php b/settings.php index a941e506..3c072891 100644 --- a/settings.php +++ b/settings.php @@ -97,6 +97,10 @@ new lang_string('settings:maxorphanedage', 'tool_objectfs'), new lang_string('settings:maxorphanedage_help', 'tool_objectfs'), 0, DAYSECS)); + $settings->add(new admin_setting_configduration('tool_objectfs/delaydeleteexternalobject', + new lang_string('settings:delaydeleteexternalobject', 'tool_objectfs'), + new lang_string('settings:delaydeleteexternalobject_help', 'tool_objectfs'), 0, DAYSECS)); + $settings->add(new admin_setting_configcheckbox('tool_objectfs/enablelogging', new lang_string('settings:enablelogging', 'tool_objectfs'), '', '')); diff --git a/tests/local/tasks_test.php b/tests/local/tasks_test.php index 6cae7841..c2670b49 100644 --- a/tests/local/tasks_test.php +++ b/tests/local/tasks_test.php @@ -16,6 +16,8 @@ namespace tool_objectfs\local; +use tool_objectfs\tests\test_file_system; + /** * End to end tests for tasks. Make sure all the plumbing is ok. * @@ -32,6 +34,14 @@ protected function tearDown(): void { ob_end_clean(); } + public static function get_orphaned_delayed_count() { + global $DB; + + $orphanedcount = $DB->count_records_sql("SELECT COUNT(id) FROM {tool_objectfs_objects} WHERE timeorphaned > 0"); + + return $orphanedcount; + } + public function test_run_legacy_cron() { $config = manager::get_objectfs_config(); $config->enabletasks = true; @@ -69,4 +79,48 @@ public function test_run_scheduled_tasks() { $this->expectNotToPerformAssertions(); // Just check we get this far without any exceptions. } + public function test_delay_delete_orphaned_object() { + global $CFG, $DB; + + $this->resetAfterTest(true); + + $this->filesystem = new test_file_system(); + $file = $this->create_remote_file(); + $filehash = $file->get_contenthash(); + $objectrecord = $DB->get_record('tool_objectfs_objects', ['contenthash' => $filehash]); + $file->delete(); // This makes it orphaned. + + // Set config. + set_config('delaydeleteexternalobject', (3 * DAYSECS), 'tool_objectfs'); + set_config('maxorphanedage', (1 * DAYSECS), 'tool_objectfs'); + set_config('deleteexternal', TOOL_OBJECTFS_DELETE_EXTERNAL_TRASH, 'tool_objectfs'); + set_config('filesystem', "tool_objectfs\\tests\\test_file_system", 'tool_objectfs'); + + unset($CFG->forced_plugin_settings['tool_objectfs']['deleteexternal']); // This will be reset in parent::setUp(). + + // Update time so it is considered for orphaned deletion. It should be skipped due to delay setting. + $objectrecord->timeduplicated = time() - (2 * DAYSECS); // Time for it to be considered orphaned. + $DB->update_record('tool_objectfs_objects', $objectrecord); + + $objectrecord = manager::update_object($objectrecord, OBJECT_LOCATION_ORPHANED); + + $pretaskcount = $this->get_orphaned_delayed_count(); + + $task = \core\task\manager::get_scheduled_task('\\tool_objectfs\\task\\delete_orphaned_object_metadata'); + $task->execute(); + + $posttaskcount = $this->get_orphaned_delayed_count(); + + $this->assertSame($pretaskcount, $posttaskcount, 'No file should have been deleted due to delay setting'); + + $objectrecord->timeorphaned = time() - (4 * DAYSECS); // Time for it to be able to be deleted. + $DB->update_record('tool_objectfs_objects', $objectrecord); + + $task = \core\task\manager::get_scheduled_task('\\tool_objectfs\\task\\delete_orphaned_object_metadata'); + $task->execute(); + + $posttaskcount = $this->get_orphaned_delayed_count(); + + $this->assertGreaterThan($posttaskcount, $pretaskcount, 'There should be less files after removing delayed orphaned files'); + } } diff --git a/version.php b/version.php index 4fc19d6c..f26b9792 100644 --- a/version.php +++ b/version.php @@ -25,7 +25,7 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2022120800; // The current plugin version (Date: YYYYMMDDXX). +$plugin->version = 2022120806; // The current plugin version (Date: YYYYMMDDXX). $plugin->release = 2022120800; // Same as version. $plugin->requires = 2020110900; // Requires Filesystem API. $plugin->component = "tool_objectfs";