Skip to content

Add .lrrignore support for filtering archives #1672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sitiyou wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Add .lrrignore support for filtering archives #1672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/LANraragi.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use Time::HiRes qw(gettimeofday);

use LANraragi::Utils::Generic qw(start_shinobu start_minion get_version);
use LANraragi::Utils::Logging qw(get_logger get_logdir);
use LANraragi::Utils::Ignore;
use LANraragi::Utils::Plugins qw(get_plugins);
use LANraragi::Utils::TempFolder qw(get_temp);
use LANraragi::Utils::Routing;
Expand Down Expand Up @@ -186,6 +187,9 @@ sub startup {
# Anything else can cause weird database lockups.
$self->minion->enqueue('build_stat_hashes');

# Load ignore rules into Redis before starting any workers
LANraragi::Utils::Ignore::initialize();

# Start a Minion worker in a subprocess
start_minion($self);

Expand Down
9 changes: 9 additions & 0 deletions lib/LANraragi/Model/Upload.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use File::Find qw(find);
use LANraragi::Utils::Archive qw(extract_thumbnail);
use LANraragi::Utils::Database qw(invalidate_cache compute_id set_title set_summary add_archive_to_redis add_timestamp_tag add_pagecount add_arcsize);
use LANraragi::Utils::Logging qw(get_logger);
use LANraragi::Utils::Ignore qw(load_ignore_rules is_ignored);
use LANraragi::Utils::Redis qw(redis_encode);
use LANraragi::Utils::Generic qw(is_archive get_bytelength);
use LANraragi::Utils::String qw(trim trim_CRLF trim_url);
Expand Down Expand Up @@ -58,6 +59,14 @@ sub handle_incoming_file ( $tempfile, $catid, $tags, $title, $summary ) {
my $userdir = LANraragi::Model::Config->get_userdir;
my $output_file = create_path( $userdir . '/' . $filename );

# Check ignore rules: reject if the file would be ignored by .lrrignore rules
my $ignore_rules = load_ignore_rules();
if ( is_ignored( $output_file, $ignore_rules ) ) {
$logger->info("$filename matches .lrrignore rules, rejecting upload.");
unlink_path $tempfile;
return ( 415, "deadbeef", $filename, "This file matches configured ignore rules." );
}

#Check if the ID is already in the database, and
#that the file it references still exists on the filesystem
my $redis = LANraragi::Model::Config->get_redis;
Expand Down
191 changes: 191 additions & 0 deletions lib/LANraragi/Utils/Ignore.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package LANraragi::Utils::Ignore;
Comment thread
sitiyou marked this conversation as resolved.

use strict;
use warnings;
use utf8;
use feature qw(signatures state);
no warnings 'experimental::signatures';

use File::Spec;
use File::Basename qw(dirname basename);
use Config;

use LANraragi::Utils::Path qw(open_path find_path create_path);
use LANraragi::Utils::String qw(trim);
use Storable qw(nfreeze thaw);

use Exporter 'import';
our @EXPORT_OK = qw(
is_ignored
build_ignore_rules
initialize
load_ignore_rules
);

use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' );

# Called at server startup to scan .lrrignore files and store rules in Redis.
sub initialize {
my $redis = LANraragi::Model::Config->get_redis_config;
my $rules = build_ignore_rules( LANraragi::Model::Config->get_userdir );
$redis->set( "LRR_IGNORE_RULES", nfreeze($rules) );
$redis->quit();
}

# Load cached ignore rules from Redis.
# Rules are immutable at runtime; they only change when the server restarts.
sub load_ignore_rules {
state $cached = do {
my $redis = LANraragi::Model::Config->get_redis_config;
my $data = $redis->get("LRR_IGNORE_RULES");
$redis->quit();
return undef unless $data;

# compile regexp
my $rules = thaw($data);
for my $patterns (values %{ $rules->{entries} }) {
for my $p (@$patterns) {
$p->{regex} = qr{$p->{regex}};
}
}
return $rules;
};
return $cached;
}

# Scan the content directory for .lrrignore files and build rule entries.
sub build_ignore_rules ($content_dir) {
$content_dir =~ s{\\}{/}g unless IS_UNIX;
my $rules = {
content_dir => $content_dir,
entries => {},
};

my @lrrignore_dirs;
find_path(
sub {
my $f = create_path($_);
return if -d $f;
return unless basename($f) eq '.lrrignore';
push @lrrignore_dirs, dirname($f);
},
$content_dir
);

for my $dir (@lrrignore_dirs) {
load_ignore_file( $dir, $rules->{entries} );
}

return $rules;
}

# Rules data structure:
# $rules = {
# content_dir => $path, # Root content directory for scanning
# entries => { # Hash map: dir -> patterns
# $path => [ # Parsed patterns from a single .lrrignore file
# {
# pattern => "*.tmp", # Original pattern string
# regex => qr/.../i, # Compiled regex for matching
# negation => 0|1, # 1 = ! re-include, 0 = ignore
# },
# ...
# ],
# ...
# },
# }
#
sub is_ignored ($file, $rules) {
return 0 unless $rules;

my $entries = $rules->{entries};
return 0 unless $entries && keys %$entries;

return 0 unless is_subpath($file, $rules->{content_dir});

my $d = dirname($file);
while (1) {
if ( my $patterns = $entries->{$d} ) {
my $rel = File::Spec->abs2rel( $file, $d );
$rel =~ s{\\}{/}g unless IS_UNIX;
utf8::decode($rel);
my $result;
for my $rule ( @$patterns ) {
next unless $rel =~ $rule->{regex};
$result = $rule->{negation} ? 0 : 1;
}
return $result if defined $result;
}
last if $d eq $rules->{content_dir};
$d = dirname($d);
}
return 0;
}

sub load_ignore_file ($dir, $entries) {
my $ignore = File::Spec->catfile( $dir, ".lrrignore" );
return unless -f $ignore;
my @parsed = parse_ignore_file($ignore);
return unless @parsed;
$entries->{$dir} = \@parsed;
}

sub parse_ignore_file ($path) {
my @patterns;

open_path( my $fh, '<:encoding(UTF-8)', $path ) or return @patterns;
my @lines = <$fh>;
close($fh);

for my $line (@lines) {
$line = trim($line);
next if $line eq '';
next if $line =~ /^#/;

my $negation = 0;
if ( $line =~ s/^!// ) { $negation = 1; }

my $regex = glob_to_regex($line);
next unless defined $regex;

push @patterns, {
pattern => $line,
regex => $regex,
negation => $negation,
};
}

return @patterns;
}

sub glob_to_regex ($pattern_str) {

my $match_dir = 0;
if ( $pattern_str =~ s{/$}{} ) { $match_dir = 1; }

my $anchored = 0;
if ( $pattern_str =~ s{^\.?/}{} ) { $anchored = 1; }

return () if $pattern_str eq '';

$pattern_str = join '[^/]*', map {
quotemeta($_ =~ s/\\(.)/$1/gr);
} split(/(?<!\\)\*/, $pattern_str, -1);

if (!$anchored) {
$pattern_str = '(?:.+/)?' . $pattern_str;
}
if ( $match_dir ) {
$pattern_str .= '/.*';
}

return ( '(?i)^' . $pattern_str . '$' );
}

sub is_subpath ($path, $parent) {
$parent =~ s{/+$}{};
return 0 if $path eq $parent;
return index( $path, $parent . '/' ) == 0;
}

1;
36 changes: 33 additions & 3 deletions lib/Shinobu.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ use LANraragi::Utils::Logging qw(get_logger);
use LANraragi::Utils::Generic qw(is_archive exec_with_lock_pure);
use LANraragi::Utils::Redis qw(redis_encode);
use LANraragi::Utils::Path qw(create_path open_path find_path get_archive_path);
use LANraragi::Utils::Ignore qw(is_ignored load_ignore_rules);
use LANraragi::Model::Archive;

use LANraragi::Model::Config;
use LANraragi::Model::Plugins;
Expand Down Expand Up @@ -133,11 +135,18 @@ sub update_filemap {
my $dirname = LANraragi::Model::Config->get_userdir;
my @files;

my $ignore_rules = load_ignore_rules();

# Get all files in content directory and subdirectories.
find_path(
sub {
$_ = create_path($_);
return if -d $_; #Directories are excluded on the spot

if ( is_ignored( $_, $ignore_rules ) ) {
return;
}

return unless is_archive($_);
push @files, $_; #Push files to array
},
Expand All @@ -151,17 +160,32 @@ sub update_filemap {
my %fshash = map { $_ => 1 } @files;

my @newfiles = grep { !$filemaphash{$_} } @files;
my @deletedfiles = grep { !$fshash{$_} } @filemapfiles;
my @deletedfiles = grep { !$fshash{$_} } @filemapfiles; # contains both deleted and ignored files.

$logger->info( "Found " . scalar @newfiles . " new files." );
$logger->info( scalar @deletedfiles . " files were found on the filemap but not on the filesystem." );
$logger->info( scalar @deletedfiles . " stale entries to remove from filemap." );

# Delete old files from filemap
# Delete old files from filemap and cleanup ignored archives
my $redis_arc = LANraragi::Model::Config->get_redis;
foreach my $deletedfile (@deletedfiles) {
$logger->debug("Removing $deletedfile from filemap.");
if ( -e $deletedfile ) {
my $id = $redis->hget( "LRR_FILEMAP", $deletedfile );
if ($id) {
my ($acquired) = exec_with_lock_pure(
[ "archive-write:$id" ],
sub { $redis_arc->hset( $id, "file", "" ); },
undef, 60
);
unless ($acquired) {
$logger->warn("Could not acquire lock for $id, skipping file field clear.");
}
}
}
$redis->hdel( "LRR_FILEMAP", $deletedfile ) || $logger->warn("Couldn't delete previous filemap data.");
}

$redis_arc->quit();
$redis->quit();

eval {
Expand Down Expand Up @@ -329,6 +353,12 @@ sub new_file_callback ($name) {
$logger->debug("New file detected: $name");
unless ( -d $name ) {

my $ignore_rules = load_ignore_rules();
if ( is_ignored( $name, $ignore_rules ) ) {
$logger->debug("$name matches .lrrignore rules, skipping.");
return;
}

my $redis = LANraragi::Model::Config->get_redis_config;
eval { add_to_filemap( $redis, $name ); };
$redis->quit();
Expand Down
Loading
Loading