From 30d06fa85ed84febec2bcd9899a193d2c74a562c Mon Sep 17 00:00:00 2001 From: Kevin Robert Stravers Date: Tue, 7 Jul 2020 00:31:35 +0200 Subject: [PATCH 01/46] Use `0` as the initial log --- Cargo.toml | 6 +++--- src/lib.rs | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3fafbef..ede22e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.2.0" +version = "0.3.0" authors = ["Kevin Robert Stravers "] edition = "2018" description = "Log rotation for files" @@ -10,5 +10,5 @@ keywords= ["log", "rotate", "logrotate"] license = "LGPL-3.0-or-later" [dev-dependencies] -quickcheck = "0.9" -quickcheck_macros = "0.9" +quickcheck = "0.9.2" +quickcheck_macros = "0.9.1" diff --git a/src/lib.rs b/src/lib.rs index c026ac1..6b652ae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,10 +98,12 @@ //! assert_eq!("E", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); //! //! -//! // Here we overwrite the 0 file since we're out of log files, restarting the sequencing +//! // Here we overwrite the `1` file since we're out of log files, restarting the sequencing. +//! // We keep file 0 since this is the initial file. It may contain system startup information we +//! // do not want to lose. //! write!(log, "F"); -//! assert_eq!("E", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); -//! assert_eq!("B", fs::read_to_string("target/my-log-directory-small/my-log-file.1").unwrap()); +//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); +//! assert_eq!("E", fs::read_to_string("target/my-log-directory-small/my-log-file.1").unwrap()); //! assert_eq!("C", fs::read_to_string("target/my-log-directory-small/my-log-file.2").unwrap()); //! assert_eq!("D", fs::read_to_string("target/my-log-directory-small/my-log-file.3").unwrap()); //! assert_eq!("F", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); @@ -199,7 +201,7 @@ impl FileRotate { let _ = fs::rename(&self.basename, path); self.file = Some(File::create(&self.basename)?); - self.file_number = (self.file_number + 1) % (self.max_file_number + 1); + self.file_number = ((self.file_number + 1) % (self.max_file_number + 1)).max(1); self.count = 0; Ok(()) @@ -304,7 +306,7 @@ mod tests { writeln!(rot, "d").unwrap(); assert_eq!("", fs::read_to_string("target/rotate/log").unwrap()); - assert_eq!("d\n", fs::read_to_string("target/rotate/log.0").unwrap()); + assert_eq!("d\n", fs::read_to_string("target/rotate/log.1").unwrap()); } #[quickcheck_macros::quickcheck] From 55c743612a493b9c993739b37739fa5e3280c00e Mon Sep 17 00:00:00 2001 From: Archis Gore Date: Tue, 24 Nov 2020 10:09:20 -0800 Subject: [PATCH 02/46] Cargo fmt --- src/lib.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c2ff2a5..f3989fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -173,13 +173,13 @@ impl FileRotate { match rotation_mode { RotationMode::Bytes(bytes) => { assert!(bytes > 0); - }, + } RotationMode::Lines(lines) => { assert!(lines > 0); - }, + } RotationMode::BytesSurpassed(bytes) => { assert!(bytes > 0); - }, + } }; Self { @@ -250,13 +250,9 @@ impl Write for FileRotate { if let Some(Err(err)) = self.file.as_mut().map(|file| file.write(buf)) { return Err(err); } - }, + } RotationMode::BytesSurpassed(bytes) => { - if let Some(Err(err)) = self - .file - .as_mut() - .map(|file| file.write(&buf)) - { + if let Some(Err(err)) = self.file.as_mut().map(|file| file.write(&buf)) { return Err(err); } self.count += buf.len(); @@ -330,7 +326,11 @@ mod tests { let _ = fs::remove_dir_all("target/surpassed_bytes"); fs::create_dir("target/surpassed_bytes").unwrap(); - let mut rot = FileRotate::new("target/surpassed_bytes/log", RotationMode::BytesSurpassed(1), 1); + let mut rot = FileRotate::new( + "target/surpassed_bytes/log", + RotationMode::BytesSurpassed(1), + 1, + ); write!(rot, "0123456789").unwrap(); rot.flush().unwrap(); @@ -385,5 +385,4 @@ mod tests { fs::remove_dir_all("target/arbitrary_bytes").unwrap(); } - } From d9e115816df33ca7ffe792956e282b74a8d54cf8 Mon Sep 17 00:00:00 2001 From: BourgondAries Date: Sun, 6 Dec 2020 02:52:32 +0100 Subject: [PATCH 03/46] Add license, fix grammar --- Cargo.toml | 4 ++-- LICENSE | 21 +++++++++++++++++++++ README.md | 11 +++++++++++ src/lib.rs | 2 +- 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 LICENSE create mode 100644 README.md diff --git a/Cargo.toml b/Cargo.toml index ede22e3..27a9a61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "file-rotate" -version = "0.3.0" +version = "0.4.0" authors = ["Kevin Robert Stravers "] edition = "2018" description = "Log rotation for files" homepage = "https://github.com/BourgondAries/file-rotate" repository = "https://github.com/BourgondAries/file-rotate" keywords= ["log", "rotate", "logrotate"] -license = "LGPL-3.0-or-later" +license = "MIT" [dev-dependencies] quickcheck = "0.9.2" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5a75d5a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 BourgondAries + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..d9986e9 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +## License + +This project is licensed under the [MIT license]. + +[MIT license]: https://github.com/BourgondAries/file-rotate/blob/master/LICENSE + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in file-rotate by you, shall be licensed as MIT, without any additional +terms or conditions. diff --git a/src/lib.rs b/src/lib.rs index 4662b71..e260c64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ //! use std::{fs, io::Write}; //! //! // Create a directory to store our logs, this is not strictly needed but shows how we can -//! // arbitrary paths. +//! // use arbitrary paths. //! fs::create_dir("target/my-log-directory-lines"); //! //! // Create a new log writer. The first argument is anything resembling a path. The From a23035ed8e365cfaf3ce3548b2433aa244c5526b Mon Sep 17 00:00:00 2001 From: Kevin Robert Stravers Date: Sun, 6 Dec 2020 03:43:22 +0100 Subject: [PATCH 04/46] Fix the arbitrary_bytes test --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e260c64..d5e4149 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -369,11 +369,11 @@ mod tests { } #[quickcheck_macros::quickcheck] - fn arbitrary_bytes() { + fn arbitrary_bytes(count: usize) { let _ = fs::remove_dir_all("target/arbitrary_bytes"); fs::create_dir("target/arbitrary_bytes").unwrap(); - let count = 0.max(1); + let count = count.max(1); let mut rot = FileRotate::new("target/arbitrary_bytes/log", RotationMode::Bytes(count), 0); for _ in 0..count { From b48f906c5064e84247d64ac32e685f876731b0cf Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Mon, 13 Dec 2021 22:51:50 +0100 Subject: [PATCH 05/46] Extensible suffix behaviour (timestamp, count) through trait SuffixScheme (#7) * "allow to add timestamp suffix to log path" PR by gfreezy squashed and rebased by Erlend Langseth <3rlendhl@gmail.com> * Extensible suffix behaviour (timestamp, count) through trait SuffixScheme In the count case, changed from unintuitive O(1) renaming, to intuitive O(N) cascade of renames. * Make chrono dep optional * Fix clippy warnings and errors There were errors like ``` error: written amount is not handled. Use `Write::write_all` instead --> src/lib.rs:278:25 | 278 | file.write(&buf[..bytes_left])?; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[deny(clippy::unused_io_amount)]` on by default = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#unused_io_amount ``` Seems like a good catch by clippy. * Add timestamp_age_rotation test, fix a bug Bug: - have to use NaiveDateTime to parse timezone-less string, and not DateTime - suffix_to_string was wrong - and adjust some tests * Document how to use age as file limit Co-authored-by: Alex.F --- Cargo.toml | 8 + README.md | 82 ++++++++ src/lib.rs | 537 +++++++++++++++++++++++++++++++++----------------- src/suffix.rs | 275 ++++++++++++++++++++++++++ 4 files changed, 719 insertions(+), 183 deletions(-) create mode 100644 src/suffix.rs diff --git a/Cargo.toml b/Cargo.toml index 27a9a61..4be75fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,14 @@ repository = "https://github.com/BourgondAries/file-rotate" keywords= ["log", "rotate", "logrotate"] license = "MIT" +[dependencies] +chrono = { version = "0.4.11", optional = true } + [dev-dependencies] quickcheck = "0.9.2" quickcheck_macros = "0.9.1" +tempdir = "0.3.7" + +[features] +default = ["chrono04"] +chrono04 = ["chrono"] diff --git a/README.md b/README.md index d9986e9..fd5c152 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,84 @@ +# file-rotate + +Rotate files with configurable suffix. + +Look to the [docs](https://docs.rs/file-rotate/0.4.0/file_rotate/) for explanatory examples. + +## Basic example + +```rust +use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; +use std::{fs, io::Write, path::PathBuf}; + +fn main() { + let mut log = FileRotate::new("logs/log", CountSuffix::new(2), ContentLimit::Lines(3)); + + // Write a bunch of lines + writeln!(log, "Line 1: Hello World!"); + for idx in 2..=10 { + writeln!(log, "Line {}", idx); + } +} +``` + +``` +$ ls logs +log log.1 log.2 + +$ cat log.2 log.1 log +Line 4 +Line 5 +Line 6 +Line 7 +Line 8 +Line 9 +Line 10 +``` + +## Example with timestamp suffixes + +```rust +let mut log = FileRotate::new( + "logs/log", + TimestampSuffix::default(FileLimit::MaxFiles(3)), + ContentLimit::Lines(3), +); + +// Write a bunch of lines +writeln!(log, "Line 1: Hello World!"); +for idx in 2..=10 { + std::thread::sleep(std::time::Duration::from_millis(200)); + writeln!(log, "Line {}", idx); +} +``` + +``` +$ ls logs +log log.20210825T151133.1 +log.20210825T151133 log.20210825T151134 + +$ cat logs/* +Line 10 +Line 1: Hello World! +Line 2 +Line 3 +Line 4 +Line 5 +Line 6 +Line 7 +Line 8 +Line 9 +``` + +The timestamp format (including the extra trailing `.N`) works by default so that the lexical ordering of filenames equals the chronological ordering. +So it almost works perfectly with `cat logs/*`, except that `log` is smaller (lexically "older") than all the rest. This can of course be fixed with a more complex script to assemble the logs. + + +## Content limit + +We can rotate log files by using the amount of lines as a limit, as seem above with `ContentLimit::Lines(3)`. +Another method of rotation is by bytes instead of lines, byt using for example `ContentLimit::BytesSurpassed(1_000_000)`. + ## License This project is licensed under the [MIT license]. @@ -9,3 +90,4 @@ This project is licensed under the [MIT license]. Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in file-rotate by you, shall be licensed as MIT, without any additional terms or conditions. + diff --git a/src/lib.rs b/src/lib.rs index d5e4149..ec1d9d6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,25 +2,29 @@ //! //! Defines a simple [std::io::Write] object that you can plug into your writers as middleware. //! -//! # Rotating by Lines # +//! # Content limit # //! -//! We can rotate log files by using the amount of lines as a limit. +//! Content limit specifies at what point a log file has to be rotated. +//! +//! ## Rotating by Lines ## +//! +//! We can rotate log files with the amount of lines as a limit, by using `ContentLimit::Lines`. //! //! ``` -//! use file_rotate::{FileRotate, RotationMode}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; //! use std::{fs, io::Write}; //! -//! // Create a directory to store our logs, this is not strictly needed but shows how we can -//! // use arbitrary paths. -//! fs::create_dir("target/my-log-directory-lines"); -//! //! // Create a new log writer. The first argument is anything resembling a path. The //! // basename is used for naming the log files. //! // //! // Here we choose to limit logs by 10 lines, and have at most 2 rotated log files. This -//! // makes the total amount of log files 4, since the original file is present as well as -//! // file 0. -//! let mut log = FileRotate::new("target/my-log-directory-lines/my-log-file", RotationMode::Lines(3), 2); +//! // makes the total amount of log files 3, since the original file is present as well. +//! +//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = directory.path(); +//! let log_path = directory.join("my-log-file"); +//! +//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(2), ContentLimit::Lines(3)); //! //! // Write a bunch of lines //! writeln!(log, "Line 1: Hello World!"); @@ -28,30 +32,29 @@ //! writeln!(log, "Line {}", idx); //! } //! -//! assert_eq!("Line 10\n", fs::read_to_string("target/my-log-directory-lines/my-log-file").unwrap()); -//! -//! assert_eq!("Line 1: Hello World!\nLine 2\nLine 3\n", fs::read_to_string("target/my-log-directory-lines/my-log-file.0").unwrap()); -//! assert_eq!("Line 4\nLine 5\nLine 6\n", fs::read_to_string("target/my-log-directory-lines/my-log-file.1").unwrap()); -//! assert_eq!("Line 7\nLine 8\nLine 9\n", fs::read_to_string("target/my-log-directory-lines/my-log-file.2").unwrap()); +//! assert_eq!("Line 10\n", fs::read_to_string(&log_path).unwrap()); //! -//! fs::remove_dir_all("target/my-log-directory-lines"); +//! assert_eq!("Line 4\nLine 5\nLine 6\n", fs::read_to_string(&directory.join("my-log-file.2")).unwrap()); +//! assert_eq!("Line 7\nLine 8\nLine 9\n", fs::read_to_string(&directory.join("my-log-file.1")).unwrap()); //! ``` //! -//! # Rotating by Bytes # +//! ## Rotating by Bytes ## //! -//! Another method of rotation is by bytes instead of lines. +//! Another method of rotation is by bytes instead of lines, with `ContentLimit::Bytes`. //! //! ``` -//! use file_rotate::{FileRotate, RotationMode}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; //! use std::{fs, io::Write}; //! -//! fs::create_dir("target/my-log-directory-bytes"); +//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = directory.path(); +//! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new("target/my-log-directory-bytes/my-log-file", RotationMode::Bytes(5), 2); +//! let mut log = FileRotate::new("target/my-log-directory-bytes/my-log-file", CountSuffix::new(2), ContentLimit::Bytes(5)); //! //! writeln!(log, "Test file"); //! -//! assert_eq!("Test ", fs::read_to_string("target/my-log-directory-bytes/my-log-file.0").unwrap()); +//! assert_eq!("Test ", fs::read_to_string(&log.log_paths()[0]).unwrap()); //! assert_eq!("file\n", fs::read_to_string("target/my-log-directory-bytes/my-log-file").unwrap()); //! //! fs::remove_dir_all("target/my-log-directory-bytes"); @@ -59,56 +62,99 @@ //! //! # Rotation Method # //! -//! The rotation method used is to always write to the base path, and then move the file to a new -//! location when the limit is exceeded. The moving occurs in the sequence 0, 1, 2, n, 0, 1, 2... +//! Two rotation methods are provided, but any behaviour can be implemented with the `SuffixScheme` +//! trait. +//! +//! ## Basic count ## +//! +//! With `CountSuffix`, when the limit is reached in the main log file, the file is moved with +//! suffix `.1`, and subsequently numbered files are moved in a cascade. //! //! Here's an example with 1 byte limits: //! //! ``` -//! use file_rotate::{FileRotate, RotationMode}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; //! use std::{fs, io::Write}; //! -//! fs::create_dir("target/my-log-directory-small"); +//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = directory.path(); +//! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new("target/my-log-directory-small/my-log-file", RotationMode::Bytes(1), 3); +//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(3), ContentLimit::Bytes(1)); //! //! write!(log, "A"); -//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); +//! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); //! //! write!(log, "B"); -//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); -//! assert_eq!("B", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); +//! assert_eq!("A", fs::read_to_string(directory.join("my-log-file.1")).unwrap()); +//! assert_eq!("B", fs::read_to_string(&log_path).unwrap()); //! //! write!(log, "C"); -//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); -//! assert_eq!("B", fs::read_to_string("target/my-log-directory-small/my-log-file.1").unwrap()); -//! assert_eq!("C", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); +//! assert_eq!("A", fs::read_to_string(directory.join("my-log-file.2")).unwrap()); +//! assert_eq!("B", fs::read_to_string(directory.join("my-log-file.1")).unwrap()); +//! assert_eq!("C", fs::read_to_string(&log_path).unwrap()); //! //! write!(log, "D"); -//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); -//! assert_eq!("B", fs::read_to_string("target/my-log-directory-small/my-log-file.1").unwrap()); -//! assert_eq!("C", fs::read_to_string("target/my-log-directory-small/my-log-file.2").unwrap()); -//! assert_eq!("D", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); +//! assert_eq!("A", fs::read_to_string(directory.join("my-log-file.3")).unwrap()); +//! assert_eq!("B", fs::read_to_string(directory.join("my-log-file.2")).unwrap()); +//! assert_eq!("C", fs::read_to_string(directory.join("my-log-file.1")).unwrap()); +//! assert_eq!("D", fs::read_to_string(&log_path).unwrap()); //! //! write!(log, "E"); -//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); -//! assert_eq!("B", fs::read_to_string("target/my-log-directory-small/my-log-file.1").unwrap()); -//! assert_eq!("C", fs::read_to_string("target/my-log-directory-small/my-log-file.2").unwrap()); -//! assert_eq!("D", fs::read_to_string("target/my-log-directory-small/my-log-file.3").unwrap()); -//! assert_eq!("E", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); -//! -//! -//! // Here we overwrite the `1` file since we're out of log files, restarting the sequencing. -//! // We keep file 0 since this is the initial file. It may contain system startup information we -//! // do not want to lose. -//! write!(log, "F"); -//! assert_eq!("A", fs::read_to_string("target/my-log-directory-small/my-log-file.0").unwrap()); -//! assert_eq!("E", fs::read_to_string("target/my-log-directory-small/my-log-file.1").unwrap()); -//! assert_eq!("C", fs::read_to_string("target/my-log-directory-small/my-log-file.2").unwrap()); -//! assert_eq!("D", fs::read_to_string("target/my-log-directory-small/my-log-file.3").unwrap()); -//! assert_eq!("F", fs::read_to_string("target/my-log-directory-small/my-log-file").unwrap()); -//! -//! fs::remove_dir_all("target/my-log-directory-small"); +//! assert_eq!("B", fs::read_to_string(directory.join("my-log-file.3")).unwrap()); +//! assert_eq!("C", fs::read_to_string(directory.join("my-log-file.2")).unwrap()); +//! assert_eq!("D", fs::read_to_string(directory.join("my-log-file.1")).unwrap()); +//! assert_eq!("E", fs::read_to_string(&log_path).unwrap()); +//! ``` +//! +//! ## Timestamp suffix ## +//! +//! With `TimestampSuffix`, when the limit is reached in the main log file, the file is moved with +//! suffix equal to the current timestamp (with the specified or a default format). If the +//! destination file name already exists, `.1` (and up) is appended. +//! +//! Note that this works somewhat different to `CountSuffix` because of lexical ordering concerns: +//! Higher numbers mean more recent logs, whereas `CountSuffix` works in the opposite way. +//! The reason for this is to keep the lexical ordering of log names consistent: Higher lexical value +//! means more recent. +//! This is of course all assuming that the format start with the year (or most significant +//! component). +//! +//! With this suffix scheme, you can also decide whether to delete old files based on the age of +//! their timestamp (`FileLimit::Age`), or just maximum number of files (`FileLimit::MaxFiles`). +//! +//! ``` +//! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffix, FileLimit}}; +//! use std::{fs, io::Write}; +//! +//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = directory.path(); +//! let log_path = directory.join("my-log-file"); +//! +//! let mut log = FileRotate::new(log_path.clone(), TimestampSuffix::default(FileLimit::MaxFiles(2)), ContentLimit::Bytes(1)); +//! +//! write!(log, "A"); +//! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); +//! +//! write!(log, "B"); +//! assert_eq!("A", fs::read_to_string(&log.log_paths()[0]).unwrap()); +//! assert_eq!("B", fs::read_to_string(&log_path).unwrap()); +//! +//! write!(log, "C"); +//! assert_eq!("A", fs::read_to_string(&log.log_paths()[0]).unwrap()); +//! assert_eq!("B", fs::read_to_string(&log.log_paths()[1]).unwrap()); +//! assert_eq!("C", fs::read_to_string(&log_path).unwrap()); +//! +//! write!(log, "D"); +//! assert_eq!("B", fs::read_to_string(&log.log_paths()[0]).unwrap()); +//! assert_eq!("C", fs::read_to_string(&log.log_paths()[1]).unwrap()); +//! assert_eq!("D", fs::read_to_string(&log_path).unwrap()); +//! ``` +//! +//! If you use timestamps as suffix, you can also configure files to be removed as they reach a +//! certain age. For example: +//! ```rust +//! TimestampSuffix::default(FileLimit::Age(chrono::Duration::weeks(1))) //! ``` //! //! # Filesystem Errors # @@ -119,6 +165,7 @@ //! date is sent to the void. //! //! This logger never panics. + #![deny( missing_docs, trivial_casts, @@ -134,114 +181,121 @@ use std::{ path::{Path, PathBuf}, }; +/// Suffix scheme etc +pub mod suffix; + // --- -/// Condition on which a file is rotated. -pub enum RotationMode { +/// When to move files: Condition on which a file is rotated. +pub enum ContentLimit { /// Cut the log at the exact size in bytes. Bytes(usize), /// Cut the log file at line breaks. Lines(usize), /// Cut the log file after surpassing size in bytes (but having written a complete buffer from a write call.) BytesSurpassed(usize), + // TODO: Custom(Fn(suffix: &str) -> bool) + // Which can be used to test age in case of timestamps. } /// The main writer used for rotating logs. -pub struct FileRotate { - basename: PathBuf, - count: usize, +pub struct FileRotate { + basepath: PathBuf, file: Option, - file_number: usize, - max_file_number: usize, - mode: RotationMode, + content_limit: ContentLimit, + count: usize, + suffix_scheme: S, +} + +fn create_parent_dir(path: &Path) { + if let Some(dirname) = path.parent() { + if !dirname.exists() { + fs::create_dir_all(dirname).expect("create dir"); + } + } } -impl FileRotate { +impl FileRotate { /// Create a new [FileRotate]. /// /// The basename of the `path` is used to create new log files by appending an extension of the - /// form `.N`, where N is `0..=max_file_number`. + /// form `.N`, where N is `0..=max_files`. /// - /// `rotation_mode` specifies the limits for rotating a file. + /// `content_limit` specifies the limits for rotating a file. /// /// # Panics /// /// Panics if `bytes == 0` or `lines == 0`. - pub fn new>( - path: P, - rotation_mode: RotationMode, - max_file_number: usize, - ) -> Self { - match rotation_mode { - RotationMode::Bytes(bytes) => { + pub fn new>(path: P, suffix_scheme: S, content_limit: ContentLimit) -> Self { + match content_limit { + ContentLimit::Bytes(bytes) => { assert!(bytes > 0); } - RotationMode::Lines(lines) => { + ContentLimit::Lines(lines) => { assert!(lines > 0); } - RotationMode::BytesSurpassed(bytes) => { + ContentLimit::BytesSurpassed(bytes) => { assert!(bytes > 0); } }; + let basepath = path.as_ref().to_path_buf(); + create_parent_dir(&basepath); + Self { - basename: path.as_ref().to_path_buf(), + file: File::create(&basepath).ok(), + basepath, + content_limit, count: 0, - file: match File::create(&path) { - Ok(file) => Some(file), - Err(_) => None, - }, - file_number: 0, - max_file_number, - mode: rotation_mode, + suffix_scheme, } } + /// Get paths of rotated log files (excluding the original/current log file) + pub fn log_paths(&mut self) -> Vec { + self.suffix_scheme.log_paths(&self.basepath) + } fn rotate(&mut self) -> io::Result<()> { - let mut path = self.basename.clone(); - path.set_extension(self.file_number.to_string()); + let suffix = self.suffix_scheme.rotate(&self.basepath); + let path = PathBuf::from(format!("{}.{}", self.basepath.display(), suffix)); + + create_parent_dir(&path); let _ = self.file.take(); - let _ = fs::rename(&self.basename, path); - self.file = Some(File::create(&self.basename)?); + // TODO should handle this error (and others) + let _ = fs::rename(&self.basepath, &path); - self.file_number = ((self.file_number + 1) % (self.max_file_number + 1)).max(1); + self.file = Some(File::create(&self.basepath)?); self.count = 0; Ok(()) } } -impl Write for FileRotate { +impl Write for FileRotate { fn write(&mut self, mut buf: &[u8]) -> io::Result { let written = buf.len(); - match self.mode { - RotationMode::Bytes(bytes) => { + match self.content_limit { + ContentLimit::Bytes(bytes) => { while self.count + buf.len() > bytes { let bytes_left = bytes - self.count; - if let Some(Err(err)) = self - .file - .as_mut() - .map(|file| file.write(&buf[..bytes_left])) - { - return Err(err); + if let Some(ref mut file) = self.file { + file.write_all(&buf[..bytes_left])?; } self.rotate()?; buf = &buf[bytes_left..]; } self.count += buf.len(); - if let Some(Err(err)) = self.file.as_mut().map(|file| file.write(&buf[..])) { - return Err(err); + if let Some(ref mut file) = self.file { + file.write_all(&buf)?; } } - RotationMode::Lines(lines) => { + ContentLimit::Lines(lines) => { while let Some((idx, _)) = buf.iter().enumerate().find(|(_, byte)| *byte == &b'\n') { - if let Some(Err(err)) = - self.file.as_mut().map(|file| file.write(&buf[..idx + 1])) - { - return Err(err); + if let Some(ref mut file) = self.file { + file.write_all(&buf[..idx + 1])?; } self.count += 1; buf = &buf[idx + 1..]; @@ -249,142 +303,259 @@ impl Write for FileRotate { self.rotate()?; } } - if let Some(Err(err)) = self.file.as_mut().map(|file| file.write(buf)) { - return Err(err); + if let Some(ref mut file) = self.file { + file.write_all(buf)?; } } - RotationMode::BytesSurpassed(bytes) => { - if let Some(Err(err)) = self.file.as_mut().map(|file| file.write(&buf)) { - return Err(err); - } - self.count += buf.len(); + ContentLimit::BytesSurpassed(bytes) => { if self.count > bytes { self.rotate()? } + if let Some(ref mut file) = self.file { + file.write_all(&buf)?; + } + self.count += buf.len(); } } Ok(written) } fn flush(&mut self) -> io::Result<()> { - if let Some(Err(err)) = self.file.as_mut().map(|file| file.flush()) { - Err(err) - } else { - Ok(()) - } + self.file + .as_mut() + .map(|file| file.flush()) + .unwrap_or(Ok(())) } } #[cfg(test)] mod tests { - use super::*; + use super::{suffix::*, *}; + use tempdir::TempDir; + + // Just useful to debug why test doesn't succeed + #[allow(dead_code)] + fn list(dir: &Path) { + let filenames = fs::read_dir(dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name()) + .collect::>(); + println!("Files on disk: {:?}", filenames); + } #[test] - #[should_panic(expected = "assertion failed: bytes > 0")] - fn zero_bytes() { - let mut rot = FileRotate::new("target/zero_bytes", RotationMode::Bytes(0), 0); - writeln!(rot, "Zero").unwrap(); - assert_eq!("\n", fs::read_to_string("target/zero_bytes").unwrap()); - assert_eq!("o", fs::read_to_string("target/zero_bytes.0").unwrap()); - } + fn timestamp_max_files_rotation() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let log_path = tmp_dir.path().join("log"); + + let mut log = FileRotate::new( + &log_path, + TimestampSuffix::default(FileLimit::MaxFiles(4)), + ContentLimit::Lines(2), + ); + // Write 9 lines + // This should result in 5 files in total (4 rotated files). The main file will have one line. + write!(log, "a\nb\nc\nd\ne\nf\ng\nh\ni\n").unwrap(); + let log_paths = log.log_paths(); + assert_eq!(log_paths.len(), 4); + + // Log names should be sorted. Low (old timestamp) to high (more recent timestamp) + let mut log_paths_sorted = log_paths.clone(); + log_paths_sorted.sort(); + assert_eq!(log_paths, log_paths_sorted); + + assert_eq!("a\nb\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("c\nd\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("i\n", fs::read_to_string(&log_path).unwrap()); + + // Write 4 more lines + write!(log, "j\nk\nl\nm\n").unwrap(); + let log_paths = log.log_paths(); + assert_eq!(log_paths.len(), 4); + let mut log_paths_sorted = log_paths.clone(); + log_paths_sorted.sort(); + assert_eq!(log_paths, log_paths_sorted); + + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("k\nl\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); + } + #[test] + #[cfg(feature = "chrono04")] + fn timestamp_age_rotation() { + // In order not to have to sleep, and keep it deterministic, let's already create the log files and see how FileRotate + // cleans up the old ones. + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + // One recent file: + let recent_file = chrono::offset::Local::now().format("log.%Y%m%dT%H%M%S").to_string(); + File::create(dir.join(&recent_file)).unwrap(); + // Two very old files: + File::create(dir.join("log.20200825T151133")).unwrap(); + File::create(dir.join("log.20200825T151133.1")).unwrap(); + + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + TimestampSuffix::default(FileLimit::Age(chrono::Duration::weeks(1))), + ContentLimit::Lines(1), + ); + writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); + + + let mut filenames = std::fs::read_dir(dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect::>(); + filenames.sort(); + assert!(filenames.contains(&"log".to_string())); + assert!(filenames.contains(&recent_file)); + assert!(!filenames.contains(&"log.20200825T151133".to_string())); + assert!(!filenames.contains(&"log.20200825T151133.1".to_string())); + } #[test] - #[should_panic(expected = "assertion failed: lines > 0")] - fn zero_lines() { - let mut rot = FileRotate::new("target/zero_lines", RotationMode::Lines(0), 0); - write!(rot, "a\nb\nc\nd\n").unwrap(); - assert_eq!("", fs::read_to_string("target/zero_lines").unwrap()); - assert_eq!("d\n", fs::read_to_string("target/zero_lines.0").unwrap()); + fn count_max_files_rotation() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(4), + ContentLimit::Lines(2), + ); + + // Write 9 lines + // This should result in 5 files in total (4 rotated files). The main file will have one line. + write!(log, "a\nb\nc\nd\ne\nf\ng\nh\ni\n").unwrap(); // 9 lines + let log_paths = vec![ + parent.join("log.4"), + parent.join("log.3"), + parent.join("log.2"), + parent.join("log.1"), + ]; + assert_eq!(log_paths, log.log_paths()); + assert_eq!("a\nb\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("c\nd\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("i\n", fs::read_to_string(&log_path).unwrap()); + + // Write 4 more lines + write!(log, "j\nk\nl\nm\n").unwrap(); + list(parent); + assert_eq!(log_paths, log.log_paths()); + + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("k\nl\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); } #[test] fn rotate_to_deleted_directory() { - let _ = fs::remove_dir_all("target/rotate"); - fs::create_dir("target/rotate").unwrap(); - - let mut rot = FileRotate::new("target/rotate/log", RotationMode::Lines(1), 0); - writeln!(rot, "a").unwrap(); - assert_eq!("", fs::read_to_string("target/rotate/log").unwrap()); - assert_eq!("a\n", fs::read_to_string("target/rotate/log.0").unwrap()); + // NOTE: Only supported with count, not with timestamp suffix. + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(4), + ContentLimit::Lines(1), + ); - fs::remove_dir_all("target/rotate").unwrap(); + write!(log, "a\nb\n").unwrap(); + assert_eq!("", fs::read_to_string(&log_path).unwrap()); + assert_eq!("a\n", fs::read_to_string(&log.log_paths()[0]).unwrap()); - assert!(writeln!(rot, "b").is_err()); + let _ = fs::remove_dir_all(parent); - rot.flush().unwrap(); - assert!(fs::read_dir("target/rotate").is_err()); - fs::create_dir("target/rotate").unwrap(); + assert!(writeln!(log, "c").is_ok()); - writeln!(rot, "c").unwrap(); - assert_eq!("", fs::read_to_string("target/rotate/log").unwrap()); + log.flush().unwrap(); - writeln!(rot, "d").unwrap(); - assert_eq!("", fs::read_to_string("target/rotate/log").unwrap()); - assert_eq!("d\n", fs::read_to_string("target/rotate/log.1").unwrap()); + writeln!(log, "d").unwrap(); + assert_eq!("", fs::read_to_string(&log_path).unwrap()); + assert_eq!("d\n", fs::read_to_string(&log.log_paths()[0]).unwrap()); } #[test] fn write_complete_record_until_bytes_surpassed() { - let _ = fs::remove_dir_all("target/surpassed_bytes"); - fs::create_dir("target/surpassed_bytes").unwrap(); - - let mut rot = FileRotate::new( - "target/surpassed_bytes/log", - RotationMode::BytesSurpassed(1), - 1, + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + let mut log = FileRotate::new( + &log_path, + TimestampSuffix::default(FileLimit::MaxFiles(100)), + ContentLimit::BytesSurpassed(1), ); - write!(rot, "0123456789").unwrap(); - rot.flush().unwrap(); - assert!(Path::new("target/surpassed_bytes/log.0").exists()); + write!(log, "0123456789").unwrap(); + log.flush().unwrap(); + assert!(log_path.exists()); // shouldn't exist yet - because entire record was written in one shot - assert!(!Path::new("target/surpassed_bytes/log.1").exists()); + assert!(log.log_paths().is_empty()); // This should create the second file - write!(rot, "0123456789").unwrap(); - rot.flush().unwrap(); - assert!(Path::new("target/surpassed_bytes/log.1").exists()); - - fs::remove_dir_all("target/surpassed_bytes").unwrap(); + write!(log, "0123456789").unwrap(); + log.flush().unwrap(); + assert!(&log.log_paths()[0].exists()); } #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { - let _ = fs::remove_dir_all("target/arbitrary_lines"); - fs::create_dir("target/arbitrary_lines").unwrap(); + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); let count = count.max(1); - let mut rot = FileRotate::new("target/arbitrary_lines/log", RotationMode::Lines(count), 0); + let mut log = FileRotate::new( + &log_path, + TimestampSuffix::default(FileLimit::MaxFiles(100)), + ContentLimit::Lines(count), + ); for _ in 0..count - 1 { - writeln!(rot).unwrap(); + writeln!(log).unwrap(); } - rot.flush().unwrap(); - assert!(!Path::new("target/arbitrary_lines/log.0").exists()); - writeln!(rot).unwrap(); - assert!(Path::new("target/arbitrary_lines/log.0").exists()); - - fs::remove_dir_all("target/arbitrary_lines").unwrap(); + log.flush().unwrap(); + assert!(log.log_paths().is_empty()); + writeln!(log).unwrap(); + assert!(Path::new(&log.log_paths()[0]).exists()); } #[quickcheck_macros::quickcheck] fn arbitrary_bytes(count: usize) { - let _ = fs::remove_dir_all("target/arbitrary_bytes"); - fs::create_dir("target/arbitrary_bytes").unwrap(); + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); let count = count.max(1); - let mut rot = FileRotate::new("target/arbitrary_bytes/log", RotationMode::Bytes(count), 0); + let mut log = FileRotate::new( + &log_path, + TimestampSuffix::default(FileLimit::MaxFiles(100)), + ContentLimit::Bytes(count), + ); for _ in 0..count { - write!(rot, "0").unwrap(); + write!(log, "0").unwrap(); } - rot.flush().unwrap(); - assert!(!Path::new("target/arbitrary_bytes/log.0").exists()); - write!(rot, "1").unwrap(); - assert!(Path::new("target/arbitrary_bytes/log.0").exists()); - - fs::remove_dir_all("target/arbitrary_bytes").unwrap(); + log.flush().unwrap(); + assert!(log.log_paths().is_empty()); + write!(log, "1").unwrap(); + assert!(&log.log_paths()[0].exists()); } } diff --git a/src/suffix.rs b/src/suffix.rs new file mode 100644 index 0000000..f9704d2 --- /dev/null +++ b/src/suffix.rs @@ -0,0 +1,275 @@ +use chrono::NaiveDateTime; +#[cfg(feature = "chrono04")] +use chrono::{offset::Local, Duration}; +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, +}; + +/// How to move files: How to rename, when to delete. +pub trait SuffixScheme { + /// Returns new suffix to which to move current log file (does not do the move). + /// Deletes old log files. + /// Might also do other operations, like moving files in a cascading way. + fn rotate(&mut self, basepath: &Path) -> String; + + /// Get paths of rotated log files, in order from newest to oldest. + /// Excludes the suffix-less log file. + fn log_paths(&mut self, basepath: &Path) -> Vec; +} + +/// Rotated log files get a number as suffix. The greater the number, the older. The oldest files +/// are deleted. +pub struct CountSuffix { + max_files: usize, +} + +impl CountSuffix { + /// New CountSuffix + pub fn new(max_files: usize) -> Self { + Self { max_files } + } +} + +impl SuffixScheme for CountSuffix { + fn rotate(&mut self, basepath: &Path) -> String { + /// Make sure that path(count) does not exist, by moving it to path(count+1). + fn cascade(basepath: &Path, count: usize, max_files: usize) { + let src = PathBuf::from(format!("{}.{}", basepath.display(), count)); + if src.exists() { + let dest = PathBuf::from(format!("{}.{}", basepath.display(), count + 1)); + if dest.exists() { + cascade(basepath, count + 1, max_files); + } + if count >= max_files { + // If the file is too old (too big count), delete it, + // (also if count == max_files, because then the .(max_files-1) file will be moved + // to .max_files) + let _ = std::fs::remove_file(&src).unwrap(); + } else { + // otherwise, rename it. + let _ = std::fs::rename(src, dest); + } + } + } + cascade(basepath, 1, self.max_files); + "1".to_string() + } + fn log_paths(&mut self, basepath: &Path) -> Vec { + let filename_prefix = &*basepath + .file_name() + .expect("basepath.file_name()") + .to_string_lossy(); + let filenames = std::fs::read_dir(basepath.parent().expect("basepath.parent()")) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name()); + let mut numbers = Vec::new(); + for filename in filenames { + let filename = filename.to_string_lossy(); + if !filename.starts_with(&filename_prefix) { + continue; + } + if let Some(dot) = filename.find('.') { + let suffix = &filename[(dot + 1)..]; + if let Ok(n) = suffix.parse::() { + numbers.push(n); + } else { + continue; + } + } else { + // We don't consider the current (suffix-less) log file. + } + } + // Sort descending - the largest numbers are the oldest and thus should come first + numbers.sort_by(|x, y| y.cmp(x)); + numbers + .iter() + .map(|n| PathBuf::from(format!("{}.{}", basepath.display(), n))) + .collect::>() + } +} + +/// Current limitations: +/// - Neither `format` or the base filename can include the character `"."`. +/// - The `format` should ensure that the lexical and chronological orderings are the same +#[cfg(feature = "chrono04")] +pub struct TimestampSuffix { + /// None means that we don't know the files, and a scan is necessary. + pub(crate) suffixes: Option)>>, + format: &'static str, + file_limit: FileLimit, +} + +#[cfg(feature = "chrono04")] +impl TimestampSuffix { + /// With format `"%Y%m%dT%H%M%S"` + pub fn default(file_limit: FileLimit) -> Self { + Self { + suffixes: None, + format: "%Y%m%dT%H%M%S", + file_limit, + } + } + /// Create new TimestampSuffix suffix scheme + pub fn with_format(format: &'static str, file_limit: FileLimit) -> Self { + Self { + suffixes: None, + format, + file_limit, + } + } + /// NOTE: For future use in RotationMode::Custom + pub fn should_rotate(&self, age: Duration) -> impl Fn(&str) -> bool { + let format = self.format.to_string(); + move |suffix| { + let old_timestamp = (Local::now() - age).format(&format).to_string(); + suffix < old_timestamp.as_str() + } + } + pub(crate) fn suffix_to_string(&self, suffix: &(String, Option)) -> String { + match suffix.1 { + Some(n) => format!("{}.{}", suffix.0, n), + None => suffix.0.clone(), + } + } + pub(crate) fn suffix_to_path( + &self, + basepath: &Path, + suffix: &(String, Option), + ) -> PathBuf { + PathBuf::from(format!( + "{}.{}", + basepath.display(), + self.suffix_to_string(suffix) + )) + } + /// Scan files in the log directory to construct the list of files + fn ensure_suffix_list(&mut self, basepath: &Path) { + if self.suffixes.is_none() { + let mut suffixes = VecDeque::new(); + let filename_prefix = &*basepath + .file_name() + .expect("basepath.file_name()") + .to_string_lossy(); + let parent = basepath.parent().unwrap(); + let filenames = std::fs::read_dir(parent) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name()); + for filename in filenames { + let filename = filename.to_string_lossy(); + if !filename.starts_with(&filename_prefix) { + continue; + } + // Find the up to two `.` in the filename + if let Some(first_dot) = filename.find('.') { + let suffix = &filename[(first_dot + 1)..]; + let (timestamp_str, n) = if let Some(second_dot) = suffix.find('.') { + if let Ok(n) = suffix[(second_dot + 1)..].parse::() { + (&suffix[..second_dot], Some(n)) + } else { + continue; + } + } else { + (suffix, None) + }; + if NaiveDateTime::parse_from_str(timestamp_str, self.format).is_ok() { + suffixes.push_back((timestamp_str.to_string(), n)) + } + } else { + // We don't consider the current (suffix-less) log file. + } + } + // Sort in Ascending order (higher value (most recent) first) + suffixes + .make_contiguous() + .sort_by_key(|suffix| self.suffix_to_string(suffix)); + self.suffixes = Some(suffixes); + } + } +} +#[cfg(feature = "chrono04")] +impl SuffixScheme for TimestampSuffix { + fn rotate(&mut self, basepath: &Path) -> String { + let now = Local::now().format(self.format).to_string(); + + self.ensure_suffix_list(basepath); + + // For all existing suffixes that equals `now`, take the max `n`, and add one + let n = self + .suffixes + .as_ref() + .unwrap() + .iter() + .filter(|suffix| suffix.0 == now) + .map(|suffix| suffix.1.unwrap_or(0)) + .max() + .map(|n| n + 1); + + // Register the selected suffix as taken + self.suffixes.as_mut().unwrap().push_back((now.clone(), n)); + + // Remove old files + // Note that the oldest are the first in the list + let to_delete = match self.file_limit { + FileLimit::MaxFiles(max_files) => { + let n_files = self.suffixes.as_ref().unwrap().len(); + if n_files > max_files { + n_files - max_files + } else { + 0 + } + } + FileLimit::Age(age) => { + let mut to_delete = 0; + for suffix in self.suffixes.as_ref().unwrap().iter() { + let old_timestamp = (Local::now() - age).format(self.format).to_string(); + let delete = suffix.0 < old_timestamp; + if delete { + let _ = std::fs::remove_file(self.suffix_to_path(basepath, suffix)); + to_delete += 1; + } else { + // Remember that `suffixes` has the oldest entries in the front, we can `break` + // once we find an entry that doesn't have to deleted + break; + } + } + to_delete + } + }; + + // Delete respective entries + for _ in 0..to_delete { + self.suffixes.as_mut().unwrap().pop_front(); + } + + self.suffix_to_string(&(now, n)) + } + fn log_paths(&mut self, basepath: &Path) -> Vec { + self.ensure_suffix_list(basepath); + self.suffixes + .as_ref() + .unwrap() + .iter() + .map(|suffix| { + PathBuf::from(format!( + "{}.{}", + basepath.display(), + self.suffix_to_string(suffix) + )) + }) + .collect::>() + } +} + +/// How to determine if a file should be deleted, in the case of TimestampSuffix. +#[cfg(feature = "chrono04")] +pub enum FileLimit { + /// Delete the oldest files if number of files is too high + MaxFiles(usize), + /// Delete files that have too old timestamp + Age(Duration), +} From 23539faa9f8692c04d529866e0fc46ef46cdc3d8 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Sun, 29 Aug 2021 14:38:21 +0200 Subject: [PATCH 06/46] SuffixScheme implementations are lighter, more logic in FileRotate Now the SuffixScheme functions don't do file operations. FileRotate takes care of cascading files if necessary, removing files, keeping track of log paths etc. --- src/compression.rs | 6 + src/lib.rs | 176 ++++++++++++++++++++--- src/suffix.rs | 351 ++++++++++++++++++++------------------------- 3 files changed, 315 insertions(+), 218 deletions(-) create mode 100644 src/compression.rs diff --git a/src/compression.rs b/src/compression.rs new file mode 100644 index 0000000..b4cb3c4 --- /dev/null +++ b/src/compression.rs @@ -0,0 +1,6 @@ + + +pub enum Compression { + OnRotate (usize) + // In the future, maybe stream compression +} diff --git a/src/lib.rs b/src/lib.rs index ec1d9d6..43aa2d4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -124,14 +124,14 @@ //! their timestamp (`FileLimit::Age`), or just maximum number of files (`FileLimit::MaxFiles`). //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffix, FileLimit}}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffixScheme, FileLimit}}; //! use std::{fs, io::Write}; //! //! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), TimestampSuffix::default(FileLimit::MaxFiles(2)), ContentLimit::Bytes(1)); +//! let mut log = FileRotate::new(log_path.clone(), TimestampSuffixScheme::default(FileLimit::MaxFiles(2)), ContentLimit::Bytes(1)); //! //! write!(log, "A"); //! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); @@ -154,7 +154,8 @@ //! If you use timestamps as suffix, you can also configure files to be removed as they reach a //! certain age. For example: //! ```rust -//! TimestampSuffix::default(FileLimit::Age(chrono::Duration::weeks(1))) +//! use file_rotate::suffix::{TimestampSuffixScheme, FileLimit}; +//! TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))); //! ``` //! //! # Filesystem Errors # @@ -176,10 +177,13 @@ )] use std::{ + cmp::Ordering, + collections::BTreeSet, fs::{self, File}, io::{self, Write}, path::{Path, PathBuf}, }; +use suffix::Representation; /// Suffix scheme etc pub mod suffix; @@ -194,17 +198,45 @@ pub enum ContentLimit { Lines(usize), /// Cut the log file after surpassing size in bytes (but having written a complete buffer from a write call.) BytesSurpassed(usize), - // TODO: Custom(Fn(suffix: &str) -> bool) - // Which can be used to test age in case of timestamps. +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct SuffixInfo { + pub suffix: Repr, + pub compressed: bool, +} + +impl SuffixInfo { + pub fn to_path(&self, basepath: &Path) -> PathBuf { + let path = self.suffix.to_path(basepath); + if self.compressed { + PathBuf::from(format!("{}.tar.gz", path.display())) + } else { + path + } + } +} + +impl Ord for SuffixInfo { + fn cmp(&self, other: &Self) -> Ordering { + self.suffix.cmp(&other.suffix) + } +} +impl PartialOrd for SuffixInfo { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } } /// The main writer used for rotating logs. -pub struct FileRotate { +pub struct FileRotate { basepath: PathBuf, file: Option, content_limit: ContentLimit, count: usize, suffix_scheme: S, + /// The bool is whether or not there's a .tar.gz suffix to the filename + suffixes: BTreeSet>, } fn create_parent_dir(path: &Path) { @@ -242,35 +274,124 @@ impl FileRotate { let basepath = path.as_ref().to_path_buf(); create_parent_dir(&basepath); + // Construct `suffixes` + let mut suffixes = BTreeSet::new(); + let filename_prefix = &*basepath + .file_name() + .expect("basepath.file_name()") + .to_string_lossy(); + let parent = basepath.parent().unwrap(); + let filenames = std::fs::read_dir(parent) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name()); + for filename in filenames { + let filename = filename.to_string_lossy(); + if !filename.starts_with(&filename_prefix) { + continue; + } + let (filename, compressed) = Self::prepare_filename(&*filename); + let suffix_str = filename.strip_prefix(&format!("{}.", filename_prefix)); + if let Some(suffix) = suffix_str.and_then(|s| suffix_scheme.parse(s)) { + suffixes.insert(SuffixInfo { suffix, compressed }); + } + } + Self { file: File::create(&basepath).ok(), basepath, content_limit, count: 0, + suffixes, suffix_scheme, } } - /// Get paths of rotated log files (excluding the original/current log file) + fn prepare_filename(path: &str) -> (&str, bool) { + path.strip_prefix(".tar.gz") + .map(|x| (x, true)) + .unwrap_or((path, false)) + } + /// Get paths of rotated log files (excluding the original/current log file), ordered from + /// oldest to most recent pub fn log_paths(&mut self) -> Vec { - self.suffix_scheme.log_paths(&self.basepath) + self.suffixes + .iter() + .rev() + .map(|suffix| suffix.to_path(&self.basepath)) + .collect::>() } - fn rotate(&mut self) -> io::Result<()> { - let suffix = self.suffix_scheme.rotate(&self.basepath); - let path = PathBuf::from(format!("{}.{}", self.basepath.display(), suffix)); + /// Recursive function that keeps moving files if there's any file name collision. + /// If `suffix` is `None`, it moves from basepath to next suffix given by the SuffixScheme + /// Assumption: Any collision in file name is due to an old log file. + /// + /// Returns the suffix of the new file (the last suffix after possible cascade of renames). + // TODO remove panics + fn move_file_with_suffix(&mut self, suffix: Option) -> S::Repr { + // NOTE: this newest_suffix is there only because TimestampSuffixScheme specifically needs + // it. Otherwise it might not be necessary to provide this to `rotate_file`. We could also + // have passed the internal BTreeMap itself, but it would require to make SuffixInfo `pub`. + let newest_suffix = self.suffixes.iter().next().map(|info| &info.suffix); + + let new_suffix = self.suffix_scheme.rotate_file(&self.basepath, newest_suffix, &suffix); + let new_path = new_suffix.to_path(&self.basepath); + + // Move destination file out of the way if it exists + let newly_created_suffix = if new_path.exists() { + self.move_file_with_suffix(Some(new_suffix)) + } else { + new_suffix + }; + assert!(!new_path.exists()); - create_parent_dir(&path); + let old_path = match suffix { + Some(suffix) => suffix.to_path(&self.basepath), + None => self.basepath.clone(), + }; + + fs::rename(old_path, new_path).unwrap(); + newly_created_suffix + } + + fn rotate(&mut self) -> io::Result<()> { + create_parent_dir(&self.basepath); let _ = self.file.take(); - // TODO should handle this error (and others) - let _ = fs::rename(&self.basepath, &path); + // This function will always create a new file. Returns suffix of that file + let new_suffix = self.move_file_with_suffix(None); + self.suffixes.insert(SuffixInfo { + suffix: new_suffix, + compressed: false, + }); self.file = Some(File::create(&self.basepath)?); + self.count = 0; + self.prune_old_files(); + Ok(()) } + fn prune_old_files(&mut self) { + // Find the youngest suffix that is too old, and then remove all suffixes that are older or + // equally old: + let mut youngest_old = None; + // Start from oldest suffix, stop when we find a suffix that is not too old + for (i, suffix) in self.suffixes.iter().enumerate().rev() { + if self.suffix_scheme.too_old(&suffix.suffix, i) { + let _ = std::fs::remove_file(suffix.to_path(&self.basepath)); + youngest_old = Some((*suffix).clone()); + } else { + break; + } + } + if let Some(youngest_old) = youngest_old { + // Removes all the too old + let _ = self.suffixes.split_off(&youngest_old); + } + } } impl Write for FileRotate { @@ -352,7 +473,7 @@ mod tests { let mut log = FileRotate::new( &log_path, - TimestampSuffix::default(FileLimit::MaxFiles(4)), + TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), ContentLimit::Lines(2), ); @@ -381,6 +502,10 @@ mod tests { log_paths_sorted.sort(); assert_eq!(log_paths, log_paths_sorted); + // println!("{:?}", log_paths); + // println!("{:?}", log_paths.iter().map(|path| fs::read_to_string(path)).collect::>()); + // println!("Main log: {}", fs::read_to_string(&log_path).unwrap()); + list(&tmp_dir.path()); assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); @@ -389,7 +514,7 @@ mod tests { } #[test] #[cfg(feature = "chrono04")] - fn timestamp_age_rotation() { + fn timestamp_max_age_deletion() { // In order not to have to sleep, and keep it deterministic, let's already create the log files and see how FileRotate // cleans up the old ones. let tmp_dir = TempDir::new("file-rotate-test").unwrap(); @@ -397,7 +522,9 @@ mod tests { let log_path = dir.join("log"); // One recent file: - let recent_file = chrono::offset::Local::now().format("log.%Y%m%dT%H%M%S").to_string(); + let recent_file = chrono::offset::Local::now() + .format("log.%Y%m%dT%H%M%S") + .to_string(); File::create(dir.join(&recent_file)).unwrap(); // Two very old files: File::create(dir.join("log.20200825T151133")).unwrap(); @@ -405,12 +532,11 @@ mod tests { let mut log = FileRotate::new( &*log_path.to_string_lossy(), - TimestampSuffix::default(FileLimit::Age(chrono::Duration::weeks(1))), + TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))), ContentLimit::Lines(1), ); writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); - let mut filenames = std::fs::read_dir(dir) .unwrap() .filter_map(|entry| entry.ok()) @@ -462,9 +588,12 @@ mod tests { assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); } + // Currently not supported. May add support if it's important. + // Also consider removing `FileRotate::suffixes` and instead using more disk operations to + // check all files in the log directory on every rotation. + /* #[test] fn rotate_to_deleted_directory() { - // NOTE: Only supported with count, not with timestamp suffix. let tmp_dir = TempDir::new("file-rotate-test").unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); @@ -488,6 +617,7 @@ mod tests { assert_eq!("", fs::read_to_string(&log_path).unwrap()); assert_eq!("d\n", fs::read_to_string(&log.log_paths()[0]).unwrap()); } + */ #[test] fn write_complete_record_until_bytes_surpassed() { @@ -497,7 +627,7 @@ mod tests { let mut log = FileRotate::new( &log_path, - TimestampSuffix::default(FileLimit::MaxFiles(100)), + TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), ContentLimit::BytesSurpassed(1), ); @@ -522,7 +652,7 @@ mod tests { let count = count.max(1); let mut log = FileRotate::new( &log_path, - TimestampSuffix::default(FileLimit::MaxFiles(100)), + TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), ContentLimit::Lines(count), ); @@ -545,7 +675,7 @@ mod tests { let count = count.max(1); let mut log = FileRotate::new( &log_path, - TimestampSuffix::default(FileLimit::MaxFiles(100)), + TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), ContentLimit::Bytes(count), ); diff --git a/src/suffix.rs b/src/suffix.rs index f9704d2..740e3c8 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -1,21 +1,47 @@ -use chrono::NaiveDateTime; #[cfg(feature = "chrono04")] -use chrono::{offset::Local, Duration}; +use chrono::{offset::Local, Duration, NaiveDateTime}; use std::{ - collections::VecDeque, + cmp::Ordering, path::{Path, PathBuf}, }; +/// Representation of a suffix +/// `Ord + PartialOrd`: sort by age of the suffix. Most recent first (smallest). +pub trait Representation: Ord + ToString + Eq + Clone + std::fmt::Debug { + /// Create path + fn to_path(&self, basepath: &Path) -> PathBuf { + PathBuf::from(format!("{}.{}", basepath.display(), self.to_string())) + } +} + /// How to move files: How to rename, when to delete. pub trait SuffixScheme { - /// Returns new suffix to which to move current log file (does not do the move). - /// Deletes old log files. - /// Might also do other operations, like moving files in a cascading way. - fn rotate(&mut self, basepath: &Path) -> String; + /// The representation of suffixes that this suffix scheme uses. + /// E.g. if the suffix is a number, you can use `usize`. + type Repr: Representation; - /// Get paths of rotated log files, in order from newest to oldest. - /// Excludes the suffix-less log file. - fn log_paths(&mut self, basepath: &Path) -> Vec; + /// The file at `suffix` needs to be rotated. + /// Returns the target file path. + /// The file will be moved outside this function. + /// If the target path already exists, rotate_file is called again with `path` set to the + /// target path. Thus it cascades files by default, and if this is not desired, it's up to + /// `rotate_file` to return a path that does not already exist. + /// + /// `prev_suffix` is provided just in case it's useful (not always) + fn rotate_file( + &mut self, + basepath: &Path, + newest_suffix: Option<&Self::Repr>, + suffix: &Option, + ) -> Self::Repr; + + /// Parse suffix from string. + fn parse(&self, suffix: &str) -> Option; + + /// Whether either the suffix or the chronological file number indicates that the file is old + /// and should be deleted, depending of course on the file limit. + /// `file_number` starts at 0. + fn too_old(&self, suffix: &Self::Repr, file_number: usize) -> bool; } /// Rotated log files get a number as suffix. The greater the number, the older. The oldest files @@ -31,63 +57,23 @@ impl CountSuffix { } } +impl Representation for usize {} impl SuffixScheme for CountSuffix { - fn rotate(&mut self, basepath: &Path) -> String { - /// Make sure that path(count) does not exist, by moving it to path(count+1). - fn cascade(basepath: &Path, count: usize, max_files: usize) { - let src = PathBuf::from(format!("{}.{}", basepath.display(), count)); - if src.exists() { - let dest = PathBuf::from(format!("{}.{}", basepath.display(), count + 1)); - if dest.exists() { - cascade(basepath, count + 1, max_files); - } - if count >= max_files { - // If the file is too old (too big count), delete it, - // (also if count == max_files, because then the .(max_files-1) file will be moved - // to .max_files) - let _ = std::fs::remove_file(&src).unwrap(); - } else { - // otherwise, rename it. - let _ = std::fs::rename(src, dest); - } - } + type Repr = usize; + fn rotate_file(&mut self, + _basepath: &Path, + _: Option<&usize>, + suffix: &Option) -> usize { + match suffix { + Some(suffix) => suffix + 1, + None => 1, } - cascade(basepath, 1, self.max_files); - "1".to_string() } - fn log_paths(&mut self, basepath: &Path) -> Vec { - let filename_prefix = &*basepath - .file_name() - .expect("basepath.file_name()") - .to_string_lossy(); - let filenames = std::fs::read_dir(basepath.parent().expect("basepath.parent()")) - .unwrap() - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().is_file()) - .map(|entry| entry.file_name()); - let mut numbers = Vec::new(); - for filename in filenames { - let filename = filename.to_string_lossy(); - if !filename.starts_with(&filename_prefix) { - continue; - } - if let Some(dot) = filename.find('.') { - let suffix = &filename[(dot + 1)..]; - if let Ok(n) = suffix.parse::() { - numbers.push(n); - } else { - continue; - } - } else { - // We don't consider the current (suffix-less) log file. - } - } - // Sort descending - the largest numbers are the oldest and thus should come first - numbers.sort_by(|x, y| y.cmp(x)); - numbers - .iter() - .map(|n| PathBuf::from(format!("{}.{}", basepath.display(), n))) - .collect::>() + fn parse(&self, suffix: &str) -> Option { + suffix.parse::().ok() + } + fn too_old(&self, _suffix: &usize, file_number: usize) -> bool { + file_number >= self.max_files } } @@ -95,30 +81,23 @@ impl SuffixScheme for CountSuffix { /// - Neither `format` or the base filename can include the character `"."`. /// - The `format` should ensure that the lexical and chronological orderings are the same #[cfg(feature = "chrono04")] -pub struct TimestampSuffix { - /// None means that we don't know the files, and a scan is necessary. - pub(crate) suffixes: Option)>>, +pub struct TimestampSuffixScheme { format: &'static str, file_limit: FileLimit, } #[cfg(feature = "chrono04")] -impl TimestampSuffix { +impl TimestampSuffixScheme { /// With format `"%Y%m%dT%H%M%S"` pub fn default(file_limit: FileLimit) -> Self { Self { - suffixes: None, format: "%Y%m%dT%H%M%S", file_limit, } } - /// Create new TimestampSuffix suffix scheme + /// Create new TimestampSuffixScheme suffix scheme pub fn with_format(format: &'static str, file_limit: FileLimit) -> Self { - Self { - suffixes: None, - format, - file_limit, - } + Self { format, file_limit } } /// NOTE: For future use in RotationMode::Custom pub fn should_rotate(&self, age: Duration) -> impl Fn(&str) -> bool { @@ -128,144 +107,100 @@ impl TimestampSuffix { suffix < old_timestamp.as_str() } } - pub(crate) fn suffix_to_string(&self, suffix: &(String, Option)) -> String { - match suffix.1 { - Some(n) => format!("{}.{}", suffix.0, n), - None => suffix.0.clone(), +} + +/// Structured representation of the suffixes of TimestampSuffixScheme. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TimestampSuffix { + timestamp: String, + number: Option, +} +impl Representation for TimestampSuffix {} +impl Ord for TimestampSuffix { + fn cmp(&self, other: &Self) -> Ordering { + // Most recent = smallest (opposite as the timestamp Ord) + // Smallest = most recent. Thus, biggest timestamp first. And then biggest number + match other.timestamp.cmp(&self.timestamp) { + Ordering::Equal => other.number.cmp(&self.number), + unequal => unequal, } } - pub(crate) fn suffix_to_path( - &self, - basepath: &Path, - suffix: &(String, Option), - ) -> PathBuf { - PathBuf::from(format!( - "{}.{}", - basepath.display(), - self.suffix_to_string(suffix) - )) +} +impl PartialOrd for TimestampSuffix { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } - /// Scan files in the log directory to construct the list of files - fn ensure_suffix_list(&mut self, basepath: &Path) { - if self.suffixes.is_none() { - let mut suffixes = VecDeque::new(); - let filename_prefix = &*basepath - .file_name() - .expect("basepath.file_name()") - .to_string_lossy(); - let parent = basepath.parent().unwrap(); - let filenames = std::fs::read_dir(parent) - .unwrap() - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().is_file()) - .map(|entry| entry.file_name()); - for filename in filenames { - let filename = filename.to_string_lossy(); - if !filename.starts_with(&filename_prefix) { - continue; - } - // Find the up to two `.` in the filename - if let Some(first_dot) = filename.find('.') { - let suffix = &filename[(first_dot + 1)..]; - let (timestamp_str, n) = if let Some(second_dot) = suffix.find('.') { - if let Ok(n) = suffix[(second_dot + 1)..].parse::() { - (&suffix[..second_dot], Some(n)) - } else { - continue; - } - } else { - (suffix, None) - }; - if NaiveDateTime::parse_from_str(timestamp_str, self.format).is_ok() { - suffixes.push_back((timestamp_str.to_string(), n)) - } - } else { - // We don't consider the current (suffix-less) log file. - } - } - // Sort in Ascending order (higher value (most recent) first) - suffixes - .make_contiguous() - .sort_by_key(|suffix| self.suffix_to_string(suffix)); - self.suffixes = Some(suffixes); +} +impl std::fmt::Display for TimestampSuffix { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + match self.number { + Some(n) => write!(f, "{}.{}", self.timestamp, n), + None => write!(f, "{}", self.timestamp), } } } -#[cfg(feature = "chrono04")] -impl SuffixScheme for TimestampSuffix { - fn rotate(&mut self, basepath: &Path) -> String { - let now = Local::now().format(self.format).to_string(); - - self.ensure_suffix_list(basepath); - // For all existing suffixes that equals `now`, take the max `n`, and add one - let n = self - .suffixes - .as_ref() - .unwrap() - .iter() - .filter(|suffix| suffix.0 == now) - .map(|suffix| suffix.1.unwrap_or(0)) - .max() - .map(|n| n + 1); +#[cfg(feature = "chrono04")] +impl SuffixScheme for TimestampSuffixScheme { + type Repr = TimestampSuffix; - // Register the selected suffix as taken - self.suffixes.as_mut().unwrap().push_back((now.clone(), n)); + fn rotate_file( + &mut self, + _basepath: &Path, + newest_suffix: Option<&TimestampSuffix>, + suffix: &Option, + ) -> TimestampSuffix { + if suffix.is_none() { + let now = Local::now().format(self.format).to_string(); - // Remove old files - // Note that the oldest are the first in the list - let to_delete = match self.file_limit { - FileLimit::MaxFiles(max_files) => { - let n_files = self.suffixes.as_ref().unwrap().len(); - if n_files > max_files { - n_files - max_files + let number = if let Some(newest_suffix) = newest_suffix { + if newest_suffix.timestamp == now { + Some(newest_suffix.number.unwrap_or(0) + 1) } else { - 0 - } - } - FileLimit::Age(age) => { - let mut to_delete = 0; - for suffix in self.suffixes.as_ref().unwrap().iter() { - let old_timestamp = (Local::now() - age).format(self.format).to_string(); - let delete = suffix.0 < old_timestamp; - if delete { - let _ = std::fs::remove_file(self.suffix_to_path(basepath, suffix)); - to_delete += 1; - } else { - // Remember that `suffixes` has the oldest entries in the front, we can `break` - // once we find an entry that doesn't have to deleted - break; - } + None } - to_delete + } else { + None + }; + TimestampSuffix { + timestamp: now, + number } - }; - - // Delete respective entries - for _ in 0..to_delete { - self.suffixes.as_mut().unwrap().pop_front(); + } else { + // This rotation scheme dictates that only the main log file should ever be renamed. + // TODO: do something else than panic + panic!("programmer error in TimestampSuffixScheme::rotate_file") } - - self.suffix_to_string(&(now, n)) } - fn log_paths(&mut self, basepath: &Path) -> Vec { - self.ensure_suffix_list(basepath); - self.suffixes - .as_ref() - .unwrap() - .iter() - .map(|suffix| { - PathBuf::from(format!( - "{}.{}", - basepath.display(), - self.suffix_to_string(suffix) - )) + fn parse(&self, suffix: &str) -> Option { + let (timestamp_str, n) = if let Some(dot) = suffix.find('.') { + if let Ok(n) = suffix[(dot + 1)..].parse::() { + (&suffix[..dot], Some(n)) + } else { + return None; + } + } else { + (suffix, None) + }; + NaiveDateTime::parse_from_str(timestamp_str, self.format) + .map(|_| TimestampSuffix { + timestamp: timestamp_str.to_string(), + number: n, }) - .collect::>() + .ok() + } + fn too_old(&self, suffix: &TimestampSuffix, file_number: usize) -> bool { + match self.file_limit { + FileLimit::MaxFiles(max_files) => file_number >= max_files, + FileLimit::Age(age) => { + let old_timestamp = (Local::now() - age).format(self.format).to_string(); + suffix.timestamp < old_timestamp + } + } } } -/// How to determine if a file should be deleted, in the case of TimestampSuffix. +/// How to determine if a file should be deleted, in the case of TimestampSuffixScheme. #[cfg(feature = "chrono04")] pub enum FileLimit { /// Delete the oldest files if number of files is too high @@ -273,3 +208,29 @@ pub enum FileLimit { /// Delete files that have too old timestamp Age(Duration), } + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn timestamp_ordering() { + assert!( + TimestampSuffix { + timestamp: "2021".to_string(), + number: None + } < TimestampSuffix { + timestamp: "2020".to_string(), + number: None + } + ); + assert!( + TimestampSuffix { + timestamp: "2021".to_string(), + number: Some(1) + } < TimestampSuffix { + timestamp: "2021".to_string(), + number: None + } + ); + } +} From f34a416449bb5ef1cd43a26eaf3663ccfea1a91f Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Tue, 31 Aug 2021 16:15:50 +0200 Subject: [PATCH 07/46] test rotate_to_deleted_directory one write fails but subsequently works fine --- src/lib.rs | 75 +++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 43aa2d4..b3d64ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -239,14 +239,6 @@ pub struct FileRotate { suffixes: BTreeSet>, } -fn create_parent_dir(path: &Path) { - if let Some(dirname) = path.parent() { - if !dirname.exists() { - fs::create_dir_all(dirname).expect("create dir"); - } - } -} - impl FileRotate { /// Create a new [FileRotate]. /// @@ -272,15 +264,35 @@ impl FileRotate { }; let basepath = path.as_ref().to_path_buf(); - create_parent_dir(&basepath); + fs::create_dir_all(basepath.parent().unwrap()).expect("create dir"); - // Construct `suffixes` + let mut s = Self { + file: File::create(&basepath).ok(), + basepath, + content_limit, + count: 0, + suffixes: BTreeSet::new(), + suffix_scheme, + }; + s.scan_suffixes(); + s + } + fn ensure_log_directory_exists(&mut self) { + let path = self.basepath.parent().unwrap(); + if !path.exists() { + let _ = fs::create_dir_all(path).expect("create dir"); + let _ = File::create(&self.basepath); + self.scan_suffixes(); + } + } + fn scan_suffixes(&mut self) { let mut suffixes = BTreeSet::new(); - let filename_prefix = &*basepath + let filename_prefix = &*self + .basepath .file_name() .expect("basepath.file_name()") .to_string_lossy(); - let parent = basepath.parent().unwrap(); + let parent = self.basepath.parent().unwrap(); let filenames = std::fs::read_dir(parent) .unwrap() .filter_map(|entry| entry.ok()) @@ -293,19 +305,11 @@ impl FileRotate { } let (filename, compressed) = Self::prepare_filename(&*filename); let suffix_str = filename.strip_prefix(&format!("{}.", filename_prefix)); - if let Some(suffix) = suffix_str.and_then(|s| suffix_scheme.parse(s)) { + if let Some(suffix) = suffix_str.and_then(|s| self.suffix_scheme.parse(s)) { suffixes.insert(SuffixInfo { suffix, compressed }); } } - - Self { - file: File::create(&basepath).ok(), - basepath, - content_limit, - count: 0, - suffixes, - suffix_scheme, - } + self.suffixes = suffixes; } fn prepare_filename(path: &str) -> (&str, bool) { path.strip_prefix(".tar.gz") @@ -334,7 +338,9 @@ impl FileRotate { // have passed the internal BTreeMap itself, but it would require to make SuffixInfo `pub`. let newest_suffix = self.suffixes.iter().next().map(|info| &info.suffix); - let new_suffix = self.suffix_scheme.rotate_file(&self.basepath, newest_suffix, &suffix); + let new_suffix = self + .suffix_scheme + .rotate_file(&self.basepath, newest_suffix, &suffix); let new_path = new_suffix.to_path(&self.basepath); // Move destination file out of the way if it exists @@ -355,7 +361,7 @@ impl FileRotate { } fn rotate(&mut self) -> io::Result<()> { - create_parent_dir(&self.basepath); + self.ensure_log_directory_exists(); let _ = self.file.take(); @@ -409,7 +415,7 @@ impl Write for FileRotate { } self.count += buf.len(); if let Some(ref mut file) = self.file { - file.write_all(&buf)?; + file.write_all(buf)?; } } ContentLimit::Lines(lines) => { @@ -433,7 +439,7 @@ impl Write for FileRotate { self.rotate()? } if let Some(ref mut file) = self.file { - file.write_all(&buf)?; + file.write_all(buf)?; } self.count += buf.len(); } @@ -502,10 +508,7 @@ mod tests { log_paths_sorted.sort(); assert_eq!(log_paths, log_paths_sorted); - // println!("{:?}", log_paths); - // println!("{:?}", log_paths.iter().map(|path| fs::read_to_string(path)).collect::>()); - // println!("Main log: {}", fs::read_to_string(&log_path).unwrap()); - list(&tmp_dir.path()); + list(tmp_dir.path()); assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); @@ -588,10 +591,6 @@ mod tests { assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); } - // Currently not supported. May add support if it's important. - // Also consider removing `FileRotate::suffixes` and instead using more disk operations to - // check all files in the log directory on every rotation. - /* #[test] fn rotate_to_deleted_directory() { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); @@ -609,15 +608,15 @@ mod tests { let _ = fs::remove_dir_all(parent); - assert!(writeln!(log, "c").is_ok()); - + // Will fail to write `"c"` + writeln!(log, "c").unwrap(); log.flush().unwrap(); + // But the next `write` will succeed writeln!(log, "d").unwrap(); assert_eq!("", fs::read_to_string(&log_path).unwrap()); - assert_eq!("d\n", fs::read_to_string(&log.log_paths()[0]).unwrap()); + assert_eq!("d\n", fs::read_to_string(&log.log_paths()[1]).unwrap()); } - */ #[test] fn write_complete_record_until_bytes_surpassed() { From 802bedda8f218efb539239dbb4d180f5fafd8064 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Tue, 31 Aug 2021 16:34:28 +0200 Subject: [PATCH 08/46] Propagate IO errors --- src/lib.rs | 21 +++++++++++---------- src/suffix.rs | 29 ++++++++++++++++++----------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b3d64ea..7610627 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -331,8 +331,7 @@ impl FileRotate { /// Assumption: Any collision in file name is due to an old log file. /// /// Returns the suffix of the new file (the last suffix after possible cascade of renames). - // TODO remove panics - fn move_file_with_suffix(&mut self, suffix: Option) -> S::Repr { + fn move_file_with_suffix(&mut self, suffix: Option) -> io::Result { // NOTE: this newest_suffix is there only because TimestampSuffixScheme specifically needs // it. Otherwise it might not be necessary to provide this to `rotate_file`. We could also // have passed the internal BTreeMap itself, but it would require to make SuffixInfo `pub`. @@ -340,12 +339,12 @@ impl FileRotate { let new_suffix = self .suffix_scheme - .rotate_file(&self.basepath, newest_suffix, &suffix); + .rotate_file(&self.basepath, newest_suffix, &suffix)?; let new_path = new_suffix.to_path(&self.basepath); // Move destination file out of the way if it exists let newly_created_suffix = if new_path.exists() { - self.move_file_with_suffix(Some(new_suffix)) + self.move_file_with_suffix(Some(new_suffix))? } else { new_suffix }; @@ -356,8 +355,8 @@ impl FileRotate { None => self.basepath.clone(), }; - fs::rename(old_path, new_path).unwrap(); - newly_created_suffix + fs::rename(old_path, new_path)?; + Ok(newly_created_suffix) } fn rotate(&mut self) -> io::Result<()> { @@ -366,7 +365,7 @@ impl FileRotate { let _ = self.file.take(); // This function will always create a new file. Returns suffix of that file - let new_suffix = self.move_file_with_suffix(None); + let new_suffix = self.move_file_with_suffix(None)?; self.suffixes.insert(SuffixInfo { suffix: new_suffix, compressed: false, @@ -376,18 +375,19 @@ impl FileRotate { self.count = 0; - self.prune_old_files(); + self.prune_old_files()?; Ok(()) } - fn prune_old_files(&mut self) { + fn prune_old_files(&mut self) -> io::Result<()> { // Find the youngest suffix that is too old, and then remove all suffixes that are older or // equally old: let mut youngest_old = None; // Start from oldest suffix, stop when we find a suffix that is not too old + let mut result = Ok(()); for (i, suffix) in self.suffixes.iter().enumerate().rev() { if self.suffix_scheme.too_old(&suffix.suffix, i) { - let _ = std::fs::remove_file(suffix.to_path(&self.basepath)); + result = result.and(std::fs::remove_file(suffix.to_path(&self.basepath))); youngest_old = Some((*suffix).clone()); } else { break; @@ -397,6 +397,7 @@ impl FileRotate { // Removes all the too old let _ = self.suffixes.split_off(&youngest_old); } + result } } diff --git a/src/suffix.rs b/src/suffix.rs index 740e3c8..7064802 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -2,6 +2,7 @@ use chrono::{offset::Local, Duration, NaiveDateTime}; use std::{ cmp::Ordering, + io, path::{Path, PathBuf}, }; @@ -33,7 +34,7 @@ pub trait SuffixScheme { basepath: &Path, newest_suffix: Option<&Self::Repr>, suffix: &Option, - ) -> Self::Repr; + ) -> io::Result; /// Parse suffix from string. fn parse(&self, suffix: &str) -> Option; @@ -60,14 +61,16 @@ impl CountSuffix { impl Representation for usize {} impl SuffixScheme for CountSuffix { type Repr = usize; - fn rotate_file(&mut self, + fn rotate_file( + &mut self, _basepath: &Path, _: Option<&usize>, - suffix: &Option) -> usize { - match suffix { + suffix: &Option, + ) -> io::Result { + Ok(match suffix { Some(suffix) => suffix + 1, None => 1, - } + }) } fn parse(&self, suffix: &str) -> Option { suffix.parse::().ok() @@ -149,7 +152,8 @@ impl SuffixScheme for TimestampSuffixScheme { _basepath: &Path, newest_suffix: Option<&TimestampSuffix>, suffix: &Option, - ) -> TimestampSuffix { + ) -> io::Result { + assert!(suffix.is_none()); if suffix.is_none() { let now = Local::now().format(self.format).to_string(); @@ -162,14 +166,17 @@ impl SuffixScheme for TimestampSuffixScheme { } else { None }; - TimestampSuffix { + Ok(TimestampSuffix { timestamp: now, - number - } + number, + }) } else { // This rotation scheme dictates that only the main log file should ever be renamed. - // TODO: do something else than panic - panic!("programmer error in TimestampSuffixScheme::rotate_file") + // In debug build the above assert will catch this. + Err(io::Error::new( + io::ErrorKind::InvalidData, + "Critical error in file-rotate algorithm", + )) } } fn parse(&self, suffix: &str) -> Option { From 54b3396088227719e8434e529688a8550e97b744 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Wed, 1 Sep 2021 11:45:28 +0200 Subject: [PATCH 09/46] Compression --- Cargo.toml | 1 + src/compression.rs | 35 ++++++++- src/lib.rs | 175 +++++++++++++++++++++++++++++++++++++-------- src/suffix.rs | 6 +- 4 files changed, 181 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4be75fd..1267861 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ license = "MIT" [dependencies] chrono = { version = "0.4.11", optional = true } +flate2 = "1.0.21" [dev-dependencies] quickcheck = "0.9.2" diff --git a/src/compression.rs b/src/compression.rs index b4cb3c4..5e55093 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1,6 +1,35 @@ +use flate2::write::GzEncoder; +use std::{ + fs::{self, File, OpenOptions}, + io, + path::{Path, PathBuf}, +}; - +/// In the future, maybe stream compression pub enum Compression { - OnRotate (usize) - // In the future, maybe stream compression + /// No compression + None, + /// Look for files to compress when rotating. + /// First argument: How many files to keep uncompressed (excluding the original file) + OnRotate(usize), +} + +pub(crate) fn compress(path: &Path) -> io::Result<()> { + let dest_path = PathBuf::from(format!("{}.gz", path.display())); + + let mut src_file = File::open(path)?; + let dest_file = OpenOptions::new() + .write(true) + .create(true) + .append(false) + .open(&dest_path)?; + + assert!(path.exists()); + assert!(dest_path.exists()); + let mut encoder = GzEncoder::new(dest_file, flate2::Compression::default()); + io::copy(&mut src_file, &mut encoder)?; + + fs::remove_file(path)?; + + Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 7610627..29c47d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ //! We can rotate log files with the amount of lines as a limit, by using `ContentLimit::Lines`. //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; //! use std::{fs, io::Write}; //! //! // Create a new log writer. The first argument is anything resembling a path. The @@ -24,7 +24,7 @@ //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(2), ContentLimit::Lines(3)); +//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(2), ContentLimit::Lines(3), Compression::None); //! //! // Write a bunch of lines //! writeln!(log, "Line 1: Hello World!"); @@ -43,14 +43,14 @@ //! Another method of rotation is by bytes instead of lines, with `ContentLimit::Bytes`. //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; //! use std::{fs, io::Write}; //! //! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new("target/my-log-directory-bytes/my-log-file", CountSuffix::new(2), ContentLimit::Bytes(5)); +//! let mut log = FileRotate::new("target/my-log-directory-bytes/my-log-file", CountSuffix::new(2), ContentLimit::Bytes(5), Compression::None); //! //! writeln!(log, "Test file"); //! @@ -73,14 +73,14 @@ //! Here's an example with 1 byte limits: //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; //! use std::{fs, io::Write}; //! //! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(3), ContentLimit::Bytes(1)); +//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(3), ContentLimit::Bytes(1), Compression::None); //! //! write!(log, "A"); //! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); @@ -124,14 +124,15 @@ //! their timestamp (`FileLimit::Age`), or just maximum number of files (`FileLimit::MaxFiles`). //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffixScheme, FileLimit}}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffixScheme, FileLimit}, +//! compression::Compression}; //! use std::{fs, io::Write}; //! //! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), TimestampSuffixScheme::default(FileLimit::MaxFiles(2)), ContentLimit::Bytes(1)); +//! let mut log = FileRotate::new(log_path.clone(), TimestampSuffixScheme::default(FileLimit::MaxFiles(2)), ContentLimit::Bytes(1), Compression::None); //! //! write!(log, "A"); //! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); @@ -176,6 +177,7 @@ unused_qualifications )] +use compression::*; use std::{ cmp::Ordering, collections::BTreeSet, @@ -185,6 +187,8 @@ use std::{ }; use suffix::Representation; +/// Compression +pub mod compression; /// Suffix scheme etc pub mod suffix; @@ -200,17 +204,22 @@ pub enum ContentLimit { BytesSurpassed(usize), } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Eq)] struct SuffixInfo { pub suffix: Repr, pub compressed: bool, } +impl PartialEq for SuffixInfo { + fn eq(&self, other: &Self) -> bool { + self.suffix == other.suffix + } +} impl SuffixInfo { pub fn to_path(&self, basepath: &Path) -> PathBuf { let path = self.suffix.to_path(basepath); if self.compressed { - PathBuf::from(format!("{}.tar.gz", path.display())) + PathBuf::from(format!("{}.gz", path.display())) } else { path } @@ -234,8 +243,9 @@ pub struct FileRotate { file: Option, content_limit: ContentLimit, count: usize, + compression: Compression, suffix_scheme: S, - /// The bool is whether or not there's a .tar.gz suffix to the filename + /// The bool is whether or not there's a .gz suffix to the filename suffixes: BTreeSet>, } @@ -250,7 +260,12 @@ impl FileRotate { /// # Panics /// /// Panics if `bytes == 0` or `lines == 0`. - pub fn new>(path: P, suffix_scheme: S, content_limit: ContentLimit) -> Self { + pub fn new>( + path: P, + suffix_scheme: S, + content_limit: ContentLimit, + compression: Compression, + ) -> Self { match content_limit { ContentLimit::Bytes(bytes) => { assert!(bytes > 0); @@ -271,6 +286,7 @@ impl FileRotate { basepath, content_limit, count: 0, + compression, suffixes: BTreeSet::new(), suffix_scheme, }; @@ -312,7 +328,7 @@ impl FileRotate { self.suffixes = suffixes; } fn prepare_filename(path: &str) -> (&str, bool) { - path.strip_prefix(".tar.gz") + path.strip_prefix(".gz") .map(|x| (x, true)) .unwrap_or((path, false)) } @@ -331,31 +347,57 @@ impl FileRotate { /// Assumption: Any collision in file name is due to an old log file. /// /// Returns the suffix of the new file (the last suffix after possible cascade of renames). - fn move_file_with_suffix(&mut self, suffix: Option) -> io::Result { + fn move_file_with_suffix( + &mut self, + old_suffix_info: Option>, + ) -> io::Result> { // NOTE: this newest_suffix is there only because TimestampSuffixScheme specifically needs // it. Otherwise it might not be necessary to provide this to `rotate_file`. We could also // have passed the internal BTreeMap itself, but it would require to make SuffixInfo `pub`. + let newest_suffix = self.suffixes.iter().next().map(|info| &info.suffix); - let new_suffix = self - .suffix_scheme - .rotate_file(&self.basepath, newest_suffix, &suffix)?; - let new_path = new_suffix.to_path(&self.basepath); + let new_suffix = self.suffix_scheme.rotate_file( + &self.basepath, + newest_suffix, + &old_suffix_info.clone().map(|i| i.suffix), + )?; + + // The destination file/path eventual .gz suffix must match the source path + let new_suffix_info = SuffixInfo { + suffix: new_suffix, + compressed: old_suffix_info + .as_ref() + .map(|x| x.compressed) + .unwrap_or(false), + }; + let new_path = new_suffix_info.to_path(&self.basepath); + + // Whatever exists that would block a move to the new suffix + let existing_suffix_info = self.suffixes.get(&new_suffix_info).cloned(); // Move destination file out of the way if it exists - let newly_created_suffix = if new_path.exists() { - self.move_file_with_suffix(Some(new_suffix))? + let newly_created_suffix = if let Some(existing_suffix_info) = existing_suffix_info { + // We might move files in a way that the destination path doesn't equal the path that + // was replaced. Due to possible `.gz`, a "conflicting" file doesn't mean that paths + // are equal. + self.suffixes.replace(new_suffix_info); + // Recurse to move conflicting file. + self.move_file_with_suffix(Some(existing_suffix_info))? } else { - new_suffix + new_suffix_info }; - assert!(!new_path.exists()); - let old_path = match suffix { + let old_path = match old_suffix_info { Some(suffix) => suffix.to_path(&self.basepath), None => self.basepath.clone(), }; + // Do the move + assert!(old_path.exists()); + assert!(!new_path.exists()); fs::rename(old_path, new_path)?; + Ok(newly_created_suffix) } @@ -365,21 +407,18 @@ impl FileRotate { let _ = self.file.take(); // This function will always create a new file. Returns suffix of that file - let new_suffix = self.move_file_with_suffix(None)?; - self.suffixes.insert(SuffixInfo { - suffix: new_suffix, - compressed: false, - }); + let new_suffix_info = self.move_file_with_suffix(None)?; + self.suffixes.insert(new_suffix_info); self.file = Some(File::create(&self.basepath)?); self.count = 0; - self.prune_old_files()?; + self.handle_old_files()?; Ok(()) } - fn prune_old_files(&mut self) -> io::Result<()> { + fn handle_old_files(&mut self) -> io::Result<()> { // Find the youngest suffix that is too old, and then remove all suffixes that are older or // equally old: let mut youngest_old = None; @@ -397,6 +436,31 @@ impl FileRotate { // Removes all the too old let _ = self.suffixes.split_off(&youngest_old); } + + // Compression + if let Compression::OnRotate(max_file_n) = self.compression { + let n = (self.suffixes.len() as i32 - max_file_n as i32).max(0) as usize; + // The oldest N files should be compressed + let suffixes_to_compress = self + .suffixes + .iter() + .rev() + .take(n) + .filter(|info| !info.compressed) + .cloned() + .collect::>(); + for info in suffixes_to_compress { + // Do the compression + let path = info.suffix.to_path(&self.basepath); + compress(&path)?; + + self.suffixes.replace(SuffixInfo { + compressed: true, + ..info + }); + } + } + result } } @@ -458,7 +522,7 @@ impl Write for FileRotate { #[cfg(test)] mod tests { - use super::{suffix::*, *}; + use super::{compression::*, suffix::*, *}; use tempdir::TempDir; // Just useful to debug why test doesn't succeed @@ -482,6 +546,7 @@ mod tests { &log_path, TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), ContentLimit::Lines(2), + Compression::None, ); // Write 9 lines @@ -538,6 +603,7 @@ mod tests { &*log_path.to_string_lossy(), TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))), ContentLimit::Lines(1), + Compression::None, ); writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); @@ -562,6 +628,7 @@ mod tests { &*log_path.to_string_lossy(), CountSuffix::new(4), ContentLimit::Lines(2), + Compression::None, ); // Write 9 lines @@ -601,6 +668,7 @@ mod tests { &*log_path.to_string_lossy(), CountSuffix::new(4), ContentLimit::Lines(1), + Compression::None, ); write!(log, "a\nb\n").unwrap(); @@ -629,6 +697,7 @@ mod tests { &log_path, TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), ContentLimit::BytesSurpassed(1), + Compression::None, ); write!(log, "0123456789").unwrap(); @@ -643,6 +712,48 @@ mod tests { assert!(&log.log_paths()[0].exists()); } + #[test] + fn compression_on_rotation() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(3), + ContentLimit::Lines(1), + Compression::OnRotate(1), // Keep one file uncompressed + ); + + writeln!(log, "A").unwrap(); + writeln!(log, "B").unwrap(); + writeln!(log, "C").unwrap(); + list(tmp_dir.path()); + + let log_paths = log.log_paths(); + + assert_eq!( + log_paths, + vec![ + parent.join("log.3.gz"), + parent.join("log.2.gz"), + parent.join("log.1"), + ] + ); + + assert_eq!("", fs::read_to_string(&log_path).unwrap()); + + fn compress(text: &str) -> Vec { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + + encoder.write_all(text.as_bytes()).unwrap(); + encoder.finish().unwrap() + } + assert_eq!(compress("A\n"), fs::read(&log.log_paths()[0]).unwrap()); + assert_eq!(compress("B\n"), fs::read(&log.log_paths()[1]).unwrap()); + assert_eq!("C\n", fs::read_to_string(&log.log_paths()[2]).unwrap()); + } + #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); @@ -654,6 +765,7 @@ mod tests { &log_path, TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), ContentLimit::Lines(count), + Compression::None, ); for _ in 0..count - 1 { @@ -677,6 +789,7 @@ mod tests { &log_path, TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), ContentLimit::Bytes(count), + Compression::None, ); for _ in 0..count { diff --git a/src/suffix.rs b/src/suffix.rs index 7064802..cc55efd 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -41,7 +41,7 @@ pub trait SuffixScheme { /// Whether either the suffix or the chronological file number indicates that the file is old /// and should be deleted, depending of course on the file limit. - /// `file_number` starts at 0. + /// `file_number` starts at 0 for the most recent suffix. fn too_old(&self, suffix: &Self::Repr, file_number: usize) -> bool; } @@ -52,7 +52,9 @@ pub struct CountSuffix { } impl CountSuffix { - /// New CountSuffix + /// New CountSuffix, deleting files when the total number of files exceeds `max_files`. + /// For example, if max_files is 3, then the files `log`, `log.1`, `log.2`, `log.3` may exist + /// but not `log.4`. In other words, `max_files` determines the largest possible suffix number. pub fn new(max_files: usize) -> Self { Self { max_files } } From 213f6acd4e2e5794ebc376e5b2ce27cfa05d4f85 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Wed, 1 Sep 2021 15:31:13 +0200 Subject: [PATCH 10/46] Don't truncate log file in FileRotate::new --- src/lib.rs | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 29c47d8..dc0affc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -181,7 +181,7 @@ use compression::*; use std::{ cmp::Ordering, collections::BTreeSet, - fs::{self, File}, + fs::{self, File, OpenOptions}, io::{self, Write}, path::{Path, PathBuf}, }; @@ -282,7 +282,7 @@ impl FileRotate { fs::create_dir_all(basepath.parent().unwrap()).expect("create dir"); let mut s = Self { - file: File::create(&basepath).ok(), + file: None, basepath, content_limit, count: 0, @@ -290,6 +290,7 @@ impl FileRotate { suffixes: BTreeSet::new(), suffix_scheme, }; + s.ensure_log_directory_exists(); s.scan_suffixes(); s } @@ -297,9 +298,16 @@ impl FileRotate { let path = self.basepath.parent().unwrap(); if !path.exists() { let _ = fs::create_dir_all(path).expect("create dir"); - let _ = File::create(&self.basepath); self.scan_suffixes(); } + if !self.basepath.exists() || self.file.is_none() { + self.file = OpenOptions::new() + .write(true) + .create(true) + .append(true) + .open(&self.basepath) + .ok(); + } } fn scan_suffixes(&mut self) { let mut suffixes = BTreeSet::new(); @@ -528,13 +536,16 @@ mod tests { // Just useful to debug why test doesn't succeed #[allow(dead_code)] fn list(dir: &Path) { - let filenames = fs::read_dir(dir) + let files = fs::read_dir(dir) .unwrap() .filter_map(|entry| entry.ok()) .filter(|entry| entry.path().is_file()) - .map(|entry| entry.file_name()) + .map(|entry| (entry.file_name(), fs::read_to_string(entry.path()))) .collect::>(); - println!("Files on disk: {:?}", filenames); + println!("Files on disk:"); + for (name, content) in files { + println!("{:?}: {:?}", name, content); + } } #[test] @@ -754,6 +765,28 @@ mod tests { assert_eq!("C\n", fs::read_to_string(&log.log_paths()[2]).unwrap()); } + #[test] + fn no_truncate() { + // Don't truncate log file if it already exists + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let file_rotate = || { + FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(3), + ContentLimit::Lines(10000), + Compression::None, + ) + }; + writeln!(file_rotate(), "A").unwrap(); + list(parent); + writeln!(file_rotate(), "B").unwrap(); + list(parent); + + assert_eq!("A\nB\n", fs::read_to_string(&log_path).unwrap()); + } + #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); From 85e4619fe9fdf0abeb4942a521a66097b909b526 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 2 Sep 2021 11:47:55 +0200 Subject: [PATCH 11/46] Remove patch number from flate2 dep --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1267861..82eb9b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" [dependencies] chrono = { version = "0.4.11", optional = true } -flate2 = "1.0.21" +flate2 = "1.0" [dev-dependencies] quickcheck = "0.9.2" From f4e84135c0922ebb5fae424b68910f27a0458ae5 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 2 Sep 2021 09:47:05 +0200 Subject: [PATCH 12/46] Move tests to their own file --- src/lib.rs | 310 +-------------------------------------------------- src/tests.rs | 304 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 308 deletions(-) create mode 100644 src/tests.rs diff --git a/src/lib.rs b/src/lib.rs index dc0affc..5c2c67c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -191,6 +191,8 @@ use suffix::Representation; pub mod compression; /// Suffix scheme etc pub mod suffix; +#[cfg(test)] +mod tests; // --- @@ -527,311 +529,3 @@ impl Write for FileRotate { .unwrap_or(Ok(())) } } - -#[cfg(test)] -mod tests { - use super::{compression::*, suffix::*, *}; - use tempdir::TempDir; - - // Just useful to debug why test doesn't succeed - #[allow(dead_code)] - fn list(dir: &Path) { - let files = fs::read_dir(dir) - .unwrap() - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().is_file()) - .map(|entry| (entry.file_name(), fs::read_to_string(entry.path()))) - .collect::>(); - println!("Files on disk:"); - for (name, content) in files { - println!("{:?}: {:?}", name, content); - } - } - - #[test] - fn timestamp_max_files_rotation() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let log_path = tmp_dir.path().join("log"); - - let mut log = FileRotate::new( - &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), - ContentLimit::Lines(2), - Compression::None, - ); - - // Write 9 lines - // This should result in 5 files in total (4 rotated files). The main file will have one line. - write!(log, "a\nb\nc\nd\ne\nf\ng\nh\ni\n").unwrap(); - let log_paths = log.log_paths(); - assert_eq!(log_paths.len(), 4); - - // Log names should be sorted. Low (old timestamp) to high (more recent timestamp) - let mut log_paths_sorted = log_paths.clone(); - log_paths_sorted.sort(); - assert_eq!(log_paths, log_paths_sorted); - - assert_eq!("a\nb\n", fs::read_to_string(&log_paths[0]).unwrap()); - assert_eq!("c\nd\n", fs::read_to_string(&log_paths[1]).unwrap()); - assert_eq!("e\nf\n", fs::read_to_string(&log_paths[2]).unwrap()); - assert_eq!("g\nh\n", fs::read_to_string(&log_paths[3]).unwrap()); - assert_eq!("i\n", fs::read_to_string(&log_path).unwrap()); - - // Write 4 more lines - write!(log, "j\nk\nl\nm\n").unwrap(); - let log_paths = log.log_paths(); - assert_eq!(log_paths.len(), 4); - let mut log_paths_sorted = log_paths.clone(); - log_paths_sorted.sort(); - assert_eq!(log_paths, log_paths_sorted); - - list(tmp_dir.path()); - assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); - assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); - assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); - assert_eq!("k\nl\n", fs::read_to_string(&log_paths[3]).unwrap()); - assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); - } - #[test] - #[cfg(feature = "chrono04")] - fn timestamp_max_age_deletion() { - // In order not to have to sleep, and keep it deterministic, let's already create the log files and see how FileRotate - // cleans up the old ones. - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let dir = tmp_dir.path(); - let log_path = dir.join("log"); - - // One recent file: - let recent_file = chrono::offset::Local::now() - .format("log.%Y%m%dT%H%M%S") - .to_string(); - File::create(dir.join(&recent_file)).unwrap(); - // Two very old files: - File::create(dir.join("log.20200825T151133")).unwrap(); - File::create(dir.join("log.20200825T151133.1")).unwrap(); - - let mut log = FileRotate::new( - &*log_path.to_string_lossy(), - TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))), - ContentLimit::Lines(1), - Compression::None, - ); - writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); - - let mut filenames = std::fs::read_dir(dir) - .unwrap() - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().is_file()) - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .collect::>(); - filenames.sort(); - assert!(filenames.contains(&"log".to_string())); - assert!(filenames.contains(&recent_file)); - assert!(!filenames.contains(&"log.20200825T151133".to_string())); - assert!(!filenames.contains(&"log.20200825T151133.1".to_string())); - } - #[test] - fn count_max_files_rotation() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let parent = tmp_dir.path(); - let log_path = parent.join("log"); - let mut log = FileRotate::new( - &*log_path.to_string_lossy(), - CountSuffix::new(4), - ContentLimit::Lines(2), - Compression::None, - ); - - // Write 9 lines - // This should result in 5 files in total (4 rotated files). The main file will have one line. - write!(log, "a\nb\nc\nd\ne\nf\ng\nh\ni\n").unwrap(); // 9 lines - let log_paths = vec![ - parent.join("log.4"), - parent.join("log.3"), - parent.join("log.2"), - parent.join("log.1"), - ]; - assert_eq!(log_paths, log.log_paths()); - assert_eq!("a\nb\n", fs::read_to_string(&log_paths[0]).unwrap()); - assert_eq!("c\nd\n", fs::read_to_string(&log_paths[1]).unwrap()); - assert_eq!("e\nf\n", fs::read_to_string(&log_paths[2]).unwrap()); - assert_eq!("g\nh\n", fs::read_to_string(&log_paths[3]).unwrap()); - assert_eq!("i\n", fs::read_to_string(&log_path).unwrap()); - - // Write 4 more lines - write!(log, "j\nk\nl\nm\n").unwrap(); - list(parent); - assert_eq!(log_paths, log.log_paths()); - - assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); - assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); - assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); - assert_eq!("k\nl\n", fs::read_to_string(&log_paths[3]).unwrap()); - assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); - } - - #[test] - fn rotate_to_deleted_directory() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let parent = tmp_dir.path(); - let log_path = parent.join("log"); - let mut log = FileRotate::new( - &*log_path.to_string_lossy(), - CountSuffix::new(4), - ContentLimit::Lines(1), - Compression::None, - ); - - write!(log, "a\nb\n").unwrap(); - assert_eq!("", fs::read_to_string(&log_path).unwrap()); - assert_eq!("a\n", fs::read_to_string(&log.log_paths()[0]).unwrap()); - - let _ = fs::remove_dir_all(parent); - - // Will fail to write `"c"` - writeln!(log, "c").unwrap(); - log.flush().unwrap(); - - // But the next `write` will succeed - writeln!(log, "d").unwrap(); - assert_eq!("", fs::read_to_string(&log_path).unwrap()); - assert_eq!("d\n", fs::read_to_string(&log.log_paths()[1]).unwrap()); - } - - #[test] - fn write_complete_record_until_bytes_surpassed() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let dir = tmp_dir.path(); - let log_path = dir.join("log"); - - let mut log = FileRotate::new( - &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), - ContentLimit::BytesSurpassed(1), - Compression::None, - ); - - write!(log, "0123456789").unwrap(); - log.flush().unwrap(); - assert!(log_path.exists()); - // shouldn't exist yet - because entire record was written in one shot - assert!(log.log_paths().is_empty()); - - // This should create the second file - write!(log, "0123456789").unwrap(); - log.flush().unwrap(); - assert!(&log.log_paths()[0].exists()); - } - - #[test] - fn compression_on_rotation() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let parent = tmp_dir.path(); - let log_path = parent.join("log"); - let mut log = FileRotate::new( - &*log_path.to_string_lossy(), - CountSuffix::new(3), - ContentLimit::Lines(1), - Compression::OnRotate(1), // Keep one file uncompressed - ); - - writeln!(log, "A").unwrap(); - writeln!(log, "B").unwrap(); - writeln!(log, "C").unwrap(); - list(tmp_dir.path()); - - let log_paths = log.log_paths(); - - assert_eq!( - log_paths, - vec![ - parent.join("log.3.gz"), - parent.join("log.2.gz"), - parent.join("log.1"), - ] - ); - - assert_eq!("", fs::read_to_string(&log_path).unwrap()); - - fn compress(text: &str) -> Vec { - let mut encoder = - flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - - encoder.write_all(text.as_bytes()).unwrap(); - encoder.finish().unwrap() - } - assert_eq!(compress("A\n"), fs::read(&log.log_paths()[0]).unwrap()); - assert_eq!(compress("B\n"), fs::read(&log.log_paths()[1]).unwrap()); - assert_eq!("C\n", fs::read_to_string(&log.log_paths()[2]).unwrap()); - } - - #[test] - fn no_truncate() { - // Don't truncate log file if it already exists - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let parent = tmp_dir.path(); - let log_path = parent.join("log"); - let file_rotate = || { - FileRotate::new( - &*log_path.to_string_lossy(), - CountSuffix::new(3), - ContentLimit::Lines(10000), - Compression::None, - ) - }; - writeln!(file_rotate(), "A").unwrap(); - list(parent); - writeln!(file_rotate(), "B").unwrap(); - list(parent); - - assert_eq!("A\nB\n", fs::read_to_string(&log_path).unwrap()); - } - - #[quickcheck_macros::quickcheck] - fn arbitrary_lines(count: usize) { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let dir = tmp_dir.path(); - let log_path = dir.join("log"); - - let count = count.max(1); - let mut log = FileRotate::new( - &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), - ContentLimit::Lines(count), - Compression::None, - ); - - for _ in 0..count - 1 { - writeln!(log).unwrap(); - } - - log.flush().unwrap(); - assert!(log.log_paths().is_empty()); - writeln!(log).unwrap(); - assert!(Path::new(&log.log_paths()[0]).exists()); - } - - #[quickcheck_macros::quickcheck] - fn arbitrary_bytes(count: usize) { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); - let dir = tmp_dir.path(); - let log_path = dir.join("log"); - - let count = count.max(1); - let mut log = FileRotate::new( - &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), - ContentLimit::Bytes(count), - Compression::None, - ); - - for _ in 0..count { - write!(log, "0").unwrap(); - } - - log.flush().unwrap(); - assert!(log.log_paths().is_empty()); - write!(log, "1").unwrap(); - assert!(&log.log_paths()[0].exists()); - } -} diff --git a/src/tests.rs b/src/tests.rs new file mode 100644 index 0000000..0852189 --- /dev/null +++ b/src/tests.rs @@ -0,0 +1,304 @@ +use super::{compression::*, suffix::*, *}; +use tempdir::TempDir; + +// Just useful to debug why test doesn't succeed +#[allow(dead_code)] +fn list(dir: &Path) { + let files = fs::read_dir(dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| (entry.file_name(), fs::read_to_string(entry.path()))) + .collect::>(); + println!("Files on disk:"); + for (name, content) in files { + println!("{:?}: {:?}", name, content); + } +} + +#[test] +fn timestamp_max_files_rotation() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let log_path = tmp_dir.path().join("log"); + + let mut log = FileRotate::new( + &log_path, + TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), + ContentLimit::Lines(2), + Compression::None, + ); + + // Write 9 lines + // This should result in 5 files in total (4 rotated files). The main file will have one line. + write!(log, "a\nb\nc\nd\ne\nf\ng\nh\ni\n").unwrap(); + let log_paths = log.log_paths(); + assert_eq!(log_paths.len(), 4); + + // Log names should be sorted. Low (old timestamp) to high (more recent timestamp) + let mut log_paths_sorted = log_paths.clone(); + log_paths_sorted.sort(); + assert_eq!(log_paths, log_paths_sorted); + + assert_eq!("a\nb\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("c\nd\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("i\n", fs::read_to_string(&log_path).unwrap()); + + // Write 4 more lines + write!(log, "j\nk\nl\nm\n").unwrap(); + let log_paths = log.log_paths(); + assert_eq!(log_paths.len(), 4); + let mut log_paths_sorted = log_paths.clone(); + log_paths_sorted.sort(); + assert_eq!(log_paths, log_paths_sorted); + + list(tmp_dir.path()); + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("k\nl\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); +} +#[test] +#[cfg(feature = "chrono04")] +fn timestamp_max_age_deletion() { + // In order not to have to sleep, and keep it deterministic, let's already create the log files and see how FileRotate + // cleans up the old ones. + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + // One recent file: + let recent_file = chrono::offset::Local::now() + .format("log.%Y%m%dT%H%M%S") + .to_string(); + File::create(dir.join(&recent_file)).unwrap(); + // Two very old files: + File::create(dir.join("log.20200825T151133")).unwrap(); + File::create(dir.join("log.20200825T151133.1")).unwrap(); + + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))), + ContentLimit::Lines(1), + Compression::None, + ); + writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); + + let mut filenames = std::fs::read_dir(dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect::>(); + filenames.sort(); + assert!(filenames.contains(&"log".to_string())); + assert!(filenames.contains(&recent_file)); + assert!(!filenames.contains(&"log.20200825T151133".to_string())); + assert!(!filenames.contains(&"log.20200825T151133.1".to_string())); +} +#[test] +fn count_max_files_rotation() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(4), + ContentLimit::Lines(2), + Compression::None, + ); + + // Write 9 lines + // This should result in 5 files in total (4 rotated files). The main file will have one line. + write!(log, "a\nb\nc\nd\ne\nf\ng\nh\ni\n").unwrap(); // 9 lines + let log_paths = vec![ + parent.join("log.4"), + parent.join("log.3"), + parent.join("log.2"), + parent.join("log.1"), + ]; + assert_eq!(log_paths, log.log_paths()); + assert_eq!("a\nb\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("c\nd\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("i\n", fs::read_to_string(&log_path).unwrap()); + + // Write 4 more lines + write!(log, "j\nk\nl\nm\n").unwrap(); + list(parent); + assert_eq!(log_paths, log.log_paths()); + + assert_eq!("e\nf\n", fs::read_to_string(&log_paths[0]).unwrap()); + assert_eq!("g\nh\n", fs::read_to_string(&log_paths[1]).unwrap()); + assert_eq!("i\nj\n", fs::read_to_string(&log_paths[2]).unwrap()); + assert_eq!("k\nl\n", fs::read_to_string(&log_paths[3]).unwrap()); + assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); +} + +#[test] +fn rotate_to_deleted_directory() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(4), + ContentLimit::Lines(1), + Compression::None, + ); + + write!(log, "a\nb\n").unwrap(); + assert_eq!("", fs::read_to_string(&log_path).unwrap()); + assert_eq!("a\n", fs::read_to_string(&log.log_paths()[0]).unwrap()); + + let _ = fs::remove_dir_all(parent); + + // Will fail to write `"c"` + writeln!(log, "c").unwrap(); + log.flush().unwrap(); + + // But the next `write` will succeed + writeln!(log, "d").unwrap(); + assert_eq!("", fs::read_to_string(&log_path).unwrap()); + assert_eq!("d\n", fs::read_to_string(&log.log_paths()[1]).unwrap()); +} + +#[test] +fn write_complete_record_until_bytes_surpassed() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + let mut log = FileRotate::new( + &log_path, + TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), + ContentLimit::BytesSurpassed(1), + Compression::None, + ); + + write!(log, "0123456789").unwrap(); + log.flush().unwrap(); + assert!(log_path.exists()); + // shouldn't exist yet - because entire record was written in one shot + assert!(log.log_paths().is_empty()); + + // This should create the second file + write!(log, "0123456789").unwrap(); + log.flush().unwrap(); + assert!(&log.log_paths()[0].exists()); +} + +#[test] +fn compression_on_rotation() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(3), + ContentLimit::Lines(1), + Compression::OnRotate(1), // Keep one file uncompressed + ); + + writeln!(log, "A").unwrap(); + writeln!(log, "B").unwrap(); + writeln!(log, "C").unwrap(); + list(tmp_dir.path()); + + let log_paths = log.log_paths(); + + assert_eq!( + log_paths, + vec![ + parent.join("log.3.gz"), + parent.join("log.2.gz"), + parent.join("log.1"), + ] + ); + + assert_eq!("", fs::read_to_string(&log_path).unwrap()); + + fn compress(text: &str) -> Vec { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + + encoder.write_all(text.as_bytes()).unwrap(); + encoder.finish().unwrap() + } + assert_eq!(compress("A\n"), fs::read(&log.log_paths()[0]).unwrap()); + assert_eq!(compress("B\n"), fs::read(&log.log_paths()[1]).unwrap()); + assert_eq!("C\n", fs::read_to_string(&log.log_paths()[2]).unwrap()); +} + +#[test] +fn no_truncate() { + // Don't truncate log file if it already exists + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let file_rotate = || { + FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(3), + ContentLimit::Lines(10000), + Compression::None, + ) + }; + writeln!(file_rotate(), "A").unwrap(); + list(parent); + writeln!(file_rotate(), "B").unwrap(); + list(parent); + + assert_eq!("A\nB\n", fs::read_to_string(&log_path).unwrap()); +} + +#[quickcheck_macros::quickcheck] +fn arbitrary_lines(count: usize) { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + let count = count.max(1); + let mut log = FileRotate::new( + &log_path, + TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), + ContentLimit::Lines(count), + Compression::None, + ); + + for _ in 0..count - 1 { + writeln!(log).unwrap(); + } + + log.flush().unwrap(); + assert!(log.log_paths().is_empty()); + writeln!(log).unwrap(); + assert!(Path::new(&log.log_paths()[0]).exists()); +} + +#[quickcheck_macros::quickcheck] +fn arbitrary_bytes(count: usize) { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + let count = count.max(1); + let mut log = FileRotate::new( + &log_path, + TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), + ContentLimit::Bytes(count), + Compression::None, + ); + + for _ in 0..count { + write!(log, "0").unwrap(); + } + + log.flush().unwrap(); + assert!(log.log_paths().is_empty()); + write!(log, "1").unwrap(); + assert!(&log.log_paths()[0].exists()); +} From 21c1d89f80476bd969aab7efa785832c600458a0 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 2 Sep 2021 11:47:22 +0200 Subject: [PATCH 13/46] Expose SuffixInfo and move scan_suffix fn into SuffixScheme --- src/compression.rs | 1 + src/lib.rs | 45 +++++++++++---------------------------------- src/suffix.rs | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 5e55093..0eda4fe 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -6,6 +6,7 @@ use std::{ }; /// In the future, maybe stream compression +#[derive(Debug, Clone)] pub enum Compression { /// No compression None, diff --git a/src/lib.rs b/src/lib.rs index 5c2c67c..942bcd9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -185,7 +185,7 @@ use std::{ io::{self, Write}, path::{Path, PathBuf}, }; -use suffix::Representation; +use suffix::*; /// Compression pub mod compression; @@ -197,6 +197,7 @@ mod tests; // --- /// When to move files: Condition on which a file is rotated. +#[derive(Clone, Debug)] pub enum ContentLimit { /// Cut the log at the exact size in bytes. Bytes(usize), @@ -206,9 +207,12 @@ pub enum ContentLimit { BytesSurpassed(usize), } +/// Used mostly internally. Info about suffix + compressed state. #[derive(Clone, Debug, Eq)] -struct SuffixInfo { +pub struct SuffixInfo { + /// Suffix pub suffix: Repr, + /// Whether there is a `.gz` suffix after the suffix pub compressed: bool, } impl PartialEq for SuffixInfo { @@ -218,6 +222,7 @@ impl PartialEq for SuffixInfo { } impl SuffixInfo { + /// Append this suffix (and eventual `.gz`) to a path pub fn to_path(&self, basepath: &Path) -> PathBuf { let path = self.suffix.to_path(basepath); if self.compressed { @@ -240,7 +245,7 @@ impl PartialOrd for SuffixInfo { } /// The main writer used for rotating logs. -pub struct FileRotate { +pub struct FileRotate { basepath: PathBuf, file: Option, content_limit: ContentLimit, @@ -251,7 +256,7 @@ pub struct FileRotate { suffixes: BTreeSet>, } -impl FileRotate { +impl FileRotate { /// Create a new [FileRotate]. /// /// The basename of the `path` is used to create new log files by appending an extension of the @@ -312,35 +317,7 @@ impl FileRotate { } } fn scan_suffixes(&mut self) { - let mut suffixes = BTreeSet::new(); - let filename_prefix = &*self - .basepath - .file_name() - .expect("basepath.file_name()") - .to_string_lossy(); - let parent = self.basepath.parent().unwrap(); - let filenames = std::fs::read_dir(parent) - .unwrap() - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().is_file()) - .map(|entry| entry.file_name()); - for filename in filenames { - let filename = filename.to_string_lossy(); - if !filename.starts_with(&filename_prefix) { - continue; - } - let (filename, compressed) = Self::prepare_filename(&*filename); - let suffix_str = filename.strip_prefix(&format!("{}.", filename_prefix)); - if let Some(suffix) = suffix_str.and_then(|s| self.suffix_scheme.parse(s)) { - suffixes.insert(SuffixInfo { suffix, compressed }); - } - } - self.suffixes = suffixes; - } - fn prepare_filename(path: &str) -> (&str, bool) { - path.strip_prefix(".gz") - .map(|x| (x, true)) - .unwrap_or((path, false)) + self.suffixes = self.suffix_scheme.scan_suffixes(&self.basepath); } /// Get paths of rotated log files (excluding the original/current log file), ordered from /// oldest to most recent @@ -475,7 +452,7 @@ impl FileRotate { } } -impl Write for FileRotate { +impl Write for FileRotate { fn write(&mut self, mut buf: &[u8]) -> io::Result { let written = buf.len(); match self.content_limit { diff --git a/src/suffix.rs b/src/suffix.rs index cc55efd..cd58d46 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -1,7 +1,9 @@ +use crate::SuffixInfo; #[cfg(feature = "chrono04")] use chrono::{offset::Local, Duration, NaiveDateTime}; use std::{ cmp::Ordering, + collections::BTreeSet, io, path::{Path, PathBuf}, }; @@ -43,6 +45,39 @@ pub trait SuffixScheme { /// and should be deleted, depending of course on the file limit. /// `file_number` starts at 0 for the most recent suffix. fn too_old(&self, suffix: &Self::Repr, file_number: usize) -> bool; + + /// Find all files in the basepath.parent() directory that has path equal to basepath + a valid + /// suffix. Return sorted collection - sorted from most recent to oldest. + fn scan_suffixes(&self, basepath: &Path) -> BTreeSet> { + let mut suffixes = BTreeSet::new(); + let filename_prefix = basepath + .file_name() + .expect("basepath.file_name()") + .to_string_lossy(); + let parent = basepath.parent().unwrap(); + let filenames = std::fs::read_dir(parent) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name()); + for filename in filenames { + let filename = filename.to_string_lossy(); + if !filename.starts_with(&*filename_prefix) { + continue; + } + let (filename, compressed) = prepare_filename(&*filename); + let suffix_str = filename.strip_prefix(&format!("{}.", filename_prefix)); + if let Some(suffix) = suffix_str.and_then(|s| self.parse(s)) { + suffixes.insert(SuffixInfo { suffix, compressed }); + } + } + suffixes + } +} +fn prepare_filename(path: &str) -> (&str, bool) { + path.strip_prefix(".gz") + .map(|x| (x, true)) + .unwrap_or((path, false)) } /// Rotated log files get a number as suffix. The greater the number, the older. The oldest files From c65813379dfb5fa7158c4afa66896be43a4be046 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Tue, 14 Sep 2021 16:18:15 +0200 Subject: [PATCH 14/46] Fix bug in scan_suffixes, that doesn't register the compressed filenames And add test --- src/suffix.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/suffix.rs b/src/suffix.rs index cd58d46..96f3d41 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -75,7 +75,7 @@ pub trait SuffixScheme { } } fn prepare_filename(path: &str) -> (&str, bool) { - path.strip_prefix(".gz") + path.strip_suffix(".gz") .map(|x| (x, true)) .unwrap_or((path, false)) } @@ -256,6 +256,7 @@ pub enum FileLimit { #[cfg(test)] mod test { use super::*; + use std::fs::File; #[test] fn timestamp_ordering() { assert!( @@ -277,4 +278,18 @@ mod test { } ); } + + #[test] + fn scan_suffixes() { + let directory = tempdir::TempDir::new("file-rotate").unwrap(); + let directory = directory.path(); + let log_path = directory.join("logs"); + std::fs::create_dir_all(&log_path).unwrap(); + File::create(log_path.join("all.log.20210911T121830")).unwrap(); + File::create(log_path.join("all.log.20210911T121831.gz")).unwrap(); + + let suffix_scheme = TimestampSuffixScheme::default(FileLimit::Age(Duration::weeks(1))); + let paths = suffix_scheme.scan_suffixes(&log_path.join("all.log")); + assert_eq!(paths.len(), 2); + } } From 85eee2d757eafb676cbae3cea920178bb75be62a Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Fri, 10 Dec 2021 16:30:09 +0100 Subject: [PATCH 15/46] Make fields of TimestampSuffix public --- src/suffix.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/suffix.rs b/src/suffix.rs index 96f3d41..395b034 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -152,8 +152,8 @@ impl TimestampSuffixScheme { /// Structured representation of the suffixes of TimestampSuffixScheme. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TimestampSuffix { - timestamp: String, - number: Option, + pub timestamp: String, + pub number: Option, } impl Representation for TimestampSuffix {} impl Ord for TimestampSuffix { From a0cad14053990fd47bba4cd3c5027ea1af407276 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Mon, 13 Dec 2021 10:00:11 +0100 Subject: [PATCH 16/46] Bugfix: If log file already exists, update self.count to reflect its size --- src/lib.rs | 7 +++++++ src/suffix.rs | 2 ++ src/tests.rs | 32 +++++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 942bcd9..b593127 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -308,6 +308,13 @@ impl FileRotate { self.scan_suffixes(); } if !self.basepath.exists() || self.file.is_none() { + // Update `count` + if let Ok(metadata) = self.basepath.metadata() { + self.count = metadata.len() as usize; + } else { + self.count = 0; + } + // Create new file self.file = OpenOptions::new() .write(true) .create(true) diff --git a/src/suffix.rs b/src/suffix.rs index 395b034..0d27f4f 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -152,7 +152,9 @@ impl TimestampSuffixScheme { /// Structured representation of the suffixes of TimestampSuffixScheme. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TimestampSuffix { + /// The timestamp pub timestamp: String, + /// Optional number suffix if two timestamp suffixes are the same pub number: Option, } impl Representation for TimestampSuffix {} diff --git a/src/tests.rs b/src/tests.rs index 0852189..e9c6e1f 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,4 +1,4 @@ -use super::{compression::*, suffix::*, *}; +use super::{suffix::*, *}; use tempdir::TempDir; // Just useful to debug why test doesn't succeed @@ -222,8 +222,7 @@ fn compression_on_rotation() { assert_eq!("", fs::read_to_string(&log_path).unwrap()); fn compress(text: &str) -> Vec { - let mut encoder = - flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); encoder.write_all(text.as_bytes()).unwrap(); encoder.finish().unwrap() @@ -255,6 +254,33 @@ fn no_truncate() { assert_eq!("A\nB\n", fs::read_to_string(&log_path).unwrap()); } +#[test] +fn count_recalculation() { + // If there is already some content in the logging file, FileRotate should set its `count` + // field to the size of the file, so that it rotates at the right time + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + + std::fs::write(&log_path, b"a").unwrap(); + + let mut file_rotate = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(3), + ContentLimit::Bytes(2), + Compression::None, + ); + + write!(file_rotate, "bc").unwrap(); + assert_eq!(file_rotate.log_paths().len(), 1); + // The size of the rotated file should be 2 ('ab) + let rotated_content = std::fs::read(&file_rotate.log_paths()[0]).unwrap(); + assert_eq!(rotated_content, b"ab"); + // The size of the main file should be 1 ('c') + let main_content = std::fs::read(log_path).unwrap(); + assert_eq!(main_content, b"c"); +} + #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); From b5ee443fe4cda364476f761750cbd8cec1af838a Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Wed, 15 Dec 2021 16:01:40 +0100 Subject: [PATCH 17/46] Bump version number and publish --- Cargo.toml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82eb9b4..e15c0d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "file-rotate" -version = "0.4.0" -authors = ["Kevin Robert Stravers "] +version = "0.5.0" +authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" homepage = "https://github.com/BourgondAries/file-rotate" diff --git a/README.md b/README.md index fd5c152..ed44f7b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Rotate files with configurable suffix. -Look to the [docs](https://docs.rs/file-rotate/0.4.0/file_rotate/) for explanatory examples. +Look to the [docs](https://docs.rs/file-rotate/0.5.0/file_rotate/) for explanatory examples. ## Basic example From a6130208f00f4317b58797121691199f2364b635 Mon Sep 17 00:00:00 2001 From: Matt Pontius <68478352+mmponn@users.noreply.github.com> Date: Fri, 31 Dec 2021 09:32:29 -0800 Subject: [PATCH 18/46] Fixed bug where a log file's size was used as its line count --- src/lib.rs | 31 +++++++++++++++++++++++-------- src/tests.rs | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b593127..88d190d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -185,6 +185,7 @@ use std::{ io::{self, Write}, path::{Path, PathBuf}, }; +use std::io::{BufRead, BufReader}; use suffix::*; /// Compression @@ -308,19 +309,33 @@ impl FileRotate { self.scan_suffixes(); } if !self.basepath.exists() || self.file.is_none() { - // Update `count` - if let Ok(metadata) = self.basepath.metadata() { - self.count = metadata.len() as usize; - } else { - self.count = 0; - } - // Create new file + // Open or create the file self.file = OpenOptions::new() - .write(true) + .read(true) .create(true) .append(true) .open(&self.basepath) .ok(); + match self.file { + None => + self.count = 0, + Some(ref mut file) => { + match self.content_limit { + ContentLimit::Bytes(_) + | ContentLimit::BytesSurpassed(_) => { + // Update byte `count` + if let Ok(metadata) = file.metadata() { + self.count = metadata.len() as usize; + } else { + self.count = 0; + } + } + ContentLimit::Lines(_) => { + self.count = BufReader::new(file).lines().count(); + } + } + } + } } } fn scan_suffixes(&mut self) { diff --git a/src/tests.rs b/src/tests.rs index e9c6e1f..c855a25 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -255,7 +255,7 @@ fn no_truncate() { } #[test] -fn count_recalculation() { +fn byte_count_recalculation() { // If there is already some content in the logging file, FileRotate should set its `count` // field to the size of the file, so that it rotates at the right time let tmp_dir = TempDir::new("file-rotate-test").unwrap(); @@ -281,6 +281,41 @@ fn count_recalculation() { assert_eq!(main_content, b"c"); } +#[test] +fn line_count_recalculation() { + // If there is already some content in the logging file, FileRotate should set its `count` + // field to the line count of the file, so that it rotates at the right time + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + + std::fs::write(&log_path, b"a\n").unwrap(); + + let mut file_rotate = FileRotate::new( + &*log_path.to_string_lossy(), + CountSuffix::new(3), + ContentLimit::Lines(2), + Compression::None, + ); + + // A single line existed before the new logger ('a') + assert_eq!(file_rotate.count, 1); + + writeln!(file_rotate, "b").unwrap(); + writeln!(file_rotate, "c").unwrap(); + + assert_eq!(file_rotate.log_paths().len(), 1); + + // The line count of the rotated file should be 2 ('a' & 'b') + let mut lines = BufReader::new(File::open(&file_rotate.log_paths()[0]).unwrap()).lines(); + assert_eq!(lines.next().unwrap().unwrap(), "a".to_string()); + assert_eq!(lines.next().unwrap().unwrap(), "b".to_string()); + + // The line count of the main file should be 1 ('c') + let mut lines = BufReader::new(File::open(&log_path).unwrap()).lines(); + assert_eq!(lines.next().unwrap().unwrap(), "c".to_string()); +} + #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); From 689812bdba66ce1c0467658f1fb6d436540ed651 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 6 Jan 2022 11:11:06 +0100 Subject: [PATCH 19/46] Bump version number --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e15c0d5..fd04608 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.5.0" +version = "0.5.1" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" From c50940fdb179637cfd0442d552291e481c58b03d Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Wed, 12 Jan 2022 11:51:06 +0100 Subject: [PATCH 20/46] Improve documentation - add compression example - delete a function that was never used: `TimestampSuffixScheme::should_rotate` - expose fields of `TimestampSuffixScheme` --- Cargo.toml | 2 +- README.md | 13 +++--- src/compression.rs | 2 +- src/lib.rs | 110 ++++++++++++++++++++++++++++++++++++++------- src/suffix.rs | 19 +++----- 5 files changed, 111 insertions(+), 35 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fd04608..7f8f9da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.5.1" +version = "0.5.2" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" diff --git a/README.md b/README.md index ed44f7b..259d366 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,13 @@ Rotate files with configurable suffix. -Look to the [docs](https://docs.rs/file-rotate/0.5.0/file_rotate/) for explanatory examples. +Look to the [docs](https://docs.rs/file-rotate/0.5.0/file_rotate/) for explanatory examples of all features, like: +* Using count or timestamp as suffix +* Age-based deletion of log files +* Optional compression +* Getting a list of log files + +Following are some supplementary examples to get started. ## Basic example @@ -74,11 +80,6 @@ The timestamp format (including the extra trailing `.N`) works by default so tha So it almost works perfectly with `cat logs/*`, except that `log` is smaller (lexically "older") than all the rest. This can of course be fixed with a more complex script to assemble the logs. -## Content limit - -We can rotate log files by using the amount of lines as a limit, as seem above with `ContentLimit::Lines(3)`. -Another method of rotation is by bytes instead of lines, byt using for example `ContentLimit::BytesSurpassed(1_000_000)`. - ## License This project is licensed under the [MIT license]. diff --git a/src/compression.rs b/src/compression.rs index 0eda4fe..43bf536 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -5,7 +5,7 @@ use std::{ path::{Path, PathBuf}, }; -/// In the future, maybe stream compression +/// Compression mode - when to compress files. #[derive(Debug, Clone)] pub enum Compression { /// No compression diff --git a/src/lib.rs b/src/lib.rs index 88d190d..9050dd3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,11 +4,11 @@ //! //! # Content limit # //! -//! Content limit specifies at what point a log file has to be rotated. +//! [ContentLimit] specifies at what point a log file has to be rotated. //! //! ## Rotating by Lines ## //! -//! We can rotate log files with the amount of lines as a limit, by using `ContentLimit::Lines`. +//! We can rotate log files with the amount of lines as a limit, by using [ContentLimit::Lines]. //! //! ``` //! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; @@ -40,7 +40,7 @@ //! //! ## Rotating by Bytes ## //! -//! Another method of rotation is by bytes instead of lines, with `ContentLimit::Bytes`. +//! Another method of rotation is by bytes instead of lines, with [ContentLimit::Bytes]. //! //! ``` //! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; @@ -62,12 +62,12 @@ //! //! # Rotation Method # //! -//! Two rotation methods are provided, but any behaviour can be implemented with the `SuffixScheme` +//! Two rotation methods are provided, but any behaviour can be implemented with the [SuffixScheme] //! trait. //! //! ## Basic count ## //! -//! With `CountSuffix`, when the limit is reached in the main log file, the file is moved with +//! With [CountSuffix], when the limit is reached in the main log file, the file is moved with //! suffix `.1`, and subsequently numbered files are moved in a cascade. //! //! Here's an example with 1 byte limits: @@ -109,7 +109,7 @@ //! //! ## Timestamp suffix ## //! -//! With `TimestampSuffix`, when the limit is reached in the main log file, the file is moved with +//! With [TimestampSuffix], when the limit is reached in the main log file, the file is moved with //! suffix equal to the current timestamp (with the specified or a default format). If the //! destination file name already exists, `.1` (and up) is appended. //! @@ -121,7 +121,7 @@ //! component). //! //! With this suffix scheme, you can also decide whether to delete old files based on the age of -//! their timestamp (`FileLimit::Age`), or just maximum number of files (`FileLimit::MaxFiles`). +//! their timestamp ([FileLimit::Age]), or just maximum number of files ([FileLimit::MaxFiles]). //! //! ``` //! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffixScheme, FileLimit}, @@ -159,14 +159,96 @@ //! TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))); //! ``` //! +//! # Compression # +//! +//! Select a [Compression] mode to make the file rotater compress old files using flate2. +//! Compressed files get an additional suffix `.gz` after the main suffix. +//! +//! ## Compression example ## +//! If we run this: +//! +//! ```ignore +//! use file_rotate::{compression::*, suffix::*, *}; +//! use std::io::Write; +//! +//! let mut log = FileRotate::new( +//! "./log", +//! TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), +//! ContentLimit::Bytes(1), +//! Compression::OnRotate(2), +//! ); +//! +//! for i in 0..6 { +//! write!(log, "{}", i).unwrap(); +//! std::thread::sleep(std::time::Duration::from_secs(1)); +//! } +//! ``` +//! The following files will be created: +//! ```ignore +//! log log.20220112T112415.gz log.20220112T112416.gz log.20220112T112417 log.20220112T112418 +//! ``` +//! And we can assemble all the available log data with: +//! ```ignore +//! $ gunzip -c log.20220112T112415.gz ; gunzip -c log.20220112T112416.gz ; cat log.20220112T112417 log.20220112T112418 log +//! 12345 +//! ``` +//! +//! +//! ## Get structured list of log files ## +//! +//! We can programmatically get the list of log files. +//! The following code scans the current directory and recognizes log files based on their file name: +//! +//! ``` +//! # use file_rotate::{suffix::*, *}; +//! # use std::path::Path; +//! println!( +//! "{:#?}", +//! TimestampSuffixScheme::default(FileLimit::MaxFiles(4)).scan_suffixes(Path::new("./log")) +//! ); +//! ``` +//! [SuffixScheme::scan_suffixes] also takes into account the possibility of the extra `.gz` suffix, and +//! interprets it correctly as compression. The output: +//! ```ignore +//! { +//! SuffixInfo { +//! suffix: TimestampSuffix { +//! timestamp: "20220112T112418", +//! number: None, +//! }, +//! compressed: false, +//! }, +//! SuffixInfo { +//! suffix: TimestampSuffix { +//! timestamp: "20220112T112417", +//! number: None, +//! }, +//! compressed: false, +//! }, +//! SuffixInfo { +//! suffix: TimestampSuffix { +//! timestamp: "20220112T112416", +//! number: None, +//! }, +//! compressed: true, +//! }, +//! SuffixInfo { +//! suffix: TimestampSuffix { +//! timestamp: "20220112T112415", +//! number: None, +//! }, +//! compressed: true, +//! }, +//! } +//! ``` +//! This information can be used by for example a program to assemble log history. +//! //! # Filesystem Errors # //! //! If the directory containing the logs is deleted or somehow made inaccessible then the rotator //! will simply continue operating without fault. When a rotation occurs, it attempts to open a //! file in the directory. If it can, it will just continue logging. If it can't then the written -//! date is sent to the void. -//! -//! This logger never panics. +//! data is sent to the void. #![deny( missing_docs, @@ -178,6 +260,7 @@ )] use compression::*; +use std::io::{BufRead, BufReader}; use std::{ cmp::Ordering, collections::BTreeSet, @@ -185,7 +268,6 @@ use std::{ io::{self, Write}, path::{Path, PathBuf}, }; -use std::io::{BufRead, BufReader}; use suffix::*; /// Compression @@ -317,12 +399,10 @@ impl FileRotate { .open(&self.basepath) .ok(); match self.file { - None => - self.count = 0, + None => self.count = 0, Some(ref mut file) => { match self.content_limit { - ContentLimit::Bytes(_) - | ContentLimit::BytesSurpassed(_) => { + ContentLimit::Bytes(_) | ContentLimit::BytesSurpassed(_) => { // Update byte `count` if let Ok(metadata) = file.metadata() { self.count = metadata.len() as usize; diff --git a/src/suffix.rs b/src/suffix.rs index 0d27f4f..257450a 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -122,8 +122,10 @@ impl SuffixScheme for CountSuffix { /// - The `format` should ensure that the lexical and chronological orderings are the same #[cfg(feature = "chrono04")] pub struct TimestampSuffixScheme { - format: &'static str, - file_limit: FileLimit, + /// The format of the timestamp suffix + pub format: &'static str, + /// The file limit, e.g. when to delete an old file - by age (given by suffix) or by number of files + pub file_limit: FileLimit, } #[cfg(feature = "chrono04")] @@ -139,14 +141,6 @@ impl TimestampSuffixScheme { pub fn with_format(format: &'static str, file_limit: FileLimit) -> Self { Self { format, file_limit } } - /// NOTE: For future use in RotationMode::Custom - pub fn should_rotate(&self, age: Duration) -> impl Fn(&str) -> bool { - let format = self.format.to_string(); - move |suffix| { - let old_timestamp = (Local::now() - age).format(&format).to_string(); - suffix < old_timestamp.as_str() - } - } } /// Structured representation of the suffixes of TimestampSuffixScheme. @@ -246,12 +240,13 @@ impl SuffixScheme for TimestampSuffixScheme { } } -/// How to determine if a file should be deleted, in the case of TimestampSuffixScheme. +/// How to determine if a file should be deleted, in the case of [TimestampSuffixScheme]. #[cfg(feature = "chrono04")] pub enum FileLimit { /// Delete the oldest files if number of files is too high MaxFiles(usize), - /// Delete files that have too old timestamp + /// Delete files whose by their age, determined by the suffix (only works in the case that + /// [TimestampSuffixScheme] is used) Age(Duration), } From 2c9a2647ede4e27e82def122776132c2a8af7680 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Wed, 12 Jan 2022 12:34:36 +0100 Subject: [PATCH 21/46] More docs --- Cargo.toml | 2 +- README.md | 2 +- src/lib.rs | 2 ++ src/suffix.rs | 21 +++++++++++++-------- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7f8f9da..5d467c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.5.2" +version = "0.5.3" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" diff --git a/README.md b/README.md index 259d366..1aeac63 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Rotate files with configurable suffix. -Look to the [docs](https://docs.rs/file-rotate/0.5.0/file_rotate/) for explanatory examples of all features, like: +Look to the [docs](https://docs.rs/file-rotate/latest/file_rotate/index.html) for explanatory examples of all features, like: * Using count or timestamp as suffix * Age-based deletion of log files * Optional compression diff --git a/src/lib.rs b/src/lib.rs index 9050dd3..d1d8491 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -207,8 +207,10 @@ //! TimestampSuffixScheme::default(FileLimit::MaxFiles(4)).scan_suffixes(Path::new("./log")) //! ); //! ``` +//! //! [SuffixScheme::scan_suffixes] also takes into account the possibility of the extra `.gz` suffix, and //! interprets it correctly as compression. The output: +//! //! ```ignore //! { //! SuffixInfo { diff --git a/src/suffix.rs b/src/suffix.rs index 257450a..2bced42 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -23,14 +23,18 @@ pub trait SuffixScheme { /// E.g. if the suffix is a number, you can use `usize`. type Repr: Representation; - /// The file at `suffix` needs to be rotated. - /// Returns the target file path. - /// The file will be moved outside this function. - /// If the target path already exists, rotate_file is called again with `path` set to the - /// target path. Thus it cascades files by default, and if this is not desired, it's up to - /// `rotate_file` to return a path that does not already exist. + /// `file-rotate` calls this function when the file at `suffix` needs to be rotated, and moves the log file + /// accordingly. Thus, this function should not move any files itself. /// - /// `prev_suffix` is provided just in case it's useful (not always) + /// If `suffix` is `None`, it means it's the main log file (with path equal to just `basepath`) + /// that is being rotated. + /// + /// Returns the target suffix that the log file should be moved to. + /// If the target suffix already exists, `rotate_file` is called again with `suffix` set to the + /// target suffix. Thus it cascades files by default, and if this is not desired, it's up to + /// `rotate_file` to return a suffix that does not already exist on disk. + /// + /// `newest_suffix` is provided just in case it's useful (depending on the particular suffix scheme, it's not always useful) fn rotate_file( &mut self, basepath: &Path, @@ -47,7 +51,8 @@ pub trait SuffixScheme { fn too_old(&self, suffix: &Self::Repr, file_number: usize) -> bool; /// Find all files in the basepath.parent() directory that has path equal to basepath + a valid - /// suffix. Return sorted collection - sorted from most recent to oldest. + /// suffix. Return sorted collection - sorted from most recent to oldest based on the + /// [Ord](std::cmp::Ord) implementation of `Self::Repr`. fn scan_suffixes(&self, basepath: &Path) -> BTreeSet> { let mut suffixes = BTreeSet::new(); let filename_prefix = basepath From f82cd14e937efe6420aca7e80cd819fda7413830 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Mon, 17 Jan 2022 11:31:34 +0100 Subject: [PATCH 22/46] derive Debug for FileRotate --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index d1d8491..6fe3a20 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -330,6 +330,7 @@ impl PartialOrd for SuffixInfo { } /// The main writer used for rotating logs. +#[derive(Debug)] pub struct FileRotate { basepath: PathBuf, file: Option, From 340c4e5d9d4cc8dede61be6ed9234a25435370a5 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Mon, 31 Jan 2022 17:23:04 +0100 Subject: [PATCH 23/46] Fix bug: scan_suffixes panicked if given path only has one component --- src/suffix.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/suffix.rs b/src/suffix.rs index 2bced42..2ae7506 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -59,6 +59,7 @@ pub trait SuffixScheme { .file_name() .expect("basepath.file_name()") .to_string_lossy(); + let basepath = basepath.canonicalize().unwrap(); let parent = basepath.parent().unwrap(); let filenames = std::fs::read_dir(parent) .unwrap() From 8e14ece90a7d2b94e2dcf650ab375234722c7a79 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Fri, 4 Feb 2022 14:44:10 +0100 Subject: [PATCH 24/46] Fix bug in `scan_suffixes` (not fixed in previous commit) - bug fix: basically the previous commit only made matters worse - now a real fix + a test --- src/lib.rs | 2 +- src/suffix.rs | 56 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6fe3a20..009de26 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -109,7 +109,7 @@ //! //! ## Timestamp suffix ## //! -//! With [TimestampSuffix], when the limit is reached in the main log file, the file is moved with +//! With [TimestampSuffixScheme], when the limit is reached in the main log file, the file is moved with //! suffix equal to the current timestamp (with the specified or a default format). If the //! destination file name already exists, `.1` (and up) is appended. //! diff --git a/src/suffix.rs b/src/suffix.rs index 2ae7506..644a1b9 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -59,8 +59,19 @@ pub trait SuffixScheme { .file_name() .expect("basepath.file_name()") .to_string_lossy(); - let basepath = basepath.canonicalize().unwrap(); + + // We need the parent directory of the given basepath, but this should also work when the path + // only has one segment. Thus we prepend the current working dir if the path is relative: + let basepath = if basepath.is_relative() { + let mut path = std::env::current_dir().unwrap(); + path.push(basepath); + path + } else { + basepath.to_path_buf() + }; + let parent = basepath.parent().unwrap(); + let filenames = std::fs::read_dir(parent) .unwrap() .filter_map(|entry| entry.ok()) @@ -93,7 +104,7 @@ pub struct CountSuffix { } impl CountSuffix { - /// New CountSuffix, deleting files when the total number of files exceeds `max_files`. + /// New suffix scheme, deleting files when the total number of files exceeds `max_files`. /// For example, if max_files is 3, then the files `log`, `log.1`, `log.2`, `log.3` may exist /// but not `log.4`. In other words, `max_files` determines the largest possible suffix number. pub fn new(max_files: usize) -> Self { @@ -284,15 +295,38 @@ mod test { #[test] fn scan_suffixes() { - let directory = tempdir::TempDir::new("file-rotate").unwrap(); - let directory = directory.path(); - let log_path = directory.join("logs"); - std::fs::create_dir_all(&log_path).unwrap(); - File::create(log_path.join("all.log.20210911T121830")).unwrap(); - File::create(log_path.join("all.log.20210911T121831.gz")).unwrap(); - + let working_dir = tempdir::TempDir::new("file-rotate").unwrap(); + let working_dir = working_dir.path().join("dir"); let suffix_scheme = TimestampSuffixScheme::default(FileLimit::Age(Duration::weeks(1))); - let paths = suffix_scheme.scan_suffixes(&log_path.join("all.log")); - assert_eq!(paths.len(), 2); + + // Test `scan_suffixes` for different possible paths given to it + // (it used to have a bug taking e.g. "log".parent() --> panic) + for relative_path in ["logs/log", "./log", "log", "../log", "../logs/log"] { + std::fs::create_dir_all(&working_dir).unwrap(); + println!("Testing relative path: {}", relative_path); + let relative_path = Path::new(relative_path); + + let log_file = working_dir.join(relative_path); + let log_dir = log_file.parent().unwrap(); + // Ensure all directories needed exist + std::fs::create_dir_all(log_dir).unwrap(); + + // We cd into working_dir + std::env::set_current_dir(&working_dir).unwrap(); + + // Need to create the log file in order to canonicalize it and then get the parent + File::create(working_dir.join(&relative_path)).unwrap(); + let canonicalized = relative_path.canonicalize().unwrap(); + let relative_dir = canonicalized.parent().unwrap(); + + File::create(relative_dir.join("log.20210911T121830")).unwrap(); + File::create(relative_dir.join("log.20210911T121831.gz")).unwrap(); + + let paths = suffix_scheme.scan_suffixes(relative_path); + assert_eq!(paths.len(), 2); + + // Cleanup + std::fs::remove_dir_all(&working_dir).unwrap(); + } } } From eeafc740ac4e3a79b24c47869c7ee63ee0184fc4 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Fri, 4 Feb 2022 20:18:01 +0100 Subject: [PATCH 25/46] s/CountSuffix/AppendCount, s/TimestampSuffixScheme/AppendTimestamp - bump to v0.6.0 because of breaking change - some changes to docs --- Cargo.toml | 2 +- README.md | 6 ++--- src/compression.rs | 1 + src/lib.rs | 56 ++++++++++++++++++++++++++++++---------------- src/suffix.rs | 42 ++++++++++++++++++++-------------- src/tests.rs | 22 +++++++++--------- 6 files changed, 78 insertions(+), 51 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5d467c2..d4efe01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.5.3" +version = "0.6.0" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" diff --git a/README.md b/README.md index 1aeac63..f6cdf67 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ Following are some supplementary examples to get started. ## Basic example ```rust -use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix}; +use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount}; use std::{fs, io::Write, path::PathBuf}; fn main() { - let mut log = FileRotate::new("logs/log", CountSuffix::new(2), ContentLimit::Lines(3)); + let mut log = FileRotate::new("logs/log", AppendCount::new(2), ContentLimit::Lines(3)); // Write a bunch of lines writeln!(log, "Line 1: Hello World!"); @@ -46,7 +46,7 @@ Line 10 ```rust let mut log = FileRotate::new( "logs/log", - TimestampSuffix::default(FileLimit::MaxFiles(3)), + AppendTimestamp::default(FileLimit::MaxFiles(3)), ContentLimit::Lines(3), ); diff --git a/src/compression.rs b/src/compression.rs index 43bf536..70095ed 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1,3 +1,4 @@ +//! Compression - configuration and implementation use flate2::write::GzEncoder; use std::{ fs::{self, File, OpenOptions}, diff --git a/src/lib.rs b/src/lib.rs index 009de26..10c95ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ //! We can rotate log files with the amount of lines as a limit, by using [ContentLimit::Lines]. //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount, compression::Compression}; //! use std::{fs, io::Write}; //! //! // Create a new log writer. The first argument is anything resembling a path. The @@ -24,7 +24,12 @@ //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(2), ContentLimit::Lines(3), Compression::None); +//! let mut log = FileRotate::new( +//! log_path.clone(), +//! AppendCount::new(2), +//! ContentLimit::Lines(3), +//! Compression::None +//! ); //! //! // Write a bunch of lines //! writeln!(log, "Line 1: Hello World!"); @@ -43,14 +48,19 @@ //! Another method of rotation is by bytes instead of lines, with [ContentLimit::Bytes]. //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount, compression::Compression}; //! use std::{fs, io::Write}; //! //! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new("target/my-log-directory-bytes/my-log-file", CountSuffix::new(2), ContentLimit::Bytes(5), Compression::None); +//! let mut log = FileRotate::new( +//! "target/my-log-directory-bytes/my-log-file", +//! AppendCount::new(2), +//! ContentLimit::Bytes(5), +//! Compression::None +//! ); //! //! writeln!(log, "Test file"); //! @@ -67,20 +77,25 @@ //! //! ## Basic count ## //! -//! With [CountSuffix], when the limit is reached in the main log file, the file is moved with +//! With [AppendCount], when the limit is reached in the main log file, the file is moved with //! suffix `.1`, and subsequently numbered files are moved in a cascade. //! //! Here's an example with 1 byte limits: //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::CountSuffix, compression::Compression}; +//! use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount, compression::Compression}; //! use std::{fs, io::Write}; //! //! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), CountSuffix::new(3), ContentLimit::Bytes(1), Compression::None); +//! let mut log = FileRotate::new( +//! log_path.clone(), +//! AppendCount::new(3), +//! ContentLimit::Bytes(1), +//! Compression::None +//! ); //! //! write!(log, "A"); //! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); @@ -109,12 +124,12 @@ //! //! ## Timestamp suffix ## //! -//! With [TimestampSuffixScheme], when the limit is reached in the main log file, the file is moved with +//! With [AppendTimestamp], when the limit is reached in the main log file, the file is moved with //! suffix equal to the current timestamp (with the specified or a default format). If the //! destination file name already exists, `.1` (and up) is appended. //! -//! Note that this works somewhat different to `CountSuffix` because of lexical ordering concerns: -//! Higher numbers mean more recent logs, whereas `CountSuffix` works in the opposite way. +//! Note that this works somewhat different to `AppendCount` because of lexical ordering concerns: +//! Higher numbers mean more recent logs, whereas `AppendCount` works in the opposite way. //! The reason for this is to keep the lexical ordering of log names consistent: Higher lexical value //! means more recent. //! This is of course all assuming that the format start with the year (or most significant @@ -124,7 +139,7 @@ //! their timestamp ([FileLimit::Age]), or just maximum number of files ([FileLimit::MaxFiles]). //! //! ``` -//! use file_rotate::{FileRotate, ContentLimit, suffix::{TimestampSuffixScheme, FileLimit}, +//! use file_rotate::{FileRotate, ContentLimit, suffix::{AppendTimestamp, FileLimit}, //! compression::Compression}; //! use std::{fs, io::Write}; //! @@ -132,7 +147,12 @@ //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! -//! let mut log = FileRotate::new(log_path.clone(), TimestampSuffixScheme::default(FileLimit::MaxFiles(2)), ContentLimit::Bytes(1), Compression::None); +//! let mut log = FileRotate::new( +//! log_path.clone(), +//! AppendTimestamp::default(FileLimit::MaxFiles(2)), +//! ContentLimit::Bytes(1), +//! Compression::None +//! ); //! //! write!(log, "A"); //! assert_eq!("A", fs::read_to_string(&log_path).unwrap()); @@ -155,8 +175,8 @@ //! If you use timestamps as suffix, you can also configure files to be removed as they reach a //! certain age. For example: //! ```rust -//! use file_rotate::suffix::{TimestampSuffixScheme, FileLimit}; -//! TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))); +//! use file_rotate::suffix::{AppendTimestamp, FileLimit}; +//! AppendTimestamp::default(FileLimit::Age(chrono::Duration::weeks(1))); //! ``` //! //! # Compression # @@ -173,7 +193,7 @@ //! //! let mut log = FileRotate::new( //! "./log", -//! TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), +//! AppendTimestamp::default(FileLimit::MaxFiles(4)), //! ContentLimit::Bytes(1), //! Compression::OnRotate(2), //! ); @@ -204,7 +224,7 @@ //! # use std::path::Path; //! println!( //! "{:#?}", -//! TimestampSuffixScheme::default(FileLimit::MaxFiles(4)).scan_suffixes(Path::new("./log")) +//! AppendTimestamp::default(FileLimit::MaxFiles(4)).scan_suffixes(Path::new("./log")) //! ); //! ``` //! @@ -272,9 +292,7 @@ use std::{ }; use suffix::*; -/// Compression pub mod compression; -/// Suffix scheme etc pub mod suffix; #[cfg(test)] mod tests; @@ -443,7 +461,7 @@ impl FileRotate { &mut self, old_suffix_info: Option>, ) -> io::Result> { - // NOTE: this newest_suffix is there only because TimestampSuffixScheme specifically needs + // NOTE: this newest_suffix is there only because AppendTimestamp specifically needs // it. Otherwise it might not be necessary to provide this to `rotate_file`. We could also // have passed the internal BTreeMap itself, but it would require to make SuffixInfo `pub`. diff --git a/src/suffix.rs b/src/suffix.rs index 644a1b9..98c3101 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -1,3 +1,8 @@ +//! Suffix schemes determine the suffix of rotated files +//! +//! This behaviour is fully extensible through the [SuffixScheme] trait, and two behaviours are +//! provided: [AppendCount] and [AppendTimestamp] +//! use crate::SuffixInfo; #[cfg(feature = "chrono04")] use chrono::{offset::Local, Duration, NaiveDateTime}; @@ -97,15 +102,16 @@ fn prepare_filename(path: &str) -> (&str, bool) { .unwrap_or((path, false)) } -/// Rotated log files get a number as suffix. The greater the number, the older. The oldest files -/// are deleted. -pub struct CountSuffix { +/// Append a number when rotating the file. +/// The greater the number, the older. The oldest files are deleted. +pub struct AppendCount { max_files: usize, } -impl CountSuffix { - /// New suffix scheme, deleting files when the total number of files exceeds `max_files`. - /// For example, if max_files is 3, then the files `log`, `log.1`, `log.2`, `log.3` may exist +impl AppendCount { + /// New suffix scheme, deleting files when the number of rotated files (i.e. excluding the main + /// file) exceeds `max_files`. + /// For example, if `max_files` is 3, then the files `log`, `log.1`, `log.2`, `log.3` may exist /// but not `log.4`. In other words, `max_files` determines the largest possible suffix number. pub fn new(max_files: usize) -> Self { Self { max_files } @@ -113,7 +119,7 @@ impl CountSuffix { } impl Representation for usize {} -impl SuffixScheme for CountSuffix { +impl SuffixScheme for AppendCount { type Repr = usize; fn rotate_file( &mut self, @@ -134,11 +140,14 @@ impl SuffixScheme for CountSuffix { } } +/// Append current timestamp as suffix when rotating files. +/// If the timestamp already exists, an additional number is appended. +/// /// Current limitations: -/// - Neither `format` or the base filename can include the character `"."`. +/// - Neither `format` nor the base filename can include the character `"."`. /// - The `format` should ensure that the lexical and chronological orderings are the same #[cfg(feature = "chrono04")] -pub struct TimestampSuffixScheme { +pub struct AppendTimestamp { /// The format of the timestamp suffix pub format: &'static str, /// The file limit, e.g. when to delete an old file - by age (given by suffix) or by number of files @@ -146,7 +155,7 @@ pub struct TimestampSuffixScheme { } #[cfg(feature = "chrono04")] -impl TimestampSuffixScheme { +impl AppendTimestamp { /// With format `"%Y%m%dT%H%M%S"` pub fn default(file_limit: FileLimit) -> Self { Self { @@ -154,13 +163,13 @@ impl TimestampSuffixScheme { file_limit, } } - /// Create new TimestampSuffixScheme suffix scheme + /// Create new AppendTimestamp suffix scheme pub fn with_format(format: &'static str, file_limit: FileLimit) -> Self { Self { format, file_limit } } } -/// Structured representation of the suffixes of TimestampSuffixScheme. +/// Structured representation of the suffixes of AppendTimestamp. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TimestampSuffix { /// The timestamp @@ -194,7 +203,7 @@ impl std::fmt::Display for TimestampSuffix { } #[cfg(feature = "chrono04")] -impl SuffixScheme for TimestampSuffixScheme { +impl SuffixScheme for AppendTimestamp { type Repr = TimestampSuffix; fn rotate_file( @@ -257,13 +266,12 @@ impl SuffixScheme for TimestampSuffixScheme { } } -/// How to determine if a file should be deleted, in the case of [TimestampSuffixScheme]. +/// How to determine whether a file should be deleted, in the case of [AppendTimestamp]. #[cfg(feature = "chrono04")] pub enum FileLimit { /// Delete the oldest files if number of files is too high MaxFiles(usize), - /// Delete files whose by their age, determined by the suffix (only works in the case that - /// [TimestampSuffixScheme] is used) + /// Delete files whose age exceeds the `Duration` - age is determined by the suffix of the file Age(Duration), } @@ -297,7 +305,7 @@ mod test { fn scan_suffixes() { let working_dir = tempdir::TempDir::new("file-rotate").unwrap(); let working_dir = working_dir.path().join("dir"); - let suffix_scheme = TimestampSuffixScheme::default(FileLimit::Age(Duration::weeks(1))); + let suffix_scheme = AppendTimestamp::default(FileLimit::Age(Duration::weeks(1))); // Test `scan_suffixes` for different possible paths given to it // (it used to have a bug taking e.g. "log".parent() --> panic) diff --git a/src/tests.rs b/src/tests.rs index c855a25..38207c4 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -23,7 +23,7 @@ fn timestamp_max_files_rotation() { let mut log = FileRotate::new( &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(4)), + AppendTimestamp::default(FileLimit::MaxFiles(4)), ContentLimit::Lines(2), Compression::None, ); @@ -80,7 +80,7 @@ fn timestamp_max_age_deletion() { let mut log = FileRotate::new( &*log_path.to_string_lossy(), - TimestampSuffixScheme::default(FileLimit::Age(chrono::Duration::weeks(1))), + AppendTimestamp::default(FileLimit::Age(chrono::Duration::weeks(1))), ContentLimit::Lines(1), Compression::None, ); @@ -105,7 +105,7 @@ fn count_max_files_rotation() { let log_path = parent.join("log"); let mut log = FileRotate::new( &*log_path.to_string_lossy(), - CountSuffix::new(4), + AppendCount::new(4), ContentLimit::Lines(2), Compression::None, ); @@ -145,7 +145,7 @@ fn rotate_to_deleted_directory() { let log_path = parent.join("log"); let mut log = FileRotate::new( &*log_path.to_string_lossy(), - CountSuffix::new(4), + AppendCount::new(4), ContentLimit::Lines(1), Compression::None, ); @@ -174,7 +174,7 @@ fn write_complete_record_until_bytes_surpassed() { let mut log = FileRotate::new( &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), + AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::BytesSurpassed(1), Compression::None, ); @@ -198,7 +198,7 @@ fn compression_on_rotation() { let log_path = parent.join("log"); let mut log = FileRotate::new( &*log_path.to_string_lossy(), - CountSuffix::new(3), + AppendCount::new(3), ContentLimit::Lines(1), Compression::OnRotate(1), // Keep one file uncompressed ); @@ -241,7 +241,7 @@ fn no_truncate() { let file_rotate = || { FileRotate::new( &*log_path.to_string_lossy(), - CountSuffix::new(3), + AppendCount::new(3), ContentLimit::Lines(10000), Compression::None, ) @@ -266,7 +266,7 @@ fn byte_count_recalculation() { let mut file_rotate = FileRotate::new( &*log_path.to_string_lossy(), - CountSuffix::new(3), + AppendCount::new(3), ContentLimit::Bytes(2), Compression::None, ); @@ -293,7 +293,7 @@ fn line_count_recalculation() { let mut file_rotate = FileRotate::new( &*log_path.to_string_lossy(), - CountSuffix::new(3), + AppendCount::new(3), ContentLimit::Lines(2), Compression::None, ); @@ -325,7 +325,7 @@ fn arbitrary_lines(count: usize) { let count = count.max(1); let mut log = FileRotate::new( &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), + AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::Lines(count), Compression::None, ); @@ -349,7 +349,7 @@ fn arbitrary_bytes(count: usize) { let count = count.max(1); let mut log = FileRotate::new( &log_path, - TimestampSuffixScheme::default(FileLimit::MaxFiles(100)), + AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::Bytes(count), Compression::None, ); From cf713f750b67b376952cd4d856ead0c1252ab243 Mon Sep 17 00:00:00 2001 From: jb-alvarado Date: Thu, 5 May 2022 18:37:04 +0200 Subject: [PATCH 26/46] add rotate by date With rotation options: Hourly, Daily, Weekly, Monthly, Yearly. And timestamp from yesterday, hour before, or now. Add test to rotate by date. --- .gitignore | 1 + Cargo.toml | 1 + examples/rotate_by_date.rs | 22 +++++++ src/lib.rs | 115 ++++++++++++++++++++++++++++++++++++- src/suffix.rs | 40 +++++++++++-- src/tests.rs | 112 ++++++++++++++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 6 deletions(-) create mode 100644 examples/rotate_by_date.rs diff --git a/.gitignore b/.gitignore index 1e7caa9..e25542f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ Cargo.lock +logs/ target/ diff --git a/Cargo.toml b/Cargo.toml index d4efe01..34dc764 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ chrono = { version = "0.4.11", optional = true } flate2 = "1.0" [dev-dependencies] +filetime = "0.2" quickcheck = "0.9.2" quickcheck_macros = "0.9.1" tempdir = "0.3.7" diff --git a/examples/rotate_by_date.rs b/examples/rotate_by_date.rs new file mode 100644 index 0000000..d5ab2f6 --- /dev/null +++ b/examples/rotate_by_date.rs @@ -0,0 +1,22 @@ +use file_rotate::{ + compression::Compression, + suffix::{AppendTimestamp, DateFrom, FileLimit}, + ContentLimit, FileRotate, TimeFrequency, +}; +use std::io::Write; + +fn main() { + let mut log = FileRotate::new( + "logs/log", + AppendTimestamp::with_format("%Y-%m-%d", FileLimit::MaxFiles(7), DateFrom::DateYesterday), + ContentLimit::Time(TimeFrequency::Daily), + Compression::None, + ); + + // Write a bunch of lines + writeln!(log, "Line 1: Hello World!").expect("write log"); + for idx in 2..=10 { + std::thread::sleep(std::time::Duration::from_millis(500)); + writeln!(log, "Line {}", idx).expect("write log"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 10c95ab..1a67ea3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -281,6 +281,7 @@ unused_qualifications )] +use chrono::prelude::*; use compression::*; use std::io::{BufRead, BufReader}; use std::{ @@ -299,6 +300,21 @@ mod tests; // --- +/// At which frequency to rotate the file. +#[derive(Clone, Copy, Debug)] +pub enum TimeFrequency { + /// Rotate every hour. + Hourly, + /// Rotate one time a day. + Daily, + /// Rotate ones a week. + Weekly, + /// Rotate every month. + Monthly, + /// Rotate yearly. + Yearly, +} + /// When to move files: Condition on which a file is rotated. #[derive(Clone, Debug)] pub enum ContentLimit { @@ -306,6 +322,8 @@ pub enum ContentLimit { Bytes(usize), /// Cut the log file at line breaks. Lines(usize), + /// Cut the log at time interval. + Time(TimeFrequency), /// Cut the log file after surpassing size in bytes (but having written a complete buffer from a write call.) BytesSurpassed(usize), } @@ -352,6 +370,7 @@ impl PartialOrd for SuffixInfo { pub struct FileRotate { basepath: PathBuf, file: Option, + modified: Option>, content_limit: ContentLimit, count: usize, compression: Compression, @@ -384,6 +403,7 @@ impl FileRotate { ContentLimit::Lines(lines) => { assert!(lines > 0); } + ContentLimit::Time(_) => {} ContentLimit::BytesSurpassed(bytes) => { assert!(bytes > 0); } @@ -394,6 +414,7 @@ impl FileRotate { let mut s = Self { file: None, + modified: None, basepath, content_limit, count: 0, @@ -434,6 +455,9 @@ impl FileRotate { ContentLimit::Lines(_) => { self.count = BufReader::new(file).lines().count(); } + ContentLimit::Time(_) => { + self.modified = mtime(file); + } } } } @@ -521,7 +545,6 @@ impl FileRotate { self.suffixes.insert(new_suffix_info); self.file = Some(File::create(&self.basepath)?); - self.count = 0; self.handle_old_files()?; @@ -593,6 +616,52 @@ impl Write for FileRotate { file.write_all(buf)?; } } + ContentLimit::Time(time) => { + let local: DateTime = now(); + + if let Some(modified) = self.modified { + match time { + TimeFrequency::Hourly => { + if local.hour() != modified.hour() + || local.day() != modified.day() + || local.month() != modified.month() + || local.year() != modified.year() + { + self.rotate()?; + } + } + TimeFrequency::Daily => { + if local.date() > modified.date() { + self.rotate()?; + } + } + TimeFrequency::Weekly => { + if local.iso_week().week() != modified.iso_week().week() + || local.year() > modified.year() + { + self.rotate()?; + } + } + TimeFrequency::Monthly => { + if local.month() != modified.month() || local.year() != modified.year() + { + self.rotate()?; + } + } + TimeFrequency::Yearly => { + if local.year() > modified.year() { + self.rotate()?; + } + } + } + } + + if let Some(ref mut file) = self.file { + file.write_all(buf)?; + + self.modified = Some(local); + } + } ContentLimit::Lines(lines) => { while let Some((idx, _)) = buf.iter().enumerate().find(|(_, byte)| *byte == &b'\n') { @@ -629,3 +698,47 @@ impl Write for FileRotate { .unwrap_or(Ok(())) } } + +/// Get modification time, in non test case. +#[cfg(not(test))] +fn mtime(file: &File) -> Option> { + if let Ok(time) = file.metadata().and_then(|metadata| metadata.modified()) { + return Some(time.into()); + } + + None +} + +/// Get modification time, in test case. +#[cfg(test)] +fn mtime(_: &File) -> Option> { + Some(now()) +} + +/// Get system time, in non test case. +#[cfg(not(test))] +fn now() -> DateTime { + Local::now() +} + +/// Get mocked system time, in test case. +#[cfg(test)] +pub mod mock_time { + use super::*; + use std::cell::RefCell; + + thread_local! { + static MOCK_TIME: RefCell>> = RefCell::new(None); + } + + pub fn now() -> DateTime { + MOCK_TIME.with(|cell| cell.borrow().as_ref().cloned().unwrap_or_else(Local::now)) + } + + pub fn set_mock_time(time: DateTime) { + MOCK_TIME.with(|cell| *cell.borrow_mut() = Some(time)); + } +} + +#[cfg(test)] +pub use mock_time::now; diff --git a/src/suffix.rs b/src/suffix.rs index 98c3101..6ba1b49 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -3,6 +3,7 @@ //! This behaviour is fully extensible through the [SuffixScheme] trait, and two behaviours are //! provided: [AppendCount] and [AppendTimestamp] //! +use super::now; use crate::SuffixInfo; #[cfg(feature = "chrono04")] use chrono::{offset::Local, Duration, NaiveDateTime}; @@ -140,6 +141,16 @@ impl SuffixScheme for AppendCount { } } +/// Add timestamp from: +pub enum DateFrom { + /// Date yesterday, to represent the timestamps within the log file. + DateYesterday, + /// Date from hour ago, useful with rotate hourly. + DateHourAgo, + /// Date from now. + Now, +} + /// Append current timestamp as suffix when rotating files. /// If the timestamp already exists, an additional number is appended. /// @@ -152,6 +163,8 @@ pub struct AppendTimestamp { pub format: &'static str, /// The file limit, e.g. when to delete an old file - by age (given by suffix) or by number of files pub file_limit: FileLimit, + /// Add timestamp from DateFrom + pub date_from: DateFrom, } #[cfg(feature = "chrono04")] @@ -161,11 +174,16 @@ impl AppendTimestamp { Self { format: "%Y%m%dT%H%M%S", file_limit, + date_from: DateFrom::Now, } } /// Create new AppendTimestamp suffix scheme - pub fn with_format(format: &'static str, file_limit: FileLimit) -> Self { - Self { format, file_limit } + pub fn with_format(format: &'static str, file_limit: FileLimit, date_from: DateFrom) -> Self { + Self { + format, + file_limit, + date_from, + } } } @@ -214,10 +232,22 @@ impl SuffixScheme for AppendTimestamp { ) -> io::Result { assert!(suffix.is_none()); if suffix.is_none() { - let now = Local::now().format(self.format).to_string(); + let mut now = now(); + + match self.date_from { + DateFrom::DateYesterday => { + now = now - Duration::days(1); + } + DateFrom::DateHourAgo => { + now = now - Duration::hours(1); + } + _ => {} + }; + + let fmt_now = now.format(self.format).to_string(); let number = if let Some(newest_suffix) = newest_suffix { - if newest_suffix.timestamp == now { + if newest_suffix.timestamp == fmt_now { Some(newest_suffix.number.unwrap_or(0) + 1) } else { None @@ -226,7 +256,7 @@ impl SuffixScheme for AppendTimestamp { None }; Ok(TimestampSuffix { - timestamp: now, + timestamp: fmt_now, number, }) } else { diff --git a/src/tests.rs b/src/tests.rs index 38207c4..5a36272 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -363,3 +363,115 @@ fn arbitrary_bytes(count: usize) { write!(log, "1").unwrap(); assert!(&log.log_paths()[0].exists()); } + +#[test] +fn rotate_by_time_frequency() { + // Test time frequency by hours. + test_time_frequency( + "2022-05-03T06:00:12", + "2022-05-03T06:59:00", + "2022-05-03T07:01:00", + "2022-05-03_06-01-00", + TimeFrequency::Hourly, + DateFrom::DateHourAgo, + ); + + // Test time frequency by days. + test_time_frequency( + "2022-05-02T12:59:59", + "2022-05-02T23:01:15", + "2022-05-03T01:01:00", + "2022-05-02_01-01-00", + TimeFrequency::Daily, + DateFrom::DateYesterday, + ); + + // Test time frequency by weeks. + test_time_frequency( + "2022-05-02T12:34:02", + "2022-05-06T11:30:00", + "2022-05-09T13:01:00", + "2022-05-08_13-01-00", + TimeFrequency::Weekly, + DateFrom::DateYesterday, + ); + + // Test time frequency by months. + test_time_frequency( + "2022-03-01T11:50:01", + "2022-03-30T15:30:10", + "2022-04-02T05:03:50", + "2022-04-02_05-03-50", + TimeFrequency::Monthly, + DateFrom::Now, + ); + + // Test time frequency by year. + test_time_frequency( + "2021-08-31T12:34:02", + "2021-12-15T15:20:00", + "2022-09-02T13:01:00", + "2022-09-01_13-01-00", + TimeFrequency::Yearly, + DateFrom::DateYesterday, + ); +} + +fn get_fake_date_time(date_time: &str) -> DateTime { + let date_obj = NaiveDateTime::parse_from_str(date_time, "%Y-%m-%dT%H:%M:%S"); + + Local.from_local_datetime(&date_obj.unwrap()).unwrap() +} + +fn test_time_frequency( + old_time: &str, + second_old_time: &str, + new_time: &str, + test_suffix: &str, + frequency: TimeFrequency, + date_from: DateFrom, +) { + let old_time = get_fake_date_time(old_time); + let new_time = get_fake_date_time(new_time); + let second_old_time = get_fake_date_time(second_old_time); + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("log"); + + mock_time::set_mock_time(old_time); + + let mut log = FileRotate::new( + &log_path, + AppendTimestamp::with_format("%Y-%m-%d_%H-%M-%S", FileLimit::MaxFiles(7), date_from), + ContentLimit::Time(frequency), + Compression::None, + ); + + writeln!(log, "a").unwrap(); + log.flush().unwrap(); + + filetime::set_file_mtime( + log_path, + filetime::FileTime::from_system_time(old_time.into()), + ) + .unwrap(); + + mock_time::set_mock_time(second_old_time); + + writeln!(log, "b").unwrap(); + + mock_time::set_mock_time(new_time); + + writeln!(log, "c").unwrap(); + + assert!(&log.log_paths()[0].exists()); + assert_eq!( + log.log_paths()[0] + .display() + .to_string() + .split('.') + .collect::>() + .last(), + Some(&test_suffix) + ); +} From 8df55c6e4e2e043f6c7ef755ebb2dbf65dceef78 Mon Sep 17 00:00:00 2001 From: biggio Date: Wed, 27 Apr 2022 12:09:24 +0200 Subject: [PATCH 27/46] Add possibility to specify UNIX file permissions --- examples/rotate_by_date.rs | 2 ++ src/lib.rs | 52 +++++++++++++++++++++++++------- src/tests.rs | 61 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 11 deletions(-) diff --git a/examples/rotate_by_date.rs b/examples/rotate_by_date.rs index d5ab2f6..5775a4e 100644 --- a/examples/rotate_by_date.rs +++ b/examples/rotate_by_date.rs @@ -11,6 +11,8 @@ fn main() { AppendTimestamp::with_format("%Y-%m-%d", FileLimit::MaxFiles(7), DateFrom::DateYesterday), ContentLimit::Time(TimeFrequency::Daily), Compression::None, + #[cfg(unix)] + None, ); // Write a bunch of lines diff --git a/src/lib.rs b/src/lib.rs index 1a67ea3..e223fb3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,9 @@ //! log_path.clone(), //! AppendCount::new(2), //! ContentLimit::Lines(3), -//! Compression::None +//! Compression::None, +//! #[cfg(unix)] +//! None, //! ); //! //! // Write a bunch of lines @@ -59,7 +61,9 @@ //! "target/my-log-directory-bytes/my-log-file", //! AppendCount::new(2), //! ContentLimit::Bytes(5), -//! Compression::None +//! Compression::None, +//! #[cfg(unix)] +//! None, //! ); //! //! writeln!(log, "Test file"); @@ -94,7 +98,9 @@ //! log_path.clone(), //! AppendCount::new(3), //! ContentLimit::Bytes(1), -//! Compression::None +//! Compression::None, +//! #[cfg(unix)] +//! None, //! ); //! //! write!(log, "A"); @@ -151,7 +157,9 @@ //! log_path.clone(), //! AppendTimestamp::default(FileLimit::MaxFiles(2)), //! ContentLimit::Bytes(1), -//! Compression::None +//! Compression::None, +//! #[cfg(unix)] +//! None, //! ); //! //! write!(log, "A"); @@ -196,6 +204,8 @@ //! AppendTimestamp::default(FileLimit::MaxFiles(4)), //! ContentLimit::Bytes(1), //! Compression::OnRotate(2), +//! #[cfg(unix)] +//! None, //! ); //! //! for i in 0..6 { @@ -293,6 +303,9 @@ use std::{ }; use suffix::*; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; + pub mod compression; pub mod suffix; #[cfg(test)] @@ -377,6 +390,8 @@ pub struct FileRotate { suffix_scheme: S, /// The bool is whether or not there's a .gz suffix to the filename suffixes: BTreeSet>, + #[cfg(unix)] + mode: Option, } impl FileRotate { @@ -395,6 +410,7 @@ impl FileRotate { suffix_scheme: S, content_limit: ContentLimit, compression: Compression, + #[cfg(unix)] mode: Option, ) -> Self { match content_limit { ContentLimit::Bytes(bytes) => { @@ -421,9 +437,12 @@ impl FileRotate { compression, suffixes: BTreeSet::new(), suffix_scheme, + #[cfg(unix)] + mode, }; s.ensure_log_directory_exists(); s.scan_suffixes(); + s } fn ensure_log_directory_exists(&mut self) { @@ -434,12 +453,8 @@ impl FileRotate { } if !self.basepath.exists() || self.file.is_none() { // Open or create the file - self.file = OpenOptions::new() - .read(true) - .create(true) - .append(true) - .open(&self.basepath) - .ok(); + self.open_file(); + match self.file { None => self.count = 0, Some(ref mut file) => { @@ -463,6 +478,20 @@ impl FileRotate { } } } + + fn open_file(&mut self) { + let mut open_options = OpenOptions::new(); + + open_options.read(true).create(true).append(true); + + if let Some(mode) = self.mode { + #[cfg(unix)] + open_options.mode(mode); + } + + self.file = open_options.open(&self.basepath).ok(); + } + fn scan_suffixes(&mut self) { self.suffixes = self.suffix_scheme.scan_suffixes(&self.basepath); } @@ -544,7 +573,8 @@ impl FileRotate { let new_suffix_info = self.move_file_with_suffix(None)?; self.suffixes.insert(new_suffix_info); - self.file = Some(File::create(&self.basepath)?); + self.open_file(); + self.count = 0; self.handle_old_files()?; diff --git a/src/tests.rs b/src/tests.rs index 5a36272..540b99a 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,4 +1,6 @@ use super::{suffix::*, *}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; use tempdir::TempDir; // Just useful to debug why test doesn't succeed @@ -26,6 +28,8 @@ fn timestamp_max_files_rotation() { AppendTimestamp::default(FileLimit::MaxFiles(4)), ContentLimit::Lines(2), Compression::None, + #[cfg(unix)] + None, ); // Write 9 lines @@ -83,6 +87,8 @@ fn timestamp_max_age_deletion() { AppendTimestamp::default(FileLimit::Age(chrono::Duration::weeks(1))), ContentLimit::Lines(1), Compression::None, + #[cfg(unix)] + None, ); writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); @@ -108,6 +114,8 @@ fn count_max_files_rotation() { AppendCount::new(4), ContentLimit::Lines(2), Compression::None, + #[cfg(unix)] + None, ); // Write 9 lines @@ -148,6 +156,8 @@ fn rotate_to_deleted_directory() { AppendCount::new(4), ContentLimit::Lines(1), Compression::None, + #[cfg(unix)] + None, ); write!(log, "a\nb\n").unwrap(); @@ -177,6 +187,8 @@ fn write_complete_record_until_bytes_surpassed() { AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::BytesSurpassed(1), Compression::None, + #[cfg(unix)] + None, ); write!(log, "0123456789").unwrap(); @@ -201,6 +213,8 @@ fn compression_on_rotation() { AppendCount::new(3), ContentLimit::Lines(1), Compression::OnRotate(1), // Keep one file uncompressed + #[cfg(unix)] + None, ); writeln!(log, "A").unwrap(); @@ -244,6 +258,8 @@ fn no_truncate() { AppendCount::new(3), ContentLimit::Lines(10000), Compression::None, + #[cfg(unix)] + None, ) }; writeln!(file_rotate(), "A").unwrap(); @@ -269,6 +285,8 @@ fn byte_count_recalculation() { AppendCount::new(3), ContentLimit::Bytes(2), Compression::None, + #[cfg(unix)] + None, ); write!(file_rotate, "bc").unwrap(); @@ -296,6 +314,8 @@ fn line_count_recalculation() { AppendCount::new(3), ContentLimit::Lines(2), Compression::None, + #[cfg(unix)] + None, ); // A single line existed before the new logger ('a') @@ -316,6 +336,41 @@ fn line_count_recalculation() { assert_eq!(lines.next().unwrap().unwrap(), "c".to_string()); } +#[cfg(unix)] +#[test] +fn unix_file_permissions() { + let permissions = &[0o600, 0o644]; + + for permission in permissions { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + + let mut file_rotate = FileRotate::new( + &*log_path.to_string_lossy(), + AppendCount::new(3), + ContentLimit::Lines(2), + Compression::None, + Some(*permission), + ); + + // Trigger a rotation by writing three lines + writeln!(file_rotate, "a").unwrap(); + writeln!(file_rotate, "b").unwrap(); + writeln!(file_rotate, "c").unwrap(); + + assert_eq!(file_rotate.log_paths().len(), 1); + + // The file created at initialization time should have the right permissions ... + let metadata = fs::metadata(&log_path).unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, *permission); + + // ... and also the one generated through a rotation + let metadata = fs::metadata(&file_rotate.log_paths()[0]).unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, *permission); + } +} + #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); @@ -328,6 +383,8 @@ fn arbitrary_lines(count: usize) { AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::Lines(count), Compression::None, + #[cfg(unix)] + None, ); for _ in 0..count - 1 { @@ -352,6 +409,8 @@ fn arbitrary_bytes(count: usize) { AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::Bytes(count), Compression::None, + #[cfg(unix)] + None, ); for _ in 0..count { @@ -445,6 +504,8 @@ fn test_time_frequency( AppendTimestamp::with_format("%Y-%m-%d_%H-%M-%S", FileLimit::MaxFiles(7), date_from), ContentLimit::Time(frequency), Compression::None, + #[cfg(unix)] + None, ); writeln!(log, "a").unwrap(); From 0416ad19cf0cb59ac41f188b33c3b6b4603d92ab Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Fri, 13 May 2022 21:02:22 +0200 Subject: [PATCH 28/46] Create rust.yml for github actions --- .github/workflows/rust.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..4882eb2 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,30 @@ +name: Rust + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check + run: cargo check + test: + runs-on: '${{ matrix.os }}' + strategy: + matrix: + include: + - os: macos-latest + - os: ubuntu-latest + - os: windows-latest + steps: + - uses: actions/checkout@v3 + - name: Test + run: cargo test From be183d460992434c3ec1b8cc053ec03eef179b2e Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Sat, 14 May 2022 11:26:26 +0200 Subject: [PATCH 29/46] Fix compilation error on Windows --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e223fb3..bb88bdf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -484,8 +484,8 @@ impl FileRotate { open_options.read(true).create(true).append(true); + #[cfg(unix)] if let Some(mode) = self.mode { - #[cfg(unix)] open_options.mode(mode); } From c587c387f8d49fc176241f156e5f622a9c1e16c9 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Sat, 14 May 2022 11:28:23 +0200 Subject: [PATCH 30/46] Update README examples --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f6cdf67..3979ee8 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount}; use std::{fs, io::Write, path::PathBuf}; fn main() { - let mut log = FileRotate::new("logs/log", AppendCount::new(2), ContentLimit::Lines(3)); + let mut log = FileRotate::new("logs/log", AppendCount::new(2), ContentLimit::Lines(3), None); // Write a bunch of lines writeln!(log, "Line 1: Hello World!"); @@ -48,6 +48,7 @@ let mut log = FileRotate::new( "logs/log", AppendTimestamp::default(FileLimit::MaxFiles(3)), ContentLimit::Lines(3), + None, ); // Write a bunch of lines From 4d6014071a4e86d088db1211ab108ef24f923d7b Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Sat, 14 May 2022 11:35:24 +0200 Subject: [PATCH 31/46] Fix one test that failed in Windows --- src/suffix.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/suffix.rs b/src/suffix.rs index 6ba1b49..aded72b 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -363,6 +363,11 @@ mod test { let paths = suffix_scheme.scan_suffixes(relative_path); assert_eq!(paths.len(), 2); + // Reset CWD: necessary on Windows only - otherwise we get the error: + // "The process cannot access the file because it is being used by another process." + // (code 32) + std::env::set_current_dir("/").unwrap(); + // Cleanup std::fs::remove_dir_all(&working_dir).unwrap(); } From 7b45ca1c9af7603a587b617220177e4d6574adc9 Mon Sep 17 00:00:00 2001 From: Andreas Runfalk Date: Wed, 15 Jun 2022 13:31:54 +0200 Subject: [PATCH 32/46] Add support for manual log rotation. This adds a new `ContentLimit` that never rotates the log automatically. To force a rotation the user has to call `FileRotate::rotate()` which is not part of the public API. --- src/lib.rs | 13 ++++++++++++- src/tests.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index bb88bdf..6d23489 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -339,6 +339,8 @@ pub enum ContentLimit { Time(TimeFrequency), /// Cut the log file after surpassing size in bytes (but having written a complete buffer from a write call.) BytesSurpassed(usize), + /// Don't do any rotation automatically + None, } /// Used mostly internally. Info about suffix + compressed state. @@ -423,6 +425,7 @@ impl FileRotate { ContentLimit::BytesSurpassed(bytes) => { assert!(bytes > 0); } + ContentLimit::None => {} }; let basepath = path.as_ref().to_path_buf(); @@ -473,6 +476,7 @@ impl FileRotate { ContentLimit::Time(_) => { self.modified = mtime(file); } + ContentLimit::None => {} } } } @@ -564,7 +568,9 @@ impl FileRotate { Ok(newly_created_suffix) } - fn rotate(&mut self) -> io::Result<()> { + /// Trigger a log rotation manually. This is mostly intended for use with `ContentLimit::None` + /// but will work with all content limits. + pub fn rotate(&mut self) -> io::Result<()> { self.ensure_log_directory_exists(); let _ = self.file.take(); @@ -717,6 +723,11 @@ impl Write for FileRotate { } self.count += buf.len(); } + ContentLimit::None => { + if let Some(ref mut file) = self.file { + file.write_all(buf)?; + } + } } Ok(written) } diff --git a/src/tests.rs b/src/tests.rs index 540b99a..8d1f7d6 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -371,6 +371,33 @@ fn unix_file_permissions() { } } +#[test] +fn manual_rotation() { + // Check that manual rotation works as intented + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let parent = tmp_dir.path(); + let log_path = parent.join("log"); + let mut log = FileRotate::new( + &*log_path.to_string_lossy(), + AppendCount::new(3), + ContentLimit::None, + Compression::None, + #[cfg(unix)] + None, + ); + writeln!(log, "A").unwrap(); + log.rotate().unwrap(); + list(parent); + writeln!(log, "B").unwrap(); + list(parent); + + dbg!(log.log_paths()); + let logs = log.log_paths(); + assert_eq!(logs.len(), 1); + assert_eq!("A\n", fs::read_to_string(&logs[0]).unwrap()); + assert_eq!("B\n", fs::read_to_string(&log_path).unwrap()); +} + #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { let tmp_dir = TempDir::new("file-rotate-test").unwrap(); From 81c006d27c40c5540888d02e2533b15180f8f0fa Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 12 May 2022 11:31:58 +0200 Subject: [PATCH 33/46] Fix AppendTimestamp::parse to work with any format --- Cargo.toml | 4 +-- src/suffix.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++++---- src/tests.rs | 33 +++++++++++++++++++++ 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 34dc764..ab188dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.6.0" +version = "0.7.0-rc.0" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" @@ -10,7 +10,7 @@ keywords= ["log", "rotate", "logrotate"] license = "MIT" [dependencies] -chrono = { version = "0.4.11", optional = true } +chrono = {version = "0.4.20-rc.1", optional = true } flate2 = "1.0" [dev-dependencies] diff --git a/src/suffix.rs b/src/suffix.rs index aded72b..19772ae 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -6,7 +6,7 @@ use super::now; use crate::SuffixInfo; #[cfg(feature = "chrono04")] -use chrono::{offset::Local, Duration, NaiveDateTime}; +use chrono::{format::ParseErrorKind, offset::Local, Duration, NaiveDateTime}; use std::{ cmp::Ordering, collections::BTreeSet, @@ -278,12 +278,18 @@ impl SuffixScheme for AppendTimestamp { } else { (suffix, None) }; - NaiveDateTime::parse_from_str(timestamp_str, self.format) - .map(|_| TimestampSuffix { + let success = match NaiveDateTime::parse_from_str(timestamp_str, self.format) { + Ok(_) => true, + Err(e) => e.kind() == ParseErrorKind::NotEnough, + }; + if success { + Some(TimestampSuffix { timestamp: timestamp_str.to_string(), number: n, }) - .ok() + } else { + None + } } fn too_old(&self, suffix: &TimestampSuffix, file_number: usize) -> bool { match self.file_limit { @@ -309,6 +315,7 @@ pub enum FileLimit { mod test { use super::*; use std::fs::File; + use tempdir::TempDir; #[test] fn timestamp_ordering() { assert!( @@ -332,7 +339,7 @@ mod test { } #[test] - fn scan_suffixes() { + fn timestamp_scan_suffixes_base_paths() { let working_dir = tempdir::TempDir::new("file-rotate").unwrap(); let working_dir = working_dir.path().join("dir"); let suffix_scheme = AppendTimestamp::default(FileLimit::Age(Duration::weeks(1))); @@ -372,4 +379,66 @@ mod test { std::fs::remove_dir_all(&working_dir).unwrap(); } } + + #[test] + fn timestamp_scan_suffixes_formats() { + struct TestCase { + format: &'static str, + suffixes: &'static [&'static str], + incorrect_suffixes: &'static [&'static str], + } + + let cases = [ + TestCase { + format: "%Y%m%dT%H%M%S", + suffixes: &["20220201T101010", "20220202T101010"], + incorrect_suffixes: &["20220201T1010", "20220201T999999", "2022-02-02"], + }, + TestCase { + format: "%Y-%m-%d", + suffixes: &["2022-02-01", "2022-02-02"], + incorrect_suffixes: &[ + "abc", + "2022-99-99", + "2022-05", + "2022", + "20220202", + "2022-02-02T112233", + ], + }, + ]; + + for (i, case) in cases.iter().enumerate() { + println!("Case {}", i); + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("file"); + + for suffix in case.suffixes.iter().chain(case.incorrect_suffixes) { + std::fs::File::create(dir.join(format!("file.{}", suffix))).unwrap(); + } + + let scheme = AppendTimestamp::with_format( + case.format, + FileLimit::MaxFiles(1), + DateFrom::DateYesterday, + ); + + // Scan for suffixes + let suffixes_set = scheme.scan_suffixes(&log_path); + + // Collect these suffixes, and the expected suffixes, into Vec, and sort + let mut suffixes = suffixes_set + .into_iter() + .map(|x| x.suffix.to_string()) + .collect::>(); + suffixes.sort_unstable(); + + let mut expected_suffixes = case.suffixes.to_vec(); + expected_suffixes.sort_unstable(); + + assert_eq!(suffixes, case.suffixes); + println!("Passed\n"); + } + } } diff --git a/src/tests.rs b/src/tests.rs index 8d1f7d6..e685736 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -503,6 +503,39 @@ fn rotate_by_time_frequency() { ); } +#[test] +fn test_file_limit() { + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("file"); + let old_file = dir.join("file.2022-02-01"); + + std::fs::File::create(&old_file).unwrap(); + + let first = get_fake_date_time("2022-02-02T01:00:00"); + let second = get_fake_date_time("2022-02-03T01:00:00"); + let third = get_fake_date_time("2022-02-04T01:00:00"); + + let mut log = FileRotate::new( + &log_path, + AppendTimestamp::with_format("%Y-%m-%d", FileLimit::MaxFiles(1), DateFrom::DateYesterday), + ContentLimit::Time(TimeFrequency::Daily), + Compression::None, + #[cfg(unix)] + None, + ); + + mock_time::set_mock_time(first); + writeln!(log, "1").unwrap(); + mock_time::set_mock_time(second); + writeln!(log, "2").unwrap(); + mock_time::set_mock_time(third); + writeln!(log, "3").unwrap(); + + assert_eq!(log.log_paths(), [dir.join("file.2022-02-03")]); + assert!(!old_file.is_file()); +} + fn get_fake_date_time(date_time: &str) -> DateTime { let date_obj = NaiveDateTime::parse_from_str(date_time, "%Y-%m-%dT%H:%M:%S"); From 30f0eccb35c6cab4787fc29e6869be7f11b29aff Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Mon, 8 Aug 2022 11:09:18 +0200 Subject: [PATCH 34/46] Update chrono from 0.4.20 rc to release --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ab188dd..ecc6b2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.0-rc.0" +version = "0.7.0" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" @@ -10,7 +10,7 @@ keywords= ["log", "rotate", "logrotate"] license = "MIT" [dependencies] -chrono = {version = "0.4.20-rc.1", optional = true } +chrono = {version = "0.4.20", optional = true } flate2 = "1.0" [dev-dependencies] From ff27734438f041a6b420bb7d0ba00a54fb61f5d0 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Mon, 8 Aug 2022 11:25:33 +0200 Subject: [PATCH 35/46] Add Limitations in README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 3979ee8..50adcc8 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ Look to the [docs](https://docs.rs/file-rotate/latest/file_rotate/index.html) fo * Optional compression * Getting a list of log files +Limitations / known issues: +* `file-rotate` assumes that no other process or user moves files around in the logging directory, but we want to find a way to [support this](https://github.com/BourgondAries/file-rotate/issues/17) + Following are some supplementary examples to get started. ## Basic example From 6340dae55f390046512078610ad108bdc4990ac1 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Fri, 14 Oct 2022 12:57:44 +0200 Subject: [PATCH 36/46] Fix issue #20: panic due to subtraction overflow --- src/lib.rs | 2 +- src/tests.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6d23489..811664a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -640,7 +640,7 @@ impl Write for FileRotate { match self.content_limit { ContentLimit::Bytes(bytes) => { while self.count + buf.len() > bytes { - let bytes_left = bytes - self.count; + let bytes_left = bytes.saturating_sub(self.count); if let Some(ref mut file) = self.file { file.write_all(&buf[..bytes_left])?; } diff --git a/src/tests.rs b/src/tests.rs index e685736..c17e8af 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -536,6 +536,47 @@ fn test_file_limit() { assert!(!old_file.is_file()); } +#[test] +fn test_panic() { + use std::io::Write; + + let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let dir = tmp_dir.path(); + let log_path = dir.join("file"); + // write 9 bytes of data + { + let mut log = FileRotate::new( + &log_path, + AppendCount::new(2), + ContentLimit::None, + Compression::None, + #[cfg(unix)] + None, + ); + + write!(log, "nineteen characters").unwrap(); + } + + // set content limit to less than the existing file size + let mut log = FileRotate::new( + &log_path, + AppendCount::new(2), + ContentLimit::Bytes(8), + Compression::None, + #[cfg(unix)] + None, + ); + + write!(log, "0123").unwrap(); + + let log_paths = log.log_paths(); + assert_eq!( + "nineteen characters", + fs::read_to_string(&log_paths[0]).unwrap() + ); + assert_eq!("0123", fs::read_to_string(&log_path).unwrap()); +} + fn get_fake_date_time(date_time: &str) -> DateTime { let date_obj = NaiveDateTime::parse_from_str(date_time, "%Y-%m-%dT%H:%M:%S"); From 405f6d8dfa56159d1b21cf84e1443f5e1d08242e Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Fri, 14 Oct 2022 13:05:38 +0200 Subject: [PATCH 37/46] Bump version to 0.7.1 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ecc6b2b..65b8dac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.0" +version = "0.7.1" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" From 5f904d85fae295a00f7e1e4c086c8cdf739d58e9 Mon Sep 17 00:00:00 2001 From: Harish Rajagopal Date: Fri, 27 Jan 2023 19:29:28 +0100 Subject: [PATCH 38/46] Remove time dependency to fix CVE-2020-26235 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 65b8dac..30e8e67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords= ["log", "rotate", "logrotate"] license = "MIT" [dependencies] -chrono = {version = "0.4.20", optional = true } +chrono = { version = "0.4.20", optional = true, default-features = false, features = ["clock"] } flate2 = "1.0" [dev-dependencies] From f9967d8e9994ead6a8d32f346f4d1f34605488ac Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Wed, 1 Mar 2023 13:23:52 +0100 Subject: [PATCH 39/46] Bump version to 0.7.2 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 30e8e67..5114961 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.1" +version = "0.7.2" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" From 1f02211e513c15224094e3c22cbbcea1a78a3600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Kj=C3=A4ll?= Date: Thu, 9 Mar 2023 16:54:43 +0100 Subject: [PATCH 40/46] replace tempdir with tempfile --- Cargo.toml | 2 +- src/lib.rs | 8 ++++---- src/suffix.rs | 6 +++--- src/tests.rs | 34 +++++++++++++++++----------------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5114961..0094455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ flate2 = "1.0" filetime = "0.2" quickcheck = "0.9.2" quickcheck_macros = "0.9.1" -tempdir = "0.3.7" +tempfile = "3" [features] default = ["chrono04"] diff --git a/src/lib.rs b/src/lib.rs index 811664a..257d2ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ //! // Here we choose to limit logs by 10 lines, and have at most 2 rotated log files. This //! // makes the total amount of log files 3, since the original file is present as well. //! -//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = tempfile::TempDir::new().unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! @@ -53,7 +53,7 @@ //! use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount, compression::Compression}; //! use std::{fs, io::Write}; //! -//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = tempfile::TempDir::new().unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! @@ -90,7 +90,7 @@ //! use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount, compression::Compression}; //! use std::{fs, io::Write}; //! -//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = tempfile::TempDir::new().unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! @@ -149,7 +149,7 @@ //! compression::Compression}; //! use std::{fs, io::Write}; //! -//! # let directory = tempdir::TempDir::new("rotation-doc-test").unwrap(); +//! # let directory = tempfile::TempDir::new().unwrap(); //! # let directory = directory.path(); //! let log_path = directory.join("my-log-file"); //! diff --git a/src/suffix.rs b/src/suffix.rs index 19772ae..3e8a56a 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -315,7 +315,7 @@ pub enum FileLimit { mod test { use super::*; use std::fs::File; - use tempdir::TempDir; + use tempfile::TempDir; #[test] fn timestamp_ordering() { assert!( @@ -340,7 +340,7 @@ mod test { #[test] fn timestamp_scan_suffixes_base_paths() { - let working_dir = tempdir::TempDir::new("file-rotate").unwrap(); + let working_dir = TempDir::new().unwrap(); let working_dir = working_dir.path().join("dir"); let suffix_scheme = AppendTimestamp::default(FileLimit::Age(Duration::weeks(1))); @@ -410,7 +410,7 @@ mod test { for (i, case) in cases.iter().enumerate() { println!("Case {}", i); - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("file"); diff --git a/src/tests.rs b/src/tests.rs index c17e8af..6a6a139 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,7 +1,7 @@ use super::{suffix::*, *}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; -use tempdir::TempDir; +use tempfile::TempDir; // Just useful to debug why test doesn't succeed #[allow(dead_code)] @@ -20,7 +20,7 @@ fn list(dir: &Path) { #[test] fn timestamp_max_files_rotation() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let log_path = tmp_dir.path().join("log"); let mut log = FileRotate::new( @@ -69,7 +69,7 @@ fn timestamp_max_files_rotation() { fn timestamp_max_age_deletion() { // In order not to have to sleep, and keep it deterministic, let's already create the log files and see how FileRotate // cleans up the old ones. - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("log"); @@ -106,7 +106,7 @@ fn timestamp_max_age_deletion() { } #[test] fn count_max_files_rotation() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); let mut log = FileRotate::new( @@ -148,7 +148,7 @@ fn count_max_files_rotation() { #[test] fn rotate_to_deleted_directory() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); let mut log = FileRotate::new( @@ -178,7 +178,7 @@ fn rotate_to_deleted_directory() { #[test] fn write_complete_record_until_bytes_surpassed() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("log"); @@ -205,7 +205,7 @@ fn write_complete_record_until_bytes_surpassed() { #[test] fn compression_on_rotation() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); let mut log = FileRotate::new( @@ -249,7 +249,7 @@ fn compression_on_rotation() { #[test] fn no_truncate() { // Don't truncate log file if it already exists - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); let file_rotate = || { @@ -274,7 +274,7 @@ fn no_truncate() { fn byte_count_recalculation() { // If there is already some content in the logging file, FileRotate should set its `count` // field to the size of the file, so that it rotates at the right time - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); @@ -303,7 +303,7 @@ fn byte_count_recalculation() { fn line_count_recalculation() { // If there is already some content in the logging file, FileRotate should set its `count` // field to the line count of the file, so that it rotates at the right time - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); @@ -342,7 +342,7 @@ fn unix_file_permissions() { let permissions = &[0o600, 0o644]; for permission in permissions { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); @@ -374,7 +374,7 @@ fn unix_file_permissions() { #[test] fn manual_rotation() { // Check that manual rotation works as intented - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let parent = tmp_dir.path(); let log_path = parent.join("log"); let mut log = FileRotate::new( @@ -400,7 +400,7 @@ fn manual_rotation() { #[quickcheck_macros::quickcheck] fn arbitrary_lines(count: usize) { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("log"); @@ -426,7 +426,7 @@ fn arbitrary_lines(count: usize) { #[quickcheck_macros::quickcheck] fn arbitrary_bytes(count: usize) { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("log"); @@ -505,7 +505,7 @@ fn rotate_by_time_frequency() { #[test] fn test_file_limit() { - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("file"); let old_file = dir.join("file.2022-02-01"); @@ -540,7 +540,7 @@ fn test_file_limit() { fn test_panic() { use std::io::Write; - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("file"); // write 9 bytes of data @@ -594,7 +594,7 @@ fn test_time_frequency( let old_time = get_fake_date_time(old_time); let new_time = get_fake_date_time(new_time); let second_old_time = get_fake_date_time(second_old_time); - let tmp_dir = TempDir::new("file-rotate-test").unwrap(); + let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path(); let log_path = dir.join("log"); From f7769774433d9ace0671ff24951971ce29e1af99 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Sun, 12 Mar 2023 11:43:20 +0100 Subject: [PATCH 41/46] Chrono dep no longer optional --- Cargo.toml | 6 +----- src/suffix.rs | 5 ----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0094455..a3e5b12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords= ["log", "rotate", "logrotate"] license = "MIT" [dependencies] -chrono = { version = "0.4.20", optional = true, default-features = false, features = ["clock"] } +chrono = { version = "0.4.20", default-features = false, features = ["clock"] } flate2 = "1.0" [dev-dependencies] @@ -18,7 +18,3 @@ filetime = "0.2" quickcheck = "0.9.2" quickcheck_macros = "0.9.1" tempfile = "3" - -[features] -default = ["chrono04"] -chrono04 = ["chrono"] diff --git a/src/suffix.rs b/src/suffix.rs index 3e8a56a..ae867c4 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -5,7 +5,6 @@ //! use super::now; use crate::SuffixInfo; -#[cfg(feature = "chrono04")] use chrono::{format::ParseErrorKind, offset::Local, Duration, NaiveDateTime}; use std::{ cmp::Ordering, @@ -157,7 +156,6 @@ pub enum DateFrom { /// Current limitations: /// - Neither `format` nor the base filename can include the character `"."`. /// - The `format` should ensure that the lexical and chronological orderings are the same -#[cfg(feature = "chrono04")] pub struct AppendTimestamp { /// The format of the timestamp suffix pub format: &'static str, @@ -167,7 +165,6 @@ pub struct AppendTimestamp { pub date_from: DateFrom, } -#[cfg(feature = "chrono04")] impl AppendTimestamp { /// With format `"%Y%m%dT%H%M%S"` pub fn default(file_limit: FileLimit) -> Self { @@ -220,7 +217,6 @@ impl std::fmt::Display for TimestampSuffix { } } -#[cfg(feature = "chrono04")] impl SuffixScheme for AppendTimestamp { type Repr = TimestampSuffix; @@ -303,7 +299,6 @@ impl SuffixScheme for AppendTimestamp { } /// How to determine whether a file should be deleted, in the case of [AppendTimestamp]. -#[cfg(feature = "chrono04")] pub enum FileLimit { /// Delete the oldest files if number of files is too high MaxFiles(usize), From 5b4f7d1c182874ffc4569afa3025acea6d24f845 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Tue, 14 Mar 2023 13:24:24 +0100 Subject: [PATCH 42/46] Bump to 0.7.3 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a3e5b12..ecddd68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.2" +version = "0.7.3" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" From 08b2a432c116594a2425f91af3eb46d6b30432b6 Mon Sep 17 00:00:00 2001 From: Kevin Robert Stravers Date: Sun, 14 May 2023 16:14:03 +0200 Subject: [PATCH 43/46] Move repository --- Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ecddd68..f6683d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "file-rotate" -version = "0.7.3" -authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] +version = "0.7.4" +authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" -homepage = "https://github.com/BourgondAries/file-rotate" -repository = "https://github.com/BourgondAries/file-rotate" +homepage = "https://github.com/kstrafe/file-rotate" +repository = "https://github.com/kstrafe/file-rotate" keywords= ["log", "rotate", "logrotate"] license = "MIT" From 3acb684906950f87f3be1abe03f061365cc16434 Mon Sep 17 00:00:00 2001 From: Kevin Robert Stravers Date: Thu, 8 Jun 2023 09:09:08 +0200 Subject: [PATCH 44/46] Fix readme examples to include compression --- Cargo.toml | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f6683d2..a3e23a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.4" +version = "0.7.5" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" diff --git a/README.md b/README.md index 50adcc8..98edd92 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ Following are some supplementary examples to get started. ## Basic example ```rust -use file_rotate::{FileRotate, ContentLimit, suffix::AppendCount}; +use file_rotate::{FileRotate, ContentLimit, compression::Compression, suffix::AppendCount}; use std::{fs, io::Write, path::PathBuf}; fn main() { - let mut log = FileRotate::new("logs/log", AppendCount::new(2), ContentLimit::Lines(3), None); + let mut log = FileRotate::new("logs/log", AppendCount::new(2), ContentLimit::Lines(3), Compression::None, None); // Write a bunch of lines writeln!(log, "Line 1: Hello World!"); @@ -51,6 +51,7 @@ let mut log = FileRotate::new( "logs/log", AppendTimestamp::default(FileLimit::MaxFiles(3)), ContentLimit::Lines(3), + Compression::None, None, ); @@ -95,4 +96,3 @@ This project is licensed under the [MIT license]. Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in file-rotate by you, shall be licensed as MIT, without any additional terms or conditions. - From fc60342baf0e8510d5f5f9e3ff5471f776f025b5 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 16 May 2024 20:46:09 +0200 Subject: [PATCH 45/46] Add FileLimit::Unlimited (implements #26) --- Cargo.toml | 2 +- src/lib.rs | 2 +- src/suffix.rs | 7 +++++-- src/tests.rs | 14 +++++++------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a3e23a7..ec4fb12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.5" +version = "0.7.6" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" diff --git a/src/lib.rs b/src/lib.rs index 257d2ef..898ee34 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -595,7 +595,7 @@ impl FileRotate { let mut result = Ok(()); for (i, suffix) in self.suffixes.iter().enumerate().rev() { if self.suffix_scheme.too_old(&suffix.suffix, i) { - result = result.and(std::fs::remove_file(suffix.to_path(&self.basepath))); + result = result.and(fs::remove_file(suffix.to_path(&self.basepath))); youngest_old = Some((*suffix).clone()); } else { break; diff --git a/src/suffix.rs b/src/suffix.rs index ae867c4..b7279f3 100644 --- a/src/suffix.rs +++ b/src/suffix.rs @@ -57,7 +57,7 @@ pub trait SuffixScheme { /// Find all files in the basepath.parent() directory that has path equal to basepath + a valid /// suffix. Return sorted collection - sorted from most recent to oldest based on the - /// [Ord](std::cmp::Ord) implementation of `Self::Repr`. + /// [Ord] implementation of `Self::Repr`. fn scan_suffixes(&self, basepath: &Path) -> BTreeSet> { let mut suffixes = BTreeSet::new(); let filename_prefix = basepath @@ -294,6 +294,7 @@ impl SuffixScheme for AppendTimestamp { let old_timestamp = (Local::now() - age).format(self.format).to_string(); suffix.timestamp < old_timestamp } + FileLimit::Unlimited => false, } } } @@ -304,6 +305,8 @@ pub enum FileLimit { MaxFiles(usize), /// Delete files whose age exceeds the `Duration` - age is determined by the suffix of the file Age(Duration), + /// Never delete files + Unlimited, } #[cfg(test)] @@ -410,7 +413,7 @@ mod test { let log_path = dir.join("file"); for suffix in case.suffixes.iter().chain(case.incorrect_suffixes) { - std::fs::File::create(dir.join(format!("file.{}", suffix))).unwrap(); + File::create(dir.join(format!("file.{}", suffix))).unwrap(); } let scheme = AppendTimestamp::with_format( diff --git a/src/tests.rs b/src/tests.rs index 6a6a139..4e545d7 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -92,7 +92,7 @@ fn timestamp_max_age_deletion() { ); writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); - let mut filenames = std::fs::read_dir(dir) + let mut filenames = fs::read_dir(dir) .unwrap() .filter_map(|entry| entry.ok()) .filter(|entry| entry.path().is_file()) @@ -278,7 +278,7 @@ fn byte_count_recalculation() { let parent = tmp_dir.path(); let log_path = parent.join("log"); - std::fs::write(&log_path, b"a").unwrap(); + fs::write(&log_path, b"a").unwrap(); let mut file_rotate = FileRotate::new( &*log_path.to_string_lossy(), @@ -292,10 +292,10 @@ fn byte_count_recalculation() { write!(file_rotate, "bc").unwrap(); assert_eq!(file_rotate.log_paths().len(), 1); // The size of the rotated file should be 2 ('ab) - let rotated_content = std::fs::read(&file_rotate.log_paths()[0]).unwrap(); + let rotated_content = fs::read(&file_rotate.log_paths()[0]).unwrap(); assert_eq!(rotated_content, b"ab"); // The size of the main file should be 1 ('c') - let main_content = std::fs::read(log_path).unwrap(); + let main_content = fs::read(log_path).unwrap(); assert_eq!(main_content, b"c"); } @@ -307,7 +307,7 @@ fn line_count_recalculation() { let parent = tmp_dir.path(); let log_path = parent.join("log"); - std::fs::write(&log_path, b"a\n").unwrap(); + fs::write(&log_path, b"a\n").unwrap(); let mut file_rotate = FileRotate::new( &*log_path.to_string_lossy(), @@ -510,14 +510,14 @@ fn test_file_limit() { let log_path = dir.join("file"); let old_file = dir.join("file.2022-02-01"); - std::fs::File::create(&old_file).unwrap(); + File::create(&old_file).unwrap(); let first = get_fake_date_time("2022-02-02T01:00:00"); let second = get_fake_date_time("2022-02-03T01:00:00"); let third = get_fake_date_time("2022-02-04T01:00:00"); let mut log = FileRotate::new( - &log_path, + log_path, AppendTimestamp::with_format("%Y-%m-%d", FileLimit::MaxFiles(1), DateFrom::DateYesterday), ContentLimit::Time(TimeFrequency::Daily), Compression::None, From 8ee371fd0e50d281e87d8a755aaeaa2c824f86b1 Mon Sep 17 00:00:00 2001 From: Erlend Langseth <3rlendhl@gmail.com> Date: Thu, 27 Feb 2025 08:25:33 +0100 Subject: [PATCH 46/46] Issue 29: Consistent number of arguments across OSes - take `Option` in `FileRotate::new` --- Cargo.toml | 2 +- examples/rotate_by_date.rs | 1 - src/lib.rs | 33 ++++++++++++--------------------- src/tests.rs | 31 ++++++++++--------------------- 4 files changed, 23 insertions(+), 44 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ec4fb12..6cf5e89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-rotate" -version = "0.7.6" +version = "0.8.0" authors = ["Kevin Robert Stravers ", "Erlend Langseth <3rlendhl@gmail.com>"] edition = "2018" description = "Log rotation for files" diff --git a/examples/rotate_by_date.rs b/examples/rotate_by_date.rs index 5775a4e..80f7002 100644 --- a/examples/rotate_by_date.rs +++ b/examples/rotate_by_date.rs @@ -11,7 +11,6 @@ fn main() { AppendTimestamp::with_format("%Y-%m-%d", FileLimit::MaxFiles(7), DateFrom::DateYesterday), ContentLimit::Time(TimeFrequency::Daily), Compression::None, - #[cfg(unix)] None, ); diff --git a/src/lib.rs b/src/lib.rs index 898ee34..179e3d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,6 @@ //! AppendCount::new(2), //! ContentLimit::Lines(3), //! Compression::None, -//! #[cfg(unix)] //! None, //! ); //! @@ -62,7 +61,6 @@ //! AppendCount::new(2), //! ContentLimit::Bytes(5), //! Compression::None, -//! #[cfg(unix)] //! None, //! ); //! @@ -99,7 +97,6 @@ //! AppendCount::new(3), //! ContentLimit::Bytes(1), //! Compression::None, -//! #[cfg(unix)] //! None, //! ); //! @@ -158,7 +155,6 @@ //! AppendTimestamp::default(FileLimit::MaxFiles(2)), //! ContentLimit::Bytes(1), //! Compression::None, -//! #[cfg(unix)] //! None, //! ); //! @@ -204,7 +200,6 @@ //! AppendTimestamp::default(FileLimit::MaxFiles(4)), //! ContentLimit::Bytes(1), //! Compression::OnRotate(2), -//! #[cfg(unix)] //! None, //! ); //! @@ -303,9 +298,6 @@ use std::{ }; use suffix::*; -#[cfg(unix)] -use std::os::unix::fs::OpenOptionsExt; - pub mod compression; pub mod suffix; #[cfg(test)] @@ -392,8 +384,7 @@ pub struct FileRotate { suffix_scheme: S, /// The bool is whether or not there's a .gz suffix to the filename suffixes: BTreeSet>, - #[cfg(unix)] - mode: Option, + open_options: Option, } impl FileRotate { @@ -404,6 +395,8 @@ impl FileRotate { /// /// `content_limit` specifies the limits for rotating a file. /// + /// `open_options`: If provided, you must set `.read(true).create(true).append(true)`! + /// /// # Panics /// /// Panics if `bytes == 0` or `lines == 0`. @@ -412,7 +405,7 @@ impl FileRotate { suffix_scheme: S, content_limit: ContentLimit, compression: Compression, - #[cfg(unix)] mode: Option, + open_options: Option, ) -> Self { match content_limit { ContentLimit::Bytes(bytes) => { @@ -440,8 +433,7 @@ impl FileRotate { compression, suffixes: BTreeSet::new(), suffix_scheme, - #[cfg(unix)] - mode, + open_options, }; s.ensure_log_directory_exists(); s.scan_suffixes(); @@ -484,14 +476,11 @@ impl FileRotate { } fn open_file(&mut self) { - let mut open_options = OpenOptions::new(); - - open_options.read(true).create(true).append(true); - - #[cfg(unix)] - if let Some(mode) = self.mode { - open_options.mode(mode); - } + let open_options = self.open_options.clone().unwrap_or_else(|| { + let mut o = OpenOptions::new(); + o.read(true).create(true).append(true); + o + }); self.file = open_options.open(&self.basepath).ok(); } @@ -772,10 +761,12 @@ pub mod mock_time { static MOCK_TIME: RefCell>> = RefCell::new(None); } + /// Get current _mocked_ time pub fn now() -> DateTime { MOCK_TIME.with(|cell| cell.borrow().as_ref().cloned().unwrap_or_else(Local::now)) } + /// Set mocked time pub fn set_mock_time(time: DateTime) { MOCK_TIME.with(|cell| *cell.borrow_mut() = Some(time)); } diff --git a/src/tests.rs b/src/tests.rs index 4e545d7..257d5c9 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -28,7 +28,6 @@ fn timestamp_max_files_rotation() { AppendTimestamp::default(FileLimit::MaxFiles(4)), ContentLimit::Lines(2), Compression::None, - #[cfg(unix)] None, ); @@ -65,7 +64,6 @@ fn timestamp_max_files_rotation() { assert_eq!("m\n", fs::read_to_string(&log_path).unwrap()); } #[test] -#[cfg(feature = "chrono04")] fn timestamp_max_age_deletion() { // In order not to have to sleep, and keep it deterministic, let's already create the log files and see how FileRotate // cleans up the old ones. @@ -74,9 +72,7 @@ fn timestamp_max_age_deletion() { let log_path = dir.join("log"); // One recent file: - let recent_file = chrono::offset::Local::now() - .format("log.%Y%m%dT%H%M%S") - .to_string(); + let recent_file = Local::now().format("log.%Y%m%dT%H%M%S").to_string(); File::create(dir.join(&recent_file)).unwrap(); // Two very old files: File::create(dir.join("log.20200825T151133")).unwrap(); @@ -87,7 +83,6 @@ fn timestamp_max_age_deletion() { AppendTimestamp::default(FileLimit::Age(chrono::Duration::weeks(1))), ContentLimit::Lines(1), Compression::None, - #[cfg(unix)] None, ); writeln!(log, "trigger\nat\nleast\none\nrotation").unwrap(); @@ -114,7 +109,6 @@ fn count_max_files_rotation() { AppendCount::new(4), ContentLimit::Lines(2), Compression::None, - #[cfg(unix)] None, ); @@ -156,7 +150,6 @@ fn rotate_to_deleted_directory() { AppendCount::new(4), ContentLimit::Lines(1), Compression::None, - #[cfg(unix)] None, ); @@ -187,7 +180,6 @@ fn write_complete_record_until_bytes_surpassed() { AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::BytesSurpassed(1), Compression::None, - #[cfg(unix)] None, ); @@ -213,7 +205,6 @@ fn compression_on_rotation() { AppendCount::new(3), ContentLimit::Lines(1), Compression::OnRotate(1), // Keep one file uncompressed - #[cfg(unix)] None, ); @@ -258,7 +249,6 @@ fn no_truncate() { AppendCount::new(3), ContentLimit::Lines(10000), Compression::None, - #[cfg(unix)] None, ) }; @@ -285,7 +275,6 @@ fn byte_count_recalculation() { AppendCount::new(3), ContentLimit::Bytes(2), Compression::None, - #[cfg(unix)] None, ); @@ -314,7 +303,6 @@ fn line_count_recalculation() { AppendCount::new(3), ContentLimit::Lines(2), Compression::None, - #[cfg(unix)] None, ); @@ -339,6 +327,7 @@ fn line_count_recalculation() { #[cfg(unix)] #[test] fn unix_file_permissions() { + use std::os::unix::fs::OpenOptionsExt; let permissions = &[0o600, 0o644]; for permission in permissions { @@ -346,12 +335,19 @@ fn unix_file_permissions() { let parent = tmp_dir.path(); let log_path = parent.join("log"); + let mut options = OpenOptions::new(); + options + .read(true) + .create(true) + .append(true) + .mode(*permission); + let mut file_rotate = FileRotate::new( &*log_path.to_string_lossy(), AppendCount::new(3), ContentLimit::Lines(2), Compression::None, - Some(*permission), + Some(options), ); // Trigger a rotation by writing three lines @@ -382,7 +378,6 @@ fn manual_rotation() { AppendCount::new(3), ContentLimit::None, Compression::None, - #[cfg(unix)] None, ); writeln!(log, "A").unwrap(); @@ -410,7 +405,6 @@ fn arbitrary_lines(count: usize) { AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::Lines(count), Compression::None, - #[cfg(unix)] None, ); @@ -436,7 +430,6 @@ fn arbitrary_bytes(count: usize) { AppendTimestamp::default(FileLimit::MaxFiles(100)), ContentLimit::Bytes(count), Compression::None, - #[cfg(unix)] None, ); @@ -521,7 +514,6 @@ fn test_file_limit() { AppendTimestamp::with_format("%Y-%m-%d", FileLimit::MaxFiles(1), DateFrom::DateYesterday), ContentLimit::Time(TimeFrequency::Daily), Compression::None, - #[cfg(unix)] None, ); @@ -550,7 +542,6 @@ fn test_panic() { AppendCount::new(2), ContentLimit::None, Compression::None, - #[cfg(unix)] None, ); @@ -563,7 +554,6 @@ fn test_panic() { AppendCount::new(2), ContentLimit::Bytes(8), Compression::None, - #[cfg(unix)] None, ); @@ -605,7 +595,6 @@ fn test_time_frequency( AppendTimestamp::with_format("%Y-%m-%d_%H-%M-%S", FileLimit::MaxFiles(7), date_from), ContentLimit::Time(frequency), Compression::None, - #[cfg(unix)] None, );