From 63a0268695128bb0edb1e9d100622bd6f80c73ee Mon Sep 17 00:00:00 2001 From: Amneesh Singh Date: Sun, 16 Nov 2025 18:03:32 +0530 Subject: [PATCH] feature: add cleanup Signed-off-by: Amneesh Singh --- Cargo.lock | 234 ++++++++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 4 + README.org | 14 ++- flake.nix | 11 +-- nix/yamaf.nix | 15 ++++ src/index.html | 70 +++++++++------ src/main.rs | 104 ++++++++++++++++++++-- 7 files changed, 397 insertions(+), 55 deletions(-) create mode 100644 nix/yamaf.nix diff --git a/Cargo.lock b/Cargo.lock index e3bd2f1..8041354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -9,10 +18,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "axum" -version = "0.8.6" +name = "autocfg" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core", "bytes", @@ -68,10 +83,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] -name = "bytes" -version = "1.10.1" +name = "bumpalo" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36" +dependencies = [ + "find-msvc-tools", + "shlex", +] [[package]] name = "cfg-if" @@ -79,6 +110,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -88,6 +138,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + [[package]] name = "fnv" version = "1.0.7" @@ -215,9 +271,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", @@ -236,9 +292,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ "bytes", "futures-core", @@ -250,12 +306,46 @@ dependencies = [ "tower-service", ] +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.177" @@ -333,6 +423,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -451,6 +550,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.20" @@ -528,6 +633,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.6" @@ -567,9 +678,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "syn" -version = "2.0.109" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -704,12 +815,110 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -804,6 +1013,7 @@ name = "yamaf" version = "0.1.0" dependencies = [ "axum", + "chrono", "futures-util", "mime_guess", "rand", diff --git a/Cargo.toml b/Cargo.toml index f29e80d..5f81ac4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,12 @@ edition = "2024" [dependencies] axum = { version = "0.8.6", features = ["multipart"] } +chrono = "*" futures-util = "0.3.31" mime_guess= "*" rand = "*" tokio = { version = "1.48.0", features = ["full"] } tokio-util = { version = "0.7.17", features = ["io"] } + +[features] +cleanup = [] diff --git a/README.org b/README.org index ffab415..1e03e2d 100644 --- a/README.org +++ b/README.org @@ -18,7 +18,19 @@ | EXTERNAL_HOST | User facing domain name, used to return accessible URLs | ${INTERNAL_HOST} | | EXTERNAL_HAS_TLS | URLs in html have https if set, otherwise http | - | | MAX_FILES | This does not actually limit the number of files, and used to calculate bodysize | 10 | -| MAX_FILESIZE_MB | Determines the max filesize possible, used to return failure and calculate bodysize | 100M | +| MAX_FILESIZE_MB | Determines the max filesize possible, used to return failure and calculate bodysize | 100M | +| MIN_FILEDAYS | Minimum number of days for file retention | 30 days | +| MAX_FILEDAYS | Maximum number of days for file retention | 365 days | + +** Retention + +File retention is as per the following formula + +#+begin_src text +retention = min_days + (max_days - min_days) * (1 - file_size / max_size)) ^ e +#+end_src + +Cleanup happens everyday when midnight (IST) has passed [fn:1] https://git.weirdnatto.in/natto1784/simple-filehost/ diff --git a/flake.nix b/flake.nix index 843cff1..cbe82a9 100644 --- a/flake.nix +++ b/flake.nix @@ -47,17 +47,14 @@ cargoArtifacts = craneLib.buildDepsOnly commonArgs; - yamaf = craneLib.buildPackage ( - commonArgs - // { - inherit cargoArtifacts; - doCheck = false; - } - ); + yamaf = pkgs.callPackage ./nix/yamaf.nix { + inherit craneLib lib src; + }; in { packages = { inherit yamaf; + yamaf-all = yamaf.override { withCleanup = true; }; default = yamaf; image = pkgs.dockerTools.buildImage { name = "yamaf"; diff --git a/nix/yamaf.nix b/nix/yamaf.nix new file mode 100644 index 0000000..fe5ca4b --- /dev/null +++ b/nix/yamaf.nix @@ -0,0 +1,15 @@ +{ + craneLib, + lib, + src, + withCleanup ? false, +}: +let + cargoArtifacts = craneLib.buildDepsOnly { inherit src; }; +in +craneLib.buildPackage { + inherit cargoArtifacts src; + doCheck = false; + + cargoExtraArgs = lib.optionalString withCleanup "--features cleanup"; +} diff --git a/src/index.html b/src/index.html index d7c6c89..ce82214 100644 --- a/src/index.html +++ b/src/index.html @@ -1,31 +1,45 @@ - - {{TITLE}} - - -

{{TITLE}}

-

- Use curl to upload: -
- - curl -F file=@"[file]" {{USER_URL}} - -
-
- If key is enabled then a field "key" will be required. -
- Make sure the key is the first field in the multipart before any files. -
- - curl -F "key=[key]" -F file=@"[file]" -F file=@"[file2]" {{USER_URL}} - -

-
- {{KEY_FIELD}} - -
- -
-
+ + {{TITLE}} + + +

{{TITLE}}

+ +

+ Use curl to upload: +
+ + curl -F file=@"[file]" {{USER_URL}} + +
+
+ If key is enabled then a field "key" will be required. +
+ Make sure the key is the first field in the multipart before any files. +
+ + curl -F "key=[key]" -F file=@"[file]" -F file=@"[file2]" {{USER_URL}} + +

+ +
+ {{KEY_FIELD}} + +
+ +
+ + +

Cleanup

+ File retention is as per the following formula + 

+    max_size = {{MAX_SIZE}} // in megabytes
+    max_days = {{MAX_DAYS}}
+    min_days = {{MIN_DAYS}}
+    retention = min_days + (max_days - min_days) * (1 - file_size / max_size)) ^ e
+
+ Cleanup happens everyday when midnight (IST) has passed + + diff --git a/src/main.rs b/src/main.rs index 013b391..e8cffca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,11 +6,10 @@ use axum::{ response::{Html, IntoResponse, Response}, routing::get, }; - use rand::{Rng, distr::Alphanumeric}; use std::{env, net::SocketAddr, sync::LazyLock}; use std::{env::VarError, path::PathBuf}; -use tokio::fs::File; +use tokio::fs; use tokio_util::io::ReaderStream; struct Config { @@ -23,12 +22,14 @@ struct Config { external_protocol: &'static str, max_filesize: usize, max_bodysize: usize, + min_filedays: usize, + max_filedays: usize, } static CONFIG: LazyLock = LazyLock::new(|| { let root_dir = env::var("ROOT_DIR").unwrap_or_else(|_| "/var/files".to_string()); let key = env::var("KEY"); - let title = env::var("TITLE").unwrap_or_else(|_| "Simpler Filehost".to_string()); + let title = env::var("TITLE").unwrap_or_else(|_| "Yet Another Mid Ahh Filehost".to_string()); let internal_host = env::var("INTERNAL_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); let internal_port = env::var("INTERNAL_PORT") @@ -54,6 +55,18 @@ static CONFIG: LazyLock = LazyLock::new(|| { << 20; let max_bodysize = max_files * max_filesize * 2; + let min_filedays = env::var("MIN_FILEDAYS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(30); + + let max_filedays = env::var("MAX_FILEDAYS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(365); + + assert!(max_filedays >= min_filedays); + Config { root_dir, key, @@ -64,6 +77,8 @@ static CONFIG: LazyLock = LazyLock::new(|| { external_protocol, max_filesize, max_bodysize, + min_filedays, + max_filedays, } }); @@ -86,6 +101,35 @@ async fn main() { addr, CONFIG.root_dir ); + /* cleanup cronjob */ + #[cfg(feature = "cleanup")] + tokio::spawn(async move { + use chrono::{Local, NaiveDate}; + + let mut last_cleanup_day: NaiveDate = Local::now().date_naive(); + + loop { + use tokio::time::{Duration, sleep}; + + /* every 30 minutes */ + sleep(Duration::from_secs(30 * 60)).await; + + let now = Local::now(); + let today = now.date_naive(); + + /* if the date changed, midnight has passed */ + if today > last_cleanup_day { + println!("attempting cleanup at {}", now.to_string()); + if let Err(e) = cleanup_directory().await { + eprintln!("cleanup failed: {}", e); + } else { + println!("cleanup succeeded!"); + last_cleanup_day = today; + } + } + } + }); + axum::serve(listener, app).await.unwrap(); } @@ -108,6 +152,19 @@ static INDEX_HTML: LazyLock = LazyLock::new(|| { }, ); + if cfg!(feature = "cleanup") { + html = html + .replace("{{MAX_SIZE}}", &(CONFIG.max_filesize >> 20).to_string()) + .replace("{{MAX_DAYS}}", &CONFIG.max_filedays.to_string()) + .replace("{{MIN_DAYS}}", &CONFIG.min_filedays.to_string()); + } else { + if let Some(start) = html.find("") { + if let Some(end) = html.find("") { + html.replace_range(start..end + "".len(), ""); + } + } + } + html }); @@ -205,7 +262,7 @@ async fn upload(mut payload: Multipart) -> Result let save_path = std::path::Path::new(&CONFIG.root_dir).join(&filename); - let mut file = File::create(&save_path) + let mut file = fs::File::create(&save_path) .await .map_err(|_| YamafError::InternalError("Internal i/o error".into()))?; @@ -223,7 +280,7 @@ async fn upload(mut payload: Multipart) -> Result .ok_or_else(|| YamafError::BadRequest("File too large".into()))?; if written > CONFIG.max_filesize { - _ = tokio::fs::remove_file(&save_path).await; + _ = fs::remove_file(&save_path).await; return Err(YamafError::FileTooBig(filename)); } @@ -260,10 +317,10 @@ async fn upload(mut payload: Multipart) -> Result async fn serve_file(Path(filename): Path) -> Result { let path = PathBuf::from(&CONFIG.root_dir).join(&filename); - let metadata = tokio::fs::metadata(&path) + let metadata = fs::metadata(&path) .await .map_err(|_| YamafError::FileNotFound)?; - let file = File::open(&path) + let file = fs::File::open(&path) .await .map_err(|_| YamafError::FileNotFound)?; let mime = mime_guess::from_path(&path).first_or_octet_stream(); @@ -289,3 +346,36 @@ async fn serve_file(Path(filename): Path) -> Result Result<(), std::io::Error> { + let dir = std::path::Path::new(&CONFIG.root_dir); + let mut entries = fs::read_dir(dir).await?; + + while let Some(entry) = entries.next_entry().await? { + use std::os::unix::fs::MetadataExt; + use std::time::SystemTime; + + let path = entry.path(); + let meta = entry.metadata().await?; + let modified = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH); + let age = SystemTime::now() + .duration_since(modified) + .unwrap_or_default(); + + /* min_days + (max_days - min_days) * (1 - size/max_size)^e */ + let retention = (CONFIG.min_filedays as f64 + + (CONFIG.max_filedays - CONFIG.min_filedays) as f64 + * (1.0 - (meta.size() as f64 / CONFIG.max_filesize as f64)) + .powf(std::f64::consts::E)) + * 24.0 + * 60.0; + + if meta.is_file() && age.as_secs_f64() > retention { + fs::remove_file(&path).await?; + println!("Deleted {:?}", path); + } + } + + Ok(()) +}