diff --git a/Cargo.lock b/Cargo.lock index 662ae37..13b82b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aes" @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -69,44 +69,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", - "once_cell", + "once_cell_polyfill", "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "async-trait" @@ -121,15 +121,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -148,9 +148,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "block-buffer" @@ -172,9 +172,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" @@ -193,24 +193,24 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.19" +version = "1.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" dependencies = [ "shlex", ] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "core-foundation-sys" @@ -253,9 +253,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -365,12 +365,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.11" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -517,20 +517,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", @@ -546,9 +535,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "hdfs-native" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9a986a98854573dfbc130f42f81e92f6d4581e23060708842fede6edef0f1f" +checksum = "2b4cabd42d46aac490d01b80596dfd930a8316b4ee3e8b0c94a7cba888791c0f" dependencies = [ "aes", "base64", @@ -659,21 +648,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -682,31 +672,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -714,67 +684,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "idna" version = "1.0.3" @@ -788,9 +745,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -806,6 +763,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -829,9 +797,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.6" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f33145a5cbea837164362c7bd596106eb7c5198f97d1ba6f6ebb3223952e488" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" dependencies = [ "jiff-static", "log", @@ -842,9 +810,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.6" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ce13c40ec6956157a3635d97a1ee2df323b263f09ea14165131289cb0f5c19" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", @@ -863,18 +831,18 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.171" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.2", ] [[package]] @@ -885,15 +853,15 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -917,28 +885,28 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -961,9 +929,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" +checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" dependencies = [ "async-trait", "bytes", @@ -974,11 +942,14 @@ dependencies = [ "itertools", "parking_lot", "percent-encoding", + "rand", "thiserror", "tokio", "tracing", "url", "walkdir", + "wasm-bindgen-futures", + "web-time", ] [[package]] @@ -987,11 +958,17 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -999,9 +976,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", @@ -1030,9 +1007,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "portable-atomic-util" @@ -1043,6 +1020,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1054,9 +1040,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -1104,26 +1090,25 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.8.5" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "libc", "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" -version = "0.3.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", "rand_core", @@ -1131,18 +1116,18 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.2.15", + "getrandom", ] [[package]] name = "redox_syscall" -version = "0.5.11" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" dependencies = [ "bitflags", ] @@ -1184,28 +1169,28 @@ checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "same-file" @@ -1218,9 +1203,9 @@ dependencies = [ [[package]] name = "scc" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea091f6cac2595aa38993f04f4ee692ed43757035c36e67c180b6828356385b1" +checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4" dependencies = [ "sdd", ] @@ -1233,9 +1218,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sdd" -version = "3.0.8" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584e070911c7017da6cb2eb0788d09f43d789029b5877d3e5ecc8acf86ceee21" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" [[package]] name = "serde" @@ -1290,24 +1275,21 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1327,9 +1309,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.100" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -1338,9 +1320,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", @@ -1369,9 +1351,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -1379,15 +1361,17 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.2" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", @@ -1417,9 +1401,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", @@ -1428,9 +1412,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", ] @@ -1458,12 +1442,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -1478,11 +1456,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ - "getrandom 0.3.2", + "getrandom", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -1503,9 +1483,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -1548,6 +1528,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -1590,6 +1583,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "7.0.3" @@ -1624,9 +1627,9 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", @@ -1659,24 +1662,24 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-result" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link", ] @@ -1708,6 +1711,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -1732,13 +1744,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -1751,6 +1779,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -1763,6 +1797,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -1775,12 +1815,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -1793,6 +1845,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -1805,6 +1863,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -1817,6 +1881,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -1829,6 +1899,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winsafe" version = "0.0.19" @@ -1844,23 +1920,17 @@ dependencies = [ "bitflags", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -1870,9 +1940,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", @@ -1882,18 +1952,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", @@ -1921,11 +1991,22 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -1934,9 +2015,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index da79dc1..1c9758f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ bytes = "1" chrono = "0.4" futures = "0.3" hdfs-native = "0.11" -object_store = "0.12" +object_store = "0.12.2" thiserror = "2" tokio = { version = "1", features = ["rt", "net", "io-util", "macros", "sync", "time"] } @@ -25,4 +25,4 @@ env_logger = "0.11" serial_test = "3" [features] -integration-test = ["hdfs-native/integration-test"] +integration-test = ["hdfs-native/integration-test", "object_store/integration"] diff --git a/src/lib.rs b/src/lib.rs index 66e5968..466767e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,3 @@ -// #![warn(missing_docs)] //! [object_store::ObjectStore] implementation for the Native Rust HDFS client //! //! # Usage @@ -27,10 +26,10 @@ use futures::{ FutureExt, }; use hdfs_native::{client::FileStatus, file::FileWriter, Client, HdfsError, WriteOptions}; +#[allow(deprecated)] use object_store::{ - path::Path, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, - ObjectMeta, ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, - UploadPart, + path::Path, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, + ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, UploadPart, }; use tokio::{ sync::{mpsc, oneshot}, @@ -41,6 +40,16 @@ use tokio::{ #[cfg(feature = "integration-test")] pub use hdfs_native::minidfs; +fn generic_error( + source: Box, +) -> object_store::Error { + object_store::Error::Generic { + store: "HFDS", + source, + } +} + +/// Interface for [Hadoop Distributed File System](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). #[derive(Debug, Clone)] pub struct HdfsObjectStore { client: Arc, @@ -143,7 +152,7 @@ impl HdfsObjectStore { .to_string(); let tmp_file_path = path_buf - .with_file_name(format!(".{}.tmp", file_name)) + .with_file_name(format!(".{file_name}.tmp")) .to_str() .ok_or(HdfsError::InvalidPath("path not valid unicode".to_string())) .to_object_store_err()? @@ -152,7 +161,7 @@ impl HdfsObjectStore { // Try to create a file with an incrementing index until we find one that doesn't exist yet let mut index = 1; loop { - let path = format!("{}.{}", tmp_file_path, index); + let path = format!("{tmp_file_path}.{index}"); match self.client.create(&path, WriteOptions::default()).await { Ok(writer) => break Ok((writer, path)), Err(HdfsError::AlreadyExists(_)) => index += 1, @@ -191,9 +200,7 @@ impl ObjectStore for HdfsObjectStore { PutMode::Create => false, PutMode::Overwrite => true, PutMode::Update(_) => { - return Err(object_store::Error::NotSupported { - source: "Update mode not supported".to_string().into(), - }) + return Err(object_store::Error::NotImplemented); } }; @@ -218,14 +225,17 @@ impl ObjectStore for HdfsObjectStore { .await .to_object_store_err()?; + let e_tag = self.head(location).await?.e_tag; + Ok(PutResult { - e_tag: None, + e_tag, version: None, }) } /// Create a multipart writer that writes to a temporary file in a background task, and renames /// to the final destination on complete. + #[allow(deprecated)] async fn put_multipart_opts( &self, location: &Path, @@ -245,23 +255,15 @@ impl ObjectStore for HdfsObjectStore { /// Reads data for the specified location. async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { - if options.if_match.is_some() - || options.if_none_match.is_some() - || options.if_modified_since.is_some() - || options.if_unmodified_since.is_some() - { - return Err(object_store::Error::NotImplemented); - } - let meta = self.head(location).await?; + options.check_preconditions(&meta)?; + let range = options .range - .map(|r| match r { - GetRange::Bounded(range) => range, - GetRange::Offset(offset) => offset..meta.size, - GetRange::Suffix(suffix) => meta.size.saturating_sub(suffix)..meta.size, - }) + .map(|r| r.as_range(meta.size)) + .transpose() + .map_err(|source| generic_error(source.into()))? .unwrap_or(0..meta.size); let reader = self @@ -301,10 +303,10 @@ impl ObjectStore for HdfsObjectStore { .to_object_store_err()?; if status.isdir { - return Err(HdfsError::IsADirectoryError( - "Head must be called on a file".to_string(), - )) - .to_object_store_err(); + return Err(object_store::Error::NotFound { + path: location.to_string(), + source: "Head cannot be called on a directory".into(), + }); } get_object_meta(&status) @@ -319,10 +321,7 @@ impl ObjectStore for HdfsObjectStore { .to_object_store_err()?; if !result { - Err(HdfsError::OperationFailed( - "failed to delete object".to_string(), - )) - .to_object_store_err()? + Err(HdfsError::FileNotFound(location.to_string())).to_object_store_err()? } Ok(()) @@ -335,16 +334,17 @@ impl ObjectStore for HdfsObjectStore { /// /// Note: the order of returned [`ObjectMeta`] is not guaranteed fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + let absolute_dir = prefix.map(make_absolute_file).unwrap_or("/".to_string()); + let status_stream = self .client - .list_status_iter( - &prefix.map(make_absolute_dir).unwrap_or("/".to_string()), - true, - ) + .list_status_iter(&absolute_dir, true) .into_stream() - .filter(|res| { + .filter(move |res| { let result = match res { - Ok(status) => !status.isdir, + // Directories aren't a thing in object stores so ignore them, and if a file is listed + // directly that should be ignored as well + Ok(status) => !status.isdir && status.path != absolute_dir, // Listing by prefix should just return an empty list if the prefix isn't found Err(HdfsError::FileNotFound(_)) => false, _ => true, @@ -363,15 +363,16 @@ impl ObjectStore for HdfsObjectStore { /// Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix of `foo/bar/x` but not of /// `foo/bar_baz/x`. async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + let absolute_dir = prefix.map(make_absolute_file).unwrap_or("/".to_string()); + let mut status_stream = self .client - .list_status_iter( - &prefix.map(make_absolute_dir).unwrap_or("".to_string()), - false, - ) + .list_status_iter(&absolute_dir, false) .into_stream() - .filter(|res| { + .filter(move |res| { let result = match res { + // If a file is listed directly it should be ignored + Ok(status) => status.path != absolute_dir, // Listing by prefix should just return an empty list if the prefix isn't found Err(HdfsError::FileNotFound(_)) => false, _ => true, @@ -402,6 +403,18 @@ impl ObjectStore for HdfsObjectStore { /// Renames a file. This operation is guaranteed to be atomic. async fn rename(&self, from: &Path, to: &Path) -> Result<()> { + // Make sure the parent directory exists + let mut parent: Vec<_> = to.parts().collect(); + parent.pop(); + + if !parent.is_empty() { + let parent_path: Path = parent.into_iter().collect(); + self.client + .mkdirs(&make_absolute_dir(&parent_path), 0o755, true) + .await + .to_object_store_err()?; + } + Ok(self .client .rename(&make_absolute_file(from), &make_absolute_file(to), true) @@ -412,11 +425,10 @@ impl ObjectStore for HdfsObjectStore { /// Renames a file only if the distination doesn't exist. This operation is guaranteed /// to be atomic. async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> { - Ok(self - .client + self.client .rename(&make_absolute_file(from), &make_absolute_file(to), false) .await - .to_object_store_err()?) + .to_object_store_err() } /// Copy an object from one path to another in the same object store. @@ -438,12 +450,6 @@ impl ObjectStore for HdfsObjectStore { } } -#[cfg(feature = "integration-test")] -pub trait HdfsErrorConvert { - fn to_object_store_err(self) -> Result; -} - -#[cfg(not(feature = "integration-test"))] trait HdfsErrorConvert { fn to_object_store_err(self) -> Result; } @@ -502,36 +508,20 @@ impl HdfsMultipartWriter { mut part_receiver: mpsc::UnboundedReceiver<(oneshot::Sender>, PutPayload)>, ) -> JoinHandle> { task::spawn(async move { - 'outer: loop { + loop { match part_receiver.recv().await { Some((sender, part)) => { for bytes in part { - if let Err(e) = writer.write(bytes).await.to_object_store_err() { - let _ = sender.send(Err(e)); - break 'outer; + if let Err(e) = writer.write(bytes).await { + let _ = sender.send(Err(e).to_object_store_err()); + return Err(generic_error("Failed to write all parts".into())); } } let _ = sender.send(Ok(())); } - None => { - return writer.close().await.to_object_store_err(); - } + None => return writer.close().await.to_object_store_err(), } } - - // If we've reached here, a write task failed so just return Err's for all new parts that come in - while let Some((sender, _)) = part_receiver.recv().await { - let _ = sender.send( - Err(HdfsError::OperationFailed( - "Write failed during one of the parts".to_string(), - )) - .to_object_store_err(), - ); - } - Err(HdfsError::OperationFailed( - "Write failed during one of the parts".to_string(), - )) - .to_object_store_err() }) } } @@ -551,19 +541,23 @@ impl MultipartUpload for HdfsMultipartWriter { let (result_sender, result_receiver) = oneshot::channel(); if let Some((_, payload_sender)) = self.sender.as_ref() { - payload_sender.send((result_sender, payload)).unwrap(); + if let Err(mpsc::error::SendError((result_sender, _))) = + payload_sender.send((result_sender, payload)) + { + let _ = result_sender.send(Err(generic_error("Write task failed".into()))); + } } else { - result_sender - .send( - Err(HdfsError::OperationFailed( - "Cannot put part after completing or aborting".to_string(), - )) - .to_object_store_err(), - ) - .unwrap(); + let _ = result_sender.send(Err(generic_error( + "Cannot put part after completing or aborting".into(), + ))); } - async { result_receiver.await.unwrap() }.boxed() + async { + result_receiver + .await + .unwrap_or_else(|_| Err(generic_error("Write task failed".into()))) + } + .boxed() } async fn complete(&mut self) -> Result { @@ -584,9 +578,9 @@ impl MultipartUpload for HdfsMultipartWriter { version: None, }) } else { - Err(object_store::Error::NotSupported { - source: "Cannot call abort or complete multiple times".into(), - }) + Err(generic_error( + "Cannot call abort or complete multiple times".into(), + )) } } @@ -605,9 +599,9 @@ impl MultipartUpload for HdfsMultipartWriter { Ok(()) } else { - Err(object_store::Error::NotSupported { - source: "Cannot call abort or complete multiple times".into(), - }) + Err(generic_error( + "Cannot call abort or complete multiple times".into(), + )) } } } @@ -625,16 +619,54 @@ fn make_absolute_dir(path: &Path) -> String { } } +fn get_etag(status: &FileStatus) -> String { + let size = status.length; + let mtime = status.modification_time; + + // Use an ETag scheme based on that used by many popular HTTP servers + // + // + format!("{mtime:x}-{size:x}") +} + fn get_object_meta(status: &FileStatus) -> Result { Ok(ObjectMeta { location: Path::parse(&status.path)?, last_modified: DateTime::::from_timestamp_millis(status.modification_time as i64) - .unwrap(), - size: status - .length - .try_into() - .expect("unable to convert status.length to usize"), - e_tag: None, + .ok_or(generic_error( + "Last modified timestamp out of bounds".into(), + ))?, + size: status.length as u64, + e_tag: Some(get_etag(status)), version: None, }) } + +#[cfg(test)] +#[cfg(feature = "integration-test")] +mod test { + use std::collections::HashSet; + + use object_store::integration::*; + + use crate::HdfsObjectStore; + + #[tokio::test] + async fn hdfs_test() { + let dfs = hdfs_native::minidfs::MiniDfs::with_features(&HashSet::from([ + hdfs_native::minidfs::DfsFeatures::HA, + ])); + + let integration = HdfsObjectStore::with_url(&dfs.url).unwrap(); + + put_get_delete_list(&integration).await; + list_uses_directories_correctly(&integration).await; + list_with_delimiter(&integration).await; + rename_and_copy(&integration).await; + copy_if_not_exists(&integration).await; + multipart_race_condition(&integration, true).await; + multipart_out_of_order(&integration).await; + get_opts(&integration).await; + put_opts(&integration, false).await; + } +} diff --git a/tests/test_object_store.rs b/tests/test_object_store.rs deleted file mode 100644 index 93c5611..0000000 --- a/tests/test_object_store.rs +++ /dev/null @@ -1,318 +0,0 @@ -#[cfg(feature = "integration-test")] -mod test { - use bytes::{Buf, BufMut, BytesMut}; - use chrono::{Datelike, Utc}; - use hdfs_native::{ - minidfs::{DfsFeatures, MiniDfs}, - Client, WriteOptions, - }; - use hdfs_native_object_store::{HdfsErrorConvert, HdfsObjectStore}; - use object_store::{PutMode, PutOptions, PutPayload}; - use serial_test::serial; - use std::{collections::HashSet, sync::Arc}; - - pub const TEST_FILE_INTS: u64 = 64 * 1024 * 1024; - - #[tokio::test] - #[serial] - async fn test_object_store() -> object_store::Result<()> { - let dfs = MiniDfs::with_features(&HashSet::from([DfsFeatures::HA])); - let client = Arc::new(Client::new(&dfs.url).to_object_store_err()?); - - // Create a test file with the client directly to sanity check reads and lists - let mut file = client - .create("/testfile", WriteOptions::default()) - .await - .unwrap(); - let mut buf = BytesMut::new(); - for i in 0..TEST_FILE_INTS as i32 { - buf.put_i32(i); - } - file.write(buf.freeze()).await.unwrap(); - file.close().await.unwrap(); - - client.mkdirs("/testdir", 0o755, true).await.unwrap(); - - let store = HdfsObjectStore::new(Arc::clone(&client)); - - test_object_store_head(&store, &client).await?; - test_object_store_list(&store).await?; - test_object_store_rename(&store).await?; - test_object_store_read(&store).await?; - test_object_store_write(&store).await?; - test_object_store_write_multipart(&store).await?; - test_object_store_copy(&store).await?; - - Ok(()) - } - - async fn test_object_store_head( - store: &HdfsObjectStore, - client: &Arc, - ) -> object_store::Result<()> { - use object_store::{path::Path, ObjectStore}; - - let status = client.get_file_info("/testfile").await.unwrap(); - - let head = store.head(&Path::from("/testfile")).await?; - assert_eq!(head.location, Path::from("/testfile")); - assert_eq!(head.size, TEST_FILE_INTS * 4); - assert_eq!( - head.last_modified.timestamp_millis(), - status.modification_time as i64 - ); - assert_eq!(head.last_modified.year(), Utc::now().year()); - - assert!(store.head(&Path::from("/testfile2")).await.is_err()); - assert!(store.head(&Path::from("/testdir")).await.is_err()); - - Ok(()) - } - - async fn test_object_store_list(store: &HdfsObjectStore) -> object_store::Result<()> { - use futures::StreamExt; - use object_store::{path::Path, ObjectMeta, ObjectStore}; - - let list: Vec> = - store.list(Some(&Path::from("/"))).collect().await; - - assert_eq!(list.len(), 1); - assert_eq!(list[0].as_ref().unwrap().location, Path::from("/testfile")); - - // A list of None should be logically equivalent to / - let list: Vec> = store.list(None).collect().await; - assert_eq!(list.len(), 1); - assert_eq!(list[0].as_ref().unwrap().location, Path::from("/testfile")); - - // Listing of a prefix that doesn't exist should return an empty result, not an error - assert_eq!( - store.list(Some(&Path::from("/doesnt/exist"))).count().await, - 0 - ); - - let list = store - .list_with_delimiter(Some(&Path::from("/doesnt/exist"))) - .await?; - - assert_eq!(list.common_prefixes.len(), 0); - assert_eq!(list.objects.len(), 0); - - Ok(()) - } - - async fn test_object_store_rename(store: &HdfsObjectStore) -> object_store::Result<()> { - use object_store::{path::Path, ObjectStore}; - - store - .rename(&Path::from("/testfile"), &Path::from("/testfile2")) - .await?; - - assert!(store.head(&Path::from("/testfile2")).await.is_ok()); - assert!(store.head(&Path::from("/testfile")).await.is_err()); - - store - .rename(&Path::from("/testfile2"), &Path::from("/testfile")) - .await?; - - assert!(store.head(&Path::from("/testfile")).await.is_ok()); - assert!(store.head(&Path::from("/testfile2")).await.is_err()); - - Ok(()) - } - - async fn test_object_store_read(store: &HdfsObjectStore) -> object_store::Result<()> { - use object_store::{path::Path, ObjectStore}; - - let location = Path::from("/testfile"); - - let mut buf = store.get(&location).await?.bytes().await?; - for i in 0..TEST_FILE_INTS as i32 { - assert_eq!(buf.get_i32(), i); - } - - // Read a single integer from the file - let offset = TEST_FILE_INTS / 2 * 4; - let mut buf = store.get_range(&location, offset..(offset + 4)).await?; - assert_eq!(buf.get_i32(), TEST_FILE_INTS as i32 / 2); - Ok(()) - } - - async fn test_object_store_write(store: &HdfsObjectStore) -> object_store::Result<()> { - use object_store::{path::Path, ObjectStore}; - - store - .put(&Path::from("/newfile"), PutPayload::new()) - .await?; - store.head(&Path::from("/newfile")).await?; - - // PutMode = Create should fail for existing file - match store - .put_opts( - &Path::from("/newfile"), - PutPayload::new(), - PutOptions { - mode: PutMode::Create, - ..Default::default() - }, - ) - .await - { - Err(object_store::Error::AlreadyExists { .. }) => (), - Err(e) => panic!("Wrong error was thrown for put without overewrite: {:?}", e), - Ok(_) => panic!("No error was thrown for put without overwrite for existing file"), - } - - // Check a small files, a file that is exactly one block, and a file slightly bigger than a block - for size_to_check in [16i32, 128 * 1024 * 1024, 130 * 1024 * 1024] { - let ints_to_write = size_to_check / 4; - - let mut data = BytesMut::with_capacity(size_to_check as usize); - for i in 0..ints_to_write { - data.put_i32(i); - } - - let buf = data.freeze(); - store - .put(&Path::from("/newfile"), PutPayload::from_bytes(buf.clone())) - .await?; - - assert_eq!( - store.head(&Path::from("/newfile")).await?.size, - size_to_check.try_into().unwrap() - ); - - let read_data = store.get(&Path::from("/newfile")).await?.bytes().await?; - - assert_eq!(buf.len(), read_data.len()); - - for pos in 0..buf.len() { - assert_eq!( - buf[pos], read_data[pos], - "data is different as position {} for size {}", - pos, size_to_check - ); - } - } - - store.delete(&Path::from("/newfile")).await?; - - Ok(()) - } - - async fn test_object_store_write_multipart( - store: &HdfsObjectStore, - ) -> object_store::Result<()> { - use object_store::{path::Path, ObjectStore}; - - let mut uploader = store.put_multipart(&"/newfile".into()).await?; - uploader.complete().await?; - - store.head(&Path::from("/newfile")).await?; - - // Check a small files, a file that is exactly one block, and a file slightly bigger than a block - for size_to_check in [16i32, 128 * 1024 * 1024, 130 * 1024 * 1024] { - let ints_to_write = size_to_check / 4; - - let mut uploader = store.put_multipart(&"/newfile".into()).await?; - - let mut data = BytesMut::with_capacity(size_to_check as usize); - for i in 0..ints_to_write { - data.put_i32(i); - } - - // Write in 10 MiB chunks - let mut buf = data.freeze(); - while !buf.is_empty() { - let to_write = usize::min(buf.len(), 10 * 1024 * 1024); - uploader.put_part(buf.split_to(to_write).into()).await?; - } - - uploader.complete().await?; - - assert_eq!( - store.head(&Path::from("/newfile")).await?.size, - size_to_check.try_into().unwrap() - ); - - let mut read_data = store.get(&Path::from("/newfile")).await?.bytes().await?; - - assert_eq!(size_to_check as usize, read_data.len()); - - for pos in 0..ints_to_write { - assert_eq!( - pos, - read_data.get_i32(), - "data is different at integer {} for size {}", - pos, - size_to_check - ); - } - } - - store.delete(&Path::from("/newfile")).await?; - - // Test aborting - let mut uploader = store.put_multipart(&"/newfile".into()).await?; - assert!(store.head(&"/.newfile.tmp.1".into()).await.is_ok()); - uploader.abort().await?; - assert!(store.head(&"/.newfile.tmp.1".into()).await.is_err()); - assert!(store.head(&"/newfile".into()).await.is_err()); - - // Test multiple uploads to the same destination, default is to overwrite - let mut uploader1 = store.put_multipart(&"/newfile".into()).await?; - let mut uploader2 = store.put_multipart(&"/newfile".into()).await?; - uploader1.put_part(vec![1].into()).await?; - uploader2.put_part(vec![2].into()).await?; - uploader1.complete().await?; - uploader2.complete().await?; - - let result = store.get(&"/newfile".into()).await?; - assert!(result.bytes().await?.to_vec() == vec![2]); - - Ok(()) - } - - async fn test_object_store_copy(store: &HdfsObjectStore) -> object_store::Result<()> { - use object_store::{path::Path, ObjectStore}; - - store - .put(&Path::from("/newfile"), PutPayload::new()) - .await?; - - let size_to_check = 128 * 1024 * 1024; - let ints_to_write = size_to_check / 4; - let mut data = BytesMut::with_capacity(size_to_check as usize); - for i in 0..ints_to_write { - data.put_i32(i); - } - - let buf = data.freeze(); - store - .put(&Path::from("/newfile"), buf.clone().into()) - .await?; - store - .copy(&Path::from("/newfile"), &Path::from("/newfile2")) - .await?; - - let read_data = store.get(&Path::from("/newfile2")).await?.bytes().await?; - assert_eq!(buf.len(), read_data.len()); - - for pos in 0..buf.len() { - assert_eq!( - buf[pos], read_data[pos], - "data is different as position {} for size {}", - pos, size_to_check - ); - } - - assert!(store - .copy_if_not_exists(&Path::from("/newfile"), &Path::from("/newfile2")) - .await - .is_err()); - - store.delete(&Path::from("/newfile")).await?; - store.delete(&Path::from("/newfile2")).await?; - - Ok(()) - } -}