From e3f3ce103e6049966ae038c17a44a041131ad828 Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Sat, 5 Jul 2025 16:55:27 +0200 Subject: [PATCH 01/10] new spring-batch-s3 module Signed-off-by: Andrea Cioni --- spring-batch-s3/.editorconfig | 10 + spring-batch-s3/.gitignore | 1 + spring-batch-s3/.mvn/maven.config | 2 + .../.mvn/wrapper/maven-wrapper.jar | Bin 0 -> 49519 bytes .../.mvn/wrapper/maven-wrapper.properties | 1 + spring-batch-s3/README.adoc | 182 ++++++++++++++ spring-batch-s3/mvnw | 236 ++++++++++++++++++ spring-batch-s3/mvnw.cmd | 146 +++++++++++ spring-batch-s3/pom.xml | 224 +++++++++++++++++ .../batch/extensions/s3/S3ItemReader.java | 140 +++++++++++ .../batch/extensions/s3/S3ItemWriter.java | 152 +++++++++++ .../batch/extensions/s3/package-info.java | 28 +++ .../s3/serializer/S3Deserializer.java | 39 +++ .../s3/serializer/S3Serializer.java | 31 +++ .../s3/serializer/S3StringDeserializer.java | 66 +++++ .../s3/serializer/S3StringSerializer.java | 36 +++ .../s3/serializer/package-info.java | 27 ++ .../batch/extensions/s3/stream/Defaults.java | 26 ++ .../extensions/s3/stream/S3InputStream.java | 78 ++++++ .../s3/stream/S3MultipartOutputStream.java | 116 +++++++++ .../s3/stream/S3MultipartUploader.java | 201 +++++++++++++++ .../extensions/s3/stream/S3OutputStream.java | 116 +++++++++ .../extensions/s3/stream/S3Uploader.java | 26 ++ .../extensions/s3/stream/package-info.java | 27 ++ .../extensions/s3/S3ItemReaderTests.java | 171 +++++++++++++ .../extensions/s3/S3ItemWriterTests.java | 101 ++++++++ .../serializer/S3StringDeserializerTests.java | 70 ++++++ .../serializer/S3StringSerializerTests.java | 62 +++++ .../s3/stream/S3InputStreamTests.java | 69 +++++ .../stream/S3MultipartOutputStreamTests.java | 87 +++++++ .../s3/stream/S3MultipartUploaderTests.java | 102 ++++++++ .../s3/stream/S3OutputStreamTests.java | 59 +++++ 32 files changed, 2632 insertions(+) create mode 100644 spring-batch-s3/.editorconfig create mode 100644 spring-batch-s3/.gitignore create mode 100644 spring-batch-s3/.mvn/maven.config create mode 100644 spring-batch-s3/.mvn/wrapper/maven-wrapper.jar create mode 100644 spring-batch-s3/.mvn/wrapper/maven-wrapper.properties create mode 100644 spring-batch-s3/README.adoc create mode 100755 spring-batch-s3/mvnw create mode 100644 spring-batch-s3/mvnw.cmd create mode 100644 spring-batch-s3/pom.xml create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/package-info.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Deserializer.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Serializer.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializer.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializer.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/package-info.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3InputStream.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploader.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3Uploader.java create mode 100644 spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/package-info.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemWriterTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializerTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializerTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3InputStreamTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java create mode 100644 spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java diff --git a/spring-batch-s3/.editorconfig b/spring-batch-s3/.editorconfig new file mode 100644 index 00000000..3e1127a6 --- /dev/null +++ b/spring-batch-s3/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*.{adoc,bat,groovy,html,java,js,jsp,kt,kts,md,properties,py,rb,sh,sql,svg,txt,xml,xsd}] +charset = utf-8 + +[*.{groovy,java,kt,kts,xml,xsd}] +indent_style = tab +indent_size = 4 +continuation_indent_size = 8 +end_of_line = lf diff --git a/spring-batch-s3/.gitignore b/spring-batch-s3/.gitignore new file mode 100644 index 00000000..d75620ab --- /dev/null +++ b/spring-batch-s3/.gitignore @@ -0,0 +1 @@ +.flattened-pom.xml diff --git a/spring-batch-s3/.mvn/maven.config b/spring-batch-s3/.mvn/maven.config new file mode 100644 index 00000000..64ec47f3 --- /dev/null +++ b/spring-batch-s3/.mvn/maven.config @@ -0,0 +1,2 @@ +-ntp +-V diff --git a/spring-batch-s3/.mvn/wrapper/maven-wrapper.jar b/spring-batch-s3/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..c6feb8bb6f76f2553e266ff8bf8867105154237e GIT binary patch literal 49519 zcmb@tV|1n6wzeBvGe*U>ZQHh;%-Bg)Y}={WHY%yuwkkF%MnzxVwRUS~wY|@J_gP;% z^VfXZ{5793?z><89(^dufT2xlYVOQnYG>@?lA@vQF|UF0&X7tk8BUf?wq2J& zZe&>>paKUg4@;fwk0yeUPvM$yk)=f>TSFFB^a8f|_@mbE#MaBnd5qf6;hXq}c%IeK zn7gB0Kldbedq-vl@2wxJi{$%lufroKUjQLSFmt|<;M8~<5otM5ur#Dgc@ivmwRiYZW(Oco7kb8DWmo|a{coqYMU2raB9r6e9viK6MI3c&%jp05-Tf*O#6@8Ra=egYy01 z-V!G;_omANEvU-8!*>*)lWka9M<+IkNsrsenbXOfLc6qrYe`;lpst;vfs*70$z9UM zq%L>pFCOr$X*|9&3L2h;?VA9-IU*iR6FiGlJ=b~DzE5s^thxXUs4%~*zD#K&k>wZAU8 zpaa!M+Z-zjkfGK15N!&o<3=cgbZV7%ex@j^)Q9V`q^i;Fsbkbe6eHJ;dx{QbdCCs1 zdxq^WxoPsr`eiK3D0Ep}k$ank-0G&+lY!ZHDZBYEx%% z2FyE?Lb0cflLB)kDIj;G=m`^UO<4h(RWdF-DT>p{1J5J90!K!AgC0)?jxPbm$KUjg zJED+#7xQmAmr`(S%BQTV-c97As~r3zD$E;3S)@}p5udA@m6pLgRL5h-;m>LvCq?&Q zokC7Vnk-zBEaa;=Y;6(LJHS>mOJV&%0YfRdUOqbKZy~b z(905jIW0Pg;y`Yv2t+RnDvL4yGEUX*tK)JT6TWn4ik~L)fX#tAV!d8)+A)qWtSjcr z7s|f%f;*%XW!jiRvv9ayj@f&dc|1tKDc{O3BWcLGsn-OYyXRLXEOEwP4k?c`nIut0 z?4S;eO@EoynmkxHq>QpDL1q^wOQxrl))2qya?dk05^5hK? z{P6;WKHUaHw9B0dd&|xw&CYN2fVrn};Gq<=Z^QZk3e~HzzY~JrnPCs0XwMp#B<9Gm zw0?7h#4EY%O-ub6mi&O2vcpIkuM?st;RtEpKSz^Xr#3WHhpsZd!gh|_jGQ`KA30T- zKlz9vgB;pY^}Uh??nQKSzk>2&J+Qi*r3DeX4^$%2ag9^x_YckA-f9p_;8ulh(8j9~ zes{O#{v!m%n^el(VryTF-C%xfJJ$rZj)|Y|8o&))q9CEwg2;Wz&xzyHD=@T_B%b}C z=8G^*4*J4#jUJn{7-3^U(_uUp6E8+GDt#le)nya-Q4kL5ZGiFxT4bF+mX`whcif*? z>CL&Ryn3HHT^^QmWYr<}Q1_Jj7fOh}cS8r+^R#at-CnNl3!1_$96&7nR}gh}))7a0J&z-_eI))+{RCt)r8|7|sV9o01^9nv?aePxMqwPP!x|sNmnn&6{K$K*mVX9lxSAmcqAV1(hKA-=coeTb*otxTOGYXsh zW$31^q7L@<#y~SUYoNKP1JK?4|FQNQb$i8mCG@WhX9i_^;@M2f#!nq7_K*M!4lGz1 z5tfADkO7BZDLgVQ?k7C)f;$eqjHI&zgxhf}x$8^ZEwFfm-qY=+M+fbS)9r8fFE5H9 zv{WPU35cR8%z;(W%5<>y+E&v84J4^Y##N!$B++RI`CZ1i3IW9Nau=*pSxW&^Ov-F> zex=&9XYLVcm1Y?am>2VC`%gMev9$#~; zYwxYvMfeKFsd!OBB@eOb2QNHFcsfKm;&z{OVEUiYmQ}~L@>$Ms@|Ptf3jQO-=Q;1+ zFCw+p+Z3lK_FmIAYnk2V;o915cDM}%Ht5RH%w}P>Yg9{h1mZ}~R6tUII4X7i4-2i% z2Uiw3_uHR!d~5(s;p6btI@-xhAkRg9K|n#}PNT9Dw9P>z$3>30lP1(=mcQ|tpyv3@ ze1qU!69OAx4s7$8r7Y-#5I`m!BXq`f!6C(BtUlG-oq+liqMCS_D@0nSFc%y+N6_Zh zi%L3LhF3zZP{d1)L&SXxPD(fp@T@J;jZeNaf$zl>vAh7=tI z2;wS^QyRdZm~)Ur&!af;8eB8*7(F96K^=WbC$)#TWvB~Awo5AtPf8Il4snD}Xsqd< z>cH+gcg72nTg5tl>oFbwdT{BDyy1=f=4~h~L$)UX;FXa;NdSlyF{(YLrx&VDp`pQI zh3pQtC=d8i1V6yUmFon*LQsNYWen?eO-gSZ4cvYcdEd0klSxcBYw+|5AyCv6TT96h z{7Yh9`h}biU?3oBFn=d8>Hn`1Q*w6rgeX^QbC-WFwjY}Int0;qUny4WMjIee@#0%l z>YAWLVCNo1lp$>9L$Tx`t!dp?>5Pfbhc*!*wzfWkj_x`Q?`3Jc@9r8uq~dgb+lgeh zlA`eUal3e2ZnWQSSYB>qy#85^>j7!=uO-hG5*erp22NaC81#Ytioc>r?D9$b_JiC+ zSp)8KR$%}FjFNRkeE#c5vKbXNJDBoO< z)73Jt7Y|3v45efud1xkg2GO3OwYfsuBV`f6S_D>Aoh2%=`1Y$bHP>0kBvTSowX57H z&1nbbx=IT>X^ScKYL&&{LNq~^UNgR|at`D;SxTYpLvnj_F*bGgNV2tEl1k$ccA&NW zmX(LV*>Op)BOgoric(98mIU)$eUa&jM5bKlnOrHm$p^v@u;W0J)!@XWg+#X=9En(-tiw!l?65rD=zzl(+%<)bI{ZN;SRco{jO;>7 zlSY|TIxuN|d#YHx^^~>iYj2V>cC>wQwWzGVI!6#epjJ6tl_`7tDY17WMKMB@s*Jr& zXOs*@>EwQ6s>M13eZEBJ#q0|;8jao{wK4keesH9?$OSk~_3#*x`8fAzQa7fprQ6(Z zi$}B%m81y*S)RxaX;wW!5{{EDw8)IE3XDRO1Y^%TMr}c|Y>WBAKT=b*K&uMT(?JSl zO>gVtl_bKQ$??TeWr7wYO+Vbl?CTQj?JrW&td`|#@;R2Gca9jq^p`{@)KY97o3}Af zfTh{pUUWD;P7sq=I!lA6;*hq0Nq`F56T)x$K?BMOk}tptYw(%$?*otp2N6IF3#GgqM46Cda!qzvGZcMgcGV`bY5ZIfOB6^;US#WgRai zq#vS8ZqPY953|eFw<-p2Cakx|z#_{4pG}mk{EANI{PnK*CUslvS8whko=OTe13|It z>{O2p=mmanR2-n>LQHaMo}noWCmjFO@7^z~`Y{V>O`@rT{yBS=VXsb}*Pi_zDqM3? zjCZqWR}fEzAkms+Hiq8~qRAFvo}dVW{1gcZ?v&PdX?UG*yS}zT9g7nZ!F1WRH}sHA zJ4~B2Br~8?uhbaX!3g+7=3fVM)q^wEzv**rk5e34==NRCV z3G$G5B!DICFslm)c){oesa_0muLxGoq`xYVNURl*NhE#v2>y9vDz&vJwrB`Q>DhN# zY2GnY!Y^8E%PU0}haXL$8a5QN1-&7NWuC~{62j| z2ozmFyx8GpOzj?&KK1JF28;E8H_p4N^LMm9K0y}!lCxcK79eFGTtGm?7jy?t94Q@X zli|our1#|>f*68fyA0bSn=YisYSl8HB(dFN4Y$qb7p4DR0YQt=^eEMnJkgiM48$>QV6x5*^a|D|t zMPDk}u<^YEYrt|H&hy)DRk%rDIb{LTo;h7=fp^J9Lr&`{9`8_pS*tQ_$KXB$2#5{h z-&yPbN-zInq{7aYZuaItS8-2Mb4OQe2jD*&)0~898E|HlAq`o!M&It@vvnj z_y@))>~_oR%S8OfmFTGYIat^#8_YKMqWLac<^}RZFDcJqvSJa>&6HaLS7p-$)QyL= zHrO|t75`d41Bp37RZtKR%g^%o@9C5Ce=CjuvVQ-KI#Uw2WWa>cho;jztUt~Le*_pT zkfA2iif9QFp;vhd)|A?tdAQ?9o~?EqgL;=)eKFQ{E^u?OIP}fl^5A;$^ZVutCIqj5 z&*i+G?!Px|5~~6zTYf>~uw*kM`5p&Hju&#w!7^An3*mQwTK22wC7p^OsvMjWf`$MY zLX|ZFV#+>Uq2!QyRD9cgbI9nswteMAMWtK(_=d%r?TLrx?_rkjbjI(rbK#T9Gn}J| z5ajow3ZErpw+%}YfVL-q^{r~##xJ^_ux2yO1!LJZXg)>F70STV=&Ruwp&XP^_?$h0 zn>$a?!>N+Kt$UXzg`e+szB}*uw)Z$uL6?>*!0IrE)SgV~#a?Qgg7HuTsu3ncrcs|l z=sQSMtr}S!sQ4SriKg=M`1Y|bC`XJ+J(YT)op!Q);kj0_e)YNVNw8SI|1f%9%X?i5>$lLE(Wfc$wY?(O985d5e*)UPtF!7gG3(Kd z-^=-%-wWCEK`r4oFh^{|;Ci%W^P>K%9dBNDqi%c$Q{iY#(zbwN7~pQI=SHd%WuV7Z zO?0P;Zc6yeN;)IbJIP0=>W)EgE!76jM^?IyQ*D(T})1NGmP z~YAb6T^#R6;)Ls;cV~LWk z33lcLpbSjxStw9Z>Nv&+rPOXxCGB=?ttZs?{OF7;GYlV&w7-82POb$XrogqFpLA2`j&MLZXr=IG>PAFSb2np~x;E_kV{ zsDwbK$?iYRn7$;mHYZhQn6P2#_hXAHd?;q~!Zy}%;@%wT3u|Sa-!WxxOE_fwyFv*Db@>X;Rl+fK1oP?55*dN0#2%SuikZ)y7Kx>`8*9d?}5 zKvXF7J5&Ey6{A8qUFxrFOh<$xdSWV^dw7z|`7RVZJhAwO72V zRrM_3*wI`^ycl7~>6KaCYBr#WGR>}B)Q(V%&$MhVrU>u~ql zjGeZF&>=_ld$oY!V}5}Gb> z*iP38KOav9RHY)0uITwgz99w- zJX-0BGCdY*$c7pi@>@-`2>#>}c(DHaI62ntpKz z`c01Z#u7WuMZ71!jl7hv5|o61+uv5nG?*dffEL~328P5HlKh2&RQ;9X@f>c1x<>v= zZWNSz3Ii~oyAsKCmbd}|$2%ZN&3gc9>(NV=Z4Fnz2F@)PPbx1wwVMsUn=-G=cqE3# zjY{G4OI~2o$|*iuswTg1=hcZK$C=0^rOt-aOwXuxU=*uT?yF00)6sE}ZAZyy*$ZTH zk!P*xILX#5RygHy{k?2((&pRQv9_Ew+wZ>KPho_o1-{~I*s1h8 zBse@ONdkk-8EG?r5qof}lwTxdmmEN|%qw(STW|PFsw1LD!h_Vjo;C4?@h|da4Y;*; zvApQ=T&=jWU39Uz=_yN@Bn0{{)yn8RZ2&X!<*KBv-7tcWdkF1Ij8D0mU zwbcs}0vDaLGd@xx%S_QZ1H)GTt`~>+#z}HXJTl9S!sd9seVJc|_wUMSdD$>k`K_RG zlq(fsnR@KM^;C}}&vG2t+}_nGPuI5ovg$6TYeMPIREGxP@2r~RKd@>gV`mq0XENsh z%IRZ-ZNP+4#J`o-yRpP;w@;CrSr3wiix3e9Qc|s(WapRq950P->g|JYC$A)$YrGeH zz5dKlAHAPJ>%?llqqB&#+#VU3sp=9>Xms1J;tSYN>LMwNtU68yr!})K4X>%^IrIDp z>SHy&6fJHybwS^BW>okFeaQp6wxaVP`hy;ZX#e+=w3c?PGD&_LmeqL8oZ*YaM1+#S z5WNAKo4+99JW(+qcMjh;+c%R#R?t;(aQ`2`C=bo((ERzgAwKKazXy*0wHN;v;P|f> zBW&?`h#_I^?Bc5GX7XP@|MOiw%&-#?EQ|w+FdCl_&qPN&s$|Z17UCF9oXS#N z)px6>zm&}0osTnCGI;AXsj`q=LpIsW4x}q~70uey5N_NpdJ*Gv^@$g@f2{EB>LP7Y zE5P`jZh1vHNgk7LfMT({jLCjRZa4ubW;UA#%<@Zj?efrPdm{W3J5UEFgm`YkVqz;AMFetZuM5uQpvORb1GDX`WZGwTrF z46+&sAri5QXCfGYpdgonWR5`>ZEa;?jrKvfNvXF<&l)1uU-3q#4X16R2~?P0yg3H` zfw82QWZo^cac+%(g^_6`+2>~Fvy{pOCGnj86+=-!N`GPWAjus1ejhn6f4|mDkU6EE z&u~;xfdRMkj=h;4d~~+4(>L8weT3cz9e@E11EH!tX<IC!@kS+dsIQA`HQ2vdoS zzSD0U?mb1M0@qXu{yhZk2Y6}2B-AvvYg|tRr6z*_*2l*VLiR6G;M{O^Znq~LI%=I_ zCEU{htx&Bo+69G`p|A@R>KlY1*;;!{aWq?Pc0Cu!mT-0S`!>3<@s%Ri;utYNQ+CXDj+LC5<*$4*$-mogGg^S~3JRv{ry zPJzKJg!XKb>P}yJVc^1V@T&MV{z;@DLhvV{dG?RogCcPkROivliSr58>5Zw&&A2?n z9`JOLU;eQGaOr6GB(u{t3!+$NaLge$x#M&*sg!J;m~rRc)Ij5|?KX_4WiM-eE%t8e zqUM7eZ~ZonavR;K4g2t$4Fj=UVyEHM7LPb%8#0?Ks{~?!qhx9)2^>rg8{0npLtFKR zJB)19TFiD^T7IUXA8wt!@n5gj&@OK~EO}MR6^qd?^-?%-0~b2K9RWh+_mSEQQWsLCFOt#JlAQMgNxvv-m z;sF*r;WZ*Wi@I|6pMN+|_rLYKlWwvpKZY9rA;fo8l8hFQGI?4#kt1-r4UL;nPF@{~ z2T~a@2>yD|GuU55boxoIIe_BFo2Vq&rs&2itv|B>OC*bIeOqMBRw~y5KRMwiVHc)` zIBdliiY?Ai7*+k#NZf3MW5!hya~RZ6r7k)b?HF0e(n`ZX=iCpT7St`FDwL@SGgKlq zNnnU*3IcnYDzJg{7V$cb`xeb4(s(({&%f69XMTw-JQErS%?X_}?&y&tvHw@>1v{#R z4J@(=el^kRI+jGa;4)l#v%-jM^$~0ulxh6-{w*4Lsa>Tuc z>ElR3uM~GUChI)c{TW${73A3$vs<&iH;e?4HjW2MvSz9tp9@69+`_@x{Qte^eFo5IlAi&zw$=t6u8K%8JtjRI88PFNM7R>DaCO3rgngmk zI-RMOyt@kr-gVra=tl^@J#tI7M$dird(?aU!`&1xcm~2;dHN(RCxh4H((f|orQ!BS zu;(3Vn+^doXaqlhnjBJj-)w?5{;EEZTMx+?G>Rp4U^g<_yw_blAkdbj=5YrNhZB9@ zNmW=-!yFx5?5aF^+6*1XI|s3lIn_eyh`uv%?liNzSC#z&z^R(mqEYL@TdWzgkf>g1 zedzs*={eJavn{8vF%4nf@et<@wkOPR>NiVuYtESbFXQ;sDz_;|ITVeoW|me5>jN5P z5--{13JT{3ktkAf9M;Jty)yectg#{+9sK{C;2CvPU81tB3{8S5>hK{EXdVe?fR?sd8m`V zPM*$)g$HKp0~9Xf6#z!YJ&g!%VkCMxkt>ofE!62?#-&%|95^)JJ9 zk;GlJdoH0HwtDF(_aTv}mt$?EyRyE6@pm5DG~Gj-2%3HcZT13e)$)z99bdK_WCx|Q zQNza(R)Z>ZKTn8oIdcw%c^pFaMpFZ4HOds!BODgSBWJJYW3I_WJvoEm4xsfs%#LZ6 zdPCk{5XJ>2f7Hj-i*9lTW6BKCIuy)3L!b3(uPoSgW1WA+OEYYBRgSsJq7wjHh%c8ymMs3FU%~cprqL*084p*^T3{J%Gwq`jB30n(&y6- zII8-_r-s5&CVtsoNZ9%On?7yn;oZG03-$wx^uRk9>b*ufh15|HHk|%=MA^ioyb9CYU$7y$4R|M5HvpiCTxKSU`LUg$+ zB3IBl&{qO}agqF~BFM6&11wMeR-#Rkuh_(^j+P4{;X_w|siva$5P`dykyhfAUD%e8 z+{G0|7(Q`_U91sMKFO^rHoCWfXi0$^ev)-187G}klYv@+Rf%uZ&T4-Uhh=)pcU6O1 znXc^c5)!$X+39|4`yNHuCj0wkm+K1VN0G3_EL?-ZH$p5Y*v6ec4MV zS~1~}ZUhl&i^4`Fa|zyH4I%rXp;D6{&@*^TPEX2;4aI$}H@*ROEyFfe^RZI%;T>X> z>WVSUmx@2gGBxkV&nfyPK=JI$HxRKUv(-*xA_C;lDxT|PgX*&YYdkrd5-*3E1OSXBs>35DLsHHp%zm+n0N(Yu{lMo>_t&d1Xy zfCxl=(CNNx>ze+7w)60mp>(M``Qn$aUrVb$cJAb6=Do7VgW`Qn2;v5{9tB)jP$_mB zn{Hb_sMs4yxK|!`PI7+zO68}{Iv)dpu!+ZZl)xuoVU(oFsm<3gT{j2c*ORl|Lt+?dR^M?0 znW6rNA)cR*ci;z?BaG(f(XynY_y+kTjj~T$9{N{>ITQ4-DmZ6{cOkoea9*LpYL{Apo0hSpLqJu z9`tjP&ei;%pn9QY>-$9=<73M#X;qGb+%Bt0x>=u`eDtthI+LWB9CdAO=ulZo9&Ohs2X8GW>b7#&U|py28KTvPBl#Nqv^{AgkVXrOyS z@%3)}$I&mJOYWoG$BBb)Kb~0ptDmBxHNH^i6B8FA7NR2HfTnjP?eDnoY4NS_aYg4P zGGPw11sAf^^fTkY#j@T#6Ll*^GVaPo-1;aS6_a}{r{tWZilzse2m zc?LS=B|EWxCD|!O%|%t3C@Rd7=rKJRsteAWRoDu|*Kx-QwYZQeYpGrZ_1J%mFM;*S*u=0 z%1OC9>kmCGqBBu#-1jVPRVW*BTv%3uPI8fO?JOZD#P_W^V+K7&KVB>hzZ@PdY*%Ezo;}|5Mk`Mo2m*_K%no*jDJGp(s9j;&U`Z>z zO#SEe)k!p$VE-j2xDoX$!;Up5%8x$c`GH$l+gTA*YQaE0jwCOA<*__2NkV){z_u2=4NQ zSk$(oj$%ygio?3V8T3IyGMYvPs`t{im2IoHs7or+>>MYvG%Q?PwOLqe%73uGh6Wn; zo>e7qI$9?%cVVkvQLOLKcU5n*`~qn8pzkdu=Z4#2VnhUy>S*;kT=NqA!dQtnE?wVg zOKobxJ|QCjk`!(2*~5NQx{{=Lr=)ndyn{V|&PxUa=xQXVU?#M24F8H%C*uvs(#Va0 zSkp}0EFYq0#9xp&$O?gIInc#^^_6Ol88W%)S5A@HeE0(SR&!Yl>u=*5JEoUViDR@2 zJBjTsp=Y44W`Nb2+*CcZCkwP(QChX1s)b09DEIZCKt1$q2~;&DJ9!{bQ1Y6&T_9u1 zZM8^im8Wf#FUO6tZqc7#`z0cN_JA>#U_b7he%?cCnlV2&47y5Fc)Z7bp5xGe1zNq9 zl1VaV-tsm3fY=oIX^SPl!P;9$o?**0brq#ShM~3CXhh^SK0oOKB9O>;q3G@ z&4&h$mLSgohc^5IC|H>IGfZvVQFUT>T$|U7{znY`56<5d)07oiv*2R0+-BGPPkWJ! zIOzKF+<5o2YLWP|SGCx8w@<>u6K1o`++xJ+6kaJrt<&0Haq zyUccgxI$sR07Vo9-pF);heBva;?&NcAzC*gSSG9B3c?A;IH9J zl$j%F4*8;F0;H2Cjo*kWz4{kSh?nX}23&&KL+U(#nOAuR`wn@uwUNkWEgb*ZShKPy z`aXTJT4f*Um4`iv2KOfzf-~`#pOfH8>is*xnLBDTyx2Xuc8Y2Od6z((P2AZK@b_96 z#0V6jdw>sEDJ#uNGV|EshD1g&bYZCzCZTZ)286HLHc8Eyy_HPi;d#%;Wx}d6tUUxq z_VB$+898z_{9-A<*v6VI7?(dC04o!8$>DQ$OdbrA_@<6auiBNp{Dw$Hs@@gcybIQT zAU7Pc5YEX&&9IZ~iDo&V`&8K$-4o$)g?wF8xdv1I8-n}1bc7tviIBqt z#iIl1Hn;W?>2&#bU#VZ1wxq(7z=Q15#0yoz)#|r`KSPKI-{aN%l61^?B4RMDt?Vk` z)G#K6vUN?C!t{Q<@O4$0(qI>$U@@TI2FVF;AhSSb5}LtXx&=k&8%MWM3wv;Xq0p~W z#ZX;QFv5G9-i6=+d;R7Dwi)ciIZ1_V!aw;K^etau+g0fOA2HXpV#LQZGzf?h#@}(o z|3w!sZ|&mp$;tmDiO=zef5C|Alz+@@4u5#yZ7yNpP=&`432%a{K#{;nsS!jwk-$Qs zZRty}+N`Y~)c8|$&ra{bOQWM2K7qa}4Y{ndK%dKp&{ zFCvX{PAy_C{xzS_-`0>JlPP7&5!5 zBQ$NQz^z#2y-VeIxnfY|RzU`w+1t6vwQ|wM)LlpuaUzYehGII;>2DYyR|~wC@l97s zgX=f*1qtfDyco%BHmN+o<2qoi`D67R+RM$$NN5-moE4kx3MCFfuip*45nComOZKQf z3!(8tkSdhY5+A%@Y=eVEZkXU3S6B2V-R$ZuRIXWhsrJg3g)p4vXY@RV60bKuG zT6T!enE<;(A{*HPQhae*(@_!maV~AWD4EOwq10tkCXq+HPoe_Pu?d4Kg=2ypcs?&f zLa>mEmPF4ucJ%i~fEsNIa{QmQU27%Abh|w(`q)s~He5$5WYQ_wNJX6Qop<=7;I1jd zNZak`}0lVm+^O!i;|Lwo}ofXuJ)*UtH4xaPm*R7?YS*<&D__=@Kki>{f_Z-XqM;Tj195+~@d;rx zh5pj8oMuupWa#E(%85**I~1Zat-Sa^_R11-CiKdd`8m(DGuzOm9lX$Dd!DX!_Al}d zS!-|}dWG80S;`jSKDH%Uv;-OJNeBI0Bp$z->{_>1KU%h&Af7nns(L=xRN1 zLvOP=*UWIr)_5G2+fCsUV7mV|D>-~_VnvZ3_>=9 z_bL6`eK%W*9eJ34&Puz^@^ZIyoF@%DTun#OOEdUEn8>N9q(}?5*?`o?!_<(i%yc`k zf!xXD6SQscHgPgiHt>x6{n{+}%azrfV4VHi#umyi0;11c816`E??2`$;Rc`)qA2H( z5L|{o=ut7Te=^~@cR0_#cah0?w0Me$&>}ga8xxy=?DDl#}S~Y z4o2n`%IyGjQEP%8qS|v(kFK&RCJbF1gsRVJ>ceSjU`LuYJu%C>SRV#l`)ShD&KKzv ztD<9l0lcW0UQ8xjv|1NXRrCZhZh3JFX_BNT@V|u9$o~8M=cjOX|5iBS|9PAGPvQLc z6sA~BTM(~!c&V=5<}ZIx}O7A;|&bd7vR_y)t+ z?Vm7kb^gJ88g;!fRfMTSvKaPozQz4WcYD8l#0WxQ${P%0A$pwhjXzyA0ZzErH{1@M z22-6b1SQ!SMNyqj_7MXE2cwcEm)W)YwB)ji`3Y^5ABx--A11WB3mBQB<7K!~``j&@ z8PKJ^KSa>#M(rar$h}aBFuNI9sB5uAquDlzKW+hYB&WKf9i&+q$j5P;sz2u$f`uHS zaX8$!@N2b81<<0w<{CpXzQGqSZRpfVb3R%bjsw-Kl}2UH>}1M?MLA#ojYaagiYL!P z$_@7yOl~PbidzJ8yx{Jz9&4NS99(R5R&lf~X_{xjXj|tuvPgvzbyC}#ABy^+H+FN0 z8p5U!{kxOvdv3fr35|Kb`J(eXzo*GvF6`_5GI)&6EW}&OGp=!8n`W0mr_o~Xq-t?% z_pDDfIW#L^DmX?q#mA%Jz-f86KG`^7V|1zdA#4#<=}91g$#@J`gOqMu+7H&yMdNIt zp02(*8z*i{Zu;#S#uP#q!6oNjQzC|?>fgzorE(d+S#iv4$if+$-4$8&eo zuSZJ1>R2HJ^3T9dr{tn+#JMGv#x@&C$EZapW9)uhp0`rDsISKrv`~3j)08JZlP&}HwA!z^~-?Ma(x0_AS{@r z8!(Z}5d8+5f7`r3pw_a=Z`!0r6r4%OAGYBoq3T7^xI@9xG3prNo>`}k>@VAQk>(=DIy(szD&6@u?YVdC|pJLT@lx{=IZ; zIkO4)YWp*Dpp$`H$Ok#yf;yBmHvTb@)4j)jVNF-O?$nD25z7)I!cWQ|Yt zeS<_C{i|BS4HICD=}T(|)@vd(v!?P4t4>APo7`K5RJvcTpr_KgWeB~zMLknrKMgpx zyN-EI%es5e)FNho=}qGu$`98v(QDPUMUGrY4tq>?x$md>qgNO0@aAQLMLr8XD8z%; z2Osn1D>N^22w4Xb8{~fi^i~SthAo7%ZjNb)ikgj0_AsXqF_0+W6E_doOUi0uV6Lvg z98Xk#>IK|-YHx!XV64==b(nYKMEyqPF?D)yxE=~;LS?LI_0)|1!T3ZtLa?(qd|YlXdI-e$W z(3J*FbOe3cSXvDaTHU^Hqpf2i8aH+ZzqY$cFFIH;fxMtW^(AmiMkBtb9esujw?rte zoo&0%Afb~VBn6A1@R1!OFJ0)6)Fn72x{}7n z+b#5gMommvlyz7c@XE`{ zXj(%~zhQne`$UZ5#&JH0g={XdiEKUyUZwIMH1rZTl%r@(dsvBg5PwEk^<+f_Yd~a@ z%+u%0@?lPzTD>!bR(}RQoc>?JwI|dTEmoL`T?7B zYl^`d{9)rW)|4&_Uc3J=RW25@?ygT$C4l-nsr+B0>HjK~{|+nFYWkm77qP!iX}31a z^$Mj&DlEuh+s(y*%1DHpDT`(sv4|FUgw5IwR_k{lz0o=zIzuCNz|(LMNJwongUHy#|&`T5_TnHLo4d+5bE zo*yU%b=5~wR@CN3YB0To^mV?3SuD~%_?Q{LQ+U){I8r*?&}iWNtji=w&GuF9t~=Q2 z$1cFAw1BTAh23~s$Ht$w!S2!8I;ONwQnAJ;-P4$qOx-7&)dWgIoy-8{>qC8LE?LhJ zR-L4qCha@z*X+j|V<+C(v)-UZmK0CYB?5`xkI)g2KgKl-q&7(tjcrhp5ZaBma4wAd zn`{j>KNPG>Q$xr7zxX}iRo=M#@?>}?F`Sv+j6>G9tN!g@14LUf(YfA4e=z+4f zNpL4g?eJK`S${tcfA{wbn({8i+$wMaLhSJo`-Yp@G2i0Yq~@wdyFxoVH$w9{5Ql2t zFdKG?0$ zV7nmYC@PSsDhnELrvd8}+T=C6ZcR?`uapdWLc2eaww5vKtjQQgbvEr^)ga?IF;@1(?PAE8Xx5`Ej&qg|)5L}yQA1<^}Y zp7WZpk%}L9gMMyB^(mFrl&2Ng$@#Ox3@Z6r%eJ`sGDQbT0a9ruO`T|71C;oCFwTVT zaTnu)eVKURM`1QuvrBhj;1e>1TEZW54sKUfx0Z=N*;Jpdh~Aj-3WB zR|EYVGDxSvnjeA?xxGF41Wj?~loVahklw|zJ=v3pOEVZFJG^TvR z-tJN5m;wZp!E7=z;5J*Oaq%2bc|Jw!{|O+*sja+B(0D2_X`c2)nVkzP1S~LOj~xs!@>aN z3$K2^pW}@R-70K!X&s4DHHoV&BmGWTG4vi9P1H$JxmD|t_V{GlHZv(`yJ234IVuSr z~!;~#ublS8qdL8SJG@XRCwWhkZyg_EKH(sB2}QQSv4W}|CT0ntD_4Eyp519d1%yKvc33|`yW9QzeJ4*XLP7@l=td+bwxSL~jCf-ny)IDC^~u5s)E-y^FdtU?)hkN{82Y{Lo)bCWcBOx;Jbw;)Pg9bWQQTY-3RWehpok!>D>Sa2EcEOS@ua)#G3I+GxL_ra^92Y!}tMX zwAp*Fv-aAarn`ME7N#Uyim%ynre6u?KS15L#$#rKZSgLnXx;g8TP9suMpO055p278 z%o-6eT(3gdIVFN}Gb3k$zbTyrHYel1x6OxETsk&h0E?&}KUA4>2mi0len7~*;{Io~ znf+tX?|;&u^`Bk-KYtx6Rb6!y7F)kP<5OGX(;)+Re0Y;asCLP;3yO#p>BRy*>lC$}LiEEUGJHB!a=&3CddUu?Qw>{{zm)83wYRy%i}UV2s| z9e>ZXHzuMV#R1yJZato0-F|Jl_w2sUjAw@FzM=DxH}vM>dlB&bQ!>51aGc}&WAH`b z6M6iG$AyJIAJ7-c0+(;pf=2=!B=%yoM1i9r==Q+}CK3uW%##U1rP~mwjUb8PLsi8Q zq!aTLLYK4HQ$vN1sU;d3XW{oFA{u@1$tduWmdOqc(~AqWq+`V)G&?YOOwAK20x>{q zOgII2&A_FXPzVtgrD80Y5J+_SEmyUcdM2N%q);|ZF_m z)6PBcOcAAy3kN*`8ac%zPH3^61_zn6_2FT#NCOWYx>ezqZzCC;tzM%pJC^gFAFcTs ze6C3WE-a*=nt8tErPG9zfPRn$QHqB7aHe8x3w&rWT(0F54<2uBJDYtbB}y|@9V6T( zmM!t}T5SuwxyTCma14&l|yiQRw5Pn|OiDBkx z?4tUGrIVsC9zs=F{W>zl9XeknEc+~Mz7zCnefUPUF8iF?A)QJK8=84#-TLLxq?BTM z=VYjYW%TOhrBp>3D@K{vStlEUt%e{HRc=766AQ+s7V_F|1A!)P3?y*=gUgbZO;O39 zX*BC((-XbnoaRGxxhRQRVKCDG9|qC6?7TwCz{A{OZp$Wu(~0DFo(w^P3f>4gr8@P^ zl8`!vA=_fvwTZc%-Z42}m>Q;KQ~&v;ipZzbA2;}Peg*v}TlKRmU%4WNN<%qb!cLo= zoSx;XBrv4}ErykT!)z)Qar4o?(q6!mpWLNFe~Nz0S@yI{1)Lxt<0K=Q$~>*HH+Wbp zQ~fx0aup_lZb|e6*@IJOJjw~Ypiwdq69&Y2vthfGq6u1!Joy%;v;~4`B@B*S(}}i- zmZc^*aHOK(dd(geOKg)P+J4+*eThk;P@wRjvm}e)h|#EpsV9YoqqRW{)ABhRlvGA* zL$&k5w*_-X1ITCwXiH=)=5lzjxY5tQJTBrv<{dM7$98pdK%i;RGZtiJKaSGCji7w)aNrHu_9_IPGHS-mMN5AheTn_ia^YdunCzcp2ap8eI-RQEm zj(q7_CT)o|w_noPm@MVqIjv%H4Bdo6*9*!Zj)bLx!p9POp(`$dj1QW`V=;=|`Gx8QST=OnK5jlJX3!KBz>v7j$&5b5YrhIArRVL)1C^o{@DJ}*mk*s=< zDK{e2f%fG)mK_Mz*x@#ahOO)cQQ#VH+8Wef>NKWcu4J>PIc3iz8y6PwCmY|UQ(O3!B;HtsE&jvyv^XjL7Env5#i zH4-k5GzPr-%36#%+Hvw1*UiOIk3b7F^|1dPi!-i7C^ZWp~_KI%D!sGYb@@zXa?*{XfjZ~%Y^mT!kaK_>K8 z_jL78^ zS0eRdqZ0v~WWow1CE;vDBh#{w9R4JgB!})W9N{{D=p-RMnehZ#pH*ABzDP46ryZkt z4ek|LHS{CDhTTMQa3a5fO9OLg?y$+#Gi2}Fv>QD-+ZEQKX2Fv{jr~miXz1ZpPcXvJ zNvQT@kQbBz_Y4Kg)*`E2t;tPh5_7tSGvL-|-A`lgHX3uVG4jLev9>YCZUeNNzioL? z;OBD{z+=Gs3+*ph)#bO#7IHl|rOFfvpK%cF>W??Q!Nh&B@hByD&}g|>a?GJ4uhX3g zPJXKKAh&zWv&wITO66G{PuGLsxpWSqaadFsv>_vQt?LVslVob7wylsa+O`IYWySoO z$tw#v7=&7ZGZqS}N!c##5-bC%>ze*s0H9J%d|!JgE#uZ|k1_bAn*x(Y%r{c=(HLwNkPZOUT#@j4{YfG#@=49YJ{?7? zddbK}G-@Dod&^Vf`GOo)G|`n@kq?Z=o84x{889+?F*dQz(kr@9lQ-TXhGN`)^-Li1 zb}xO2W(FvB2)EA;%qAkHbDd&#h`iW06N1LYz%)9;A&A25joc!4x+4%D@w1R+doLs= z#@(A@oWJq?1*oT>$+4=V=UnuMvEk;IcEnp4kcC<_>x=Hw9~h+03Og7#DK(3y3ohIp z-gQ$-RQIJTx%0o@PDST|NW41VgAR?CH`Sj-OTS0)?Y*M_wo|92;Oz)aya`^I0@?S{ z<%^epAw!Tw(bvSmU_k~Im^%#|0`Xkcmxj;31jX2Gg?PbzdXp9Dg~P)PW+Xi%iWiCr zV-Vv9IR5guDS2lGV!lfTWxkD8w%yz=UB`2j2Zb0eg~arRA*Q6>`q=8#4&OC|L6O}8 z)!w(idG0yk-BF#~k@Avk>an9z_ibOP*Rb;db_PsakNWYdNoygT?yRG=+5>ud<6Vxhk?P9rk!+8?xMg!x5kD*f2XOd^`O3U zlO;ImEy0SYI_J05cMW{dk@%d@iZFCNhIVtOm8$viM>=zM+EKJG%c0)dZ0D$4*-psQ zW+Fq|WmbYkBh5|^-l$w-`Uy8#T#<+3=}z!(6RadEpFlr1f6OFuQ5sG735YicWaoYR z`wuEZT2dntHGC7G*Kzk$tsm?Fd25LTHJj?Zo2RH;9rW9WY1`;@t_O3NC};dayX;Ib zgq6afb4!50qL-o5%yzgcR-1Xm-l4SE!rE>o!L=E`Jeug(IoZ36piq6d)aek0AV)EJ zaha2uBM!>RkZHRN0#w07A=yf4(DBmy(IN6NdGe$?(7h?5H)*?(Li#GjB!M{nq@C3# z^y{4CK_XQKuO>(88PRb&&8LbRDW1Ib>gl6qu(7g}zSkf<8=nFPXE1~pvmOT3pn^sa z+6oK0Bn$TBMWYTmhJzk_6)$>>W)nF^N$ld9 z8f^Y^MLVz@5b}F0fZID^9%hRL#()Xw*%yhs&~|PK|MGI8zuO!f!FqbmX9icd zXU(JOCwac|Z|=Yr(>Q3)HsXl!^$8VSzsgI#)D2XkpZ2=WOBcFF!2&d;*nF%h0I!`mRHl$91jYzqtLfNHUoYzrMzjR)u zP_|Hti4^){G?Ge6L_T^zVdS@KHwtq^+*+aBNl=hVc6#KB-It()qb&8LhnVW9Yxn&S z&^s^u1OzB(d_ByXz=xm4cpJzNzV+Txh`~H(176n4RGlY6( zg?ed(a!J?4(oL}@UfBpgPL*)KrGtM_hMIdu!RywK@d!b-{YAY?(?w3yB@Fi3g|G)| zho%)<=%Q$Lo7S-BxEjTL;M74{y+`Q^Xg#j}VvF|Y>X7s+Ps~aqT--tJNd9U6;Ej&o zj@|!`{Xy90t_Zdb>+m8tCFJ@X(Y$mR>%)gv4Vt;oGr`idhQ7H1^L3v4<_2}-UoguorcscRfdgumUVa0mK7-Wm~#vbrnX9ro}@82q=9t;lM9nH<} zLL#=1L7*f+mQWfyFnETMi*fe8AI+gdY6BM7CkRS&i4$ZRv$v*=*`oo>TjZ84sYD&T zI!DgZ4ueeJKvjBAmHNu|A?R2>?p{kQCRy zRnGg@C%oB#-;H-o-n##G`wcPWhTviRCjB{?mR20|wE9Kn3m6(%Sf_oNXWP^b;dz7( zb{blETKwpl`AT#W7E6T|0*bl?%r{}-BYdwrn0zN(DZXM1~53hGjjP9xzr$p z>ZH?35!~7LHiD7yo7-zzH18eTSAZjW>7-q5TYzDvJ$$S$Z@q)h)ZnY(3YBl+_ZK~* zd6T1UEKdrzmv2xc>eFj2^eQPu;gqBdB@TLqWgPk|#WAS0c@!t08Ph)b>F3 zGP}9_Pfp;kelV05nUfnb%*Oa{h;3Yi^B5xyDM~1r@o%v#RYi-%EYfSYY&02eW#bGb zu8(H8i9zhyn%?kx5Txx^6 z2i}CK(HeQ_R2_u?PFp#6CK zjr}k8Cx#C?DFgP`uN<;}x*Gd$-JgG3J_i3s>fk@_Po}b|JNz=Dm+<{^51m=mO;n4B&azYm{>+VhB{iyxuW+j>w@>VHcJyoSBQi=hu0;p zPw3Aj?%Ai^UeD{ySPIqsf|v0L&f_fmE7oh(s|jwbkK5^AQ9F|;a5V}EdSE?fyxdgf zHTq!f0;+-V{0oF+l_~>rMGk?f~m^wDXlxqt1@+)6Zv?BNR$+%$i z*NF93f}~4d9H2C7@?IibyqUtLL!XZW2ap4fkkxMqDZuZ>`+AfWJQ%~O2WR}NoA=OP zieg@q!mP z?=qU=EE6L0_UpzXt0qwX2tF~}c|;`#MUY2TMz6k({hpkiSz>Dxt*4-PtkAdAA*0hn zk~CK6#V=*^m5 zg$tB6rSO-=9l>GAl^DjJBHdk0wD0(L!OrcZ?qmtYbl+}s(@rtE-O=RTx*1cZq~u~5 zQPVt(IB=*?Pm;Le%#i1SFxHY|>=Y$^RF-FGAUSkBpn`|+p!4RHyv-Q(XgZ5Xg5W}J z8RcT?+4FdVQ>z~9kP5By8eM95f_LDnsnA%K;i6`OpcuJS=^n|6nH-B2EhH=dLbO@Z zuw=Ug>7gsu33`Pzy3Lji0x8OCH={?VRqFEi;@oDIS<*?dG@9X1*tlYCm4YUIMhyfo zJ~=K@-X$D z<-4dH<-5o#yMj%f@U{nfWYVdrREJ}_o4&|c*_+M6gk z-Up9-i~jM-bwR;Bf0&C5wteli>r7ZjGi+mHk3aC4mS5 zPC^{w+G%menlWun+&<#i&DJ41thvk;OKZEB`S%sZ6 zzYpO2x_Ce@fa0LuIeC=7gRHN#os!MQ7h}m9k3@u68K2$&;_mSe2`>uvV<`RgC)TKX z`J}&Kb%*f{Oznj$%-QafB}Zb$Pi%@D&^ZTcgJ0+Bk6-iOJ-P|Q10)5ie2u0JzKb2r z2C@{f?ZBcPw5%h&aKG+6%Qvhw(t1Y{hZ82YE4(Tlk`2VCgE&1x;AUt+5U*$%>P|iWLeb_PJL!VX=b4#>#QM;TGjFHBNRy+d{v>2cVXFyqaLd300 zFHWrc8lB1KSOH3dkJClJ%A5oE^31WrQZ3^-3`Zk?1GqoV7Wr62=V9C=(;#R zhzXAT03)d z9OdZ|;CjSnqQeqF-CUNR=x9x76JYnpr|T+6u#$y=7cMVG72k4f*BJIG>l1NNvyv6NQzr4U`r;= z&%W1Ri2sI5p|8%q5~zM-AMptHj_eX7FzJN7t(%+2dA)efyFbePBsClxY_yMqWbEdT z+jm?SZgH3mCzU?e^psnyd8UK zfZ$^_^}C1WYB1-$m4qwT@#=wsAq$9Xj=%IRvc#V?1azEi|RSc;M zQn;3%Gjk3D)R+3`gZplB>Pt;g?#EiwRzxON;% z#P5IK*YAh1Md<$o21R}j^8Y#t#`fP`nErnb@&CkI{`XNXulcVIXwLcS%VE4i4-!8a zpj-q)#TqXkFg&z4G9pG45A-$B_Lfacr)H85ge*yqTLAb(oY1$6Xu7Rc%^aVOmzsKd z=WEXA40~hm@7FKD9t14nSRt)m0XWkP1YbAE009nIupf`md=v&J;C}estaY0%^Z;;lf>5AF-y%Xf1QEK(}4n+ zhKsTx^bQSpwM=UWd3WRcpEQfw>P%zuhLeEdY}s%cGitMZa14Ui*Mzm%=(7<#b2gHmJ?kdeymT7H+Z8k8tgd zp-dhC)R!P!)w(n%RgOi%^)LGZX)yxC%@f@d4x@IRbq{elrCHyIuphEE6qd6l6O`;B zi0WQg;j`hcu51uYTBSSYNvY{Lkn$iu=Ae0g6o1cSTRwXmEvNcNI zv;)Z_?g>?aG`Zp}*gY8%LGI}{>J#`x;v=*ykuY@z2Erz>@b*)tMp2>=C20MI8|{Z2 z9hbyDJ7d#MdWK&fyZB>Jdm!#x_uRw%>`OuM!&QMim}baa76{L|VAuq%1UpXVHsClm zPD4}hjj{lj`)aaD;x|PJ9v@?8gZ!t5hER6!b~HJ_l9P|(h&R6js3mAfrC|c+fcH^1 zPF*w*_~+k%_~6|eE;-x}zc%qi-D-UpTcAg|5@FCEbYw6FhECLo+mVn^>@s-RqkhuDbDmM~lo<4sa`|9|$AltN_;g>$|B}Qs zpWVSnKNq69{}?|I`EOT~owb>vzQg|?@OEL`xKtkxLeMnWZ@ejqjJ%orYIs!jq3 zTfqdNelN8sLy2|MAkv`bxx`RN?4Dq{EIvjMbjI57d*`pO?Ns{7jxNsbUp=rF$GCut z7#7Dm#Gvh}E8~2Tyhj2reA%=ji|G6yr%@QV{(90cE{JYOW$0F|2MO+TM^`cAu$B7s zmBV^{IqUIbw5~muv}st`dDdIxSU@Eb>xf3$qwEcg;H+vp1^ArN@A)RtQ4hrid2B{9 zb~pG8?SC3#xctpJXWRGXt=cx6Cw!IqoJrK)kuLL&`UYYB{R6Dw)k9nKy>R#q_X|V* z%zVsST$=d(HozVBc|=9<175^~M$v$hL9azT^)TL7BIA#qt>N2^iWvMQgt;!YZt~cv zn!x^OB!3mOVj>^^{mloGiJhLI4qy3Vt-148>9j~d8coH)q|Cg5P89Xj>>hjtzq5iT z%go41Nhi}x7ZztTWj|deVpj>Oc#IrI{NxIm;qhnuNlvNZ0}d=DVa}=H0}Vi-I+wKK z*1uD=0_)b-!9S^5#(%_>3jcS-mv^;yFtq$1)!wGk2QP%=EbpoW++nvbFgbun1Eqri z<%yp)iPo|>^$*IHm@*O74Jve%nSmDeNGrZ&)N9 z)1rSz4ib+_{4ss2rSXRiDy zgh(descvk^&W|y)Oj#V@#)C658!**J#=ckpxGniX#zs0tA~NG>E#Hn3Q3wdKBfMG& zK}2y#|FLt}E`UQ6t3jK#G&e22bMBc3=C)LyqU706frdCAqa;~Q0L5)KJ4?@h*FFu4 z!s=hOC;G?Q)BRKJ1q_XJ9W5LLejp1L*187&5Bo4Of)k>T=WpQl3v#4iX$574fW`p+ z3m}r-F8Gjv1m3yTia=+2An1+E&psbXKjH2{<1xMb37`|D<%7c`0`~m0r>AQD^%nUJ`%PxS>)*{i zg?VHw)ju!$@$>xGszUyM_BsCF3*%>rxVZ8vrYB?PvDBBHQWz04T&UpxKU7{ zrb~8R4W>e)){FrKo^O5ts8O^r^t70=!se(2-(8&aTdaFU2;SR=dyECLBp|MVU@JIt z)z$TAHMKRnyX*5;O<*xm+(>Fo41G;Tk0w01ilh#uFJa{teQne`QCOHZp`&du5gkAWr@9Ywz%@P@KB0bD{lXo7PmrPC%J!A z%orlB>F}qRa$`XC2Ai_4L56#h2GWm;>sScPxhMO5a*guk2 z+56H}PZnq-sxASPn!B~W#8B1W=OQPf-lEbhOh%>%{AND;w%w;t<8%a%HNk`LQ0GpT z6au2l)=Brql2Fq{Kw316jHdW-WF<{46(Xad0uxi%3aEARVi*dKaR^jjW)$<$7QEiF z0uK-~dQ@|hxT5M|t$pBl+9IJig2o;?4>qY%<|sZ4Rk0Dc{ud;zd`g$&UcwLjY))aV z4jh&lc(;hjQaWB)K9EB@b^I)LQ~N_;SFEEWA&}`)g!E7-wzF%J8)yZaSOeR=igBiM zaU=T>5*oyz3jYaqv-RSC;r$%d^Z(cbLGwTQiT+3KCMt*OBOD@rPZ}8;)1_*l<5aBp zjl{A?HiE$Y6$NWUgPY(x@k^9)A|CC#nqZ?B&q-ceGE;Y7F{@0{lQuPnsj0~YX(VoZ zdJ})6X8821kH4_0vt$gocDeSve(SuROm_bM98&+q72$1m(x?A;;)@TWyuVXQV!{#( z41CN;(vq_a|56Yny*sb>5`lt+>?dvF0++3L!wQ_eJmXi)z_1UAmNi80_bG^|J$GZs zK^|0X@8jq9pyPt$dpiWWAG)mNg7X_BME=&UYoq>nc0gtk_YoXNb5hYb!hG ztf(P(6Bcy6`wroiv-5NLLjVBx&|;W6WwKMmB+ph%7$AJfV95||OktlFlTMqdKP0i#Y*rj`(XeYUz=adk`3hA(LvO`y z|0%R3GMWC#x}RbCNX_Cf;_wEOS}%lqj#-CXQDIpi8Qis%Radz>q0vjbY&8DdR>jXU zmvR%au!=9lMN?P=hzQpNGOJRw?Cn8@B@kEp4r5$bgdM0?Fdua~*H~mGTf}17rZog% z!Kj#>m=l>Po$A`_fcT-pHy*aya+n%rXmG0CJ6a{nF%>TfyzKC2Dit7a;!8r;X^G$~ zS03MClV}lI)S^Py2I2rLnpjR64L!#Fl!mCP0td}~3GFB3?F31>5JCwIC zC~8VAun2Z}@%MZ{PlIWpU@CJ06F_<61le-_Ws+FSmJ@j>XyyV(BH@K!JRR^~iGjAh zQ+NnRD1C)ttcyijf*{xky2tyhTpJvac8m%=FR-LL@s>rN`?kMDGf2yMliwkYj= zwEEJ0wlFp%TmE6|fiti_^wVrxJ#gh7z@f0+P!kS>c>;BHH)N`PW0JHTqA?B~fz6H+ zdQq>iwU2Kne+4kR2e~l2`>(-^qqujX*@|w7k>s=e)Y-lwoI{$Tx_2}&y$9LZzKG-w z{TH06d?a9;01ze%EvqDCEt;qAaOYdf@X)zT)ScQs**7gQ**A5+o9p#P*X5~lMpNl2 z6p=Ecy7#f++P2sk;I2Nd`w-!5Y^3QHV0RVy2<55pqQ z&Q&b+JIKTf&6N(UjwrECT(BwKhkdpc#(Aq= zyG*N2frC~4B2Ko7O)bOHP8(}XKc;_(GP&+{?#dJ;Y$YXT$y<%YZmc>C?Sik?i?6E1 zk~VKGMLlNws0d#wk-11tBrAf?Tbes4F)oqxr_*7R-?Yn4IlyyP_ce6(J&tXSFI~P^ zYG1K1&Y@OY%nE}Gsa8~iq!!=l4a+yi7?Rxi#owl|2CnVfey<;AkI<2^CN^r`;-)ob zX7Ccao0G6Ic0ENcm7#3(8Y>}hb9aL6Gi?llW(Kss_CW07Z*0rgVhbod7+2-z3EC%( zq7QLJy|>bn^fyDVwISg;I%*4-lpnL5wLoe=B5sV^!Vdseg%7piW`#>KU*HD}MZ&J=jCFG;)9zqX;~A15Xsg;+mAtJruykiiD4Qc5$;lWT@^-j>F$$|0*{U zmrM6Kwy7I0>uJ&DC#8>dW7&)!1!_uGQ@Mvr)n^bH?_w|*J_E0?B{C&x%7+%$9&Umb zMv=?f8jwV=X`(6MfQLkyXGt_A~#T^(h~B7+v?~%F6k&ziM^m_Cqb!a zf0y+(L*8N@-&FfWsxPx%V97(F{QW`L&>2NJyB_}HBTWa|xRs*TT-y}_qovhF=%OCJ zf)sDf8#yYtG3ySQ*(qqz9dXI;CfS6yLi>4H9w9ii-!j5NwHL>oEN83>IsEP+V_1~u z`?}q?(o8RjDY5V?z9HC@t*0V_hFqA|HyZ8k)T!UJQ`KEKMLlNlIq<$2s!x;)o#SW0?w*zVYU?yc(v(2qyZg z0(^T!7Qzhpm)`?PLS7z|(>s+ZUO?_>f0y8LjB9{7he}@4-%l99L!vhyLW=yQr!);4vCSd-wC1QX-%H=?#UM-D_Wg8t3W z0*rY0Q4xwb5i(lBSOs^u(IgRSP$j!PkhbcIr^rh}e})V_kU5jW{q)m0CALP$`wKi& z?444cDxl;D;SqSw0^h%eA6Ro@BhxmD!}qpGb6OxRi6;iFai!)ctW|gmF3jQz2*O}Z z*TPvZAxFr1-Dd!53U_WQMQh$aauyVf;O60e>&G;Mg83(TOZt!6;s2KT{}By>k&-_m zA1YA0q3ID6fx`!qxy=@dYO@Rn%rEb~7P_%;Dxvl(WAfiJUtti0?~ah#_1`K#A}P2n z7^D~GQL#`hC}2w`btD`i%)VBWnn*jWF=d!kI*6T5-wBdsT)$EZD=mrn&EhxJQ^3>1 zbLeDA3&BIDAv=kWsp0t6>a3lITA;khMX^(B8Ecb^U%P-|RNGB@XLq*Q5a zR9aZ8RFNDYvD`dcva-5ti*`CcV%ltLG;emYG)5Hvo^Boe6!Fu0ekZ(k<<5G3_4>Mg z-?ILGT9yB`Gy?Cnu(PO#(bsKyf9>@F_MJQFZFaBE?dA7x40K@HNwA20g&JE&q z6&$MUcmsL)Sq;;@a9!*!?ct(XynVCJutm{pZ5w3Xci1lQ!9oB`xCdL! z6i6sX5X8iljX<8L4KC)P_hyjfBo3W=8BfQ5^inG|_NhXI*k)fvrDRq;Mtl#IdM%t^ zo(9yQnnQj}I{C__YBGYykMvG(5)bL%7>X@vm&+vnDMvZ(QMVC;#;@DZ9#6!r74JA`7phVA#`JE` z>BU^K@B>jj8Maz2m^>t$!%J^m)e|Ylem4L>e=OHtOVBCDy{0or$Np^VjdNl=g3xT8 zqsE*&O{Q9{>LhP;F2vpR<1t@fO4^Fbd{cO753U@l zLFAlS*(cze1w03?ZyLxG9S&n_udo?=8ddzgt#cv5fKd+uyogyl;44IK1&z^wj=!YK zzUD&kgK%`pt9A4nks?WMImECKCAt*xUXcPbo9e1&PmWU$X9~!}HO|j@r(`+=V^^Lc zcLMKF*Yj`EaS|pmb1uaDbkZvx6m%4{=z+MdgTuv?mT=4T&n?h7T_tQNFYhz$`~(DF zx4T%9nS-@(gWPm3?tZwJIpHDGWzAJ__zZKP;Hw>~%&n=s$Pn?6CaJ>bJzY?o)(O#~ z1fxWpkgP7ukZGyitR1C364Jp*?#{WzBom;9o=XrY;V#_Y5@5*}T5v*hcW#I;Sb)H; z6^g4&{fOcGP0zWCURc5J$ExdSY5s?r-^r#;|BS)8NjQH2--6b}!Q-Aa$mx_pNnz4q z(1_zCdqOu|4b4oo+-*jjTTV_j3WmL9=u`0(l@>00B5Vg?4f?fqwWRCX*2JwC(Yd+i z5A-Rm0r4e~4ceSJnEmWF6Nk>Q;(7sYyQ<-CgPa1fO8m6_pu=Maf0e2hd92Q#i7j?U z-VR;%F~r=@Xs>J2`Nx))UK=X`Shhg3AWzbwE<#%hM+KSQ)y~F!~7j*2}qu zgT9Z6kE4Z|n9Leb=N0%JnFI$AeNrV+!>E(WT7dyOjN~44BhNVL4(%Eo(1JGjS^)Oc zjSPsu`3wT8k`$>Na;G3pMU(9;+ov}PpiRt6*)WNMy(rEUak-14^(K`73yJ1#LZna? zS)ypsH=xt_ z1V%Pk;E@JqJeE1&xI}|JylZJSsu+mw#r=)G*5DBGv*`Q|1AC+!MW979QEZ{H5*8ZW z_U8EI1(M1LDjG^#yy~(OGH)?SdmR~=ma_^2Q#k>)`v#$t=~Ih|79!ZutXQTK^S&w` z1)ONotPDL(cz!_@bFBBOo6W@;7Zz--d9JaOs{)ss4P|Mr%>FaiMR=(fn-Y3SA->6~ zp`5h}dOcY_YfweZB*^el7qqa$&_r-Lg-I+9~U z`JxVCD<$VmoiR$g^3dU%7Sij)XYi*?$#ihSxCBHGOaRRr|Lo9+E}O~M>I}tnokI`}F32Aty#b8rpABEKl|B;*o8ge^^)Kyk z0!(>gFV=c)Q2Y%>gz+sa3xYTUy_X`rK5ca{{erC9WJ3EPKG{|Nng_-78kAD{oh_=K zn*wopK3cG}MBJf%6=}9YouD;zyWbjRt%A#pWc1zb3@FB`_Q~~UI!uvse(FQfl zUt=Qy2DSjwpzAUJ048~^;@Yo{C56R_8nZEeF}vm)0xoYe0y|tYI!>Y(d}mSro0`z; zeb6Eg*(a2{5Ypj8S$-_~L)+IlozZn|Iak`$jQKd63hldhts0=m>k~HC&`@|~;XaG6 zLVxC))8>^?13P*mV#ydlkC0V6AWK(BjWpqu| zbh7#bkKuL<kv5;Emm4zkF;X>rfbzAc7!Z)i};f=*bypYUD zho5-B5n;)FP(nzq8FG3TH?7l0vS{G}G9@~zxY>CqbX^mb$|JncS3I_2RD@?I9bz>LbX13A0N_LQmd(!3AxqmR_;3bJavc81%v z)Q~pDm0d1VrVe~>X?GOUOz94e6Nbt|fe6(S@cN64Gy6{i*TPukTmfvgPR>+qe>)@w z8mS6=rvR0~cqVfEWFsL|kZ3t~m-iV}va(IjJ;Hh4R9uISa6;@9d{D+7CwskGx!7MGZ6|rdE_I{cMD}-` zoi0%doDSznN-Evavf!_d@UNJt*Fl;hNrnVT2Fal8iBh(LU^l>8I1%x!q=6A@zO6O} zs0R@~z(6E;t~6L7tclb6A}zwwIvS;W`?F>>P)INWt6N9r4JbH*;&^6B!lHNAY+v3R zwCVoTTSL`1XtRZ_9vWH*(HcV?PImcNBOtbC4{U(v-HA~xMdpP8<);Xv0y_e1i%t|f zdyL`MtgjoC^Z-wGt@&6(9Wx>;qYcYwopK7H4iejT?T|>BSm)-fV&7yB;ANW4ZRzzc z?^;uh#-bDq@QjjBiIf-00TSw~)V;r?BHNEpDb(dLsJ_Z!zT7<{oC-V^NTEs|MeD0- zzuH~jmz>@&JaYIW>X&?~S>~+R!;wQOq|+{tI&#vV^n%|7ksh!vXzONlSb4zc!X;}> zMaUjix==sr4oMiHxL@~MPL%PrMzU{DPuz`9zWln9XnqKqNo3TZc;22OZ{ zy(90FLmd!qHIv!b-q){c(0@VYnzE(k5#rf~N5m{u-X za_J$`vM`7Bh@_`N%&n~35!O^m^pyWGR65?W@EH_fG}veT4I>@L72iny$1yuwBopv> zsSxe4Htw2+2f`M-+7|iva$OjEp*e=6r{J`{W_IyMTo#x0Yayp+V8z~17Hx&~6G%t? zN=#7bc$BWFl&qzMvU^iRl>Rvj(_`fR9T%ZBYX1?fg((%9FgbGrBl_7^rRQW9GA*@E zLN~c4F@W|oNmH$kHZ)4U$u(P4S;GSPDy671d;6L8z}?RfSb0PHN)PsKViOm_PLB-7 z+-+jjpC&oGWj(BQ{|L#DFOC3+-%fvGOOx^u^Ysxsq)Ox4^;}rM$!;(?`m@wtkXb~%u$Zx% za#IBD9hq=no-2H90jB}1^>TfWp)=Sb1v9w#UAHvYbn1PpHFbB+hwSXWK(ta=^8VN< z^j!PhT^ZXf#;?$ZWkn?(vJ20u-_SsGO1os)z;s=hI)d6iN-4mC9>EtcU@Mybflo@| z82lRHB)FEu4k@P9W+a)>t{^Jl;)gL&tWZBy(gWmfXX8XiUdnU>LtbceRd2RogiprV zK3KHRpSd5n#Hy5wQ!-Fg;{(9?K%pRuAEZwPR-E)JGeljq?MUmP=K$zkEO46*td&DL z%C4c|+^C204zq3rsTdE?%Y;lc1vKitClZ79P)GU-k`VCL5(kX_>5D{)C18r$^duj) zab$~pZ#$FLi^ihhytr80x6p2DsA3IsHPguaQ&s4izcL;7qGj1rPQM)4uc!I=d^j7S zs{`eqUlX0}s<8@_Iij-NBLD<2BE3VJ&k4Z6H;z?!7!7-XeeC-aX{Tl6ml!93m*cFJ z#Z5Q7fr}UC|2wXN*{|KEWPZ(V^*agnsVlrYkAd651IAl&yHxt9OnMCJBht5xn*lR2&NabYN zSWC^|d16K9!d@LjLiX4uEhz;%>2G#@i;bdI;t=8bK>y@P)WT!mDr~z}pG- zRg0M$Qpz0mbKF!xENTw8!Wwu{`9|04Gou}nTQ_L@`rl58B6UT^4~-?*}V`fYfKSaDIH zavlsK6XsL9-WmdH$C72oMpwJp)?;)Z4K6Es0B$SXP*QhM!gvpdUyI?}p1c2yYhY~r z_VvRqI~hi$_97U@cE5#Z{Zhy&EqB*`vAMpf?Ya?h{;uuk-}E1T!ah4kx_Q*9mOjl* zv62c1x-eMCSfQ*b3b|P6*~#_2>fN2y=iJQy-I$q_TIV>AHLGvxzY#v#{w}OBR>mny zZ+4AXVq%F7d*h&{U!c8&&KUXS@X->Bu@pTF71|eeQVYw8ns~h`7|n?)2@d35c_1Jn zeG)5*kFZ<}MejgYN(?7Nw?Mod)k5v*wm{$@osr)Ywv-QvXpeI;3Qku^T}zo`go?co z|65!$tORilITCe4GfhNoqaj~NtO|@obiA%Tub@&qQ)*Sn14oz#=<2osGcxe*+@PL< zyx=_nR&*Un8g$Iu#el1FV8xS6kKlqt6Q_nLmsoyCCicctlpM=xVMApO3V7u00mxNJ zn8H5H7~1cY0)_}KJSfc2QSG+HDoQlkX^Iwi_%Qb4&1XPlDw$%cwf-dlhzTK+<_D-) z&P@=34aLr)@%x%0WcLNFBZ4im4biAYc zX48#WytT#YP@@jEfGgaR&J#HZzJa@HjxyMYHe{pLPnxkn;~Nj*Rk*wS5*frI0o^@# z&G3U*-hF=Y_v1Euf&ZeY$+hsoi~%M`iq}OU5nnKjI6qCo7#tk{_f3pIO(8(pMmgCr#+;(8d(-5n@oY{gBKSFB;sfY zEGd8%M6}wgw88w$*dURSw+YzI2N!gycd}~V$*T@AlPt*-f=web80-YsRGL; zIurEoITNgt(oy6p0G%)TAq})jmI~qDOTd#8SWUAuE(*k}kk&NIGfR#?MWZ&@WgOiL z>$#C7>im5ft}NgVUz#o-;GS~3h`u>vuPTQ6J_?slXE&+uSm7V8X2xqGN*g32wQVF? z60uDVd}|BtzXW}IHl+O9$Y${gL@oN<={bc5POfF*UaM4*ulAX=jeCFG9716kCF{ap z+Aa!D*;gIV6MjhUJ)8P&!?O}G@h+kF9lXMn@bE1hm7VR%NpI0p(h7q@gb zs40V7?1#wanDpa((WWtV447#&s#OHJWeK>i<+;H67mI#8cP#nvB-$#8&oY@Q_cX1> z#729EG?sBvSe1t$UC3o?5BSvkVN@w(QQ4cW%3w&{E71?HvJrUEs@C5uiGi2-#9RzC zw0R)RSq1PMNN=!DdusVZwDksjyaAQbNru6UwUWxld@ldSWo?0&)`;Xs$LTI|<=N_s z*4BCzi%Pnt37TSLENizfSMFGy!FQt!OTgaGufi;Y{r$=cJS)FXBg|11{Y)6 z&FoDw-n6}+505Cb=XILmcU3v0TbML}3&IJnbKY?t6@!3@-XG)E17_uq1tu zz$~wy7yG89CHH-vtG}q6Z~ttOmW){@%R~RrHPL3}aSux$jl5%aPq}sjvD-AQns@b7 zY@Oc;tRc(`c(&eQsK@oDdmBD-*rPabNn z(VZVY5nz7{q0q`4KJLomsMOu|s7*#%-xXTM-Iq0IbER!m(6>i7*+fAfS`~--GwXqM z4ca)XqKhhrI<(1CRvrYaF?C+w%ux-FklJA!x)gsK+>>%M>?Cm`XxbwUj;EAE@Q-G= z5cFv(Qwcw7h#q)bu5EK58r1nZ6^FodqAYE;KnPkOE*EDluO!khZFyZZGn4S2qu$k&M8jDj8T_CbL0QU?r8R{_G)Wt1$pHq>0cP3sbJb9fA#aCxY+I-RDFonr20^=HoUCZRYU z3;Wx@Q{b+BZ2dl{1zxcqS5d}TP9^VEZo``(0%P+4>^Ho?uXD2Rd}SjDvjSCkh2VrA zKWEMFMooUWGVS_sQoH(GX9QMhVu*UMH=Y!B(2b48^*fnH@gfxbGf<8rF%}3qZBgv? zh(JU+*63i>>V+rSOX()d6M}awEy>N7L-;9D0cY+eL%cJ})#Owz>4SDuWjsapJukYm z#U|itkDzOryOj(#d47LERC;) zr?00mlOxu-u}_c>)3d=1nWQ1_>F0k02%Z<)U=_eaKsaOFH4zrLYa*;@;Akf7-~g~P z1n-xT%i0(jSUv$dfNPE!IynMu{+t&lDe21Kfn)7m%JJ%C)HSiGPUMys&0o#k$Pl1AFx2#-J9Qk{BW?yJ&d`)AH4#W6I1ps&M36?pz z;*EEoPlL}Wyd}~t&>61YcyLUW`L*Z@r$ihqOO<>>P87W7%w)RnriPH5#PubXD(#Qt zb=`}6I@RDHQpY=kNa_A{ANlk2h1!-L-XsS9{Yde^7JZx&lBt*$XJa_U*{MPcyegB@ zLiCqy>-sZ1zHFGjnK%FwzcjhG6;2~wQj-;X$(393Gf(VA30y8mnsPt6v5LGPJu3eu zY%}lS@YZ2aSN!T?5YGnE75@r$2_iPZ7L`-9i-c%-06Byv)+f~T;|Gd|m55Y+$g%Bm zPj}UPswtB5NxC%9CW$b6C5-v-S_M4W{9XsSP#qo;3y`eTAPWR3Kpk!&Td%m;xeD(J zkgb$2pVc5gT>4^o<`c@;15!fPdzkh}4{kYM1SD4KDK~XdJLN?dXcN3q2h=!JPqqSs`ZYWO$j+JfDLj)AlVFaGoLZ`FsNhYa`KNgLG*%}AYs=;H z-Q%gTlisM@(w$LOiPoC~Zg644D-NihWG4QGg)6mba_C<| z;@RIbtg|gW6G~C0*G;5-D_|-`wZ2&m1fZD<%P|7sCJmNjGcn=gW2)16WU#O`laDax zK8Ni+Aoi>@VK=3s;#}xhR^9Jzw%MFc&x8*v?<7KQc~eC$6!C7}T1I4g>`)FZ;6Rnwc-Ku+?+S~*U6eo2GC z#py)*DBdbx(@JH~ypn7wmCD#+D?O9fB53UEWb`Rx5qG*P9;QEqBx0pe!g%R;g<1|W zMu{%gG1KRqtpu76i)yF|p#XiLn}Zmhwi8>MGujfX&N?{@xCESOraYg32W<;>eAK%n z={*s@RQHJgpeK#FTvnKc6_gCq#JuoUie}W< zt!_}JcJdvs(L`=w;$Bzoa@0VGU*b&#h-6ubG#6sWaT z*4e@S?>9bJF?xvi88VQ^@r zKb^NY2to+SU}2lC7kk*#5^CKI%J*psqC;BRr_+8)Xi7@g5@;Nvy3eEf#ln6AX4h~MMTk5c4t}yc06aIsgVKpin*eIuxsE?F&)z#b;yzjfuy#dfqX{bNPrN@_B>{_9E zTA9)oOozvwO4b|3^;LmSq(^Y$uRpK4e~~g3$WV`$-BNHg_JV8Bv@!_>w9>pL(8W8T zSG4bRrDxA@u=P5Iq+vU_@wG*u!cg_2hU(^|WjF(DGEeyX?=kLU(a;!+whGaG=fSNk z*d?J`ge}AuLkq8o<>B87rYJ=#c@W4vb7cAbZL+a|P3JNNTkMid`+4ty!bj+3z=Hu0 z2k~HtdJ9WD2XZ{)`#7phzt{sp23-LLii+4_=Z+?tI+p-T*MNe$odqR$OZ^4Ug5CuT z>i1p^xbmEkI^S@5AhehRFD01*!L@ABtj*r?4~-95ub}R0(7Iwut*5`#qILDD6W_+Y z7)hdJCyOScg7TgL3J2FgP@G{DM3nY%3J5%E4=gG53uob>YW;S3YOCMKEWp2y_pULd z=p=qD$*^aBEj`$6MpY$1=Rss08VHvfrz0aIPuO$uvA14Y@(@0v%R)ODP2>dYu%KdV z3le_(DM~MIPhf?ZG*^A{jL?E72-d;zxY6Q_sWG>^d_+41@mMh)5P!H8)>l(`oU75yjMi=)QZ5O0~QIy0S`KRD5!4!wV>5V?kFP{XPF5va? z8WGZv+8|*>b6RX+2UjA5NFOwz5p0Xk%wVPkH~B_fO|%-3SAXru`l;Bvj)VC1llyI#qf&7Wa-Y(RzE&hY z#c`VnHONe7V=Y8iCAFyTYmIZ+o7?S*PF%lCmTuSQ%Jo#!vaWf%RI1FfrKD#hkY^wk z>Ol?BIebHZxO^o#6XIxE5=%gk`%B3fsR3KJd{z1=UolnL zxVJG*lrB{j4QrEo1?2fkWeE@8QtFVo#bYKD-BTwXlsAn+NIb#ykk;2~i}Z^tL*(2) zDEj^l>+ymTQdwjrNTKb<0x2!h66mc&hT9y_TjZ^<6q!w3JlFH^F9%r}bVg%n`#$SA z&?V##X#;j9KdvHYJ;nlu*FKt&fVUnaw~l6VR7w7Mh6<%OUk2tF0U`-YdRCIEo2*N0JceWvAO{% z05P^$9S&j+i1P&7jd02s11a{qeAFhKXYn|Z#^q<%L~&7E#{x}TCh%f9zL9B;_`cnq%wnr{i$aybv{USMj{H&n;e zC~91brnUfLfZ$-d$uYF~3IP{V_iN_BMk)+?D8L>gm}S$!?t& zQlV)1kc4Sz^kx9=TMR`7EF>s4=Y{5@Phqsy>A;-)7co^s1!;p=U*}pMhm{+p@Vufq zatXMEDqvV#Y82v96zT<7!oqk$@r_WmroUiUA0ETO)P?^L+pKL?*#5@C#oGCq1U=5Q zA0g$CZ~r`Dhx2h-IFJTaeCVSSfwE;Ai~U4%Mq7m$8A^hr2vx1wxKsjlVJ*taD2inZ zTzJ!$3*)*Mowg_q)qb6JF*!R=E}uk`Izeuu4*gX`kp(D<1DCh^tm&)Ddt~J}Qxsnjwv(tX8 zvyX!L<$1uTZ4B=@8GX|K7p-NHRI&kObG=6SV0YmbkOV-TRnI zO|*+T>1{%)>Y&?HHZ}6B)M-B$(%6o>e)DT`N>B^fzZz(E#-_Zl+AUBz!y!nVaDOy2 z$3u6pg1+`qnWld>CufRs*74%yV;3YT)s1-)(cMSoXga~Vsd(BP^rPAa)$jC(-*v@% z37zH!198UphLe}-S3Rsm`BEDOKWWc0w{xqA*NctylQ_1U7V-~4#VrQ*?E^Rv8KvWdt1NJtqcSn{#j*j6w z_1fbstu}x`G<;}0Qkh1vRW!SfaI804LpSoumU$ORzJWX)cqNKhju>)fk(kqM3Ml&A z!2Gp=M0KTb2SOfg6AZ!n)LNnKv9DJsEvO069M7@{505>ElahKg5amp<}T8K&fK;h(?6 zD8mw1UY2+wk3w(U>HbZF1W!;bJwh(oaCX7syZ3Sf5xDMzI?8(|Toe&WF(R&fcQ+c3yu={`!G8FXR6UiyIUh!wW8&E1JhsV_F+0ryRogcJ z=mjDX`rf1N0|SyXNpzx^Ga$E{xZ0rjA#wUl`H)|yF6#O1-j|5DzIW3t#yt+7 zcNg7}SUGs7>rG7>bWO7Kff`(5%~@f&g(PraPAi=D6r5Zft>_!#dM0X0J+$2_BNH?R zoa|$Frq!Oc@hvp^n3_f=wL8pkIYe%I^NNz0o<~a;t!-9IusL$bf5@y~j^P}uJSmA`P$b6?hqshH+!(Lfw%ZzV&R@ zSeM4K%Zh$TpIJvl3*Y+435$*J^=n5yy{_hfE7>NG#EjgVvP#5-e(CKh=sppX^maAE zNX<@{IQl-T&J*XUGd?M*u+U5u(r+=mRT<)1Vz2x=5(;T>kq3-Km|}E3Yx(Hz7#Fh- zz1n~3Ra5b{ZofBz<>0=~(tV~a7j=@I={B{}SvEEpZ~--V8|+jXB-+>wb+%*PSrdZd z7M{LZGk~yc&-P~2ym$d(y&q9q~N)W7GI1>>$$4YC(l9;BI13c~kj3e=Ud&dSCF}&uf?M zQd!GHyq=ro4Wh7xiYat>cl(8HtY7Wh&9m~CO^d~rM$q3WUk>W0gg4=VV7}+B=s|xE zyE2=a+GER^wZ<-ONb~odKoM*{ON^<6vCMC38HjZPl4594l@+cg4VO?`I&Mo&us#aV z&!-u6$QGLAU*#cd%#fN1kMNt$1mqiRebD;4A5quK z7G|4$JX+^DnL|IBlVhRQcziEzlnlzG*w-%kD?5Go)@k3XN?84TAp`fR>uYF~{~Kf29!G+~dPVdddEX}m_7oomyD(yDIatk7$|^h&!doNXehDBkck zGHZHZw^gsxnR%8Mcd6cQ*_(*8?TI!o8~%Cr!~0;J=2knihLxO6xsTalBrM@Q^UNyj zVZwsht9y$YVubn_ZZF&fuy~>$Y6f9uA@PKi>23z+Q7{K@vT87eZ_m5Z9YJQD%FARh zv|zV|_NH?_O}CC$;*4S~@fX=kPp}X**M^)lUdx}$t*&sF_aybYoUtxbJ6e@BL}bl1 z!gT6u4CD@44+*4-XGo_UwnuSDFq<3Yni%th`w)asPuN!fv`@Vk1Q{p(l+*v!dyUnU z@o%Of@J0AD0uM(%Sh-G71j(L& z#P>w2frh%`Q@B-Vy)lew@)RRbW1*xiX#VUh!RrokQKezDMl(Pi7&LpTQ4WmY{j%mR z>8x+w^%Q|N=rgn$>1|JlTu_p;q~`Q0G8B^T$>eeq+Te)oVD#ZgMAFQ$_)mrzjB|g` zYS5--U%iJr+>7rW=v1SQV+cxz6!kgQ!XCkoVvHC1QeKbF9MWkg!Dv_QAffz)dg8!k zQuE^sz}g^`R)c``sZ6UDkCt|Y0SPUFV}87$sgh-)j|KOnk>d17D!hRm^A=XVt5jh> zMLY7^-f@~ojO8e$4?w2mp$dkaKo?OHsn3i~zb0SkIrsVb$m2nO#Xx9kGwk)6!4yOg z?W?Bf8f3#FIu_n8C|AH{1iDH6^kk#6ZboKqIJf=jSvq;s`D^5j0A?78kZwAX1j!|? z(Ro#^<*qj68no=MqN`!UyC{&DG>|2Urxzf2d<_NMv`I8MT!f0TR}vyyIanCmY~t>P zuspc1JS|BN^x{Pmr{`zp?V)1mH{!WDQe>FU)D^N4h_)qgYCDy(NQI`tsiKN* z^<&J-v3;7VsAjVwtwbGO<*WB+#)?m0!8ba$B{?vfrtw>+A=x918Gc4%Rzxucj&tQS!w@i}(J^sJ zKFQ=gIFhUdz7R;=5Xpcxr~b0W)oYr+jId!P$MPYlSqn4GDWT{fvr(V(8v(p~mc2vF$K-#w&EfsA&V3V^Wqp-ulGl!{yL& z*6TF`2H;Ub8CW7d@LsE;%sohS2y_ToSXhW%SYPqNs&~`YVE;h_*ne>CCHR$Y^xYq} z`k!q?Y-}9CTk!_A*Ac49jt2IQ|2xup8^BHXJ?B^ONKpX~Fu`BA4}xL;7T~&H2^(HR z7&+d^l?!%KID`Ac-+?`)t!-Zg4^(p`2neZPz*xZRrGEwXZxT`6mhqYRh@di9xu#$_ zf0Z!|>@>d<_J(Z2_NGo&;M_i9u0{acpH7(DVB_Q{?2=%xI`Arx^A{QAkpDf{KPa-E z>5xbYY@f%75D?cHjepWP_`&pVCAygu@wOOpFpM@Iz-%9YMY-NQ_(_@Ikdc3j@S}bf zIrEQ2>}?Dx#Y-9;u$uD0&*5LYLnHQYV+fmoyPY`D-oa7X$?#9J{WUBq$T_qO+!a{C zU0(R7T;QuW`2P*|haw&R8qQ9&^BFd{(}#mQz4R||W#B0E-_)cCz{JKL@UO(w4)}~-B+Zuo!lK*p3+_vwbLeSM9 zcxy@@0|Mf@B<)XPqWbL?$lOuy@HX&zPIW>NSoCf%_^&E=1;_UPrpo1j4h~>pf7lrO z5CA_;9RYuB>T>q|-DWWEG8p$)fs?_x)_xQBPe2y~d%%xjbO-RwTI*sz)eOFx1i#V$ z6YxJ7_h!-V>mu$yiH7?>LjI$eH>)52I&zhH|0Cv)p8VJ5yjeWw7Fg;&-9{+J-k1 z3jc}_r}+;Ee<<$%uLN*ghMP%NuM-phq-O@di*VN)`DQ*($)6zLs{-SH!uj_JTyINv zGm|9PBsVD6m-#wDbwr@(7#Ptd0VKP$@Z?ZKK`T%;BWE2 zE#lwhfV|y+n;CnqbNc-xb<5vrz+djm-u0AN@MNdN!< literal 0 HcmV?d00001 diff --git a/spring-batch-s3/.mvn/wrapper/maven-wrapper.properties b/spring-batch-s3/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..be4fea70 --- /dev/null +++ b/spring-batch-s3/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip diff --git a/spring-batch-s3/README.adoc b/spring-batch-s3/README.adoc new file mode 100644 index 00000000..87a9b908 --- /dev/null +++ b/spring-batch-s3/README.adoc @@ -0,0 +1,182 @@ += spring-batch-s3 +:toc: +:icons: font +:source-highlighter: highlightjs + +Spring Batch extension for Amazon S3 (even other S3 compatible may be supported) which contains `S3ItemReader` and `S3ItemWriter` implementations +for reading from and writing to S3 buckets, including support for multipart uploads. + +*Note*: these writers are based on the *AWS SDK V2*. + +== Installation + +To use the `spring-batch-s3` extension, you need to add the following dependency to your Maven or Gradle project: + +=== Maven + +[source,xml] +---- + + org.springframework.batch.extensions + spring-batch-s3 + ${spring-batch-extensions.version} + + + software.amazon.awssdk + apache-client + ${aws.sdk.version} + +---- + +=== Gradle + +[source,groovy] +---- +implementation 'org.springframework.batch.extensions:spring-batch-s3:${springBatchExtensionsVersion}' +implementation 'software.amazon.awssdk:apache-client:${awsSdkVersion}' +---- + +== Known limitations + +* The `S3ItemReader` and `S3ItemWriter` are designed to work with the synchronous AWS S3 client (`S3Client`). They do not support the asynchronous client (`S3AsyncClient`) at this time. + +== Pre-requisites + +In order to set up these components you need to provide some additional beans in your Spring Batch configuration: + +* An `S3Client` bean to interact with AWS S3. +* In case you want to use the `S3ItemReader`: an instance of `S3Deserializer` for the data you want to read. +* In case you want to use the `S3ItemWriter`: an instance of `S3Serializer` for the data you want to write. + +There are two examples of implementation for both `S3Serializer` and `S3Deserializer` provided in this project: + +* `S3StringSerializer`: take a `String` as input and writes it to S3 as a UTF-8 encoded byte array. The write functions add a line termination character at the end of each string. +* `S3StringDeserializer`: takes a UTF-8 encoded byte array from S3 and converts it to a `String`. The implementation of this deserializer is *stateful* because lines may arrive in different chunks. + +More details in the JavaDocs of the classes. + +=== Configuration of the `S3Client` + +To use the `S3ItemReader` and `S3ItemWriter`, you need to configure the AWS S3 client. +This can be done using Java configuration or XML configuration. + +So far only this synchronous client is supported, you can't use the `S3AsyncClient` with these components. + +==== Java Config + +[source,java] +---- +@Bean +public S3Client s3Client() { + return S3Client.builder().build(); +} +---- + +=== Configure `S3Serializer` + +`S3StringSerializer` is a simple implementation of `S3Serializer` that takes a `String` as input and writes it to S3 as a UTF-8 encoded byte array. You are encouraged to implement your own serializer if you need to handle different data types or formats. + +==== Java Config + +[source,java] +---- +@Bean +S3Serializer s3Serializer() { + return new S3StringSerializer(); +} +---- + +=== Configure `S3Deserializer` + +Similarly, `S3StringDeserializer` is a simple implementation of `S3Deserializer` that takes a UTF-8 encoded byte array from S3 and converts it to a `String`. You can implement your own deserializer if you need to handle different data types or formats. + +In case you don't want to implement your serializer checkout the "Alternatives readers" section below. + +==== Java Config + +[source,java] +---- +@Bean +S3Deserializer s3Deserializer() { + return new S3StringDeserializer(); +} +---- + +== Configuration of `S3ItemReader` + +Given the `S3Client` and `S3Deserializer` beans, you can now configure the `S3ItemReader`. + +=== Java Config + +To configure the `S3ItemReader`, you need to set up the AWS S3 client and specify the bucket and object key from which you want to read data. +[source,java] +---- +@Bean +ItemReader downloadItemReader() throws Exception { + return new S3ItemReader.Builder() + .s3Client(s3Client()) + .bucketName("bucket_name") + .objectKey("object_key") + .deserializer(s3Deserializer()) + .bufferSize(1024 * 1024) // Default 128 Bytes + .build(); +} +---- + +There is also an additional option to set the `bufferSize` which is the size of the buffer used to read data from S3. The default value is 128 bytes, but you can increase it to improve memory consumption The bast value for this parameter is the average length of the lines in your file. + +=== Alternative reader + +Instead `S3ItemReader` you can also use `FlatFileItemReader` with `InputStreamResources` to read files from S3 as well. +To do so this package exposes a `S3InputStreamResource` that can be used for that purpose. Below an example: + +[source,java] +---- +@Bean +ItemReader itemReader() throws Exception { + final var inputStreamResource = new InputStreamResource( + new S3InputStream(s3Client(), + "bucket_name", + "object_key")); + + return new FlatFileItemReaderBuilder() + .name("itemReader") + .resource(inputStreamResource) + .lineMapper(new PassThroughLineMapper( )) + .build(); +} +---- + +== Configuration of `S3ItemWriter` + +Given the `S3Client` and `S3Serializer` beans, you can now configure the `S3ItemWriter`. + +=== Java Config + +To configure the `S3ItemWriter`, you need to set up the AWS S3 client and specify the bucket and object key to which you want to write data. +[source,java] +---- +@Bean +ItemWriter uploadItemWriter() throws IOException { + return new S3ItemWriter.Builder() + .s3Client(s3Client()) + .bucketName("bucket_name") + .objectKey("object_key") + .multipartUpload(true) // Default is false + .partSize(10 * 1024 * 1024) // Default is 5 MB + .contentType("text/csv") // Default is application/octet-stream + .serializer(s3Serializer()) + .build(); +} +---- + +There are several additional options you can set for the `S3ItemWriter`: +* `multipartUpload`: If set to `true`, the writer will use multipart upload for large files. The default is `false`. +* `partSize`: The size of each part in a multipart upload. The default is 5 MB. +* `contentType`: The content type of the uploaded file. The default is `application/octet-stream`. + +== Links + +* https://github.com/spring-projects/spring-batch-extensions +* https://spring.io/projects/spring-batch +* https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/home.html \ No newline at end of file diff --git a/spring-batch-s3/mvnw b/spring-batch-s3/mvnw new file mode 100755 index 00000000..f13e1380 --- /dev/null +++ b/spring-batch-s3/mvnw @@ -0,0 +1,236 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # + # Look for the Apple JDKs first to preserve the existing behaviour, and then look + # for the new JDKs provided by Oracle. + # + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then + # + # Oracle JDKs + # + export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then + # + # Apple JDKs + # + export JAVA_HOME=`/usr/libexec/java_home` + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + local basedir=$(pwd) + local wdir=$(pwd) + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + wdir=$(cd "$wdir/.."; pwd) + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# avoid using MAVEN_CMD_LINE_ARGS below since that would loose parameter escaping in $@ +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/spring-batch-s3/mvnw.cmd b/spring-batch-s3/mvnw.cmd new file mode 100644 index 00000000..bb9bb461 --- /dev/null +++ b/spring-batch-s3/mvnw.cmd @@ -0,0 +1,146 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +set MAVEN_CMD_LINE_ARGS=%MAVEN_CONFIG% %* + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR=""%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# avoid using MAVEN_CMD_LINE_ARGS below since that would loose parameter escaping in %* +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/spring-batch-s3/pom.xml b/spring-batch-s3/pom.xml new file mode 100644 index 00000000..f0931103 --- /dev/null +++ b/spring-batch-s3/pom.xml @@ -0,0 +1,224 @@ + + + + 4.0.0 + + + spring-boot-starter-parent + org.springframework.boot + + 3.5.3 + + + org.springframework.batch.extensions + spring-batch-s3 + 0.1.0-SNAPSHOT + + Spring Batch S3 + Spring Batch extension for Amazon S3 + https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-s3 + 2025 + + + repo + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + andreacioni + Andrea Cioni + https://github.com/andreacioni + + + + + git://github.com/spring-projects/spring-batch-extensions.git + git@github.com:spring-projects/spring-batch-extensions.git + + https://github.com/spring-projects/spring-batch-extensions + + + + 2.31.77 + + + + + + bom + software.amazon.awssdk + import + pom + ${aws.java.sdk.version} + + + + + + + + s3 + + + + netty-nio-client + software.amazon.awssdk + + + software.amazon.awssdk + apache-client + + + software.amazon.awssdk + + + + + spring-batch-core + org.springframework.batch + + + + + assertj-core + org.assertj + test + + + junit-jupiter + org.junit.jupiter + test + + + mockito-core + org.mockito + test + + + slf4j-nop + org.slf4j + test + + + + + + + io.spring.javaformat + spring-javaformat-maven-plugin + 0.0.46 + + + maven-compiler-plugin + + + -Xlint:all,deprecation + + + org.apache.maven.plugins + + + maven-javadoc-plugin + + + + jar + + attach-javadocs + + + org.apache.maven.plugins + + + maven-source-plugin + + + + jar + + attach-sources + + + org.apache.maven.plugins + + + maven-checkstyle-plugin + + + checkstyle + com.puppycrawl.tools + 9.3 + + + spring-javaformat-checkstyle + io.spring.javaformat + 0.0.43 + + + + + + io/spring/javaformat/checkstyle/checkstyle.xml + + true + + + check + + checkstyle-validation + true + validate + + + org.apache.maven.plugins + 3.6.0 + + + flatten-maven-plugin + + + + ossrh + + remove + + + + flatten + + flatten + process-resources + + + + clean + + flatten-clean + clean + + + org.codehaus.mojo + 1.6.0 + + + + diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java new file mode 100644 index 00000000..1acf5b98 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java @@ -0,0 +1,140 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3; + +import java.io.IOException; +import java.util.Arrays; + +import software.amazon.awssdk.services.s3.S3Client; + +import org.springframework.batch.extensions.s3.serializer.S3Deserializer; +import org.springframework.batch.extensions.s3.stream.S3InputStream; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStream; +import org.springframework.batch.item.ItemStreamException; + +/** + * An {@link ItemReader} that reads items from an S3 object using a specified + * deserializer. It uses an {@link S3InputStream} to read the data and a + * {@link S3Deserializer} to convert the byte array into the desired item type. + * + * @param the type of items to read + * @author Andrea Cioni + */ +public class S3ItemReader implements ItemReader, ItemStream { + + private static final int DEFAULT_BUFFER_SIZE = 128; + + private final S3InputStream in; + + private final S3Deserializer deserializer; + + private int bufferSize = DEFAULT_BUFFER_SIZE; + + public S3ItemReader(S3InputStream in, S3Deserializer deserializer) { + this.in = in; + this.deserializer = deserializer; + } + + @Override + public T read() throws Exception { + T item; + + //before reading more bytes from the input stream get all of the items + //that may be buffered inside the deserializer (deserializer is stateful!) + while ((item = this.deserializer.deserialize(new byte[]{})) != null) { + return item; + } + + int bytesRead; + byte[] buffer = new byte[this.bufferSize]; + while ((bytesRead = this.in.read(buffer)) != -1) { + item = this.deserializer.deserialize(Arrays.copyOf(buffer, bytesRead)); + if (item != null) { + return item; + } + } + return null; + } + + @Override + public void close() throws ItemStreamException { + try { + this.in.close(); + } + catch (IOException ex) { + throw new ItemStreamException(ex); + } + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + public int getBufferSize() { + return this.bufferSize; + } + + public static class Builder { + private S3Client s3Client; + + private String bucketName; + + private String objectKey; + + private S3Deserializer deserializer; + + private Integer bufferSize; + + public Builder s3Client(S3Client s3Client) { + this.s3Client = s3Client; + return this; + } + + public Builder bucketName(String bucketName) { + this.bucketName = bucketName; + return this; + } + + public Builder objectKey(String objectKey) { + this.objectKey = objectKey; + return this; + } + + public Builder deserializer(S3Deserializer deserializer) { + this.deserializer = deserializer; + return this; + } + + public Builder bufferSize(int bufferSize) { + this.bufferSize = bufferSize; + return this; + } + + public S3ItemReader build() throws Exception { + if (this.s3Client == null || this.bucketName == null || this.objectKey == null || this.deserializer == null) { + throw new IllegalArgumentException("S3Client, bucketName, objectKey, and deserializer must be provided"); + } + S3InputStream inputStream = new S3InputStream(this.s3Client, this.bucketName, this.objectKey); + S3ItemReader reader = new S3ItemReader<>(inputStream, this.deserializer); + if (this.bufferSize != null) { + reader.setBufferSize(this.bufferSize); + } + return reader; + } + } +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java new file mode 100644 index 00000000..6ac59ea8 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java @@ -0,0 +1,152 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3; + +import java.io.IOException; +import java.io.OutputStream; + +import software.amazon.awssdk.services.s3.S3Client; + +import org.springframework.batch.extensions.s3.serializer.S3Serializer; +import org.springframework.batch.extensions.s3.stream.S3MultipartOutputStream; +import org.springframework.batch.extensions.s3.stream.S3MultipartUploader; +import org.springframework.batch.extensions.s3.stream.S3OutputStream; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemStream; +import org.springframework.batch.item.ItemStreamException; +import org.springframework.batch.item.ItemWriter; + +/** + * An {@link ItemWriter} that writes items to an S3 object using a specified serializer. + * It uses an {@link S3MultipartOutputStream} to write the data and a {@link S3Serializer} + * to convert the item into a byte array. + * + * @param the type of items to write + * @author Andrea Cioni + */ +public class S3ItemWriter implements ItemWriter, ItemStream { + + private final OutputStream out; + + private final S3Serializer serializer; + + public S3ItemWriter(OutputStream out, S3Serializer serializer) { + this.out = out; + this.serializer = serializer; + } + + @Override + public void write(Chunk chunk) throws Exception { + for (T item : chunk.getItems()) { + byte[] serializedData = this.serializer.serialize(item); + if (serializedData != null && serializedData.length > 0) { + this.out.write(serializedData); + } + else { + throw new IllegalArgumentException("Serialized data is null or empty for item: " + item); + } + } + } + + @Override + public void close() throws ItemStreamException { + try { + this.out.close(); + } + catch (IOException ex) { + throw new ItemStreamException(ex); + } + } + + public static class Builder { + private S3Client s3Client; + + private String bucket; + + private String key; + + private S3Serializer serializer; + + private boolean multipartUpload = false; + + private String contentType; + + private Integer partSize; + + public Builder s3Client(S3Client s3Client) { + this.s3Client = s3Client; + return this; + } + + public Builder bucketName(String bucketName) { + this.bucket = bucketName; + return this; + } + + public Builder objectKey(String key) { + this.key = key; + return this; + } + + public Builder serializer(S3Serializer serializer) { + this.serializer = serializer; + return this; + } + + public Builder multipartUpload(boolean multipartUpload) { + this.multipartUpload = multipartUpload; + return this; + } + + public Builder partSize(int partSize) { + this.partSize = partSize; + return this; + } + + public Builder contentType(String contentType) { + this.contentType = contentType; + return this; + } + + public S3ItemWriter build() throws IOException { + if (this.s3Client == null || this.bucket == null || this.key == null || this.serializer == null) { + throw new IllegalArgumentException("S3Client, bucket, key, and serializer must be provided"); + } + OutputStream outputStream; + if (this.multipartUpload) { + S3MultipartUploader s3MultipartUploader = new S3MultipartUploader(this.s3Client, this.bucket, this.key); + if (this.contentType != null) { + s3MultipartUploader.setContentType(this.contentType); + } + if (this.partSize != null) { + s3MultipartUploader.setPartSize(this.partSize); + } + + outputStream = new S3MultipartOutputStream(s3MultipartUploader); + } + else { + outputStream = new S3OutputStream(this.s3Client, this.bucket, this.key); + if (this.contentType != null) { + ((S3OutputStream) outputStream).setContentType(this.contentType); + } + } + + return new S3ItemWriter(outputStream, this.serializer); + } + } + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/package-info.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/package-info.java new file mode 100644 index 00000000..ebe91421 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/package-info.java @@ -0,0 +1,28 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides support for reading items from Amazon S3 using a stream-based approach. This + * package includes classes for reading items from S3 objects, deserializing them, and + * handling the input stream efficiently. + * + *

+ * Classes in this package are designed to work with the AWS SDK for Java and provide a + * convenient way to read large datasets stored in S3 without loading them entirely into + * memory. + */ + +package org.springframework.batch.extensions.s3; diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Deserializer.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Deserializer.java new file mode 100644 index 00000000..57079d06 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Deserializer.java @@ -0,0 +1,39 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.serializer; + +/** + * A functional interface for serializing items to byte arrays for S3 storage. + * Implementations should provide a way to convert an item of type T into a byte array. + * + * @param the type of items to be serialized + * @author Andrea Cioni + */ +@FunctionalInterface +public interface S3Deserializer { + + /** + * Deserialize a byte array into an object of type T. It is not guaranteed that the + * call to this method will always return a non-null value. This can happen if the + * byte array is either empty or it doesn't represent a valid object of type T yet. + * For this reason the implementation of this method should be stateful. + * @param buffer the byte array to deserialize + * @return the deserialized object + */ + T deserialize(byte[] buffer); + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Serializer.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Serializer.java new file mode 100644 index 00000000..10a18275 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3Serializer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.serializer; + +/** + * A functional interface for serializing items to byte arrays for S3 storage. + * Implementations should provide a way to convert an item of type T into a byte array. + * + * @param the type of items to be serialized + * @author Andrea Cioni + */ +@FunctionalInterface +public interface S3Serializer { + + byte[] serialize(T item); + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializer.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializer.java new file mode 100644 index 00000000..19d8ef6d --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializer.java @@ -0,0 +1,66 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.serializer; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Simple deserializer for String items from S3. It reads lines from a byte array, + * handling both \n and \r\n line endings. + * + * This is intended to be used with S3ItemReader to read text data from S3 objects. + * + * @author Andrea Cioni + */ +public class S3StringDeserializer implements S3Deserializer { + + final Charset charset; + + private StringBuilder stringBuilder = new StringBuilder(); + + public S3StringDeserializer() { + this.charset = StandardCharsets.UTF_8; + } + + public S3StringDeserializer(Charset charset) { + this.charset = charset; + } + + @Override + public String deserialize(byte[] buffer) { + String incoming = new String(buffer, this.charset); + this.stringBuilder.append(incoming); + + int newlineIdx = this.stringBuilder.indexOf("\n"); + if (newlineIdx == -1) { + return null; + } + + // Handle both \n and \r\n line endings + int lineEnd = newlineIdx; + if (newlineIdx > 0 && this.stringBuilder.charAt(newlineIdx - 1) == '\r') { + lineEnd--; + } + + String line = this.stringBuilder.substring(0, lineEnd); + this.stringBuilder = new StringBuilder(this.stringBuilder.substring(newlineIdx + 1)); + + return line; + } + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializer.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializer.java new file mode 100644 index 00000000..743f9223 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.serializer; + +import java.nio.charset.StandardCharsets; + +/** + * Simple serializer for String items to be used with S3. This serializer takes a String + * item, appends a newline character, and converts it to a byte array using UTF-8 + * encoding. This is intended to be used with S3ItemWriter to write text data to S3 + * objects. + * + * @author Andrea Cioni + */ +public class S3StringSerializer implements S3Serializer { + + @Override + public byte[] serialize(String item) { + return (item + "\n").getBytes(StandardCharsets.UTF_8); + } + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/package-info.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/package-info.java new file mode 100644 index 00000000..3bf6ba15 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/serializer/package-info.java @@ -0,0 +1,27 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides support for reading items from Amazon S3 using a stream-based approach. This + * package includes classes for reading items from S3 objects, deserializing them, and + * handling the input stream efficiently. + * + *

+ * Classes in this package are designed to work with the AWS SDK for Java and provide a + * convenient way to read large datasets stored in S3 without loading them entirely into + * memory. + */ +package org.springframework.batch.extensions.s3.serializer; diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java new file mode 100644 index 00000000..20d703c2 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java @@ -0,0 +1,26 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.batch.extensions.s3.stream; + +final class Defaults { + static final int DEFAULT_PART_SIZE = 5 * 1024 * 1024; // 5MB + + static final String DEFAULT_CONTENT_TYPE = "application/octet-stream"; + + private Defaults() { } +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3InputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3InputStream.java new file mode 100644 index 00000000..f7445906 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3InputStream.java @@ -0,0 +1,78 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.io.InputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +/** + * An {@link InputStream} that reads data from an S3 object. It uses the AWS SDK for Java + * to retrieve the object from S3. Is safe to use this stream for reading large files as + * it doesn't load the entire file into memory. + * + * @author Andrea Cioni + */ +public class S3InputStream extends InputStream { + + private static final Logger logger = LoggerFactory.getLogger(S3InputStream.class); + + private final S3Client s3; + + private final String bucketName; + + private final String objectKey; + + private InputStream inputStream; + + public S3InputStream(S3Client s3, String bucketName, String objectKey) throws IOException { + this.s3 = s3; + this.bucketName = bucketName; + this.objectKey = objectKey; + } + + @Override + public int read() throws IOException { + if (this.inputStream == null) { + this.inputStream = openS3InputStream(); + } + return this.inputStream.read(); + } + + @Override + public void close() throws IOException { + logger.debug("Closing stream"); + if (this.inputStream != null) { + this.inputStream.close(); + } + logger.debug("Stream closed"); + super.close(); + } + + private InputStream openS3InputStream() { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(this.bucketName) + .key(this.objectKey) + .build(); + return this.s3.getObject(getObjectRequest); + } + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java new file mode 100644 index 00000000..accbadeb --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java @@ -0,0 +1,116 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * An {@link OutputStream} that writes data to an S3 object using multipart upload. It + * uses a {@link PipedInputStream} and a {@link PipedOutputStream} to allow writing data + * asynchronously while uploading it in parts. This stream is suitable for large file + * uploads. + * + * @author Andrea Cioni + */ +public class S3MultipartOutputStream extends OutputStream { + + private static final Logger logger = LoggerFactory.getLogger(S3MultipartOutputStream.class); + + private final PipedInputStream pipedInputStream; + + private final PipedOutputStream pipedOutputStream; + + private final ExecutorService singleThreadExecutor; + + private volatile boolean uploading; + + private final S3Uploader multipartUpload; + + public S3MultipartOutputStream(S3Client s3Client, String bucketName, String key) throws IOException { + this(new S3MultipartUploader(s3Client, bucketName, key)); + } + + public S3MultipartOutputStream(S3Uploader s3Uploader) throws IOException { + this.pipedInputStream = new PipedInputStream(); + this.pipedOutputStream = new PipedOutputStream(this.pipedInputStream); + this.singleThreadExecutor = Executors.newSingleThreadExecutor(); + this.uploading = false; + this.multipartUpload = s3Uploader; + } + + @Override + public void write(int b) throws IOException { + if (!this.uploading) { + this.uploading = true; + + startUpload(); + } + this.pipedOutputStream.write(b); + } + + private void startUpload() { + this.singleThreadExecutor.execute(() -> { + try { + this.multipartUpload.upload(this.pipedInputStream); + } + catch (IOException ex) { + logger.error("Error during multipart upload", ex); + throw new RuntimeException(ex); + } + finally { + try { + this.pipedInputStream.close(); + } + catch (IOException ex) { + logger.error("Error closing piped input stream", ex); + } + } + }); + this.singleThreadExecutor.shutdown(); + } + + @Override + public void close() throws IOException { + logger.debug("Closing output stream"); + + this.pipedOutputStream.close(); + + if (this.uploading) { + try { + if (!this.singleThreadExecutor.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)) { + logger.warn("Multipart upload thread did not finish in time"); + } + } + catch (InterruptedException ex) { + logger.error("Multipart upload thread interrupted", ex); + } + } + + logger.debug("Output stream closed"); + super.close(); + } + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploader.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploader.java new file mode 100644 index 00000000..b7464030 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploader.java @@ -0,0 +1,201 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +/** + * A utility class for performing multipart uploads to Amazon S3. It reads data from an + * input stream and uploads it in parts to a specified S3 bucket and key.
+ * Reference: Uploading + * streams to Amazon S3 using the AWS SDK for Java 2.x + * + * @author Andrea Cioni + */ +public class S3MultipartUploader implements S3Uploader { + + private static final Logger logger = LoggerFactory.getLogger(S3MultipartUploader.class); + + private final S3Client s3Client; + + private final String bucket; + + private final String key; + + private int partSize = Defaults.DEFAULT_PART_SIZE; + + private String contentType = Defaults.DEFAULT_CONTENT_TYPE; + + public S3MultipartUploader(S3Client s3Client, String bucket, String key) { + this.s3Client = s3Client; + this.bucket = bucket; + this.key = key; + } + + /** + * Reads from the input stream into the buffer, attempting to fill the buffer + * completely or until the end of the stream is reached. + * @param inputStream the input stream to read from + * @param buffer the buffer to fill + * @return the number of bytes read, or -1 if the end of the stream is reached before + * any bytes are read + * @throws IOException if an I/O error occurs + */ + private static int readFullyOrToEnd(InputStream inputStream, byte[] buffer) throws IOException { + int totalBytesRead = 0; + int bytesRead; + while (totalBytesRead < buffer.length) { + bytesRead = inputStream.read(buffer, totalBytesRead, buffer.length - totalBytesRead); + if (bytesRead == -1) { + break; + } + totalBytesRead += bytesRead; + } + return (totalBytesRead > 0) ? totalBytesRead : -1; + } + + @Override + public long upload(InputStream inputStream) throws IOException { + String uploadId; + long totalBytesRead = 0; + + try { + CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder() + .bucket(this.bucket) + .key(this.key) + .contentType(this.contentType) + .build(); + + CreateMultipartUploadResponse createResponse = this.s3Client + .createMultipartUpload(createMultipartUploadRequest); + uploadId = createResponse.uploadId(); + logger.debug("Started multipart upload with ID: {}", uploadId); + + List completedParts = new ArrayList<>(); + int partNumber = 1; + byte[] buffer = new byte[this.partSize]; + int bytesRead; + + try { + while ((bytesRead = readFullyOrToEnd(inputStream, buffer)) > 0) { + totalBytesRead += bytesRead; + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(this.bucket) + .key(this.key) + .uploadId(uploadId) + .partNumber(partNumber) + .build(); + + RequestBody requestBody; + if (bytesRead < this.partSize) { + byte[] lastPartBuffer = new byte[bytesRead]; + System.arraycopy(buffer, 0, lastPartBuffer, 0, bytesRead); + requestBody = RequestBody.fromBytes(lastPartBuffer); + } + else { + requestBody = RequestBody.fromBytes(buffer); + } + + UploadPartResponse uploadPartResponse = this.s3Client.uploadPart(uploadPartRequest, requestBody); + CompletedPart part = CompletedPart.builder() + .partNumber(partNumber) + .eTag(uploadPartResponse.eTag()) + .build(); + completedParts.add(part); + + logger.debug("Uploaded part {} with size {} bytes", partNumber, bytesRead); + partNumber++; + } + + CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(this.bucket) + .key(this.key) + .uploadId(uploadId) + .multipartUpload(completedMultipartUpload) + .build(); + + CompleteMultipartUploadResponse completeResponse = this.s3Client + .completeMultipartUpload(completeRequest); + logger.debug("Multipart upload completed. Object URL: {}", completeResponse.location()); + } + catch (Exception ex) { + logger.error("Error during multipart upload: {}", ex.getMessage(), ex); + if (uploadId != null) { + AbortMultipartUploadRequest abortRequest = AbortMultipartUploadRequest.builder() + .bucket(this.bucket) + .key(this.key) + .uploadId(uploadId) + .build(); + this.s3Client.abortMultipartUpload(abortRequest); + logger.warn("Multipart upload aborted"); + } + throw ex; + } + finally { + try { + inputStream.close(); + } + catch (IOException ex) { + logger.error("Error closing input stream: {}", ex.getMessage(), ex); + } + } + } + finally { + this.s3Client.close(); + } + + return totalBytesRead; + } + + public int getPartSize() { + return this.partSize; + } + + public void setPartSize(int partSize) { + this.partSize = partSize; + } + + public String getContentType() { + return this.contentType; + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java new file mode 100644 index 00000000..ba58340a --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java @@ -0,0 +1,116 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * An {@link OutputStream} that writes data directly to an S3 object with a specified MIME + * type (default is application/octet-stream). This stream load the data in-memory and + * uploads it to S3 as it is written. It uses a {@link PipedInputStream} and a + * {@link PipedOutputStream} to allow writing data asynchronously while uploading it + * directly to S3. Is it not safe to use this stream with large file uploads, as it does + * not handle multipart uploads or large data efficiently. For this use case, check out + * {@link S3MultipartOutputStream}. + * + * @author Andrea Cioni + */ +public class S3OutputStream extends OutputStream { + + private static final Logger logger = LoggerFactory.getLogger(S3OutputStream.class); + + private final S3Client s3; + + private final String bucketName; + + private final String key; + + private final PipedInputStream pipedInputStream; + + private final PipedOutputStream pipedOutputStream; + + private final ExecutorService singleThreadExecutor; + + private volatile boolean uploading; + + private String contentType = Defaults.DEFAULT_CONTENT_TYPE; + + public S3OutputStream(S3Client s3, String bucketName, String key) throws IOException { + this.s3 = s3; + this.bucketName = bucketName; + this.key = key; + this.pipedInputStream = new PipedInputStream(); + this.pipedOutputStream = new PipedOutputStream(this.pipedInputStream); + this.singleThreadExecutor = Executors.newSingleThreadExecutor(); + this.uploading = false; + } + + @Override + public void write(int b) throws IOException { + if (!this.uploading) { + this.uploading = true; + runUploadThread(); + } + this.pipedOutputStream.write(b); + } + + private void runUploadThread() { + this.singleThreadExecutor.execute(() -> { + try { + RequestBody body = RequestBody + .fromContentProvider(ContentStreamProvider.fromInputStream(this.pipedInputStream), this.contentType); + this.s3.putObject((builder) -> builder.bucket(this.bucketName).key(this.key), body); + } + finally { + try { + this.pipedInputStream.close(); + } + catch (IOException ex) { + logger.error("Error closing piped input stream", ex); + } + } + }); + this.singleThreadExecutor.shutdown(); + } + + @Override + public void close() throws IOException { + logger.debug("Closing output stream"); + this.pipedOutputStream.close(); + logger.debug("Output stream closed"); + super.close(); + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } + + public String getContentType() { + return this.contentType; + } +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3Uploader.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3Uploader.java new file mode 100644 index 00000000..9fa22947 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3Uploader.java @@ -0,0 +1,26 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.io.InputStream; + +public interface S3Uploader { + + long upload(InputStream inputStream) throws IOException; + +} diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/package-info.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/package-info.java new file mode 100644 index 00000000..8d671d71 --- /dev/null +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/package-info.java @@ -0,0 +1,27 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides support for reading items from Amazon S3 using a stream-based approach. This + * package includes classes for reading items from S3 objects, deserializing them, and + * handling the input stream efficiently. + * + *

+ * Classes in this package are designed to work with the AWS SDK for Java and provide a + * convenient way to read large datasets stored in S3 without loading them entirely into + * memory. + */ +package org.springframework.batch.extensions.s3.stream; diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java new file mode 100644 index 00000000..553e7218 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java @@ -0,0 +1,171 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.batch.extensions.s3.serializer.S3Deserializer; +import org.springframework.batch.extensions.s3.serializer.S3StringDeserializer; +import org.springframework.batch.extensions.s3.stream.S3InputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; + +class S3ItemReaderTests { + + private S3Deserializer mockDeserializer; + + private S3InputStream s3InputStream; + + private S3StringDeserializer stringDeserializer; + + @BeforeEach + void setUp() { + this.stringDeserializer = new S3StringDeserializer(); + this.mockDeserializer = mock(S3Deserializer.class); + this.s3InputStream = mock(S3InputStream.class); + } + + @Test + void testReadReturnsDeserializedItemWithStreamMock() throws Exception { + byte[] data = "test".getBytes(); + // given + given(this.s3InputStream.read(any(byte[].class))).willReturn(data.length, -1); + given(this.mockDeserializer.deserialize(any(byte[].class))).willReturn(null, "item"); + + S3ItemReader reader = new S3ItemReader<>(this.s3InputStream, this.mockDeserializer); + + // when + String result = reader.read(); + + // then + assertThat(result).isEqualTo("item"); + then(this.s3InputStream).should(atLeastOnce()).read(any(byte[].class)); + then(this.mockDeserializer).should(times(2)).deserialize(any(byte[].class)); + } + + @Test + void testReadReturnsDeserializedItem() throws Exception { + byte[] data = "item\n".getBytes(); + + // given + given(this.mockDeserializer.deserialize(any(byte[].class))) + .willReturn(null); + given(this.s3InputStream.read(any(byte[].class))).willAnswer((invocation) -> { + byte[] buffer = invocation.getArgument(0); + System.arraycopy(data, 0, buffer, 0, data.length); + return data.length; + }).willReturn(-1); + + S3ItemReader reader = new S3ItemReader<>(this.s3InputStream, this.stringDeserializer); + + // when + String result = reader.read(); + + // then + assertThat(result).isEqualTo("item"); + then(this.s3InputStream).should(atLeastOnce()).read(any(byte[].class)); + } + + @Test + void testReadReturnsNullWhenNoData() throws Exception { + // given + given(this.s3InputStream.read(any(byte[].class))).willReturn(-1); + + S3ItemReader reader = new S3ItemReader<>(this.s3InputStream, this.mockDeserializer); + + // when + String result = reader.read(); + + // then + assertThat(result).isNull(); + } + + @Test + void testReadReturnsMultipleItems() throws Exception { + byte[] data1 = "item1\n".getBytes(); + byte[] data2 = "item2\n".getBytes(); + + // given + given(this.s3InputStream.read(any(byte[].class))) + .willAnswer((invocation) -> { + byte[] buffer = invocation.getArgument(0); + System.arraycopy(data1, 0, buffer, 0, data1.length); + return data1.length; + }); + given(this.mockDeserializer.deserialize(any(byte[].class))) + .willReturn("item1") + .willReturn("item2") + .willReturn(null); // No more items + given(this.s3InputStream.read(any(byte[].class))) + .willAnswer((invocation) -> { + byte[] buffer = invocation.getArgument(0); + System.arraycopy(data2, 0, buffer, 0, data2.length); + return data2.length; + }) + .willReturn(-1); // End of stream + S3ItemReader reader = new S3ItemReader<>(this.s3InputStream, this.mockDeserializer); + String result1 = reader.read(); + String result2 = reader.read(); + String result3 = reader.read(); + // then + + assertThat(result1).isEqualTo("item1"); + assertThat(result2).isEqualTo("item2"); + assertThat(result3).isNull(); + then(this.s3InputStream).should(atLeastOnce()).read(any(byte[].class)); + then(this.mockDeserializer).should(atLeastOnce()).deserialize(any(byte[].class)); + } + + @Test + void testReadReturnsMultipleItemsInSingleDeserialization() throws Exception { + byte[] data = "item1\nitem2\n".getBytes(); + + // given + given(this.s3InputStream.read(any(byte[].class))) + .willAnswer((invocation) -> { + byte[] buffer = invocation.getArgument(0); + System.arraycopy(data, 0, buffer, 0, data.length); + return data.length; + }).willAnswer((invocation) -> -1); + + given(this.mockDeserializer.deserialize(any(byte[].class))) + .willReturn(null) // buffer is empty + .willReturn("item1") + .willReturn("item2") + .willReturn(null); // End of stream + + S3ItemReader reader = new S3ItemReader<>(this.s3InputStream, this.mockDeserializer); + String result1 = reader.read(); + String result2 = reader.read(); + String result3 = reader.read(); + // then + + assertThat(result1).isEqualTo("item1"); + assertThat(result2).isEqualTo("item2"); + assertThat(result3).isNull(); + then(this.s3InputStream).should(times(2)).read(any(byte[].class)); + then(this.mockDeserializer).should(times(4)).deserialize(any(byte[].class)); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemWriterTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemWriterTests.java new file mode 100644 index 00000000..007c0802 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemWriterTests.java @@ -0,0 +1,101 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3; + +import java.io.IOException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.batch.extensions.s3.serializer.S3Serializer; +import org.springframework.batch.extensions.s3.stream.S3MultipartOutputStream; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemStreamException; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.BDDMockito.willThrow; +import static org.mockito.Mockito.mock; + +class S3ItemWriterTests { + + private S3Serializer serializer; + + private S3MultipartOutputStream outputStream; + + @BeforeEach + void setUp() { + this.serializer = mock(S3Serializer.class); + this.outputStream = mock(S3MultipartOutputStream.class); + } + + @Test + void testWrite_success() throws Exception { + String item = "test"; + byte[] data = item.getBytes(); + // given + given(this.serializer.serialize(item)).willReturn(data); + + S3ItemWriter writer = new S3ItemWriter<>(this.outputStream, this.serializer); + Chunk chunk = Chunk.of(item); + + // when + writer.write(chunk); + + // then + then(this.serializer).should().serialize(item); + then(this.outputStream).should().write(data); + } + + @Test + void testWrite_throwsOnNullOrEmpty() { + String item = "bad"; + // given + given(this.serializer.serialize(item)).willReturn(null); + + S3ItemWriter writer = new S3ItemWriter<>(this.outputStream, this.serializer); + Chunk chunk = Chunk.of(item); + + // when/then + assertThatThrownBy(() -> writer.write(chunk)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void testClose_success() throws Exception { + S3ItemWriter writer = new S3ItemWriter<>(this.outputStream, this.serializer); + + // when + writer.close(); + + // then + then(this.outputStream).should().close(); + } + + @Test + void testClose_throwsItemStreamException() throws Exception { + // given + willThrow(new IOException("close error")).given(this.outputStream).close(); + S3ItemWriter writer = new S3ItemWriter<>(this.outputStream, this.serializer); + + // when/then + assertThatThrownBy(writer::close) + .isInstanceOf(ItemStreamException.class); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializerTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializerTests.java new file mode 100644 index 00000000..80304615 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringDeserializerTests.java @@ -0,0 +1,70 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.serializer; + + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class S3StringDeserializerTests { + + @Test + void testDeserializeSingleLine() { + S3StringDeserializer deserializer = new S3StringDeserializer(); + String input = "testString\n"; + String result = deserializer.deserialize(input.getBytes(StandardCharsets.UTF_8)); + assertThat(result).isEqualTo("testString"); + } + + @Test + void testDeserializeMultipleLines() { + S3StringDeserializer deserializer = new S3StringDeserializer(); + String input = "line1\nline2\n"; + String result1 = deserializer.deserialize(input.getBytes(StandardCharsets.UTF_8)); + assertThat(result1).isEqualTo("line1"); + String result2 = deserializer.deserialize(new byte[0]); + assertThat(result2).isEqualTo("line2"); + } + + @Test + void testDeserializeWithCarriageReturn() { + S3StringDeserializer deserializer = new S3StringDeserializer(); + String input = "line1\r\n"; + String result = deserializer.deserialize(input.getBytes(StandardCharsets.UTF_8)); + assertThat(result).isEqualTo("line1"); + } + + @Test + void testDeserializePartialInput() { + S3StringDeserializer deserializer = new S3StringDeserializer(); + String part1 = "partial"; + String part2 = "Line\n"; + assertThat(deserializer.deserialize(part1.getBytes(StandardCharsets.UTF_8))).isNull(); + String result = deserializer.deserialize(part2.getBytes(StandardCharsets.UTF_8)); + assertThat(result).isEqualTo("partialLine"); + } + + @Test + void testDeserializeEmptyInput() { + S3StringDeserializer deserializer = new S3StringDeserializer(); + assertThat(deserializer.deserialize("".getBytes(StandardCharsets.UTF_8))).isNull(); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializerTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializerTests.java new file mode 100644 index 00000000..a11e9844 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/serializer/S3StringSerializerTests.java @@ -0,0 +1,62 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.serializer; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +class S3StringSerializerTests { + + @Test + void testSerialize() { + S3StringSerializer serializer = new S3StringSerializer(); + String input = "testString"; + byte[] result = serializer.serialize(input); + + String expected = "testString\n"; + assertThat(result).isEqualTo(expected.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + @Test + void testSerializeEmptyString() { + S3StringSerializer serializer = new S3StringSerializer(); + String input = ""; + byte[] result = serializer.serialize(input); + + String expected = "\n"; + assertThat(result).isEqualTo(expected.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + @Test + void testSerializeNull() { + S3StringSerializer serializer = new S3StringSerializer(); + assertThatCode(() -> serializer.serialize(null)).doesNotThrowAnyException(); + } + + @Test + void testSerializeWithSpecialCharacters() { + S3StringSerializer serializer = new S3StringSerializer(); + String input = "test\nstring\r\nwith special characters!@#$%^&*()"; + byte[] result = serializer.serialize(input); + + String expected = "test\nstring\r\nwith special characters!@#$%^&*()\n"; + assertThat(result).isEqualTo(expected.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3InputStreamTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3InputStreamTests.java new file mode 100644 index 00000000..8f09e910 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3InputStreamTests.java @@ -0,0 +1,69 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; + +class S3InputStreamTests { + + private S3Client s3Client; + + private final byte[] data = { 1, 2, 3, 4 }; + + @BeforeEach + void setUp() { + this.s3Client = Mockito.mock(S3Client.class); + } + + @Test + void testRead() throws IOException { + InputStream mockStream = new ByteArrayInputStream(this.data); + ResponseInputStream responseInputStream = new ResponseInputStream<>( + GetObjectResponse.builder().build(), mockStream); + // given + given(this.s3Client.getObject(any(GetObjectRequest.class))).willReturn(responseInputStream); + + String key = "test-key"; + String bucket = "test-bucket"; + // when + try (S3InputStream s3InputStream = new S3InputStream(this.s3Client, bucket, key)) { + for (byte b : this.data) { + assertThat(s3InputStream.read()).isEqualTo(b); + } + assertThat(s3InputStream.read()).isEqualTo(-1); + } + + // then + then(this.s3Client).should().getObject(any(GetObjectRequest.class)); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java new file mode 100644 index 00000000..c3a93d6d --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java @@ -0,0 +1,87 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.io.InputStream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.services.s3.S3Client; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.mock; + +class S3MultipartOutputStreamTests { + + private S3Client s3Client; + + private S3Uploader multipartUploadMock; + + @BeforeEach + void setUp() throws IOException { + this.s3Client = mock(S3Client.class); + this.multipartUploadMock = mock(S3Uploader.class); + + given(this.multipartUploadMock.upload(any())).willAnswer((invocation) -> { + Thread.sleep(100); // Simulate some delay for upload + return 1L; + }); + } + + @Test + void testWriteSingleByteTriggersUpload() throws IOException { + int testByte = 42; + + try (S3MultipartOutputStream out = new S3MultipartOutputStream(this.multipartUploadMock)) { + // when + out.write(testByte); + + ArgumentCaptor captor = ArgumentCaptor.forClass(InputStream.class); + + // then + then(this.multipartUploadMock).should().upload(captor.capture()); + assertThat(captor.getValue().available()).as("InputStream should contain one byte").isEqualTo(1); + } + } + + @Test + void testConstructorWithDefaultPartSize() throws IOException { + S3MultipartOutputStream out = new S3MultipartOutputStream(this.s3Client, "bucket", "key"); + out.close(); + } + + @Test + void testConstructorWithCustomPartSize() throws IOException { + int customPartSize = 10 * 1024 * 1024; + var s3Uploader = new S3MultipartUploader(this.s3Client, "bucket", "key"); + s3Uploader.setPartSize(customPartSize); + S3MultipartOutputStream out = new S3MultipartOutputStream(s3Uploader); + out.close(); + } + + @Test + void testConstructorWithS3UploadOutputStream() throws IOException { + S3MultipartOutputStream out = new S3MultipartOutputStream(this.multipartUploadMock); + out.close(); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java new file mode 100644 index 00000000..c5934c15 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java @@ -0,0 +1,102 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; + +class S3MultipartUploaderTests { + + private S3Client s3Client; + + private S3MultipartUploader s3MultipartUploader; + + @BeforeEach + void setUp() { + this.s3Client = mock(S3Client.class); + var s3Uploader = new S3MultipartUploader(this.s3Client, "bucket", "key"); + s3Uploader.setPartSize(5); + this.s3MultipartUploader = s3Uploader; + } + + @Test + void testUpload_SuccessfulUpload() throws IOException { + byte[] data = "HelloWorld!".getBytes(); // 11 bytes, 3 parts, 2 of 5 bytes each + // and one of 1 byte + ByteArrayInputStream inputStream = new ByteArrayInputStream(data); + + // given + given(this.s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))) + .willReturn(CreateMultipartUploadResponse.builder().uploadId("uploadId").build()); + + given(this.s3Client.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))) + .willReturn(UploadPartResponse.builder().eTag("etag1").build(), + UploadPartResponse.builder().eTag("etag2").build(), + UploadPartResponse.builder().eTag("etag3").build()); + + given(this.s3Client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))) + .willReturn(CompleteMultipartUploadResponse.builder().location("url").build()); + + // when + this.s3MultipartUploader.upload(inputStream); + + // then + then(this.s3Client).should().createMultipartUpload(any(CreateMultipartUploadRequest.class)); + then(this.s3Client).should(times(3)).uploadPart(any(UploadPartRequest.class), any(RequestBody.class)); + then(this.s3Client).should().completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + then(this.s3Client).should().close(); + } + + @Test + void testUpload_AbortOnException() throws IOException { + byte[] data = "HelloWorld".getBytes(); + ByteArrayInputStream inputStream = new ByteArrayInputStream(data); + + // given + given(this.s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))) + .willReturn(CreateMultipartUploadResponse.builder().uploadId("uploadId").build()); + + given(this.s3Client.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))) + .willThrow(new RuntimeException("Upload failed")); + + // when/then + assertThatThrownBy(() -> this.s3MultipartUploader.upload(inputStream)) + .isInstanceOf(RuntimeException.class); + then(this.s3Client).should().abortMultipartUpload(any(AbortMultipartUploadRequest.class)); + then(this.s3Client).should().close(); + } + +} diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java new file mode 100644 index 00000000..da244530 --- /dev/null +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java @@ -0,0 +1,59 @@ +/* + * Copyright 2006-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.s3.stream; + +import java.io.IOException; +import java.util.function.Consumer; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +class S3OutputStreamTests { + + private S3Client s3Client; + + @BeforeEach + void setUp() { + this.s3Client = mock(S3Client.class); + } + + @Test + void testWriteAndUpload() throws IOException, InterruptedException { + byte[] data = { 10, 20, 30, 40 }; + doReturn(null).when(this.s3Client).putObject(any(Consumer.class), any(RequestBody.class)); + + String bucket = "test-bucket"; + String key = "test-key"; + try (S3OutputStream out = new S3OutputStream(this.s3Client, bucket, key)) { + out.write(data); + } + + // Wait for the upload thread to finish + Thread.sleep(200); + + verify(this.s3Client, times(1)).putObject(any(Consumer.class), any(RequestBody.class)); + } + +} From 3fe11bb57382ce9638a8ac969b6975ebbce93a43 Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Tue, 8 Jul 2025 15:52:11 +0200 Subject: [PATCH 02/10] license header updated using maven plugin Signed-off-by: Andrea Cioni --- spring-batch-s3/pom.xml | 41 ++++++++++++++++++- .../batch/extensions/s3/S3ItemReader.java | 2 +- .../batch/extensions/s3/S3ItemWriter.java | 2 +- .../batch/extensions/s3/package-info.java | 2 +- .../s3/serializer/S3Deserializer.java | 2 +- .../s3/serializer/S3Serializer.java | 2 +- .../s3/serializer/S3StringDeserializer.java | 2 +- .../s3/serializer/S3StringSerializer.java | 2 +- .../s3/serializer/package-info.java | 2 +- .../batch/extensions/s3/stream/Defaults.java | 3 +- .../extensions/s3/stream/S3InputStream.java | 4 +- .../s3/stream/S3MultipartOutputStream.java | 2 +- .../s3/stream/S3MultipartUploader.java | 2 +- .../extensions/s3/stream/S3OutputStream.java | 2 +- .../extensions/s3/stream/S3Uploader.java | 2 +- .../extensions/s3/stream/package-info.java | 2 +- .../extensions/s3/S3ItemReaderTests.java | 2 +- .../extensions/s3/S3ItemWriterTests.java | 2 +- .../serializer/S3StringDeserializerTests.java | 2 +- .../serializer/S3StringSerializerTests.java | 2 +- .../s3/stream/S3InputStreamTests.java | 2 +- .../stream/S3MultipartOutputStreamTests.java | 2 +- .../s3/stream/S3MultipartUploaderTests.java | 2 +- .../s3/stream/S3OutputStreamTests.java | 2 +- 24 files changed, 64 insertions(+), 26 deletions(-) diff --git a/spring-batch-s3/pom.xml b/spring-batch-s3/pom.xml index f0931103..9f31df97 100644 --- a/spring-batch-s3/pom.xml +++ b/spring-batch-s3/pom.xml @@ -1,6 +1,6 @@ - 4.0.0 + xmlns="http://maven.apache.org/POM/4.0.0" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 3.5.3 - - + + org.springframework.boot + spring-boot-starter-parent + 3.5.3 + + - org.springframework.batch.extensions - spring-batch-s3 - 0.1.0-SNAPSHOT + org.springframework.batch.extensions + spring-batch-s3 + 0.1.0-SNAPSHOT - Spring Batch S3 - Spring Batch extension for Amazon S3 - https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-s3 - 2025 - - - repo - Apache-2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - + Spring Batch S3 + Spring Batch extension for Amazon S3 + https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-s3 + 2025 + + + repo + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + - - - andreacioni - Andrea Cioni - https://github.com/andreacioni - - + + + andreacioni + Andrea Cioni + https://github.com/andreacioni + + - - git://github.com/spring-projects/spring-batch-extensions.git - git@github.com:spring-projects/spring-batch-extensions.git - - https://github.com/spring-projects/spring-batch-extensions - + + git://github.com/spring-projects/spring-batch-extensions.git + + git@github.com:spring-projects/spring-batch-extensions.git + + https://github.com/spring-projects/spring-batch-extensions + - - 2.31.77 - 0.0.47 - + + 2.31.77 + 0.0.47 + - - - - bom - software.amazon.awssdk - import - pom - ${aws.java.sdk.version} - - - + + + + software.amazon.awssdk + bom + import + pom + ${aws.java.sdk.version} + + + - - - - s3 - - - - netty-nio-client - software.amazon.awssdk - - - software.amazon.awssdk - + + + + software.amazon.awssdk + s3 + + + software.amazon.awssdk + netty-nio-client + + + - - - spring-batch-core - org.springframework.batch - + + + org.springframework.batch + spring-batch-core + - - - assertj-core - org.assertj - test - - - junit-jupiter - org.junit.jupiter - test - - - mockito-core - org.mockito - test - - - slf4j-nop - org.slf4j - test - - + + + org.assertj + assertj-core + test + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-core + test + + - - - - io.spring.javaformat - spring-javaformat-maven-plugin - 0.0.46 - - - maven-compiler-plugin - - - -Xlint:all,deprecation - - - org.apache.maven.plugins - - - maven-javadoc-plugin - - - - jar - - attach-javadocs - - - org.apache.maven.plugins - - - maven-source-plugin - - - - jar - - attach-sources - - - org.apache.maven.plugins - - - maven-checkstyle-plugin - - - checkstyle - com.puppycrawl.tools - 9.3 - - - spring-javaformat-checkstyle - io.spring.javaformat - ${spring-javaformat.version} - - - - - - io/spring/javaformat/checkstyle/checkstyle.xml - - true - - - check - - checkstyle-validation - true - validate - - - org.apache.maven.plugins - 3.6.0 - - - flatten-maven-plugin - - - - ossrh - - remove - - - - flatten - - flatten - process-resources - - - - clean - - flatten-clean - clean - - - org.codehaus.mojo - 1.6.0 - - - com.mycila - license-maven-plugin - 4.6 - - - - + + + io.spring.javaformat + spring-javaformat-maven-plugin + 0.0.46 + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:all,deprecation + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + jar + + attach-javadocs + + + + + org.apache.maven.plugins + maven-source-plugin + + + + jar + + attach-sources + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.6.0 + + + checkstyle + com.puppycrawl.tools + 9.3 + + + spring-javaformat-checkstyle + io.spring.javaformat + ${spring-javaformat.version} + + + + + + io/spring/javaformat/checkstyle/checkstyle.xml + + true + + + check + + checkstyle-validation + true + validate + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.7.1 + + + + ossrh + + remove + + + + flatten + + flatten + process-resources + + + + clean + + flatten-clean + clean + + + + + com.mycila + license-maven-plugin + 4.6 + + + + - - src/**/*.java - - - - - 2025 - - - - - - check - - - - - - + + src/**/*.java + + + + + 2025 + + + + + + check + + + + + + diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java index fb1a3045..cd0a0719 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemReader.java @@ -35,13 +35,13 @@ */ public class S3ItemReader implements ItemReader, ItemStream { - private static final int DEFAULT_BUFFER_SIZE = 128; + private static final int DEFAULT_BUFFER_SIZE_BYTES = 128; private final S3InputStream in; private final S3Deserializer deserializer; - private int bufferSize = DEFAULT_BUFFER_SIZE; + private int bufferSize = DEFAULT_BUFFER_SIZE_BYTES; public S3ItemReader(S3InputStream in, S3Deserializer deserializer) { this.in = in; diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java index 303f625f..7d9922cd 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/S3ItemWriter.java @@ -25,6 +25,7 @@ import org.springframework.batch.item.ItemStream; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemWriter; +import org.springframework.util.ObjectUtils; /** * An {@link ItemWriter} that writes items to an S3 object using a specified serializer. @@ -49,7 +50,7 @@ public S3ItemWriter(OutputStream out, S3Serializer serializer) { public void write(Chunk chunk) throws Exception { for (T item : chunk.getItems()) { byte[] serializedData = this.serializer.serialize(item); - if (serializedData != null && serializedData.length > 0) { + if (!ObjectUtils.isEmpty(serializedData)) { this.out.write(serializedData); } else { diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java index e177c87a..ed5e61fb 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java @@ -22,6 +22,7 @@ import java.io.PipedOutputStream; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,7 +101,7 @@ public void close() throws IOException { if (this.uploading) { try { - if (!this.singleThreadExecutor.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)) { + if (!this.singleThreadExecutor.awaitTermination(10L, TimeUnit.SECONDS)) { logger.warn("Multipart upload thread did not finish in time"); } } diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java index e2ddceaa..af60049f 100644 --- a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java @@ -27,7 +27,6 @@ import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.then; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -60,7 +59,7 @@ void testReadReturnsDeserializedItemWithStreamMock() throws Exception { // then assertThat(result).isEqualTo("item"); - then(this.s3InputStream).should(atLeastOnce()).read(any(byte[].class)); + then(this.s3InputStream).should(times(1)).read(any(byte[].class)); then(this.mockDeserializer).should(times(2)).deserialize(any(byte[].class)); } @@ -84,7 +83,7 @@ void testReadReturnsDeserializedItem() throws Exception { // then assertThat(result).isEqualTo("item"); - then(this.s3InputStream).should(atLeastOnce()).read(any(byte[].class)); + then(this.s3InputStream).should(times(1)).read(any(byte[].class)); } @Test @@ -133,8 +132,8 @@ void testReadReturnsMultipleItems() throws Exception { assertThat(result1).isEqualTo("item1"); assertThat(result2).isEqualTo("item2"); assertThat(result3).isNull(); - then(this.s3InputStream).should(atLeastOnce()).read(any(byte[].class)); - then(this.mockDeserializer).should(atLeastOnce()).deserialize(any(byte[].class)); + then(this.s3InputStream).should(times(2)).read(any(byte[].class)); + then(this.mockDeserializer).should(times(4)).deserialize(any(byte[].class)); } @Test diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java index d96ab6b7..82419d5b 100644 --- a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStreamTests.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -44,7 +45,7 @@ void setUp() throws IOException { this.multipartUploadMock = mock(S3Uploader.class); given(this.multipartUploadMock.upload(any())).willAnswer((invocation) -> { - Thread.sleep(100); // Simulate some delay for upload + TimeUnit.MILLISECONDS.sleep(100); // Simulate some delay for upload return 1L; }); } diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java index a274ae56..0d4cd7f5 100644 --- a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3MultipartUploaderTests.java @@ -54,8 +54,7 @@ void setUp() { @Test void testUpload_SuccessfulUpload() throws IOException { - byte[] data = "HelloWorld!".getBytes(); // 11 bytes, 3 parts, 2 of 5 bytes each - // and one of 1 byte + byte[] data = "HelloWorld!".getBytes(); // 11 bytes, 3 parts, 2 of 5 bytes each and one of 1 byte ByteArrayInputStream inputStream = new ByteArrayInputStream(data); // given diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java index fce72586..e62f6f6d 100644 --- a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/stream/S3OutputStreamTests.java @@ -27,6 +27,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -50,9 +51,7 @@ void testWriteAndUpload() throws IOException, InterruptedException { out.write(data); } - // Wait for the upload thread to finish - Thread.sleep(200); - + verify(this.s3Client, timeout(200)).putObject(any(Consumer.class), any(RequestBody.class)); verify(this.s3Client, times(1)).putObject(any(Consumer.class), any(RequestBody.class)); } From 75849c9f96e0782b9d8ba7e7a6ec78c5ccc973d6 Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Thu, 10 Jul 2025 15:44:29 +0200 Subject: [PATCH 05/10] add ability to change the underlying executor service in OutputStream's Signed-off-by: Andrea Cioni --- .../extensions/s3/stream/S3MultipartOutputStream.java | 10 ++++++++-- .../batch/extensions/s3/stream/S3OutputStream.java | 7 +++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java index ed5e61fb..802c4735 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java @@ -44,7 +44,7 @@ public class S3MultipartOutputStream extends OutputStream { private final PipedOutputStream pipedOutputStream; - private final ExecutorService singleThreadExecutor; + private ExecutorService singleThreadExecutor; private volatile boolean uploading; @@ -57,7 +57,6 @@ public S3MultipartOutputStream(S3Client s3Client, String bucketName, String key) public S3MultipartOutputStream(S3Uploader s3Uploader) throws IOException { this.pipedInputStream = new PipedInputStream(); this.pipedOutputStream = new PipedOutputStream(this.pipedInputStream); - this.singleThreadExecutor = Executors.newSingleThreadExecutor(); this.uploading = false; this.multipartUpload = s3Uploader; } @@ -73,6 +72,10 @@ public void write(int b) throws IOException { } private void startUpload() { + if(this.singleThreadExecutor == null) { + this.singleThreadExecutor = Executors.newSingleThreadExecutor(); + } + this.singleThreadExecutor.execute(() -> { try { this.multipartUpload.upload(this.pipedInputStream); @@ -114,4 +117,7 @@ public void close() throws IOException { super.close(); } + public void setSingleThreadExecutor(ExecutorService singleThreadExecutor) { + this.singleThreadExecutor = singleThreadExecutor; + } } diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java index d1babddd..1042dd6f 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java @@ -54,7 +54,7 @@ public class S3OutputStream extends OutputStream { private final PipedOutputStream pipedOutputStream; - private final ExecutorService singleThreadExecutor; + private ExecutorService singleThreadExecutor; private volatile boolean uploading; @@ -66,7 +66,6 @@ public S3OutputStream(S3Client s3, String bucketName, String key) throws IOExcep this.key = key; this.pipedInputStream = new PipedInputStream(); this.pipedOutputStream = new PipedOutputStream(this.pipedInputStream); - this.singleThreadExecutor = Executors.newSingleThreadExecutor(); this.uploading = false; } @@ -80,6 +79,10 @@ public void write(int b) throws IOException { } private void runUploadThread() { + if(this.singleThreadExecutor == null) { + this.singleThreadExecutor = Executors.newSingleThreadExecutor(); + } + this.singleThreadExecutor.execute(() -> { try { RequestBody body = RequestBody From 2843d92d85bf9955028f06c459f743aea25ffa08 Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Thu, 10 Jul 2025 15:47:46 +0200 Subject: [PATCH 06/10] fix check format failed Signed-off-by: Andrea Cioni --- .../batch/extensions/s3/stream/S3MultipartOutputStream.java | 2 +- .../batch/extensions/s3/stream/S3OutputStream.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java index 802c4735..86ccac4f 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3MultipartOutputStream.java @@ -72,7 +72,7 @@ public void write(int b) throws IOException { } private void startUpload() { - if(this.singleThreadExecutor == null) { + if (this.singleThreadExecutor == null) { this.singleThreadExecutor = Executors.newSingleThreadExecutor(); } diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java index 1042dd6f..2117f24b 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/S3OutputStream.java @@ -79,7 +79,7 @@ public void write(int b) throws IOException { } private void runUploadThread() { - if(this.singleThreadExecutor == null) { + if (this.singleThreadExecutor == null) { this.singleThreadExecutor = Executors.newSingleThreadExecutor(); } From 5e3793ab8e13d9d927a71a301afb4c39bc6ac71c Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Mon, 14 Jul 2025 15:36:21 +0200 Subject: [PATCH 07/10] Update spring-batch-s3/pom.xml Co-authored-by: Volodymyr Signed-off-by: Andrea Cioni --- spring-batch-s3/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-batch-s3/pom.xml b/spring-batch-s3/pom.xml index beedab20..2573f694 100644 --- a/spring-batch-s3/pom.xml +++ b/spring-batch-s3/pom.xml @@ -88,7 +88,7 @@ - + org.springframework.batch spring-batch-core From 498185393aa2f592da5ebabd3c3d6a980a229ad7 Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Mon, 14 Jul 2025 15:36:33 +0200 Subject: [PATCH 08/10] Update spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java Co-authored-by: Volodymyr Signed-off-by: Andrea Cioni --- .../springframework/batch/extensions/s3/S3ItemReaderTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java index af60049f..3e8f1350 100644 --- a/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java +++ b/spring-batch-s3/src/test/java/org/springframework/batch/extensions/s3/S3ItemReaderTests.java @@ -158,6 +158,7 @@ void testReadReturnsMultipleItemsInSingleDeserialization() throws Exception { String result1 = reader.read(); String result2 = reader.read(); String result3 = reader.read(); + // then assertThat(result1).isEqualTo("item1"); From bf39daf74e5ba0ff79f1c8a0b2d50125fa0c5553 Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Mon, 14 Jul 2025 15:36:53 +0200 Subject: [PATCH 09/10] Update spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/builder/S3ItemWriterBuilder.java Co-authored-by: Volodymyr Signed-off-by: Andrea Cioni --- .../batch/extensions/s3/builder/S3ItemWriterBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/builder/S3ItemWriterBuilder.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/builder/S3ItemWriterBuilder.java index 198e24fe..e376ef16 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/builder/S3ItemWriterBuilder.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/builder/S3ItemWriterBuilder.java @@ -36,7 +36,7 @@ public class S3ItemWriterBuilder { private S3Serializer serializer; - private boolean multipartUpload = false; + private boolean multipartUpload; private String contentType; From ade7ad2b6d30aadcf17ae4906945587facbd4b5a Mon Sep 17 00:00:00 2001 From: Andrea Cioni Date: Mon, 14 Jul 2025 15:37:35 +0200 Subject: [PATCH 10/10] Update spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java Co-authored-by: Volodymyr Signed-off-by: Andrea Cioni --- .../springframework/batch/extensions/s3/stream/Defaults.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java index 2231383a..eea4061b 100644 --- a/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java +++ b/spring-batch-s3/src/main/java/org/springframework/batch/extensions/s3/stream/Defaults.java @@ -19,7 +19,7 @@ import org.springframework.util.unit.DataSize; final class Defaults { - static final int DEFAULT_PART_SIZE = (int) DataSize.ofMegabytes(5).toBytes(); + static final int DEFAULT_PART_SIZE = (int) DataSize.ofMegabytes(5L).toBytes(); static final String DEFAULT_CONTENT_TYPE = "application/octet-stream";