From b3a969245285e1947f383ceea4c1a3176ad89f4d Mon Sep 17 00:00:00 2001 From: Jici Gao Date: Mon, 5 Jun 2023 08:58:33 -0500 Subject: [PATCH 01/20] add 5G On Arm --- .../5g-on-arm/_index.md | 40 ++ .../5g-on-arm/_next-steps.md | 27 ++ .../5g-on-arm/_review.md | 45 ++ .../5g-on-arm/example-picture.png | Bin 0 -> 63167 bytes .../5g-on-arm/how-to-1.md | 122 ++++++ .../5g-on-arm/how-to-2.md | 384 ++++++++++++++++++ .../5g-on-arm/how-to-3.md | 190 +++++++++ .../5g-on-arm/how-to-4.md | 14 + 8 files changed, 822 insertions(+) create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/example-picture.png create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md create mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md new file mode 100644 index 0000000000..4ad788f8cb --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md @@ -0,0 +1,40 @@ +--- +title: 5G On Arm Handbook + +description: This + +minutes_to_complete: 10 + +who_is_this_for: This is an introductory topic for 5G software migrating to Arm. + +learning_objectives: + - Chooce the proper Arm server model to develop/deploy 5G stacks and 5GCN server + - Suggest the best practice and configuration for setting up Arm servers for 5G stacks + +prerequisites: + - Arm Neoverse server + - 5G stacks + +author_primary: 5G Solution Team/Arm Performance and Solution Engineering Team + +### Tags +skilllevels: Introductory +subjects: 5G ORAN stacks +armips: + - Arm Neoverse IP +tools: + - Arm development toolset + - Virtualization software +softwares: + - Any ORAN stacks + - L1 with or without HW accerlators +operatingsystems: + - Ubuntu 20.04+ + + +### FIXED, DO NOT MODIFY +# ================================================================================ +weight: 1 # _index.md always has weight of 1 to order correctly +layout: "learningpathall" # All files under learning paths have this same wrapper +learning_path_main_page: "yes" # This should be surfaced when looking for related content. Only set for _index.md of learning path content. +--- diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md new file mode 100644 index 0000000000..c4ae77cc92 --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md @@ -0,0 +1,27 @@ +--- +next_step_guidance: PLACEHOLDER TEXT 1 + +recommended_path: /learning-paths/PLACEHOLDER_CATEGORY/PLACEHOLDER_LEARNING_PATH/ + +further_reading: + - resource: + title: PLACEHOLDER MANUAL + link: PLACEHOLDER MANUAL LINK + type: documentation + - resource: + title: PLACEHOLDER BLOG + link: PLACEHOLDER BLOG LINK + type: blog + - resource: + title: PLACEHOLDER GENERAL WEBSITE + link: PLACEHOLDER GENERAL WEBSITE LINK + type: website + + +# ================================================================================ +# FIXED, DO NOT MODIFY +# ================================================================================ +weight: 21 # set to always be larger than the content in this path, and one more than 'review' +title: "Next Steps" # Always the same +layout: "learningpathall" # All files under learning paths have this same wrapper +--- diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md new file mode 100644 index 0000000000..9b1bfb96fe --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md @@ -0,0 +1,45 @@ +--- +review: + - questions: + question: > + PLACEHOLDER QUESTION 1? + answers: + - PLACEHOLDER ANSWER A + - PLACEHOLDER ANSWER B + correct_answer: 1 + explanation: > + PLACEHOLDER EXPLANATION 1 + + - questions: + question: > + PLACEHOLDER QUESTION 2? + answers: + - PLACEHOLDER ANSWER A + - PLACEHOLDER ANSWER B + - PLACEHOLDER ANSWER C + - PLACEHOLDER ANSWER D + correct_answer: 4 + explanation: > + PLACEHOLDER EXPLANATION 2 + + - questions: + question: > + PLACEHOLDER QUESTION 3? + answers: + - PLACEHOLDER ANSWER A + - PLACEHOLDER ANSWER B + - PLACEHOLDER ANSWER C + - PLACEHOLDER ANSWER D + correct_answer: 2 + explanation: > + PLACEHOLDER EXPLANATION 3 + + + +# ================================================================================ +# FIXED, DO NOT MODIFY +# ================================================================================ +title: "Review" # Always the same title +weight: 20 # Set to always be larger than the content in this path +layout: "learningpathall" # All files under learning paths have this same wrapper +--- diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/example-picture.png b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/example-picture.png new file mode 100644 index 0000000000000000000000000000000000000000..c69844bed44b65c7f5bc6cf511f93987fdcd7b95 GIT binary patch literal 63167 zcmeFa2UL^U+6EeXMMVUZIs+pJ3K&D^keLz00^~3ugw9BmUWCvCWE@9n0!I-cw2Yt$ zBoKrEAw)o=Nl9XeA@m-4?}T>aIsboV=3oC^_ny1%x@+CD+#$)E@7wuy`QH7O=Y4nB z>)aazoVk7D)(ybEeE`5d;RCQYy#Laz>(`zC_7n8REi;pUoY;RJAS?&30sua~fdN06 z{`ifJt=%_Yj{SGJi$p#6Z~A|E7hZR*QAlB>1>rL=K-jpj zv-gDbIk$h(py+Yc+ z$2;gtTVKkT9-qAL3x^5cZG=x501EIEz!dPKkp6$Q{4t+?IRJppDgbcc?|&R~!2$r( z#{j_j?mv!w_d5V^`ZoZeob}*tz}7u}=WNar^)7=pWzx z<6iuqjP)DgcVvX)^b>R&We2t{K4`k|Kf9+cLN^2s%0NhxCR2342er9!hT)&(Iw|YyX1G*pD$YNbpuWc z2fAPMfG7Y0SXxBI8koEvu3o9F9q37tmzIAprB%>XMjXhA4}>KLsNmCG$d;nQBKz!U zYNUm-z_d@U;^SIB>%#Vf`Jxs_=rrOCKi7ppD}!qz$W({{}+t$RX6@e=*G_Y zXfZ#EbdqXKPu5b}BChF_VEfJI-7^1EW}&Q+=*Xy_rZ9FD(U2>VTtoKYPU9wnWf$@N zh4lCTLJI%?PaI476P`EP^mJDzgvVDWtde3-B1x5mN&{(~0bSZOH0A*43$*>8QuYBy zAc?Z8n|JbeJcniea_{dJuUoB~K5~;D*EaN61h{(rr;Pc_IFt*Cl6*a65!jEw&DQYuWqt=E0h z)CcR66Up?;>|G_)@52`S2MrtW7smdJP`lc>MMUS;QWBOkvP(C90fB${4;r`FpBcOP z&k}A-T0RIG7^>x8jY4(c&K!1q`yVvy{eNNXzeu<`I(m3~1~44;nV$ zFO2;c376*m57piERdxU8-0!cd`>X2ys=B|b?ysu*KR~8`Ro#d)|DM+^#ymKiB^J6E z;x(Jq^mxq`o9D*yQfdk$Cr%pIuLb3$5h0p&-|qpQ?Ey?RZgf=L$8L>Zme1&FosLtl zZ)@{vCNR$$WViyu_{5^SQ+Nes3)}3%a0T06zy~(J2@F{AaqFkRpRZ)Mj)_Hr;}a_#>h3zDPZnzd@`U-16{#~a9!#;Nm!Y~;hBms3><*e zsuAn-W1I-??jpj2VgmLu43Fpzd(tpcG6N1Jm9aHBQp{w%T^rbfZ2kBD&dS*r5Dr{% z95NNxfyUC;UDXLF%c|65*UAD{Mx|aUq9xm^J}k?A^TY$OJ-{y=JNFUWa!!x3ztNn3 zfr@%Iz+JnW=2AFJwlryuiOo(Kcz!R;XS=eY^`%NWKHb&TMc2&)z58SD!9BnrxW*Q~ z2RL+ZGO|`JsWt)WGdz=FTR!Vq(Iu(DNq*KBxhifc3Ixe!r9h|L1xe_ffsi%ma9E8I z$L(U1iyzV>hmQibHiIT!KIyo(aZ(8uFdMJQv3d~ZqM$tcWA-nZDqj*K|0rQ^XDu=6 z4e^-*syfVgX<-RF%0@>X4n5{rc{vMWzy7&?r=7HWa+(u0F}w#zGG#t#MeYH9@LNz^ zQ2ImoUcpSCj2^D;Jft zSK#OV$Jpj}z@hpcY-zU%j%KkQpO=QN0@o6!^a$@ck(jiau5jQwjI9~GdHyilpn$&~fzUf3_ zloT}~CAMz8(>l$(z%=b^RS8$$0~h)>&c zY;0m6*K7}9m!9L|5>b4$GevscICxG@eW+WS$gT8f{tOIwPy`nh?Z7jv;AX6BOegQe-y&S5+ zT5OmbU#0ijKW`d@)L;#$3FfndQ$req(>V<r`EP+m`3EotXa&HJRz#w_i*M2(G= zhzaj<$cn@sU_Zn~u?(_NLmH&)Cen%_?A`IdAH3yy4gFd>No{p$&bj?gcBSM^KRNqp zes9g|R71N^=Zf4)13CY;h^6^L#M582q`v=4tAF`$;)l7&(4LO!@98URc86Ik)O2Hi z+0vl_Ldhr11MU$5^sAs5?V)o5?x{QN^!f09Ol-HHfod1vlrn(sSl+hab*RxQH1w#` zrcb}SP&2f?bGznBN#~)Q<=_9(4B&vZK!3gu6qET)Pdf9^lxJ0Zv=@{?WewOhQ;aPg z%))e+;&KOxvfn<-)EVyB0}L4voCwA1hh{ERwbDmesx!_p2C$*Aj5DowRZ&H2e2nO{n~b3 zcm<`v4rPmM3nwT3s#^_IPGPdAC{`-!Q~R~TQQPhfZcj5gE{BLzpS9G!6q}@`$@W!d zAnXjTZfuR?*DtI`iOg!4RS{;}(2JT}r!3E+5?$yMh1no8<$?a=gZY=GZ+&53VGU zX+lYrKMn#7VNKILH7|FQy+1S~rZEnaH z=GtHfHQ1I=YV=-{22xPs3Av9gDq%9k43SC`ALg#&9v9>@&kczx#WDJFMyxL6}(Nggf#h>ZFB+C|r`* zp~=f7Unp-?e1E?}Rei{3!81KacRb_PG}iz9*#^&l4gLQX@&7zrwC}SXa6{0AQDJXg zBzhP8NeaHwhmM~Mueo54Gp$C3@HW2RuZb(BqcEA_MI*f3=LANi8e`XgswT-UhGM3QFG1?LLhPJv~f6OGAOssN^_AncelX2$7-Q;Jn4FV z^5DYe6ZK3jh4KBWqp=((rwB=8Z+#mfe+&>V|pajy;{)?m1~2p6OxNN6L}b?WMCJ zcUsv&-gm-mY&*P~K2id7IRSl^EqOS`mAsLnV`b_hi>DzTe3)Y>9NMMlYht-$qy{s* zwK4L7gQzpxK2?s$gr&i?rU;L$Pyx#yz@_NRPNu#Y9n(W>gJeCA*DFEGw1rdM3P$9t z(sk>igbS4xU0js^m}^>9(>W7;q@D3htt4)K^O)_&5b!2HcF( zqM%rJZ6lt|NlF5{r~8HV6=GZUEz2aqqz=+-SpveyW+(eNX@R>NeQCiuXHw2NmL5-$ zB*SWX8T7(~tZUmww(*&2@yX6dvI9A{rpB{MdSgT4S{sdCZ-jSjvydygzGruBkD~jMc6u zvGS1X6Jno%kUT({4OpIqAtvzm*G7JKmQ_-4o$E>ZHR$*rAeepkm32E(DNG@4EsD&Q zK9s_df`eJ*MrPfgD^L3`L#d8Eb~jS8&V>QXq?9~2aw^_1sfYui^pZj@CSYO9wt)iG zC9Z1emsKKJ;GSui=)$rWY{^A67M~XzJClaHJlD^0@xT?CnW67K*#oe5_gCzSwB+L~ zlI28)8`ychhza_K?8~l+J~B8;V7+ zH*M57&uAvftdiFC2eSQx>l1T}I))gLNmOD#7 zZ@S>zLG4aV+?ZHcp?x}4qG`~X78d@*{N0t4_FcM?#5lDsiZRbIC@d#!7Q2xx`L~HW zkW6Hk0bvg?Wx1^CQiX~ugERB6y`3Lyo?0lxgY+KY6(c61|KQT(KNf%a8RI=b_~l&( zza0tZ9X-k(U;?*;9BVY_mFd$~9Xpj9jgeGpTlKZA@^vT2E}MgTPVZ+2?sNsY_nNh@ zr^H)Y7!AIU4^dLw1GpCh&0!E(ar>RKYkJpK3~4n$Aa!1>(7EB0pJ`w%C&#sKW-21} zGEPK-(A-*HboyQZcK(KkYvx5Adh%#Vr94E^$<4{$MF(cKynG`|%+MiO{*Ygf z^yLsg2iD-uoAOmzr#*lX%tFZ!p;z-5D3iZ2c9$1u@mMCRL(OEm2YtkBe8-_zphARz zW&MO|hn+<)*IHw@&$`o*CUu;v-t{IDKp&Gzwk+xLZ$5(3>3Jc+TC( zt(%#7iiY^04->B;mrZEtH89u{DXmmVvmJMQWb(3if9J{a%0MbSkQD)~eR|>i=J`}o zV@q)WqX)$$_qDjD=i@Z%*XkO%{Z}5Gj+XhEL+7DBPmJp73XU1o$TpcuL@gt-`8eS!IgsTUNsZcY^fB+ehi zmjzn!F+1p6@mN8z85&~M-~XQf0`?{d0U8ipaMh~$H2wPUBjes(C0HI?6l^wlzncz} zf5cS9#VKYy_p&=oqk=p+)Wb;`c)sNjCDnxl-;N;0WILa@FS;}SHaO%=X-2yGVozr{ zUaiGdLBl?lJWywX@iE>8UhS%z1%J%aV2jBmLg#zgjXQEfQ+T%{!7ky%c4UOH8M<>r zfBsf>e0TJr5|v(h_|B}Ur>oOkpaV^2{w@OIU{6TF}n$>Wq{}&$w}0FU`{n5nSBCRnM4gb1V!KsBzDH z7UJW7TB1p}iH5qte`blUygw(TDP(La0ulIf(70cPW?^r)c1*r$|3PU_q)q6a70 zTo=rroEZPfwEdG7bGXlTWirF-I+J^Cir>P#7)R#HY*WtFFC{%e%aU?UO&ExT3(k#> z4FPKzrT)GQ@&4SBwV;m+Jrt(CO42l9FzFyFJJRt;7y5|X!tzb(tC_w7Vt49u6qK&8 zTj1sSQ_G`wNL-UWcr@b9CV7+^XEa&qOI}6+0;@l+2f#p=ZbgzGDP+4+KB)x=cRQGwtg>67k;)+JKnPz)ZL89{V z-|Z`f!^|x;xuQT|i%Y~?@^-`onzE6akX)f6Z2FZ`T~2Ztlm-}wOL_>jyH3(+8Za<)ONnQV`_cKxmvqn9H+=^gDnP;mb+ zXKrvxFS3jB%w;r~Z55RtyHMYBufD!+>Gyic&{q5K2uj8sUA zZq3d4xqY8d^@p>T68+ls1to3ULI==D<42(fK4gRW3AfL*itR_=QeHF|z_RHY2Xcr$03Ca5xbJS>pq%#=gDO*X=HbKO-ptI57u^(Bt{6Pz zGBM0Vvp%@3jinA{y3H83O`bk6Ibo0VpL<$S+mkL4MR*2kr41vw!5E0O^7Eku(C=jJ z&JB9TQ{H7Vzr}?wB_1azHhrnE)8dxN(h$*H;`2!#srqPd9|nQIe&dsM`pBJns;H>j z*#@o2nM8;27s0GL!@M&`+!6wbb?1!0w#563ed=k54@d}}{Bhjs{!}U?R z=Js4A$!+RIPp1qyt({21&L{l_x??*5YuZujLtL4Cr*`K>oAKT89-#McNRC5f1{m4Jxu4P5Swxc?r&FPP}YdEN9Oa^|j-Y-9FsJ(B2=0q#baZ(|ov z7w%w}qja^za~W=oG2O5-RqyYmb`U1hdlz((w6RzSVpyS|Esl0(^TdD?j|aq z7F=^kvuP5`7iGGB8xi`$C(@Ix5oI2M^pc%8ls>%X=(nu5nn;}B*bo*Q9 zId)&jE7nbsQ+kUOmCqhA zwN{}UFK?*E@)wDzQi5Y1QVE#}m`Bu+1J`tU)=9|$t_=gHb;<(J?l>?QmGmUl`ABK7 zpP6Np4v5E+&jF^-C|y2b)?3SMG!8_;XC`ogf^+WZ%8q!FPaop)bzb1~hSU@BX3DdK zsMs8OP}6f4%c6qKmMjV<7jLuIxiGLr`bUsQiA!+t7D#AnF1_QjE<)P;(5ICtUH5KW)7}Om8U~ zHnEEolNm19CRt+;cAiPBV`|k8-vyClh{STCTiRo8_t;@%#XilwViYb^8`(GUhRMsf zW@k$~AxsF--H`=C}jI6;@VXOx6hD(P>h5OB`2(ATLe$i44 z3HOEGV$W+TG0V$k;D9G@Pt(zN;_y*bK>d;7Cd|YhK!Xf<>?pJ`%|at=YQh_4yvl4o zQXEMypsIOfES~7b02&=x;2F+;P?z?XmMZWWqO8mah_Nv>S~>A6W}6-~sA>*IUo-ss znZm3(rmE)_XJv++JxSG}rg61~v2i)z z>4b?Jm=a!zs=`WdTpqSvPuXeyh0uZda0Now^1J7VXT-yMX%0*HI29~0U$`V5qja=C zUBCHxr9gyTU&va!pE-T*Uf26}-vFxCG+q_&Sp)H5Dk=BZM~JIwwGJtZRG?ilq_4Y9 zrbk-UqpHkfh^sI;TN*+SkI{p~cV7X=ru)=$^>Z;38TaWTNsTZ`4X(r_&U-HDDMitD z;q=oUa<9{;4Zp&EssYV`p46*?W1&dL<}yA-Cv`t2s`q_bX}d&9t!n=4(_p#SVCeed zlpzCO&s|fA8GxL_v)H@N#uy1c99m)|Hq6A0iZ?-a*1A_8W9Sh+&aa$rim38}%F3^< zo?dyddIAm?sHWRF0XhF#riSQ4DR3!co?PeVwbQ{5>S8ja+&20s;7Ikj2yMR+xI|4) z5?a81!Ww95YN1#g!Q(>;5i-OaJ`fHztrHtUFo7#qhgfI)m%xTNmku5epH9qq=c()B zT$Fmxp-kRO7UyHC?l2Yx9}K#hamSZ7IIJ=#K)%#QZRFBkzIURS^}g>N+5@}{Q|>b0 z%(5S%^F@vLJtW=DmqvyD!?fDZ~a0CYDpX3E?=kY@%rn?74awkI=C zzM~$%W}v|WKoFsJgtj{3XScb3p3~(n4KZXDNr&@3N=9Y3e*a12)$-)rW8T|=-(b^L zx*E6VtER(tmA$4V`aGJ$5s~xutVMlioAR;2cT)p}a##QEx(A3JSXJFFY5iFup83lc zow8Oss{!k{mV4qBr-Txq8cVz~VoT(sM0-G7yi(Zpsr7$BE~%~)mq9Y&O|uJ=wz!g) zNs61#`nG*2Y#G26YQAHkicX z4A>5)4cAB1S#Hu%S9mwl1-78Rgr~Vjbm$?jGFqgPr_Q~fvm#2su|X~kF8Lv z*CkAg)1ESWU4UT!;PH$yAa5ecY|hOo;Zt%_Wm&qP^QY2TE!ADJ?<-@tyo_ZB8e4Tp zXED~r&#v)P$rN4*ot0oZ0No1Y$QS13#_HdC%5xMNyl)oY+B(2qGs0J=<&|#wS)5pe zMwgn;4wQfmPKX**f6Q=fXYF1T)7&jpE0y%ItMVlw3ONCBBPmu~86k2ERHe3`c;!3L zSJ9}hGa)!FBUQ88^-@C{?&K=6*0UzQpY>Al!_?2^iScD7=hK7?oYC_+act{I zh=|b7g@fxcPoj7w$mi(NB+rLQ$iH!x0}EynKtDCm6H;S?N!l}gcn_gZ?IvtdY%nb? zEqqeb;@#UKS0&|JfT$XIH0A0%~WtBO~7>~=8V3NPDt`ZVQ>MU76AMg^7rbhX-J;;yU0&=8Q z|5vZd?jK&&xrWGAC5AO=iuG%FnK4wwyVWrA)6tj_^$Y{L+xNvQyo@m45No47fS9I7 zGl)4hzV)r&#agCn1_g2DM!E-U*81nO=guFgdF5``IL=)Rl&!%Jus(x)O$s8==YmKy zt4#&S@`5o$$wkpN5@yB&nga`3KSi1!@SOAECa=kL3oU|tbfz&oHo>tgDTyT3x=f}X z);l_3=NK1bbF0gyQ(1iUk+=q!BrwI_-%w^19j-WF^u*KM@pSn2)9;A8 z-&x>{n{-GJb#4*2BRzLG?bfA<3r%(taYT}@B}lfEn+In?OlVKy6HV-IJ;jd3A-f=& z*+TC=urUShu!S5t*~gjyH@#E6OlD@}j+uzEUK-^>b!V1L{BQ50g&X-3JZwqI4nvI; zUxtzNOYfa$!u6$vC;zQObluOx--^Y zsVpFt4;SZXa?LjJ%I|lj_|-a)ItxW4Jc+&*oRtN)1e)*iCFa?zR`ZrHxcnPzFS?6< zJ$l=!qdcma)!ND_?Rn8nEJ)8G&h}5$^ZW`$W!>>^sYVW)^-a&dX+_0{CBTefK&u;7 z{#8d=Yj$fU#!kcvVr-UTPSD)u9o#YX4?~SB!@*Rk!_A)pz*oqY@a7y{dx|kjy{cur zpACgvX<1r;y;5ouM)#4^z1PVdqa71`2c$VaA&$FzU4ZCpGq-77bt1h9;fFhUHiz0O4-s6nQ{p4-As?-c3dl-L%6up-44$NB4isAJBccM2=>hZBm1Px%9-A{ zpbkw>w=Ol^Ik`+j_(EV-5XUJtDPFE7MtWX|u7?ZJb<7#B>$8o$9%G49k3&2Yy~0KCyCva?p$P1W*p{m_bu8T z4lDVmh5NO66tRpC=(diiHaUENA&#qXQZup@qBN$sq*St{FfWGeK(m6GoNLl0y48xx z*+zB^t?k6}^%b3CH?`JF1Tu0l*;l+f)5hiNI_06)9i=XieXQ*rH5>G%CxgrVT-{lP zy)<|;;So@#dwI)d{m*dCP2r`md}&2ztT50bQZ-Y4CIItK6!xsMAjnM!J3|CbjQBD( zyb>QbneK@XGcyyyMxe#j^@z4PhOa9c^IJgFOTx!HlUXho5eVr1d_vy7l})t3dVp8f zKfQe{$5Sf$0lzP*p^-cf!x=gH7C_Y_2&oXw>gYe@o4p?@+ifOtRc1?1fSIYLee8f3 zkq*TW>n+LYfdcRpbjdNEn(?!HBRt$Ay=;`@$sIZsLA-9%zo8!CpV;Hq2S1bJ986*(8}bGt-SmQ>}9T;4jZw3hV@ zwIltaA$Vkkjan&vApL{RI}T=VdtpyjMs!j)P?JT=%eJ#ds`VIG*Bofsq=Rc{VE z#}NY=J`?3fGLlO|G|sH6%W=BHeZndeeIo5H5lYfkcFTgegZQ+>F6@HlorhUs<;ur^ z-c#X?Mn;3Z13pecTMn}}xS38WLWhvLHa-B2rqs%s)oD^vIjftV+3vfWujD?ig@|V( z3=rHL-KlKfiD}$ipyg}|o0NbSFy2+2M!MS9(`6=_8EK7FtFU2TGqYXy7lLe|k3x(g zd-l9&YceXB5=SLHDjCq|VEdn^l~xc5Q^v-A8wL8(`(CmwQay?J(;bR^ghtyR8l3ho zNlnN^H!{izPAB3OT=n&lB@u7Oy{L2XuI%j#C4n2Hf%S+Uu4%en zd5-d0z5?u}i!btOX+Mpnw;VD-){54kq>O=wyv-PzlPp%Ns8Qlux!uRVPwV3l4WEK{ zGcb_9F=!G6$j;nC#}-1f=pF#~>$f$abB!elv21O&Yg1 zW@Ak8MO0AdAmIle+V5RWyb4-C0`ezfx>tfK7j?d3d5oNw9pY+q%4!W`r*EBln_Zf4 z;&^{njz?&E)Wtl$5DeGs&cVf{yL>C%=p&l}dtcXdJbW{E6&g_WsD9WZKCQc=B0Dj> zs!DWz&b2Z2Q`75H6yx3^4;xPI9$*<&XbIZP$ENlrK8!$RJIwp!ZLIz?cGXPs045te z`vO|0C;ZwnnV_0J-Xnk1)wS%)C6fFq?YphP=a(8Kf{0@fFwBhRG~P*Xy+3N9KGQA8 zP7Y9=YU&E*_Tck3RMYr~v6ppK=FnU1s|EV$!Nmlmn@d05gXX5d)xJbjZr0?Ao$Lrs zUj=R3DInR+%bqUPNe@z*cFMZHM6=|NA$6)V{U*RvZIO7v9^mL!b^;M5j<8M*plyjM zL>S~@$F+z{#*u_avk|Ib@`N6)Ia|v1-gyO4+x@h`NN;73k$p``(TG!?OQk-T$lnf0csNIQysI`CniC)PadrA zR6CXe2Z>u&V}=>8Ue|mK5#H&YSfYSAzYQJLXYT=a>$#C`A-fs_{burTVK(T-PAvm% z<%ip#xO-?U$qMbrJ%Ed6HH9urg^(Pn{>0Xjl+;%L)brA}n-kbZ^g*(NM|+9su6^QA zJwiF!^X1$Lt>PrIPj_sGqLcG3&9<6>yk@W2U(e7pn`{OdRGO`t{&a9@`2M{?zAdA! z#_{P>^ozc-;5ZuGB#@;epQZGp+j0Iu_LeP5 z#7%S$TXe^6Sa(}Y-1NC@*bA8>!*g$H6TiLdTVO++6Jc$t6B6WCI?K$0XqY$9OTx^# zv?Y}1%lLvxbo|XaUa476Sg6$FxNH7JJvg*5=Fh|?F^rDQK90<~-+vwR*f{bwtxE3k zP)XSMY=rXu86kF=cx$D}#Hk+#Mfdp|rMx?v`&34sj=oiouWJ3h!8ZK9CA6jrs<9;G ztJ6*jN{T`sWH2_>Tc*uWu)y=6`(>X!>3gaJa@t91#mn51Fg+l%(9oSvjQf+KOdW1n z=@^V&JQ8ktMr&8xYQjE!7CsPnX+4d`>2Wm7dM3WcgVyn4W`6U_aXUzefA?dqHlk_T zr>_eBuIhTB+}cjWW>1qabI-*U!^RgGT=7Jm0N*tzawg$x2vP6i8ry=K=aZ=nW&n?{0I@P=jneKdML8r~a0vKS<{NjTMP1u5YWD2YHbJEG%EfC=^5~{>q z!V57$zNl!6f;@3vJhn(OoWxybt>>(?K9DGBQ#)^O@VRkQ2wWs$n$*=i9HfSJgQuCq zxMkJYCJvA|yQQd_m;DALx?{r!!;Rc=S3ZO!Cnr2k2myf<`r;)n5odd-s&VdA6Cl%* z=vJbjpyYZ#O;%~6FS#-C8>f~J^x%Z5*9@CVIF=X&H-=W%07r?V_0VjUIONB^mTQ|$ z1MNk{GL)q?nm#o9#+^sl984_YBILqa=mcM|ODE1n`sw?7VT10CPa&q8o9T!r^c1LW zw$sgJ<6vV$fK9Qena6QeB3_unl;vttIB}r?Hz?AD{V;29biq&Dbi?hX-Uwb>%K=f4oe?3&3ARqIK!J+*Q-UY5l0_cpB?RmK^8tKEaV|pD zgKuSKsyhyB2@6O)I1p)6kzB|Nk-q2N9o64}P0N{WiE3`1+6E!s#ct$pPslz^y$}@` zrJ# zF-b?BizW^2gezxyg>CNvE)?ym7wT&SAg3K-vS*{NCOQa=DdLz@^=;vvF&ohM`-hz# z5Z)+@S0ITDsamTMOD{dK z90KL~=QS5wt-(dDf;m=UsWFx`b#BSJNE(rOpO&GWW4H_6h@A^@-|)b3WQNg7KNT}`Dhs*dY&|pm zR3FQz-fo%{2l%?_RibpRuO}ViKV@%kU)+mYDac5pPA&yakWzJ)>ooc(~6)|PmLhK&=&t?)qr(jZo|m>y={X0wcHD1JVV=ssQXcoZahMx6n=kUIPmyz z$kAXwNnmM&AwMB%HB0($xt0_Y;&2*Wv%bEY##GL#%!ynvRmrw&xeLmPQ?oRu2MR5E zW0<*S)1{cCgT|28$e3P}?q0R-E_-_L_zfdDNL5Z8+Si#TjMVl)I*4IWUD=jmSBkYY z9{Aj#aAkMQJ@e~^T;G}7!3rWht3-K5zh$~5?oReKMAqWMv#@@reI~0)dvY=~2R;hp z@VtPEwAk(Qzky6AgW3<6wqz1RW@nhC%fxr1UI*_Al@2qmzehIoda5$+K`09O~a%CX+*Yxw%$f z--aSkQ;qyVa;MftT&}KPr7Ced+wWd^KT_aottk40WaqZk9;tA;fq8mnSJQ{6#hlMo ziVybVJWEQ6g`c};LpCtHO~mKL5cQRmlrtlMVqy;y`&wq)lVVb9?UM zEG^7(K5=(8LEnA0K&O#Y+Ip^&V(MlW7V1TeD1#eJ5hExdjgr;n<%QCBG6{DqTc`=~ z@xCNpkpqcjUa6881P6`A#Qc=45xCZI`Lo}|_BWj3nY&#?_XK+_ULY+TDd?DmR^f5l zc1msxxx@XEhho~ivem)U*o{^!)^$TBj$sfnDf5{l+S<&OwVZj}f5AWGP4vwZLX75v zSiN_*=X4M~MXS)HXiXv8AxpTUW8S<9_Q3a~c}4>%^qKXmfQ<9$<%ShyqQUumbM3q& zv?!=W@ADEL)FVSjVnit$bcwN159=(4-BVgee3p8BCMZC-n1L|e+hY#P4gfqzmTc{l z_Pg0HjkfF-tf}crMG=ymz*|AaP(9B?hY{Sb$P$-`WYH_nhEFkBLu+dt<}?H4JxE05ygT5rbrL!JnZNND-GHut8C+J9&x_cha!;F^6@ zxOju5Cc_da*40`&h`n7O95+R^LKsh8&Fm{P*$T4zrvz}g^XXHdnzFZhg+2Pm57bz~ zy zv~gKaLM_t(1Qffa9c^I&o|9?iy6dLtqYqd6n>=;!^hF4BdV6VN2D|6^i+VN&B=I!U z3&sIKto4?`v5Z@(_mU;lnPsLM9>QPkor>Arl+wGl2e>{d4BuY=IJa1%;bz=sZoOPQ z>x4%#dZ0SnoU`RwGL?=V4vtJJh3YGkrHFzXGa&)Qs3zuAlU`0DikZG?%(4z!PCB@$ zsIH;daQ9M1@*7K8^9YLfwHfA&O@eTvMulB~K*7Z=wZ`P>zQ)@rN%*4u1@k@x=d){T z=P;*zwm{YnmZD-QkD)#e+pCs6RuQu9von00$nSK|zAboXl?k|qX2x~|o9Ymibj|K4 zg$}*-9KwlfOEB@-HXB$J>TFm*Y!gj1fTsab^^9{HiZxsFlh(eJ2}Z4jA?Yg{#DuCA zkK*8`A3l4CFTn@cMx8vT(Q^l1IhH2>J{}{Y&zBi^9kLGiQ}(&7YgAZB2Tt;p7G|LU zk`MhQw||1#x0Ae)?Z+y8r)oGCA34wki5=VCGB1c`O{{I)!ItBG?VBeo3pd-m{rupQ zMf8r;!kT()d_9Bz(7dlT-!8sF%a_C2(3i`bnZX_E&IxPV@Q95=zF4JoIK>*MP#fgb z?luyVBm*P&r+r?|$Cc5zMEw-n@+yhXG$`f3EG*4)+EjDRF4D%wHzPVMK8yxB@vhZK zP3l?7Wv5qVvL|^&Ktv&zw&Ivd1Cl_rQ;(&Am;r`bu)}0 z_J=B`WUeT&vT^5|Zfb!02fHDdEJ)9*PYPOtaw}c}F3oVB#Ygo`!<+QHs8(=u%LTLG zJ0Cw*J!~L<`=H4x*Q%>019ZwIk2q$~o4z<-kFn;{*2_$%m(4Vja_41M5_P>dxihwe z(}?%P45{pMa2CxWZCsqqPgmuG#1_oLWVTzC=&77WXf+Kn5P2UtU0IzMDzV{##R#{N=N=!5ZCUUAQkF1cZZx7DjCJ zLpMR_0<*t!SVCfl`{jX@mwoS>#t&JGKgLp?E(|ij1^IM$l7N zo{y8DC)Uq}m_B0aCJA?25Eq}Vm#eq%LSyNgeOcv;Tjf!2!qqHUB^iMmp5*)`;qD(0 zXe_$c`P4(1OJ%vU4*i!s)7WdiG`LN8AXm10QH|SDnakkO%$gc}4Q4L9ymnP5jLu=m z?g0iaqrPn+GJ&6mYU)^4tH!ALu_QBk)8UOD%5{LHSA4SeGbFF6#`$_);Fw;qMKuGKNpf6$g3zvR6)y5u}i-MSIfKc1*cgQuwmbW_94sp6fk<|VAZRo zy{az04r#)mDVA@3jw4$J2X%=aht!|KIb7#b=In;%k{|y*xzg4^$7lzh zLTx!0jJ9QmjwNT_I`?nIZ~XmCo2sIN^KuGdP^zW_g(cv2&&K7lxO^kI<^}B{@3m`O zgG=q?N&b4h?s-RAOR=OvpMxrRlBfs`Y?({4H~(C^gM>D60}EPpL&Jjet6X-{hl%Qz zR;alAw1xV*t_Nen(2SXaOJ0YJ{2R#|I~q^z!{>%R`TLB_i(G4Cs!h45Y-#lw)*ySA zH^Wn)zoHGquJCGH*1*$gU0S5FU@pw0HAslqn^+p)m6U6NMmIiz)(OtXd}hrT6Y6~w zCG!b$QKk1Afb;H5yy42WYZ}DAUe>dL1ffJmf=}a}ZWU9OJm};RB zhBxD;H&pRS`o^LnLa8_uc9AwJU@khEgG8EYzyI|>O-V}%2VPf9&)Qz5RT`>ta(OX^KxP399$D!y*fR8Z) zC9{1#Henf9M6t%69Q-6eEe_nN@vSobTYC^%b7|VH`F&1eX_Pp*fFAW>{rg~Rl2)HrL2+RHp}60Q)evJM3&@NNMp$)uxXL_ZMh-%qJit z3KDnvD)djgdg<3$64Q!b5A`;uVfj3wVn1h*i?GHqKW3q9n7w+;;Hvz-44N=Ow2idd zw&6w2Ct@7P5VN_z>t7Fiuzb5~+o$PsHf211Ah5+m7yso#8v{N1Fa4< zhhddEilq9|5{ibx`PGS@f)x4RiPg^;?d_GGcVuB__-MV+CJk-%hi$fg3a%) zrxm?EDU#1Q?JL|H%8|dY4xT~!6gF2BIO10m(3aGI*l}|!K0-`v0Sf+YVzYe@fSwg; zdPLobCExQx`>FGFR{L9viZDkqm*QXa;v(TdK0ZI2bwu5<;0D8~%$Vm}K}6=7Yolte z>PJ4V?~k{MM;QxuPfW}8-)z?|)xl^YOj=ZE!dh`TszT+QX$Jk(#kpMzr_*%X_xR9a zNVF~D!P`la|HMmDYx^8WPEc*hq>GKm4eKjzIod8$PPw?aDk&-HD_t!0emyB&cE!6| z+jsWJ*_d3ptRDjkIHD~oIXjyDamc~M$=2nITcCJi9!UgYdL;smX+LalBIYM=>;W>6 zsxnBgpNmak`tG+Wlz7xDSfmz4JPetfDHXwGkMk0$yhcydK=1$Ve)!Yo%KDGfoqiuw zR*5pHZ$kQ4-w6Y?WvlVNrK44stJe$1H?oQf>`@P#g8Vv?&ff%K>gEM0QMUKro-i5z z{tM!1WASUX(PH)Oww)KHk(siGA|gj3Qm|Vp?t&ww*2A8{hx7X#X1; z_S8=O@FRNu$4m#28y&gEHe?lIWnNs4Pg@(bO1OnJmfwO#D=CTHt)m8zgovL-^0CKT z5{p`6+|CZ{cx-5~fAQPm!n;GARF{T0l`>DRFobt0CyjrOBii-lUt*lToV+;w`o`x$ z>lD|Z6XH{L{gLAENBC~CN_u`QzfT?B^EnsiW&{HPfudao`N5Nu#Z5kS&!Pf%P-52` z=%CNVk77;r{dX_=!%-02vd~D!i6;`C=4cYg;SMMN58mE9s>yS0`=;Ag_jaPlq>Njv zARy4lJf>BKR)ds=5M-=Shy)08n4zszCV{QY2_#$41QG}$1V}=l3W74l5CSq!hA@SB zNFa&tZJ+mfw|njNt#7UOThEvO?zO_bLhhC8IAEc`}X>r5LSpLyR(9nfGd!j$8)dLYU5fKlLNiHiWh+YQr=|5h7EU# z>Fl%4IDPdd(Z|X{F+ksCy}v1vwm32@E zafsAFR%ivxlsNCjshY3$XYq@0sCD)af??=QdHI`WL&47dvAdOg4Pj~WH)rhsRxMGI z?n?@{k|hSHDqhMb7v%q9c_~+H;lHk5hKhbdG}AwuFLoCFrJLh>=Y}owt*jb7{I!xD zX1ltfElGVaFKlF1e&xrSf9mwW-gkAs?t4wm%nV|TtdBS|(2E2liN28F0Fs+6E#yhI%S}CL-z6#;86`vRN8ItT6bhbBHv^CT$?{$>Z@4-C zR|oYwVtt=maq9QhehE$S47``VS~}$ zQUuW$n()F0a~c~j->GVPO(pC?l^V;eOXU<@0%JR)D)M&8m|*d0 zCU+My84c?jj@9S-^bE8|YwyMasUtdYnn;6C%$*IvVFui1jbYdh3F{vD*07PJv6as3 z?%VD5`mQ_ui~znFI#pNf$)h82zNe$rrhfhw(Yd1X%M1Ey{=-hW)lNBq} z7w*;d4t>WBVRB~Gqy#QJbg!@ZYY)|y!+g{*rZc36w;<}$yx5?;3HqpPsBCSqykiZ8 z{dK%ogGZ#}gUnkY(qmh5OP_s^30v~50ivYGp!?zHMsA8k7pB`s;XzD5ebI5)caREJ z(_R_3r4-uS6jpcD9(F-;taXwX=HA1fvZJQVBVJXo-1RX*loUVNvF>}H--w#9jfWU0 zp`yE5C%0-ER%2^eAX-*K-O*Z;5&R=9Tifr`j>D^?h#OZYuI#xxuQ1jPEoPPm3CX|^r`+zv5wqwuhlQN^1$Fbhl!~cpJ!}vRk=L&&V2Oz_JS_i8vkD!>kT=cD=6#-_#XkH{~S1Nc|%j zbt*LLN^GbnI7g*lVQPrN(#0wWgS!dIL!yLuB>#N|Z~PdMykh0}>#uQcBc?V!;M0B4 zv^4Eqvs$3GbL+HQpQ2?k%=LLBkBt(qb0`(8Qss5syfs8S&Ua>Z{4OC*CgvojK6uhn zr91`=SrW!hR&K*V3akad5kRavf}{6 zyY5;!NweKd_DMQ<+6#INA6To+Tk{rEeM>gds;W`5gX7u2W~=rMhuV)VeE5(!8|=Pm zwO<26CJ|#O^vXgXW_i3IVrO5E3M*5m>*r{nzah}jn$kC6qPdyQuY%Eq8MgzP5&x-SyA+iQNBKWdSr$1CpNzi{YK z0(-f2Bl?Zh=y&b$6A#(1q?~q@zAe{94lZf;eeTlAXG(JjIg`_htNqs!*Q{IXpH=Zt z{igQbuNk%lo|X9fB;t-eM5yJ8yhP%;I>^=ZbTQD`3eMyCPeCeuyY?65K@evc#NH{i zPo$|7*VdcOd>NOTshl|JOWe3eX$G4cHs34p_E@TNI^KK zU~b$om}o{%nuo!iW8iWyQ>e|Hf3o};X{q%~sdc2z@AUE-W_`;Qy^gFo@&U@BhvGk- zciCCqHox~^Xscwqk$krm{?_8n^T@dS_XfBy$Y|dy5+!!-JydSEB6)LMUY?hk8P#yP zV$JVqw2dBuxbX13wd;MBc35YGp_9?kgxEa$oweU*{ph!15<2I)h8cA9-bJN--|QfA z5vd|Cr97yl6ss5|z_d2H;2U1p7JYu_cD?e~&He%!FFi!_*M`#D%N>3>!Q{!MJO~Kw z|IyuEQUZiZFKkX61Vjt~M*3H!g~R}dTB9g0sbW`FK%t2@DagoGQ!&ysYRe&FHrre_ zwal4>;`D>atoT*+_><1@A!R##-75mV4B|g>?*AWPnSjocYS-k~6O;M|iPirk)_&(E zNAyaguFIXzUN4#4m3}FcqcbDiH_v5LecUN)wHjHAfSVjSK$Vx*I9sw3@5kC40UW$Z zG2znGk;cQhmE|Hst7)Wp+go9Kw+~h#V7H47ec9sJ(e|lHRlvhR6{z{7{1`dRaAoe5 z1`ZO}f1ASRx$c;aAwGSNvE~!0uXdcqey;CnSQ%E?_zuQ!l%~jri&ShBoqm3jWa{iT z7cva)*!*37=`TFR8G~2G^1DC;8CooZA$=Q}cw%!ZBI>Yr?ZN1wrXH#84jeTyid+wa zgBZz$Xf7lzq*lGbO_$T3UR+4zy<@%nsGo-vOf{hYv;f*H{i&ay8S0w7)O`zUy0e|A zQ_JZwwx&56mcb~w@hiC{(t`vqD@(YOp~>tfIbC?b922l>mn7akUK+T8+Qr=7hnl05 z3$+@-d<`dt1bL!i)%$hcz}PXK%;HlN8Wt<$}*Ac?tIWTM$7vO4HQ>e z=Pm!*9%T?|CGm;n42sJxW_CL21?OJ5aJTMe*?|mS95FYQ|DyaZmSM~&TpjS~xY?xFlu!;jQ4{1GOeJHkSjAOmMsFYEMXrrey*&6 zsv0vnV(2^(eh#)U(fWWnt?(Cl`F+nN0Ow|)u zE=!?1#n0eY4?Fial>x`6f4u&KX>1bg+2n3YiDpf~xQrf~sRMya;`Fy`c|&mL*gKw2 zUuW`7xYd(9u%>RN1XbNZITt#mum}?{^PY0={u-xMLt3Ordbu!k6+ye2L@V8ZLZUX% zmgm|c?|0u8k|V8FqMH<(vW-jdsdveAtG6pD1Qor6K zRWY|W51*$(^Gcz`^mIo}O@GY^wsFfI)umUo6-^YLg$3fxib#9W_{SO-@Y>&!bQIX})k zWKmSS%#V^DwH@QLVD3w$lB)YdUKymx0i4H%r>0ZfGREi~?#|4B#6S7`?l<v(9eIYWSI^AB9;aIAfp%1^Emqmr`Vh9KEHte0tV^I zA>N*aYTUtCHw&10;`*%+xqvV1oou=q1QG>1?z~=C&`1#CQRz%~<=r>)HF;x565tXJ z^7l^wKu8ybRvGiREdIubYGxfn6BY*Q>Iw^q9ESrj)thRJA`xdxD-hd6AfFkhz2_v; zcn}UB&bGQANU}SgC>EvC;&y#s4B5xKlcanrDEZ;PPzuP7sZfD=k&zJ8B9i#BU zS=z2GInX!d7u*A4p3P~yeA&XKz?GNDE^jH^C%3&yR|!~SlL->|INz6&8k}dG%gniE zCMDwYtKM^mnT3greH|7I0AsBbQ>YPp0kTM}+qV6k?NoC`Uvqh1t+qr0B7SeD91gz5 z1)0|BYquu%sqXyj!LZ-L7`lQ?SWD?uSNiO`9t~$=5FY%H^;71Bx#vUGk0l1M6&!2t zp(HW(+sod(9(tOot6S>NrM~nN$Y!d>=N{e6s1T_vH>$phZ*$Oi2(07MCpCJMu3Hc6 zv>2}a@ty~?%wOGRrN{LmG7NE{>#6UU*qp2C130tcX@%@=ZGG=eB*(%{3r7>bSzWz9_GG) zfhclpuYx(7LeTpsKwM(+i(zcG%>q+3riCq3e7DsR+hU&H zE}6u>V-&Ky4`~|rUq562-W%$fn-!3}#cFeK+`k&v=7^`F7hZJIOgJ?^Pq2EIS)t?g zatcuJ=OfXzo8$LrkrcC?+hb)>BN3*87cIMGCtA)nWTS@F#+IFJgUR^z3IskJ|K?(4TI}61Yd7iXWLJjSI;5TZ&8T}jA%b(bPHR6%o5c6?I$YM< z7Jx4n0EU#LZv2!4rNab zR0~Rp#F+0PyRbD&+@Fq41AZKgl>WT8Nk6SRPd^Dau#QLRk9ykqwD!8M?Vo-c>uPgA z1%9>#LY-pWZHVb+!tgOUe}*UPvWtE5^T|g8Uwia=AKcVOE$D_8S!S(oZjTAJxpJe5=a$-X z-%ZEb)SG*cRJur>J)RW4`s!Sc@&&P*6Ftbr1n5}*AR}|E;XmcGfSdmj(3Km*i~YD8 zw?4=?dB4L0*2L)7fBCB6ob8}$chP|;&S>c(%jd)=jIdf8qY2kZ4Oxo?J3vk*clmmI zhlprq&o;wg=-q=QDk>}46y~;-k(V3zEa^h1|7tD1LZD^{ML7LYpYH4`K|Pg&a5wup zS^PlaXD0aBHj~w|zg5=Ruap#{pmv@H6Isi7YzVbfc`Uo(WphYd>IwP9^;2>VV#s#i zD{C5}OGeJ2fHj3vyZL6q=4$usloF7FJ}T&Sn`n119*w@jvspl}xD~+}+Yl}v_P#kg zQQ}3+JS0cJKFBm9MR48RWD#^yTQ@_frFH&{=h+i-_d8rsRu`rRx7(s0kiYJp?1WjZ zxGhbpJzSFsqhLo!JwRhOdk3vGsy3n zL6kIS@WeOuNdHS*gTS9A`Cu;A6t21M^^}kWP*E;BJovdEJeQLyKf^%lsJa{J5itB) z6+w8!S6z$cws#-ykVoD3+mAFWT{H_?a4$ugLVSJC(hyAv$uZ86qRGf*%`J{laj$FR#G|B<7Ag#mSRZ%>t0 zr`4*r5-9xe?KtZgNU8$kxp{}Pkx)L}%kMjkRpIT$ckX(bxll!DPzHYF%p<)CpV9UV z%KM!@=DDKVm4Ew<>JRRKI5CQ+^z)q8^m(g>svcu>Lm1lBb|k7bfrg^MJOOA!I0O=} zO`$3z!(a;@3voMp6Y`|UN=-)siHME8{4zM*q7d-oh0Xl5#RDR@KJBY7Z{J-@iNI*bSxKhx zmMn_NtGspQuIBTMJ1bU+iSgUbTiQA-x*!ygET;~2W-`Ox+w=Umnu<}M*+Tzk*?^aI zzV*|LpS~EB+T&?AaQYKv<;a+eO`f}*KK7^f<9OfXzo$Z8flOf_IrWAs5(kSqk{{c` zhq3zrNpF7Ecv+ks%_`QbuJ`Uj=2DzxLS9Oigm?VZb$j}^?P z?#90B=^;%inx?Vm*tT9KTU$x*GVwNjtir$A*j5ZTqp)rc3foQ>`kU9`nGD!Uek}~r zxBvQ(p&Ez8o`_qr?r>t51P0%7``NW8Nk0RhVlP&N4(QAjztp5cKW5JQjj(ZgRBym? z1?p876udPpbMAf|%((lPph&9C4a=egVPf&6tR)FlHs$q!4zf8mHIH#sa@Y{XD+_tY zg|O^gQIa0rygmP7Z&*pTxASuCE~INI(fUZnR;vL+Lh_I6sBtP{Gcsqh%Hwxt{-@}l1-i)B8xJJ8`1&0Upf>abAl_}m@ll`?rVlGs@E$G*} zgV8zheP=f&$P(gg7Mv-`Ya@f=f+PoPE|6aKC`#k&5^Hx|AczIhnDt7G-1IUEh~TRt z7HP0|L(PG`MIj&8E3b}bE|yt8gCUSE3H?L1FXvm)6Hw>4Et5C%ouNZ)ld@1$o!tO4 z8n%RIDeLC-pAzTEo_!Dqf}2FGE$lDO2~w?UPQ$#%%k{|D1IAfvneZ=_w6cNaN&}K6 zdR5m!+-nWj28-uLW;us`k0iP1-lgg2&Gr|S3mxwg58@`o;Tv~E&-N5ewwWN8(O-)t zMs`92uJmHwM>s#0LW%v`>j@pCGS!k}zM{4p7q3t{( zNX-0oXUCNNFj-@>p&`}xkAp;&#rt7SQ{!uR#oC>?g%Qb}4H<`FeYNH^(P(XQW&RG; z?#2orr*AiSikx(okjaJhPNV4$U`1`iFFPUiT4KFZPL{jV~)@Ahi8Tu#plGP}-w#km3qG87htOy?$Uj#WKg=`pZ06`!pnVrYpvV^u8{^)wbZEJ=WSm6rYFkvp84F!dDt! zueTY66r0x0k^!Bfalr0WH6}7LcngXGNW7Pi9bfewyxK-3xfDCY@3ZA4iQ>=F53?aHE)*I|Q?CTV_rd{BE;*QB}FgN(qpO z-1_E7+r3{T1PqyUsiKrbA{GEU(QA{A$GCxJs)gc{Kp3Jrzs{RM@-59v0l|E{(y-MI z#+s@$yJNCDJtkq9%`q4Qj=g`dM^j>B_a(kF$v@4{d>U55Xh@aB=@0@JSDR0rg{YGi zKFHiJ$&0&GX}9RR_xjBQBRsvxFskcCC)xdV2Ciq_heVFC7n^WU)p?s26UsER%fH-1 zoQ-=b5l~R&akAQyYFGJozTgOO*9@7Ev4m`ii^3==Q)qd}?ux(ZM=`AL77vrAC-)Q^Vd@7qx zooH=69`5Q;hMS)tXBRCu+oEda?2Z$Xw5|PBO2qslSsiesc5KP}_qf!&B;uo0N_g?S zz#mD_(k&TTaWXeKq{hAd@@ZrB)MC&CwfGr&cqPj}JeXb^54B+=l^;Ugq$p}u+V%U5 z4TJ&38cH6$e+3_wS)87j_i}BBbEsbwOk`L`g}eyn8f$816^)ZYA!e9)u^Yz@v>_As zkGCHykWXKbPsi~6Tj1&cZLrVj$LzfB;a<_ryio&aAi@!>>1D*T0{={P{@YlZMI~3* zGjU+Mt=iW9Qbz8HBxk%x%Upvjr&eJcF8Qi0WC<0%od**q0*8Ay5jiNYLDt0QFnUVn zAmHPtI~KBdm$rZZ*Ditl5#(r{_Hcdls`Cxi+McZX)G#&`y1F7>ppdzpUoHQeVe zC(&`69g(;DnQvNZp+4xf@r*iIWxSOEdhZns4Rb@IcLR!<9)$@w025yiHJg8AX}Tg6 z73X&IMz&U)F31_zLSJ$kJ}y9YEB4_i3O zM18cOaR9i6E+L=;$WEl`pL39r?H(CRa`Bs1O+OJ5mSVMlep=<7?H@Nou_H`2#!D+=Qn#haI)AG(S|cbbvZVV?2DA zm;iMEH+{5*0oqOHRZ}_Bn-2%@+>fp?n3cWVKc0AB-~ZP1MkzeP0^S6wnd2BkkSevt zKD`emBm*c&Awg%_+3)1FoaGcl`qkw^$kd$rp+etw7O4L6tqh?h+up?}*QeLo`T@h` zc&+X_&JSIcYJ`evHHA8;=}AnwwM8ZSrl0>Tb^0#Vg6B)x9nf8~Al038E%#B7-XZv( zkFz?160lo^(5mI6h6u#UIrJ`Lueu$ZO6GE$qNsrE1T_XAP6_eWft$H+3}-?&xm@{4 zvY`w0T{2(seN!!0{>eZC@#;&Nxu>Kgz`Xp&C^$Dr+S1Ox0n`GvY)IT9876s+6bSN*mtfy#1ER)f^BY7TaU z9d3WX>KtvuoGz^Tfc5pOhu6$PXk30TUN0yCglTWWr6z&)7<{dQNIP$-K`)a_K#U%PVcDG z;4kubV)t8emy-kcez`WBZO>8&$#cX2uA2G_C$1=AFh=F@|;RC-(|GsJG)+c)V z4B2bE-P_j4JO*$*@79lxXeX zT;K1Sk>GICeD~63Lo+_G!i~0YbUO~7>4ROCb;-XSBy%AUcHKNcaUix#<&9oxINrW? zH!#jn?oLbNNDuk&{!m}k2Ir=&2tQ)d9)cF&P!I26p1aqnuTN2GIF8Ny}f#={j-T11+~0DiIf!ibx6 zw*636Q{0}MJP|&9vRDMP(8yB#0^Pdp2TRh-DBD<juCVq?%dSNI?tE1^mB?xc9cQve6bgKBA~Xne?U?9((8!0XJ{lrVg+GUlHXFA zSSBxnTF7x<7j9JO4kl<@#^PFjITm{w*x{1?gF)A2@}G8dIrZufGOr|lU*LCQ`!l6^ z`!>I5-2mlJo^MSEuPcs{KJwmuOTQMsHSK4}lYB*6=Dd`=QfJR>=LYbLQ;@BJ@90_{ z)U+pNgM?i0s%6H z*)_r@ga%hB)CIO>zd36@E&&-s@ai$q6{Rr}JG(lGb|{xUuR{NgHdq%ajfVhn@LH z=+Fn5N$xW1!FkQ^mDeP68pqSzVi9@Rz1d+S;;MsWIw4(_y8d+liMEs1TziM=b+ z=1cl5f_G@tu;q66eOK6U_p z+1zhX>8Q@~7$wfB%d_k6YJcJGIw#$=zUDF9u+B#WE}bav03E0en0) zcd{Z3ZsKCUM6>LedfYUw&M49pMxZWBj1>%vxszM-;Pu{m)ajbb?(Lz zFE07)GsD51f)xL8>~GuT)W-Oi-(sbjX+z|#UV~pejjTRa&VP9sqqw=Bv*nIheOkFX zv#3A6uKK=)HG0G8t?iTT*Si68w5Agq)w}$c$LyBA7=3YSvsc>Ddte!(V3ZRNL#k;^ z31hrgM&iPkE1_o^|HG9DyV%r>u`D z3q^j;P?`ntTJrUJx{Ux^2xG05_A=CQe?8RVfu$oM?&p4A$^KIe5~)D~OHz^)gx?US z7O4t1r$*u0B`Z*DDa55n>rV3k*e&(m3#eaAfOgd%?@A)0pOeS#jaBnzO9-9zQaaZ1 ztB|~pN+397;ljoCI#HU5?$;;I+5eApb_&<-3_0}s9i8+&{yYo{##JST1F)B~2f2p! zT!mNUA^1SDTu)WOUP1v=tn08LUw5-c(eeil4_Vj%dFJySFSGE zhCv_z9;}-9D8aWRGc)I>ARS0T>5@7l4G=n(-Br+i0~ibQdMkUB^@zWJS+50Lu$^}b zl#R+8w|`=74Bn4TY?%Kmc<^%wuOM+#gpAmbFZ}q>Z2%a-cD7%-iy(gLzq>W<7Z240 z#hf<&eA->7{`60Z0@FNtcOSwLrGiZ_zF2Kq%wN0t^tDz!ds*qj{8V>VCP z>3c1%_tzb;XJoQ#+$}$C%M=e!evsh@QMYnk28TrY{vXeKzK1=G_n+-`(Wm&f#(QQW za>3Thj1#y&AsXg1vBg=Tb};)%EIW#0!%bVTIzs^l{{DnFC7RyiahR`>_g6pFi}vh% zkId!LX}8y(G;BP0|1u@6oQCt87_$wHX}{0e-a`Cv0>tMF{=He#ac=S7jfrvw)eBT$ z%PrlBW_{dpZT~5!i0h%`!%@ETQ+pJi_k!v5#k?#`aE8S7wEt4d#n$BYv_gvXb+p6N zyW2^rHSspQ(yjTr#NGJUM*=Hs1*a#?sZ2YrUx+V^gqXw3E;4h>wLqq7-ZXN-EvCa8 zoVO^7qyuOnv@46T2)qihTRXm*AE(+dI@_;(9Ib6KG!EG(ckJ1^(TNH z`R}j)`PE6sL!$zAJp8^3%W3=Tj8;;+T7 zUw574q6=nmJpZ>=KXN3p5Za;6f|&;SmXOK~cU?iIEMOd20wnos5tC=pvB-!`BmlkZ z#>Gz0neO47jqRKXC7pC|2N_RF{mugump?CCA>b}!rEjKO9@u3!!%5+XMs=5{V>uhb zt;;T3%ah=*E<0QIM5eRR%e&4WWCFMLWo6?Q20NOo*2=N9!DpLM`>Jsb5vW@g zP#qp_WL^MsXuprK)+bB8Q3)zFguy^gw=jV9;}afb;s3oi|>q08)5iiFu{D| z&8Uq&)3|i3{Iyv^|KOhc6xC#QsmOoYw9#gZU*?M^~cae|(Ip)bve-A0&j?K*9g$j$)1MNtRHCoHE z^Ch0TxzRl~4R(LcyWDqeXjFA1d{Z`L1|_L0-E%Cj@>&@u1r)mVx#(6@No!p}ps$HO z))tfE3ss&o3ln|(oO>`!6>f}6U|g%&YINoXRO6sZI>eEbI$6nWwxQkEa4Ksjt8-K% zCo?)I^a|as0bUPY_BZT9IhWa3J7siP5zj*wsGXGU@H`-~bAH6EssCJGpt#I$6w}Vy zTEx3ls2o4?dr6hK^V;WW3cJ#SLxe2(Nk|+WQ%;US2L!}^edLjuM8mzxR654qs|etf$lbF7Q>`(HX-E1eQg+<`aZK7YqzI=hNN)x02A zc}*_Cf5bw(_teqC>17zV%V08(x;K0g>g0*pj@Bb26{+W@9`K?M5}>oBr1PXhm(V3I zqtK-!7kjGNMogJke34eNv9m&FG=P?HxsT{IzLG64)XSjWRMA<1tk_PII)TX=Jddo+ zUB?1@XV6k9$H%%7&)ogxni3s%STP-8<}zQ}eq}JvNER5gcI0QNC3f53($>q@$uIN5 zA5;k%#mMzoa1k)h?!Pf3M59mmBQc==^1aD_>U+y`@;=P0O4W~BxS_IgnAm3dpADha zKN~`DiI$|s9B`ykg+R9KXyVljBU<`#22iJBai18zpazM$DH~X$&EEEJBdfo2 z$Cf|3F5}QHyHqMm48M8oL?VJM2%Uhs%A6l!ltPcOZC-LO#cfnCJb4z zRPZYN`ti4?OMTR+z8B`_H_(WPkUsonzFTQXva*1z?Q>n(ejzjF< zLTVs2YS@y5cO#Kom$@e1RCHzqT`%t>Fpoxl z3V1`cu`y%^;?iaLbnaNg%Ded8w$4< z7YJJBj=vQhR?sD2N#EGs8r1lMj6RF$R0OW9n2@Gz;UCA8=5yqeU70QX8rPgu!?`;q zu9fdF$!daVY&*_yBnr$;_e0e5PlUqR5Qy0BQrm?8yleqCoIuN%v|^<*JAZK*s{Hq?es3Vi~T^ofaiw=^0s^{tgGKOcvxr8L0C;&!_swjg z8Bh50d{>RcT5m#&s5Yacx0}^jU!IA^HeCthpt&)uKsOiw@tn`R4>DQAUlvFM!y;_$Qwi|%QWj&M-I{Mug+tt20v32J}Nm?mfM&wjPBH;jUh5-<&NeCPwRRMEpZvG?J)d8-T zbtxw(p~7-l^Pi^&+R2;O+$Q%gZ+E_pS+0E*t#LAYH}(!s`cyd$`b`<9a{XAng?>kk zPu*nLdt`#*Ku~qQJ=*&XNlP1(%=GlvD%svD1!nS|E}|HRdIv#qZn7KAM|XEgo%G zH|b(-zOfTN{4V9~WPa*YT~mB!9+Ff8)coJQA?fZn!X2PdLw%yAU3NA{V2N~(ZE;g{XxoAc z(7>yoR;zEvj=hdFnk)14auB8x_tocj??fCXt*^fRW0V2^k-N5AfgvhukgA%EC3NX< zh7ehjE`UMpBS_`vw^5UE$Si}-K9+q{kDVdnZwE)F*Ia8O@46Ev_AN;{7hGt=CFEj6 zEK|K4KU#?g3lbF8T^k(j9_RZHMO92LFA{9nijZsYQ@Qoo6w>8^R&~>5UQDFO&iFDF zwWf!YmG#j|5Sj3Eb5r-JO%b}>IPLtWBQBn0eJf|THR35elMgZr?LUgI?BmrR7%N?BEZnzTsa!N%h(3|ElRF7hTkLjT}z`<8nbu1I7ShdI#>a;t#x*U{5oa zjfq5Jo^_g*fAS$^Y9L|#dFo$tr&8r5ZerN$&v$Rs{?CqAcvk)|`oN)u)p6lrmq_1u z???8IY_QFit%r~j_M2EQC1K;&nSqY(p@`=uV!9UNoIAN8s#HoO>4 zWYB}hFk3^jCsmq;hpKsnPZp+LGB&s<8X?H;Z@d9J^EW=YQl<*EKgqo=+Xg&H;ZHG9 znAozDiB=;)cL-W#*)dLGzI$4F8G_YJp3O4vx;`)_hKu4Uy86hQaGhOy`~h|Tg!N|+ zDpi9Fp#Wc&Yh=s61)Hc;hGjq8jXh`>QP-?{DsE@EQ~VeX*>29T3DPA(3BZTi%1AqL z?d^VLZau@?OjbS)T^y~7T7qA=a5Q_T-xTp4I&RN_bcc};O~?&Mr+xJP#B={m_YE{b z@MdBE;Ovz?`d)A*@l=o{0+(1x07}2wmnl!JmG3h`g5ju348l@Vm>NA|zn752{s-D|F$%&Ov;#4~+B;9&Ibn5YvN z{pcVsnoT3Rm zKY!t#V9oF{ukqayrD{^G0U*-L+$@-f9vDd05B1-R4m^S3tDj!D&nGYNfoWb2JiFTg zL&U8=Sm7~8x(%^gU*r*5-xeh${Zqp-X8IqOQ%w0#MDm{XQ;qSumpa2wZ++uZK=u+E zQAQ_PFOI47XDzg}SI%)@&{@o4vjSYYYpR{adTW%-Zn1DJY&4o)2@ES6t0DkPxb0fp zCH&rDO=@5XOn7LV-Z_Fw)Q*cu!ezXvkId;^ozBGs_zK_SQ!8?l9%Hp6IcdObIgNg3 zfRPeYN1GJ9-{{>8gB$d*a3mC5Zo|vh#3^gSj2+)lPc?W*iLRIml!ImObX(Q0bH-5u zu!1n4S02Rsxfc}qTPFvdKqJ!Am>$vpX9;JiZ}zH+m^E4QH!p2uCr70F(jKp$(&Ax) zsvm@o>0;`iFXnyWzOvzhxL>-@=4D(Rc4ibz67(x#Gk<{BOms|O;lUVWZagmHj5@xf zf4hDA!m*$*=NoX^aF!lfRfAMJUMihrHCEWE5Y8XMV%TbW#P5>Fn|;PAvLlU+*=o=8 zPL#eJ_*FT~e}od51SjYd`eEaPF^6S9d;sS&y#HH5T2L-)r!^}Yi_EqxULDVM8)v{w z#I9Gk9RW4|D(Qm1;* z4F8d5`aveBs%X6QRO!Du)S@J>A1d9)tsKeW#ZcoTnp!|^A2v>?lGIgIlXA5EeT$nx zFgef=fYN))pZEc^rvm^U996p zm3#4VORs3&;}zoZl4h9eECDdKi)D{&`}j4;t1W$+ya%<1B6sho=3gHPUWw*3ga$gf zC%Cjy^j0OOa;Q<+!&OXV3*}=;OtO)epD6$%b#dx+ru>Fynp14&XuA`{*|<&Tz4Q+U zck7E5^A$HI=Z|gmMEX=sqM|L6 z7^b4QHp}%eIW9rmy{BE0g)GT^Lo5>}>C*dq>rLjK4Z$Hh!M1=O6ET_sfDLFtErX_| zc@W5!%l_bzmYuN0(Uz7MEs>@v2(JoH8P4kCn!v7|I6f~0I-q214Du#j!84fI&MmDuRbt?R}!U;`7t zcs>VWVEn2_L$?2vnJLNOR)f7)vx8g@&_4$t!|yZFAA0y3-P97mkxQk;kDqyXMtFq8 zBNM7Ka^8Ld6mYlnPD~U&pl&Vb^wCHEv}9TyItg!H;1i`nS!Na;*T#Yss&^Xz8=axSz&|=3{wE1xVZu8 zWoCMe=#e**HqUmnujLTG?ydF9S{Hs=g^!LvtYF}EjM_eksGPT*k09pc=hRIZHEV5_*^#=WG%la4{E>SSrN}95{Knd#C zUlU9T+h?A%w6s0$2WU!t^MpSfpP%|;9$!rC-<8P+#+CNuiRUdLsQ}+PqimNhD1`%J zq^%L7J2A!N(Xc0p5bD_SO+(^x?7`TX@4j$8yj%DI?(-0d$?l0}Qbj#g7lxEX>h*Rp z)8ViHYO|f59@2A;k#%*VO8=TRJVvOW7^5yvdoL2y!oE{2Ut>Qro45W|%ewKnIaq&? zO*cBFAX#rDogWNH)c3bE zqMo+K0nKJGW4;WHmADNy?jE7X!co+T(6GXoy;zE!!yrh}^+tL)cNx~%d~s&0(v?wz z+}l#82J=A3a4xtc-^-+{Yo{JC#1G$y$?1k>CJml7Hj#!?*P|;XzQt3s95Gv4_0Vj9 zwtKu)Zz|;awuDZLB^;o+DbA^34Bel*6nL5`wSRd>eVbLyuuM&jvPz}wI7;>SQAIx)Xa+CSSA<=2<<5XbW!!Bn>6ZQjr}` zhpW#t5X=?m!!l**)C;RzsE^~m{_3)A4fIOyvO$HBETU)i#;ay#wY61hFVr;HjiT&N zf*9u_2Kzjx5P)7~FFD&>A@pEf=iuk=H(~)-w`KjdAE2QGQ?+5vY^=R{5B^qd9pf>X z{>YoNfBp4rrJ6k@Um4XTPd@~+?q#{tz+B5{!4Lx^k?_o5fEFBm?X$LVk@xb(wMy%! zT)71LN_XpCUx>FyJvIYHLj=JAYO=5{fV}fJK0#f6V9d>BrR|!E$dTbQ4)!@@UTRED zgT)L;L)fvTd)^4#_)X>Vudr+n32AG?E)P7BXAZTPt`8a^N;}CcfiY0`FzD7u&#Bvc zqun#(IGsDl#aiFNcG|6oyyEzgIa07IIj4Z^&QZFhd92eTtuXl2c=iDi-a!zEBw&O> zJEvBK>XDr_f=Ew5O3uq&YvC$-1evqLE7U^$of}5)l{$An@1n{E0wm6wB?6Ej`6m_d z3;3ND<|`~2p%uCQa&+*-=%{xN$JcK=d-7YXCB}X!mKPVqtKRI}dyS6h7WL?DXq#!6 zxL3#E;)v~V5T8)G-0INb{c2Ao%?@VS>e!w6Av_F*&MT?wQ^TEi+`wY`@3};0d9tZE z8gT$XF}H@fHgUI=%ToN-*P_FeF!Oh#&OW#vP*92F200*180kW+Rp@jcLA8>7@#!Rf z&s%|b{C;s{>Ya1inXT>+WrNLG8dB-FUmz0SEK*Jr_Z97%Ykq`->3a_gj5H3p;qf)$ zaR#=XOAyty(E`7Fd$F2fi#*rvVxkwUo0g+iORA4>u#7@cq9b4mPkw1=x^lp(ixT}B zGDyoHiqcdx^cH3_(9!Yt+&6;z(JD{KSupTEn@do6(d`UwlFVhoIc$zOh&$?=H+}rG z)QA9KUQ&v8iGiTl(xwSdw3AlU3QRqlyZK~=!iiw+0>pIu2)f-sN~7)j)3A&1Buq){dR9;VhDnvp3sRwZoVb9(;aF~^xX6O;e5*L zKzpEK1PW8i)JsUoHZxT7Nz7}IXwPl3x~yKKg^7C*f*aKkL8s za!H~sFy3Q0!*uBiv{5!H0pvZmIg)&1JgyHosy|+%gl^w?Bj!=b{PZNzqHfwB&wM4%Yn=Xy^V-IQU9DyJ`0ppfjI1i}?1P7j7o9#lryO z@YjKff1w{(;Ieey^SVI&b9~kxgq)!oQmc(U>y3Oz7L{I_S|5$hj-@NaVtNGznDkKA zXv>bne3)IwM3OL%ZA_0SkeQ<)4D>*dZpZf1yl3kC+CBQF{nDTugq(4D!|Tvm8x*0! zlr;%&ay57!Y#x+uEXv0$Yf|r3OgPw?;_~2e^HrXkGpk%w*)n{GSgY{m&kezR9Xn8k z!Ph6rxq&PRS@oU<4MBP?&tk+RyMT@VVaUnOB&txC?JJP#BhrRR2YME_(I{;1tfAh5 z*8sqe?#nAMEKAWIrrN>)J%sC)fR2Jq-jtH8t~-4z%XX`~Jo{qp#i7;|<)WApmFLG_ z@nWgL3-QM_F~JqCZAAn=r;nDf;f^0Cs5HZ2<{7paAg>t*pqasP-Y!0rh9HhXzo{jr ztHeEaf}r$4ls_prf5|QFD>5dH;6jQG79iwH1&YqZ+Olo+?C%9W*L|l(nVod_@Q$no zEkXe6Cd~u#NUUa%4zWiweYa9R@+YCCd+TfcSyErtJ;Zg6<#sTl^GN{t0cOqS0`|Dc z)V#QAr|>;**F&Cr^~~HIrJCp|KV!N!ba`e93w7U%G5=U>L7DmaJ3Q6T-zic!xfBDu zy~NfTyW)LKi80`w=#_WGr$MIuYFj;RJlPwrk$_$#f8LoEkg3 z|C;!VmVx{(4S@+b2MV1XCVtUDR{D-}>B6fjG&lA(3 zITNW~LVM8qY{Y|Vx^l@&r82nkcBhnp;N|l9Oq}#&uxLD8cql*Iuc9KMsyzBD5lQmu zM%FE31e-OSZr}XLS`*e?Rj;VQHkZAx#y-MQp+-dZ>g8Iu8-AZR=_3U-o)#rBlau`l zulwE;Oec?ZLnS?r(l+EV~#=1$T z_i$jd6ntglmOLX{IaSg^PlauIeA&W|P|UaBGhG|}YD0vP#^lLV2dK#TMU_)X8b!Y# zlRw@=tv0c$@4L9Z7abS=Rm8ocQLx9;sRf8@vu6;Z?Mn%a)SNeBE1E*YyxlE4vSX3; zQGH(Vl`7xXa&xe_eybi=l6YIg^vd=wfE3giIr!^za88dCDPXdD3*ZJVJ(QuzPegAk z%d`FX;Qt61%%&6K|98Of@4~y=xXm!zQnqzl+)Q!d2#wb@m%Sz6R7*@Q6||(&678Ep zEN(Th$!b5&+tc^bW|A}7wW4g{BPVx5W&AFe6?yNi1WuodP$b^*cHle-nwk)}5kv)j zb19|uRu0`7$=DON;Dyzh)*g=9ou@7X!)r9(AeYKRXr>hiDKbUjMjiN+e#juf%$rJP zi4a>~_OIKqA5PBGBH^vQPxl*QHqoJ!)-Z&Ue6;rqJ$~{+7ADuHAH7#7H#xTs^_-)x z+HRdJ8cY${XEBC|1V|{?LiV*aHS%P{u>(R%WHzg(CZ&Sml$z(<+mB9KgLw|ab{?IV z(?`IiUJ}A`yaXg&*zfL>ncmq_c=A#0 zQ=XDy!q9y%O54g4(DbZLFd90m@m*@{Kr;lDmtt}8{j|-# zw%4mhstbLgPnl*iCL>J=6bDwjot{Ds9CF!9)vx`8BgX)GEVs|?H=cLlH|fzi1?XV& zb2ny8hI4KT-I_U1=M6kx(b0goJVTU+a{!nxbCX2bjw{tE`8j8z?vyUc&yf1eJ7{;0 zfmslKswE49pA_X2S@7?1VL7uV_x~1;o`8k-s%3J^ClIqUh;|Cjw}mQ%pM+}_}%+o{M1lsjnjr6-zF00 z^Qdz~YZi9=z$sUh1VD^63!kmSPxlK2(w0%KS=bD3UBx~R?Kxc+x}`hyt=3Hd!MhNE z#Cu)4FYd#(d6W(5XlvsWr9hf#pX4pZ_!1GOjW2{2Y~<>ePQpjDn8F<>T|_V__6hg$ z?$Bzhu>prOx}^^84{1hyL_CsGbb%C3h*6FqvLUleo|62Qu9jZCAit|VuI+8FN_1S` zCQVWw*aAe{Z&g3QkJml}cLuAf8r&-*&B{5|-;T8HdphF2Z^q`+E!3J52nDm_#*N6y_VH6svUAgYC;&} zXOn*1Uz&pvzcU=8`8In#ayi7-HafbWRRd{x+00pRNWGQpC&#Q4*wG9pQv9*{mJwff zS?GAwM~b+hG1I0%e)+Y-=_F!Wx@f-&wMS1}s1@G)F`$>& zA(%@dQ_7aD^rS#qfm{s&9eSxjBbh+n0+v|S_kieAzqkj{n4gu2zPK_EG-u2^Af4t3 z0%z4cGF{lIE442Wn=AY>q zhnJ7=`Yug{#MAJ%)QNuORCHIhpt( zol~vTy*_ob)R?f6DgR2x^?)U8z?b*dtis$&3bN;bhXKStYK6AZ{DV(5%T7!reh{G~ zyN(JA65-oyUGm81G5*{m(>&G*%hmKy@Ii`TXINQvBu$4>!;Mm+`319VlrA~LtX%`3M3&4Hr^4~`gHQP*yC zkDT4Doyb>J)w(`HU?vyeEeMr{wi=+Fi_~m-o+_BOW&Ao@*N||^!2F8wKwnu+Fb{(A zCn$fmXaZ;*MRA@QYt{zz8_B?BK_M_l|J`uu7SGOl#mnC9wwUyqgb##5Kn{m;sCkic zf#h2@gMD3nyYq>e0g`HGIGY~5!vb%aiv4OXA9FdlDDk0PbnKWDrQA*+MRdUST=T~w60Ni&NV zk)5gNFn~wBK0q~@l$s6XPa9?yfEJtnB%#9N3D+9<{DWa4&d+W(LQT^86K3RMJixUZ zi_b=Ci}T#i@a;D{DlNhmPRf4QSwHAu7xk1AQ<*LRlZtx|jcr;*kJ^yr08CI^Y07L8 z-sqUXf%kX`Eh?#7i-0;+9(5R2bWWztMU7~;6SMR2gSttpfiJrd4}p{wL%*fIEtTDQ ziZ4vOXfb-jNkI zw^wzy>TdeU6y@A?OPe1)V12xu`n1lo_KA`GGFO{F;gU??N$D9~(n05?M$9gA-B*`w zngJ?JpEWCKcanOEL0&wMW|glhXz$4t>V22MIic|)6%x9aLvK>D0zE_jEF$o6AO zO1|fuXMN<=#eKQP-c?rTK*?sY=|N3e{>)xOG>}iBHh*y(E{{sIYN7)jo^iz%SKLS@ zp6|ZW7zrsHdF#JEBX()UJUDbdD_Z;^O2A2YtMQ!w$-AYOxxghHBMd(rzHPbNOTO^o zYF0Szew_Q#m#ta~o{$5_8~G0skIz*NcN7?Op=q55AC=@k+XsxTxWj^Hb*AsAwfT_F z5X~0ISiq(19J?^dZ(Rw!_@IUh>rdN;&eS~vO+*Aa&Yp&1dNy45yicjU@ZHL(U)ytS zBC{UsjSd0mFocsTn9vpAh+(+L@vjE@vfZ9{jqAZ4N!vf|TC?1Mmc(;L>G;;vthA!E z0FbzHpSAg}vVdGmPMM)3r-#Bp3Yk)h&e7jxQGp=X+}zEkCmg^lc~A*nb7vxfCveu# zpB7vPFd3?z@2)nzxGZkEK1P2jQp_i|`{s*ahfHb91fs*)pfOs&;#}@x-d*~>qLNf+ z0?#0_1)7TUFH($B!uQ*aD9Q%SP|{&`h?}!Cz)*Vy2*I2YTd=M8>51`)K}NMDn!qC| z|AVt*M~i)W%hKldivl1hZaS1seBwM<7S9GkWW}Z;?Z0w$UE4U9R(1PNyVZCJ!nu$S z2PlDU${QW^ZY`bI zSgQ^~ta3g+%K)001vC(8pH5Cr%Eg6hOd7x7{?rPWL&G~2$9~=!jO)42(kbyupi7?S zD(22^GIZaFDOK9IzFrSv5JuTs$Fh=3$ER$Z77gL~!NU}Y94h(VGm{LERQcxyFlkGU zW56d9gS=C_=E7DuH)Kb1gLVDdf0_%KmU3cynH2igc^H=kGmrqWNRlI;?=r4 z5w~Jw(gkgqx$RXRtXenDMVF=T+;Ho-DMRA3y^@NZAeRdkZ%+=i@+p0c)~aA8DZC6& z9|RyuHa9C@eEe(Y?BczshG8Xb__}ndYiRYMH#BaWq)eB^reAIUO=W-8-oXMYH5MI)cc zF>TVic5^_(SIfapat2T$d~zXDCdS(*Anw{QwbrmK3tPQ8jYrYA9Jo`{PZOw=;C`fe z=E!a)ASdD=$R{ZKJ@A+}0{S(r_TW~I0E;65D5=U%xSe@+bVT7aJTmi++*agM^vL|l ze1KR&!gK{A2@rU36w|`7-g^i|T%(G1Tz&GIPC2@6ePiWpfS?PS7rv3TBcg1_1u5~3 zu}&_%?{`J-?EM#7>b4ljXmR>Ey+;RKKtcTdG3HNKP|+2Mh!?)`@@o@`_yg9BzgYP_ zDSk-B7A>vTj18W;@eeQ|JEA3z%ClmDPrrCigP-{E22Lg8uaW?$h_Z5=r2cYsIk~emHM?;nc9J}m zyMSZ2K_ddaAaj-LEYC>KEebv<=j1l=iE-txu*!uBRG&|2QtGDP=vW&iu?#)3$$=>S z7>!BJ6JFfEpvi_Hv1o6-rtxW7@GXH9-?^CE!Nt&^}-zmn0@Awdd_UMD6MUw z<0|vkxs6AbLUzk4;}=b5Nve6=HNb(FQW&DXaII}@i3IbK_J(M}miID#`v0yp^*-#S zXx`>j)U98aV)Mg!5q9TnG8TimUO-&OD0pj$M2{~11|^X9-% z39WZtubrhgzTh*sX`xOve4MrH<>xY$EZA(JEUn$YahQTJ4}mUXg=g3Yr>A2^i+SoF za}RF+un;Spo;lnRdIBz^oyB?HVQ88X+ujf05N7XB%2|{YvfMa(kCY__j88Ed6a1EN z;rnCjoK@u5?>ACBro4=w3r>UBQ)&eFri3SW8vP8o4Q!Yzh&im!j2ENcv zl-C?o@MwWae|duJoFAT4;+H)4HMTdUKn^AVOnN1Cx+ZCP%Oaap*>jnzQBau_AE9$ zf{Th~BV2KD(_s1fu82)^#cfZiJ2m(5sdTyKSlzBr6Xc?WI0BFhn8Df4PW_|j@)_%I z4@UI+v+f#+H?O_B@EWt(*YskfkZ$pPm4~5K{szhfZspKfy`OWPaTj2;@lrLrW>?9~ zuN)!E@xom42|*lWDBO|5dZOy3QnIih;3?ITq!>-=5cIClWl*yg)t=iE?I zMlASQ<>ja0r%tXXPpBQe0M{B<7sW$2Ns$b>WgqVffM*A_~{(45LnX%CP^&^(+Wsf!v>exlDqp9M5FoZw2EXbnjsO95P{Fq>$sn0Us~mx zyv96j2Zt*iH+6K5cO4v2sPC&lEfr_qk1A?lhHzfsZ`*{S_oiVws-dxop%|w$MC^Rd zrM(BNf$gU@7zQJG)s3oqL+lmQL6%BC6T`YB$T1yn#+2N#WVakD=Y zwnV-jXs2{zeppHc(cTj}DFeFDTQU|-h5E?P7GOHOa1MN2T>T@9cYc7Mbya@;e1sH0 zN_SajF+Z)T*=a2Z8!lP;+&W{!9~a`?u%0=qoyO=g7e6k0V}xDGUL2ztxOb(lMFd;v zbv|%_>L5Muz2tu}*31|2gIdWB55G=LV|aFjbgM}@)pX}~XUMaOHlJqQtd4c#Gd_N5 zz7lCi{tj=`aX-0@u!JtaZF=XpJy>1dc_e*SvqT4`4`+h!%TW5g1V%TCRP&P4xBuZ5 z?fBak{rUGT%I%GReV_?~b~IC8K5IZV`O?Ceu5@q-3dm9c*zRpdK(?uC=SBTb&j(%v z$f$EX5?qR9H7|9C)<5GIcWk>WgiR$00ri^nUye;?>)gG!czf2=8W0RMxJlKS;(HA- z@+T3*vG5Bi(Qm}81XGVDVY0WX(vniw=ZQvIjN8O);hC-y6#2O4@jHZP8DQRuChd2t zh~H_`PqZ?@QHfiZE%lqOj+-g&>@PH3#1o|5yd}(FGQKv&JehgR$zAf2fTUi1VB+aH4le@%-1?Um8dQuE|egrapBvYb3zN_CPAg*iGN(P5sy+hg^Q1-$mh2y#IdyPT!9 z%3`WC2YCejRBe}v2m_#`kP0w+;oXN|eDB-fjNHP6BHE^EFTJGg-lnHpgD;?D!lUKr zxR(;io%PV-Vz>5AY7({_0{iLZNNR0aPw6@<(oNCGzZ(3q)j80Ld4p`{ecKjePnWC& zi<#OVDPRA7k8kKXnUnD2?=z@4JM`|cg52#e`Q$Qed8@$vJsmoQ9sWTJha+= z^pHbY?hl*U*aqpxb$TdzoP$TlC4W8j7GDf?-7!;}A=!v*IY^Mg&4LN_lm#xJEe*Z> z*=C<`#LUcvF=FQF+`%!BcWvlgR9=FxsaJ*`LST=O?*g&tr{YKbnPo^LZYrC^! z$qcu7(NUb{V2p1|_;lqSjQHJX0~aB30q`K2dLt`UJKHhYlbweT)<|(P#!mg%y4*7MpKhMZO`Imgf0& z@9NY21ylo2VhDkwMD=SNfPO<OQKrN^e(~RmRg<+L1llt4gVzc&S)B?ulGluLXRfW@SCk7U zr1?>zg z<>{4f^$5GT`c+{^^eQqoVTO`n)o^18mbc`kvPie6|Ncv2>ho(W4WYjoOA8D+Tj)H3 zzcUrS9OW9&F2$r2u9ye>Lwor~>~99M_&*KiXA?3DMLUh*JfX40HGm4!Z>Q9-G*{B!YJX`UHhL80_aSt0CW%NoQQGn-S&Od}=lc=&JQk?m>8Z z$NrjI<5rW}5|2fvW(;CL-_WFC780TL=Ed9Tw34u{nS=7dMJhF3bcw=RjUF)aCm(d6 zuzg)st@P>v&FTt0{VgcVv09vZmZE5Ez%T;6VpAjJBuJh&XpN^+riN5^Sr>#H?g z*ZSn<3aqs`rdz#>8xr|#(poFY#1hJIXFMmzE+xAOlVga>joTyj`4!uFi($OVEV3R1 zfPCW|9h7c=d_SEctJ8dSCy@PcW378VK#!%jQ>m&k)fJAho|zgI&b$}JEB68w)0qllG*zUu#Rc# zf{236IWkLTevWG!r+|8J6;tKnBy|?7`2fAa`23An{Ngn7X>>M1(kitWL5lVsV=l0X zau&7;*!&vYJSK)qZ&w_GFJSBQ~90 zbHm+}lT(6yxz^gH#hORj@m|LMb}#aZ*Fsrbvr`jIx>KHR#>9>0`* zcf#Va`iiz=SagD5cEvEzHkypXz*&ERHsz%!d~i?wIEq^kp+4?PP&WSnrzV?t4jnw9 znNOO}>wPbavLr7(8{Yzv@){W4O*_44PWR3-TwbNv8CA_JY}Hnx^HRLPr@7KF#c%FB z94a!|J`ZHVtg90phj*lHnuxkG!ES_vk3`}1jA?9gp?79V`_0@2LPJ@PAHl@JvOs5O zgr!}g?cn)Q?sN(sY5pv83FRI}p4ripwvl?e2$SCn9K*g5^PZe90yS9N=$oLK)gG1s z64qD8d87p0#M6v<->W@D-m7+M(c0h+GQ>wF=;mCNQkwr7QD;#s>i5Rq+@*Z81&PTk$9V92NCGd zU``BkNG;0|0kA7MLUF3f2pb8W%nRZtOp_7qSCz`z99a^T#5v9N)Q#7Z3V33&w(x!E z-n+)=)_*ZYW1qZxQaLsLRo=6@jd!q*srse=p&Rj|Zh7zi3|T@FlWc@NQ|-dBOvRR-~nv>8W$1_3%kwqxY$IBW5~= zso^OY{oQ3R=Cjf&ecO?eMyd&clIY)=<9El)qR`uB9hNsAlNk2!>!Re2$GNn^pgBNn zZHHEO65sWt69T5=V2wh-eRBXFz1`URAVI#z1$fGc*X#gRy+R;|K zUn2hXeYvUyeyG_E4{A)Vz#}134{opz@~uzC-pj%)TSlGEb*_6xJDIjO7agO86PM{}VrM~fz3$t%=pSbpoFIq!bi6=fP47QjR7=VT@+&KQohwb4?W zk&m(pvh?(m_a_MlS>pTfH zZ>7Jwm*8F>4K}BET;e}@rt?YTC8jAfZr`T@I|S|oRkwIi0_+=RAL%i9AODA2ISbaB z6>rt~c%-DSZ$z5M@uJkkc!Zb@WJT{dfB+Pw7F)k-C%XKwd+V#sMx;!62tA{i$V-WK zf5i#rWCW9*tysmomSJxfUDh!fC?|iIij(dziJAO4v2hEd0}QE2xKM?e+kk4t8cc8A z;H0S#ENqyrtnhj|i8Ckcte?K-L8#pFCt0}7C2HWXjhRo!_BBVFwHSsGe&V2ZxX04Z zUqX9Mc}qSVE{V^4Bc@t3S9EdLe706~f~fEJ>1g#xFb0pcxGZqmAPR3Uud zr=6BWL6Ks7!^M08($`*4a@}qCP6lbmzfUnzz2&8x8kHBRTF`d^J2ylh)>E3{N&`Au z@@K9#-K|N%--;=^M*Wd7fu4UBAyZGNv}rRT4zcM~9Q==R0+~{7C2B}|cf#zU&qTP0 z4e@{v=L6?R^0w4RrcPmTlw6W=5z!KgV{R9-^)I~+)f_DU&8hXC%5fdu???%w~U)b%NT1$Q$@i zdiTx!^GQHeT^uiZxQ*QYdtK+lu!%^QH@5{>hv3Sq3IUbY5Z-XWoSDf>Ms|d40 z!CAKw(0VIXrTOG}CJ5?E%YvHVV0u95b{ze2MEL;qUWxWlnzAI=L+8+RVXOsi05`uR z*3lOhECoV6Id(N8D%aEwBOS=lkqCT;02b1zr zv?e_%q{a^fHuoldUQZ`1@Wc1uv5ody2VZuDaZ>T&o~xem!o?p0PZ~0vYOVXY28FnR z)I5y6cD`S`MIWg$HOoS-Y{l=Pg;X6Qf^3F&U6gZ2^iv;Qvm9zI@qPhztK z+&9H%vfjxQbMnG&&tj6>c&P!tUV#BaPz0ne$VC=3vE_5RU%{E6&Ed~)NAE~K3vg|J z?4+6cz~LdoNNb;W!k+6p&}raIk90+)y*-^4emL%+kiY?xlGiTJPwbVrT0^u8qxnwP zw=~1pTOP%qqu+>)lo@kc$F;nlPR{n*2fL&v2w~blzQ;!V)%^JK)Bc~;3XmofG5wLh zCueNljcNT4Up$X6Ut0@Lj9aS&7@7#hKijo11@d2y4J~e_SuC9C!)SKg_pj%fJ%Ij* zqh1h>FR%V|T>*6=HLMkoR!}yG5Fp4T0j9jGivTlJRvn6e(Cb^rcw)+O?WGbTd zG-riW+S@YzcpC^-B_U=zJgs9PX`O2BK&;AUog-$Bu;ZIJO+UKU$joy6`_^N*==nBl zdH0gMea&#F@&4;P*I7^Y+=e+SD({xk(Wsr%toho`YweZ3kF)HbG95(NpX~xds!dfc z&z4kPXZ%#7(U?`FC}Ox$RA<+=4RyN?NlHgtXRDqoD3=l1_^jTYD9Fp_^?DZDMNTM* zj)(wpsZKadgw335dU&Fvv?u?B)@S*{-+OwK7U>EbJ6FmMP!saGE5(dcyZeuN z4{JhCU8~x1+_JgtcTQ+z?!O}D8}Nq@{;3y#E|C?NHg@Pg@2PhEKpINfIoGt$>1(ua z-_^*NPYgEW6pTe8!m7HCK3@CcPtD=)UHs=bASb?#i@TSzj-Jtp?-6ggyS*c!J0oDc zJUCETnRqY_@r&dFzMAR#z0uGA-dKPBz5nOftUoiBDa%T?QOf7gByvW_Ks{4ld^AQ^ zBC48dgyZk2Y@1zqMFHrWt3<7lCU&IRbjL0RWC-6mj$9IqJ;#a`B@i~XNupyCl1VKv zI7!vpK7Ge!Pcnb}v9V_MijhCTmmZeK$^6DS=M_L}IE|}WjexQp`AwZox0HrQ6e-xC zHL8-m{RgOXiLmD_hZhlvYNs{|9J~KwX&3IGr%>(b7g!hOllzLKmQvIn&-=B5kW!*| zdJTHT{0EoIfB7@Z{JYowv7yU#-D|>kkZ*K)VNo=^>3-GWb_N!d>Z^n3r>Nxk880lj zEB(IcyTSo2YZd9{8p|$!`B5&pDzEnV(QIb9)sR2=4(MTb(Z^NqZWEv~n3MRv1+~r` zsW^1Lm6BH|KPnFz8Aezh%2!%AaYDSjTf5Gbq|?Jz(`%3kuS6pB@=oxjf?nInDTII>@$L);)Ip}pG@<;~_-b6~sN z7L&US9*n1WvhY9ppjzI`EC16s|8M+(KN!0|s5@L7*dxW_UsR{972uw3{{S-h;lJ@R z{_NELJyzw<&gx%&{r?Mz<$w9y{*?p#VPpR(*^+;C8H2I7=ISh)W=x-oslA0P95{C~mj{b#56KS#G~ q-Mv3pxc`&l*gyNt{xlEs;Qs>q{#983 literal 0 HcmV?d00001 diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md new file mode 100644 index 0000000000..4ccf280cd3 --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -0,0 +1,122 @@ +--- +title: How to Choose Right 5G Servers +weight: 2 + +### FIXED, DO NOT MODIFY +layout: learningpathall +--- + +## How to Choose Right 5G Servers +--- + +#### We have done extensive 5G development on Arm servers with various HW configurations and SW components: + + 1. 5G-in-one-box (software only) + + 2. 5G with Genevisio L1 Accerlator + + 3. 5G with Nvidia A100X (A100 GPU + BlueField2 NIC) + +#### We have evaluated variety of Arm server platforms from Foxconn, Gigabyte, SuperMicro and HPE + + 1. Gigabyte (Mt. Snow 1P) + + 2. WIWYNN (Mt. Jade 2P) + + 3. Foxconn (Mt. Collins 2P) + + 4. SuperMicro (R12SPD 1P) + + 5. HPE (ProLiant RL300 Gen11 (P59870-B21) 1P) + +#### Things to consider: + +1P vs 2P systems: + +For evlaution purpose, 1P system should be sufficient to run any 5G software (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would add more overhead introduced by numa system. So for simplicity, we would recommend 1P system. + +1U vs. 2U systems: + +1U would be sufficient to run 5G CN Core software while 2U is definitely required for using additional HW such as PCIe card for L1 processing, e.g. the Nvidia A100X and Genvisio. + +#### Accommodate PCIe Accerlators + +Please note due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accerlators. + +For full length PCIe cards, you need at least 2U server, however not every 2U server will support full length/full width PCIe devices. Also some PCIe full profile devices requirea little bit higher voltage to run properly, SuperMicro ARM server seems to accommondate Genevisio card better, the second Arm server we tested and it was working is FoxConn ARM server, but it is an 2P system, a bit overkill to start with. + +For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one devices, we need to make sure its NIC ports face out. At this moment, only SuperMicro ARM server has designed its server with A100X support in mind. + +#### Available ARM Servers + +1. Gigabytes/WIWYNN servers + + In genearal they are able to do what intends to do, supporting Nvidia A100X, but it was not designed for A100X for the NIC ports outfit. + + We have used Gigabyte servers to do CORE server, edge server or 5G network in one box (Altran), which is so far great + + However certain Gigabytes/WIWYNN models have issue to support the requirement of supporting NXP Genevisio PCIe card, Intel QAT and two Intel NIC X710 cards due to its limited estate space + + the GPU version G242-P3X claims to support up to 3 GPUs, but we tried, it is only one A100X can work + + the GPU version does not work for Genevisio card + +2. Foxconn Mt Collins 2P is a powerful 2P system to accommodate more than 4 PCIe slots, however + + It can't accommodate A100X for having its NIC ports in right orientation + + It has three riser cages, but the right one (from its back panel view) is useless and in the both left and middle ones, there are only middle and bottom slots work, the top one slots never work + + the bottom ones can't fit a full length PCIe card as its wiring and other stuff is in conflict with the PCIe card on the bottom slot + + When inserting A100X or Genevisio card or Intel NIC 710 card, the server will generate tons of PCIe AER warning messages (even though it claims it is corrected, but not sure what extent of its impact) + + It never be able to detect Intel NIC 710 4-port card + + 2P server might not be necessary in our use case here. Additionally it introduces the overhead of numa system. + +3. SuperMicro server + + SMC initially has PCIe power issues with A100X and GeneVisio cards. After put down extra power cables from SuperMicro (both the GPU power cable and the Y-shape cable to boost the 12V to its PCIe riser cards, both GeneVisio and A100X can be functional. + + It can detect Intel NIC 710 4-port, make it only Arm server being able to do that! + + Also it is tricky to play with SuperMicro to host various PCIe card + + It uses OpenBMC as its BMC, we have trouble to login into its BMC UI, however its motherboard has Password printed and use that password we can use ipmitool or its UI to operate/access the server + +4. HPE server + + Its BMC needs to enable IPMI i/f from its UI, however ipmi SOL is "not supported", but you can get serial console by... ssh-ing into the iLO and then using "vsp" (virtual serial port). There's also another one for physical, and both seem to work, but "vsp" is recommended by HP. + + We had issue when boot with USB Ubuntu from VGA or remote access, after adding "console=tty0" to the kernel boot argument, we are able to access via VGA + + this server can have only two PCIe slots, so not much we can play with for 5G server. Probably ideal for 5GCN Core server. + +#### A few thoughts about Arm ServerReady + +1. current ServerReady is not strengthened enough to make sure the ODM/OEM have 100% ready for deployment in 5G space. For example, in the case of Foxconn, we were struggling to make it accommodating to support our 5G PoC requirement and it's quite fragile. + + i. We have to try all combination of PCIe cards with different PCIe slots to finalize only one combination work + + ii. the motherboard layout design was bad for accommodating full profile/full length PCIe card, especially for the slots in the bottom. + + iii. and the top slots never work, wasting two precious locations + +2. must make ServerReady pass PCIe card detection as required (not necessary to test its driver to work), but at least the PCIe slots on server should bind to PCIe standard in voltage spec. verify by "lspci -t -nn -v " + +3. must require ODM/OEM who apply for ServerReady to have a support team ready for at least one-year period after passing certification, in case the customers run into any issue with their server during evaluation. + +4. ArmServerReady should also mandate standard IPMI interface must be supported because it is important to have integration of IPMI into automation testing + +#### 5G ServerReady +We like to develop our own methodology to qualify any server from ODM/OEM to be 5G ServerReady: + + 1. Using existing PoC example to run the test to see if the server can pass (hopefully with automation) + + 2. Using Performance PoC example to test to see if the server can meet our performance goal (hopefully with automation) + + 3. Developing General Testing Scripts to check the requirement for configuration and compliance for 5G ServerReady + + 4. Incorporate Testing equipment like Keysight and Viavi for 5G ServerReady tests + diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md new file mode 100644 index 0000000000..927525cf19 --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -0,0 +1,384 @@ +--- +title: How to Setup and Configure 5G Servers +weight: 3 + +### FIXED, DO NOT MODIFY +layout: learningpathall +--- + +## How to Setup and Configure 5G Servers + +### Firmware Maintenance + +It is important to have server's BIOS firmware updated, it is important to be in the loop of any ODM/OEM updating with its firmware. We need to have close relationship with ODM/OEM's FW team to address any issue we have encountered + +It is also important to have server's BMC firmware updated, it is important to be in the loop of any ODM/OEM updating with its FW. We need to have close relationship with ODM/OEM's FW team to address any issue we have encountered + +BMC should always support IPMI standard protocol which allows automation of controlling the servers + +### How to update firmware + +#### Via Web UI + +Most of Arm servers provide web UI for managing the firmware. + +Log into BMC IP via browser, + + 1. check its current FW version by clicking its Firmware information + + 2. go to its Firmware maintenance section to click Firmware update and follow up its interface to update BIOS/SCP and BMC software + + +#### Via ipmitool command line + +The benefit of using ipmitool is for automation of controlling the servers remotely. + +To update the firmeware, take the example of Foxconn server: + +```bash + ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin raw 0x32 0x8f 0x3 0x1 + ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade MtCollinsBmc0_43_4.hpm force + ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin raw 0x32 0x8f 0x3 0x1 + ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade 0ACOD009.hpm force + ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade Mt.Collins_MB_CPLD_v06_20220607.hpm force + ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade altra_scp_signed_2.10.20220531.hpm force +``` + +### BIOS Setting + +Typically SR-IOV enablement is required for Arm server to support 5G deployment in container environment. Through BIOS options, you can enable SR-IOV. + +### PCIe Setting + +This is really depending on the server ODM, for example for SuperMicro server PCIe slots need to be configured. + +### CPU Frequency Setting + +Following script can be run on each boot to temporarily force the cores to run at max CPU frequency + +```bash +#!/bin/bash + +for ((i=0;i<`nproc`;i++)) +do + { + #echo ondemand > /sys/devices/system/cpu/cpu$i/cpufreq/scaling_governor + #cat /sys/devices/system/cpu/cpu$i/cpufreq/cpuinfo_min_freq /sys/devices/system/cpu/cpu$i/cpufreq/scaling_min_freq + cat /sys/devices/system/cpu/cpu$i/cpufreq/cpuinfo_max_freq > /sys/devices/system/cpu/cpu$i/cpufreq/scaling_min_freq + echo performance > /sys/devices/system/cpu/cpu$i/cpufreq/scaling_governor + } +done + +for ((i=0;i<=79;i++)) +do + { + cat /sys/devices/system/cpu/cpu$i/cpufreq/cpuinfo_cur_freq + } +done +``` + +#### Change frequency scaling + +There should be a way in the BIOS to disable frequency scaling. There's also a Linux option, described in the following. + +cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor + +return "ondemand" in the current setup. + +We may want to change the CPU governor from "ondemand" to "performance". + +The following instructions show how we do it on x86. + +```bash +sudo apt install cpufrequtils +echo 'GOVERNOR="performance"' | sudo tee /etc/default/cpufrequtils +sudo systemctl disable ondemand +``` + +#### Disable freq scaling + +set in the boot argument with cpufreq.off=1 + +cpufreq.off=1 is a kernel parameter in Linux that is used to disable CPU frequency scaling. CPU frequency scaling, also known as CPU throttling, is the process of adjusting the clock speed of the CPU to conserve energy and reduce heat generation. This is typically done by the operating system based on the load and performance requirements of the system. + +By setting cpufreq.off=1, you are telling the Linux kernel not to use CPU frequency scaling. This means that the CPU will run at its maximum clock speed all the time, even if it is not needed. This can have performance and power consumption implications, as well as potential stability issues, so it's generally not recommended to disable CPU frequency scaling unless you have a specific use case that requires it. + +### Linux Requirements + +#### Boot Parameters + +##### Hugepage setting +default_hugepagesz=1G hugepagesz=1G hugepages=32 + +##### isolcpus=cpuX-cpuY +The setting "isolcpus" is a kernel command line parameter that is used to isolate specific CPUs in a multi-CPU system, such that they are not used for general purpose processing. Instead, they are reserved for real-time or other special purpose tasks that require dedicated and predictable processing resources. + +The "isolcpus" setting is specified as a list of CPU numbers, separated by commas. For example, if you wanted to isolate CPUs 0 and 1, you would specify "isolcpus=0,1" on the kernel command line. + +By isolating specific CPUs, it is possible to ensure that real-time tasks or other critical processes have access to dedicated and predictable processing resources, without being impacted by general purpose processing or other system tasks. This can be useful in a variety of applications, including real-time systems, virtualization, and high-performance computing. + +##### nohz_full=cpuX-cpuY +The setting "nohz_full" is a kernel command line parameter that is used to control the behavior of the Linux kernel's tickless system feature. The tickless system feature is used to reduce the amount of time that the CPU spends executing the kernel's timer interrupt, which is known as the "tick". + +The "nohz_full" setting is used to specify a list of CPU numbers that should be designated as "full tick" CPUs. Full tick CPUs are CPUs that continue to receive the tick even when the system is idle. + +When the tickless system feature is enabled, the CPU will stop executing the tick on all but the full tick CPUs when the system is idle. This can reduce the amount of CPU time that is spent executing the tick, which can result in reduced power consumption and improved performance for certain workloads. + +##### rcu_nocbs=cpuX-cpuY +The "rcu_nocbs" setting is used to specify a list of CPU numbers that should be designated as no-callback (nocb) CPUs. Nocb CPUs are CPUs that are used exclusively for RCU read-side critical sections, and are not used for callback processing. + +By designating specific CPUs as nocb CPUs, it is possible to reduce the overhead of callback processing, as callback processing can be a significant source of overhead for the RCU subsystem. Additionally, it can also reduce lock contention, as callback processing often requires the use of locks. + +##### rcu_nocb_poll +The setting "rcu_nocb_poll" is a kernel command line parameter that is used to control the behavior of the RCU (Read-Copy Update) subsystem in the Linux kernel. The RCU subsystem is a mechanism used to safely update shared data structures in a multi-threaded environment, without the need for locks or other synchronization mechanisms. + +The "rcu_nocb_poll" setting is used to enable or disable polling for callbacks from the RCU subsystem in the no-callback (nocb) CPUs. When this setting is set to "1", polling is enabled, and when it's set to "0", polling is disabled. + +Enabling polling for callbacks in the nocb CPUs can provide a performance improvement for some workloads, as it reduces the amount of time that the RCU subsystem spends waiting for callbacks to be processed. However, this performance improvement may come at the cost of increased CPU utilization, as polling for callbacks can consume additional CPU cycles. + +It's important to carefully consider the impact of the "rcu_nocb_poll" setting before using it in a production environment, as the performance impact can vary depending on the specific workload and system configuration. Additionally, it's recommended to thoroughly test the setting in a controlled environment before deploying it in a production setting. + +##### irqaffinity=cpuX-cpuY +irqaffinity is a kernel parameter in Linux that sets the CPU affinity for IRQs (Interrupt Requests). IRQs are a mechanism used by the kernel to handle hardware interrupts, which are signals generated by devices to indicate that they need attention. + +By setting irqaffinity, you can specify which CPU or CPUs should handle a specific IRQ. This can be useful for optimizing system performance and avoiding IRQ-related bottlenecks. + +The value for irqaffinity= is a list of CPUs, separated by commas, that should handle a specific IRQ. The IRQ number can be specified either in decimal or hexadecimal format, depending on your system. + +##### nosoftlockup +The setting "nosoftlockup" is a kernel command line parameter that is used to control the behavior of the Linux kernel's software lockup detection mechanism. + +A software lockup is a situation where the kernel is unresponsive and cannot process new tasks or respond to system calls. This can occur if a task enters an infinite loop, causing the system to hang. The software lockup detection mechanism is used to detect and recover from such situations. + +The "nosoftlockup" setting disables the software lockup detection mechanism. When this setting is used, the kernel will not detect software lockups and will not attempt to recover from them. + +It's important to carefully consider the impact of the "nosoftlockup" setting before using it in a production environment. Disabling the software lockup detection mechanism may result in the system hanging or becoming unresponsive if a software lockup occurs. This can cause data loss, corruption, or other serious problems. Additionally, it can make it more difficult to diagnose and resolve software lockup issues. As a result, it's generally recommended to leave the software lockup detection mechanism enabled, unless there is a specific reason to disable it. + +##### iommu.passthrough=1 +The "iommu.passthrough=1" setting is used to enable IOMMU pass-through, which allows virtual machines to access physical devices without any virtualization of the I/O memory. + +This setting is used in virtualization environments where it is important to have high performance I/O for certain applications or devices. For example, some graphics-intensive applications require direct access to the physical GPU, which can be achieved by enabling IOMMU pass-through. However, enabling IOMMU pass-through can also make the virtualization environment less secure, as it provides direct access to physical devices, bypassing any virtualization or security features. As a result, the setting is typically used with caution, and only in specific cases where the performance benefits outweigh the security risks. (from ChatGPT) + +##### cpufreq.off=1 +The setting "cpufreq.off=1" is a kernel command line parameter that is used to disable the CPU frequency scaling feature in the Linux kernel. CPU frequency scaling, also known as CPU speed scaling, is a technique that allows the operating system to dynamically adjust the clock speed of the CPU based on the workload. This can help to conserve energy and reduce heat generation, while still providing sufficient performance for the current workload. + +The "cpufreq.off=1" setting is used to disable this feature, which can be useful in certain scenarios where it is important to have a consistent and predictable CPU performance. For example, some real-time or embedded systems may require a fixed CPU clock speed to ensure consistent and predictable performance, which can be achieved by disabling CPU frequency scaling. + +It's important to note that disabling CPU frequency scaling can have a negative impact on energy efficiency, as the CPU will run at the highest clock speed at all times, regardless of the workload. Additionally, it can also generate more heat, which can be a concern in some systems. As a result, it's important to carefully consider the impact of this setting before disabling CPU frequency scaling in a production environment. + +##### kpti=off +"kpti=off" is a kernel parameter that can be used to disable the Kernel Page Table Isolation (KPTI) feature in Linux operating systems. + +KPTI is a security feature that was introduced to address the Spectre and Meltdown vulnerabilities, which allowed attackers to access sensitive data on a system by exploiting a design flaw in modern processors. When KPTI is enabled, it isolates the kernel's page tables from user-space page tables to prevent such attacks. + +However, some older processors, particularly those with limited address space, can experience performance issues when KPTI is enabled. In such cases, the "kpti=off" parameter can be used to disable KPTI and potentially improve system performance. + +It is important to note that disabling KPTI can leave the system vulnerable to Spectre and Meltdown attacks, so it should only be done with caution and after assessing the specific risks and benefits for a particular system. + +##### modprobe.blacklist=cppc_cpufreq +modprobe.blacklist=cppc_cpufreq is a kernel parameter in Linux that blacklists the cppc_cpufreq module. A module in Linux is a piece of code that can be loaded and unloaded dynamically into the kernel to provide additional functionality. + +By blacklisting the cppc_cpufreq module, you are telling the Linux kernel not to load it at boot time. This means that the functionality provided by the module will not be available to the system. + +##### tsc=reliable +The setting "tsc=reliable" is a kernel command line parameter that is used to set the time stamp counter (TSC) as a reliable source of time for the Linux kernel. The TSC is a CPU register that increments at a fixed rate based on the clock speed of the CPU. It is commonly used as a source of time for the operating system, as it provides a high-resolution and constant-rate timer that is usable even when other system timers are unavailable or unreliable. + +The "tsc=reliable" setting is used to indicate that the TSC is a reliable source of time, and that it should be used as the primary source of time for the Linux kernel. This can be useful in some cases where the TSC is known to be reliable and accurate, as it can provide more consistent and accurate timing information for the operating system. + +It's important to note that the TSC may not be reliable or accurate on all systems, as it depends on the underlying hardware and other system configurations. For example, some multi-socket systems may have processors with different clock speeds, which can result in inconsistent TSC values. Additionally, some processors may have bugs that affect the accuracy of the TSC, or the TSC may be reset or stop counting under certain conditions. As a result, it's important to carefully consider the reliability and accuracy of the TSC before using the "tsc=reliable" setting in a production environment. + +##### clocksource=tsc +The setting "clocksource=tsc" is a kernel command line parameter that is used to set the time stamp counter (TSC) as the clock source for the Linux kernel. The TSC is a CPU register that increments at a fixed rate based on the clock speed of the CPU. It is commonly used as a source of time for the operating system, as it provides a high-resolution and constant-rate timer that is usable even when other system timers are unavailable or unreliable. + +The "clocksource=tsc" setting is used to indicate that the TSC should be used as the clock source for the Linux kernel. This can be useful in some cases where the TSC is known to be reliable and accurate, as it can provide more consistent and accurate timing information for the operating system. + +It's important to note that the TSC may not be reliable or accurate on all systems, as it depends on the underlying hardware and other system configurations. For example, some multi-socket systems may have processors with different clock speeds, which can result in inconsistent TSC values. Additionally, some processors may have bugs that affect the accuracy of the TSC, or the TSC may be reset or stop counting under certain conditions. As a result, it's important to carefully consider the reliability and accuracy of the TSC before using the "clocksource=tsc" setting in a production environment. + +##### cma=on cma=###m +cma=on is a kernel parameter in Linux that enables the Contiguous Memory Allocator (CMA) feature. CMA is a feature that allows the kernel to allocate large contiguous blocks of memory for use by device drivers. This can be useful for certain types of devices, such as graphics cards, that require large, contiguous memory allocations. + +By setting cma=on, you are telling the Linux kernel to use the CMA feature. This can improve the performance and functionality of certain types of devices that require large, contiguous memory allocations. + +cma=###m such as cma=256m is to ask the kernel to allocate 256 MB for CMA memory during boot + +##### audit=0 +audit=0 is a kernel parameter in Linux that disables the audit system. The audit system is a feature in Linux that provides a way to track system events, such as user logins, file access, and process execution. The audit system logs these events to the audit log, which can be used for security and troubleshooting purposes. + +By setting audit=0, you are telling the Linux kernel not to use the audit system. This means that the audit log will not be generated and the events that would normally be logged will not be tracked. + +It's worth noting that disabling the audit system is generally not recommended, as it reduces the visibility into system events and makes it harder to detect and diagnose security problems and other issues. In most cases, it's better to keep the audit system enabled and use appropriate audit rules to control what events are logged. + +### Low Latency Kernel + +A low latency kernel is a version of the Linux operating system that has been optimized for real-time applications. The goal of a low latency kernel is to minimize the time it takes for the operating system to respond to events and processes, so that applications can run with minimal delay or interruption. + +Low latency kernels are commonly used in applications that require real-time processing, such as audio and video production, gaming, and scientific computing. They are also used in environments where the time taken by the operating system to respond to events is critical, such as in financial trading systems and other high-frequency data processing systems. + +To achieve low latency, low latency kernels typically implement a number of changes to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. They may also make use of specialized scheduling algorithms and memory management techniques to further optimize performance. + +How to install Lowlatency kernel on Arm server +```bash +sudo apt update +sudo apt-cache policy 'linux-image-5.*-lowlatency' +sudo apt install linux-headers-5.15.0-46-lowlatency linux-image-5.15.0-46-lowlatency -y +``` +then run this to verify the new low-latency kernel: + +awk -F\' '$1=="menuentry " {print i++ " : " $2} $1=="submenu " {print i++ " : " $2; j=0} $1=="\tmenuentry " {print "\t", j++ " : " $2}' /boot/grub/grub.cfg + +it will show: + + 0 : Ubuntu, with Linux 5.15.0-46-lowlatency + 1 : Ubuntu, with Linux 5.15.0-46-lowlatency (recovery mode) + +need to disable upgrade OS kernel: +```bash +sudo vi /etc/apt/apt.conf.d/20auto-upgrades, change 1 to 0: +APT::Periodic::Update-Package-Lists "0"; +APT::Periodic::Unattended-Upgrade "0"; +``` + +### RT Kernel + +An RT kernel is designed to provide a guaranteed minimum response time for certain system events and processes, even under heavy load conditions. This is achieved through a number of optimizations to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. Additionally, RT kernels may make use of specialized scheduling algorithms and memory management techniques to further optimize performance. + +At this moment, RT kernel is not ready from apt repository, we have to rebuild the kernel to enable RT for Arm + +### SR-IOV + +#### Supported SR-IOV NICs + + Intel® Ethernet 800 Series (E810) + + Intel® Ethernet 700 Series (XL710, X710, XXV710) + + Intel® Ethernet 500 Series (82599, X520, X540, X550) + + Mellanox ConnectX-4® + + Mellanox Connectx-4® Lx EN Adapter + + Mellanox ConnectX-5® + + Mellanox ConnectX-5® Ex + + Mellanox ConnectX-6® + + Mellanox ConnectX-6® Dx + + Mellanox BlueField-2® + +#### Creating SR-IOV Resources + + Creating SR-IOV virtual functions (VFs) + + Generation SR-IOV configuration maps for kubernetes + + - VM based (K8s configuration map for corresponding worker node) + - Bare-Metal (SR-IOV VFs and Configuration map) + + +```bash +insertModules: + - "i40e" + - "vfio-pci" +#pci details of the root device used and corresponding vf mappings +pciConfig: + - pci: "0000:00:0a.0" #sriov vf pci address which is dev passthrough to VM directly + targetPfDriver: "iavf" + vfs: 0 #vf creation is not needed since the pci itself is a sriov VF + - pci: "0000:00:0b.0" #sriov vf pci address which is dev passthrough to VM directly + targetPfDriver: "iavf" + vfs: 0 #vf creation is not needed since the pci itself is a sriov VF + +resourceConfig: + resourceList: + - resourceName: intel_sriov_netdevice_upf_ngu + resourcePrefix: arm.com + selectors: + rootDevices: + - '0000:00:0a.0' + - resourceName: intel_sriov_netdevice_upf_n6 + resourcePrefix: arm.com + selectors: + rootDevices: + - '0000:00:0b.0' +``` + +|Field|Required|Description|Type/Defaults|Example/ACeepted Values| +| ----------- | ----------- | ----------- | ----------- | ----------- | +|"resourceName"| Y| Endpoint resource name. Should not contain special characters including hyphens and must be unique in the scope of the resource prefix |string |"sriov_net_A"| +|"resourcePrefix"| N| Endpoint resource prefix name override. Should not contain special characters |string Default : "arm.com" |"yourcompany.com"| +|"selectors"| N| A map of device selectors. The "deviceType" value determines the "selectors" options. |json object as string Default: null |Example: "selectors": {"vendors": ["8086"],"devices": ["154c"]}| +|"rootDevices"| N |functions from PF matches list of PF PCI addresses |string list Default: null |"rootDevices": ["0000:86:00.0"] (See follow-up sections for some advance usage of "rootDevices")| + +#### Configure Device Plugin extended selectors in virtual environments + +SR-IOV Network Device Plugin supports running in a virtualized environment. However, not all device selectors are applicable as the VFs are passthrough to the VM without any association to their respective PF, hence any device selector that relies on the association between a VF and its PF will not work and therefore the pfNames and rootDevices extended selectors will not work in a virtual deployment. The common selector pciAddress can be used to select the virtual device. + +#### Configuring SR-IOV Resources +perform the following steps on the bare-metal host for enabling SR-IOV. + +Enable SR-IOV in BIOS settings. + +Execute the following command to append iommu.passthrough=1 option. GRUB_CMLINE_LINUX in /etc/default/grub + +Run the grub-mkconfig -o /boot/grub/grub.cfg script to update boot partitions grub configuration file. + +Reboot Linux to reflect the changes. + +#### Type I - SR-IOV Configuration for VM Deployment +echo 2 > /sys/class/net//device/sriov_numvfs + +Execute the following command to check if the SR-IOV VFs interface names are displayed. + +"lshw -c network -businfo" +NOTE: Execute the following command to remove the SR-IOV from the bare-metal host systems. + +echo 0 > /sys/class/net//device/sriov_numvfs +Type II - SR-IOV Configuration for Bare-metal Deployment +Example of a SR-IOV deployment yaml file used for UPF + +### SR-IOV CNI Resource Verification +Perform the following step to verify the SR-IOV CNI resource configuration. + +Execute the following command in the master node and confirm if allocatable resources were created on the worker node used for UPF deployment. + +kubectl get node -o json | jq '.status.allocatable' + +This command displays the following sample output. + +### PTP Setting +In 5G world, the time synchronization is critical to synchronize all components in same timing cadence + +#### GrandMaster based PTP +requires a GrandMaster Qulsar with GPS capability + +Also requires a PTP enabled switch, currently Arista switch seems to be able to PTP sync'ed all slave nodes like Nvidia/Keysight. + +#### SW based PTP +setup ptp4l/phc2sys on Linux: + +Issues with synchronization of NIC port +There was a problem with A100X NIC port enP1p3s0f0np0, every time to run phc2sys, the timedatectl will be set to the bogus ToD like 2093. work around: + +changed /lib/systemd/system/phc2sys.service: + +ExecStart=/bin/sh -c "taskset -c 31 /usr/sbin/phc2sys -s CLOCK_REALTIME -c /dev/ptp$(ethtool -T enP1p3s0f0np0 | grep PTP | awk '{print $4}') -n 24 -O 0 -R 256 -u 256" + +then run: +```bash +sudo ntpdate us.pool.ntp.org +sudo hwclock -w +sudo systemctl daemon-reload +sudo systemctl restart phc2sys.service +sudo systemctl enable phc2sys.service +sudo systemctl status phc2sys.service +``` +then changed back the /lib/systemd/system/phc2sys.service + +it will start sync' up with switch. + +the A100X device somehow got messed up for its time clock, above steps are to update it with right ToD. + + diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md new file mode 100644 index 0000000000..34e7030b56 --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -0,0 +1,190 @@ +--- +title: How to Tune Arm 5G Server +weight: 4 + +### FIXED, DO NOT MODIFY +layout: learningpathall +--- + +## How to Tune Arm 5G Server + +### Kernel Tuning + +Please refer to early section of setting kernel boot arguments and the CPU scaling. + +### DPDK Tuning + + +#### System level setting: + +Core isolation from Linux scheduler, Disabling interrupts, Huge page usage to minimize TLB misses are important before running DPDK PMDs. + +Make sure that each memory channel has at least one memory DIMM inserted with 8GB memory size. + +Enable cache stashing on ampere to stash the packets coming through NIC to SLC cache. + +Cache stashing feature enable/disable on Ampere dynamically (No reboot required). slc_inst_s0 script can be found in attachment: slc_inst_s0 +```bash + ./slc_inst_s0 1 -- enable SLC installation for all root ports on Socket 0 + ./slc_inst_s0 0 -- disable SLC installation for all root ports on Socket 0 +``` + +Here is the grub settings from Ampere machine: +```bash + # cat /proc/cmdline + BOOT_IMAGE=/boot/vmlinuz-5.15.0-46-lowlatency root=UUID=c0ef447a-8367-4d45-8991-47ece2fcb425 ro iommu.passthrough=1 default_hugepagesz=1G hugepagesz=1G hugepages=20 isolcpus=1-69 irqaffinity=0 rcu_nocbs=1-69 nohz_full=1-69 kpti=off nosoftlockup +``` + +#### NIC Rx/Tx threshold setting: + +DPDK Ethernet PMDs does the packet processing in a batch of 32 packets. Tuning the NIC Rx/Tx threshold through DPDK EAL config might provide performance boost. + +Refer testpmd application https://doc.dpdk.org/guides/testpmd_app_ug/run_app.html and look for "–rxd" and "–txd" options to set the descriptor threshold. + +#### Before running the 5G stack, check whether maximum no drop rate is achieved using DPDK test pmd application. + +Refer the same link above to run test pmd on Ampere. This will ensure that the Ethernet PMD is operating at optimal performance. + +### Memory Tuning +### IO Tuning + +### Profiling/Tracing + +#### Using perf tools like : + + - "perf record & report" can be used to identify bottleneck based on events on specific core + - "perf stat" can be used to statistically measure the KPIs like IPC, Front End/Back End Stalls and L1/L2/LLC Cache misses. + - Perf script to run on Neoverse based system is attached here: +```bash +#!/bin/bash +# This script must be run for every use case captured in SoW. This script captures the perf events on all the CU and DU DPDK and worker cores for 10 sec at 100 msec interval. +if [ $# -eq 0 ]; then + >&2 echo "Usage: $0 log_file (Nomenclature: numue_numcell_l4proto_direction_pktsize)" + exit 1 +fi + +sleep_sec=5 +interval=1000 + +#Note: Get the optimial core id from CU and DU sys_config.txt files for each use case +cu_worker_core_a=1 +cu_dpdk_core_a=21 #Perf stat on this core hangs +du_dpdk_core_a=27 +du_dpdk_core_b=28 +du_dpdk_core_c=29 +du_worker_core_a=26 +cu_worker_core_b=31 + +perf stat -A --output $1_IPC_core_$cu_worker_core_a.txt -e r8,r11,r23,r24 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_IPC_core_$cu_dpdk_core_a.txt -e r8,r11,r23,r24 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_IPC_core_$du_dpdk_core_a.txt -e r8,r11,r23,r24 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_IPC_core_$du_dpdk_core_b.txt -e r8,r11,r23,r24 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_IPC_core_$du_dpdk_core_c.txt -e r8,r11,r23,r24 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_IPC_core_$du_worker_core_a.txt -e r8,r11,r23,r24 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_IPC_core_$cu_worker_core_b.txt -e r8,r11,r23,r24 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_BR_core_$cu_worker_core_a.txt -e r8,r21,r22 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_BR_core_$cu_worker_core_a.txt -e r8,r21,r22 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_BR_core_$du_dpdk_core_a.txt -e r8,r21,r22 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_BR_core_$du_dpdk_core_b.txt -e r8,r21,r22 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_BR_core_$du_dpdk_core_c.txt -e r8,r21,r22 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_BR_core_$du_worker_core_a.txt -e r8,r21,r22 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_BR_core_$cu_worker_core_b.txt -e r8,r21,r22 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_ITLB_core_$cu_worker_core_a.txt -e r8,r35,r26 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_ITLB_core_$cu_worker_core_a.txt -e r8,r35,r26 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_ITLB_core_$du_dpdk_core_a.txt -e r8,r35,r26 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_ITLB_core_$du_dpdk_core_a.txt -e r8,r35,r26 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_ITLB_core_$du_dpdk_core_a.txt -e r8,r35,r26 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_ITLB_core_$du_worker_core_a.txt -e r8,r35,r26 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_ITLB_core_$cu_worker_core_b.txt -e r8,r35,r26 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_DTLB_core_$cu_worker_core_a.txt -e r8,r34,r25 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_DTLB_core_$cu_worker_core_a.txt -e r8,r34,r25 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_DTLB_core_$du_dpdk_core_a.txt -e r8,r34,r25 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_DTLB_core_$du_dpdk_core_a.txt -e r8,r34,r25 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_DTLB_core_$du_dpdk_core_a.txt -e r8,r34,r25 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_DTLB_core_$du_worker_core_a.txt -e r8,r34,r25 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_DTLB_core_$cu_worker_core_b.txt -e r8,r34,r25 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_L1I_core_$cu_worker_core_a.txt -e r8,r14,r1 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_L1I_core_$cu_worker_core_a.txt -e r8,r14,r1 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L1I_core_$du_dpdk_core_a.txt -e r8,r14,r1 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L1I_core_$du_dpdk_core_a.txt -e r8,r14,r1 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_L1I_core_$du_dpdk_core_a.txt -e r8,r14,r1 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_L1I_core_$du_worker_core_a.txt -e r8,r14,r1 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L1I_core_$cu_worker_core_b.txt -e r8,r14,r1 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_L1D_core_$cu_worker_core_a.txt -e r8,r4,r3,r40,r41 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_L1D_core_$cu_worker_core_a.txt -e r8,r4,r3,r40,r41 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L1D_core_$du_dpdk_core_a.txt -e r8,r4,r3,r40,r41 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L1D_core_$du_dpdk_core_a.txt -e r8,r4,r3,r40,r41 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_L1D_core_$du_dpdk_core_a.txt -e r8,r4,r3,r40,r41 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_L1D_core_$du_worker_core_a.txt -e r8,r4,r3,r40,r41 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_L1D_core_$cu_worker_core_b.txt -e r8,r4,r3,r40,r41 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec + +perf stat -A --output $1_L2_core_$cu_worker_core_a.txt -e r8,r16,r17,r20 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_L2_core_$cu_worker_core_a.txt -e r8,r16,r17,r20 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_core_$du_dpdk_core_a.txt -e r8,r16,r17,r20 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_core_$du_dpdk_core_a.txt -e r8,r16,r17,r20 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_core_$du_dpdk_core_a.txt -e r8,r16,r17,r20 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_core_$du_worker_core_a.txt -e r8,r16,r17,r20 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_core_$cu_worker_core_b.txt -e r8,r16,r17,r20 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_L2_RD_WR_core_$cu_worker_core_a.txt -e r16,r17,r20,r50,r51 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_L2_RD_WR_core_$cu_worker_core_a.txt -e r16,r17,r20,r50,r51 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_RD_WR_core_$du_dpdk_core_a.txt -e r16,r17,r20,r50,r51 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_RD_WR_core_$du_dpdk_core_a.txt -e r16,r17,r20,r50,r51 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_RD_WR_core_$du_dpdk_core_a.txt -e r16,r17,r20,r50,r51 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_RD_WR_core_$du_worker_core_a.txt -e r16,r17,r20,r50,r51 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_L2_RD_WR_core_$cu_worker_core_b.txt -e r16,r17,r20,r50,r51 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_LLC_$cu_worker_core_a.txt -e r8,r36,r37,r13 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_LLC_$cu_worker_core_a.txt -e r8,r36,r37,r13 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_LLC_$du_dpdk_core_a.txt -e r8,r36,r37,r13 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_LLC_$du_dpdk_core_a.txt -e r8,r36,r37,r13 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_LLC_$du_dpdk_core_a.txt -e r8,r36,r37,r13 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_LLC_$du_worker_core_a.txt -e r8,r36,r37,r13 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_LLC_$cu_worker_core_b.txt -e r8,r36,r37,r13 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_Mem_access_$cu_worker_core_a.txt -e r8,r13,r66,r67 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_Mem_access_$cu_worker_core_a.txt -e r8,r13,r66,r67 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Mem_access_$du_dpdk_core_a.txt -e r8,r13,r66,r67 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Mem_access_$du_dpdk_core_a.txt -e r8,r13,r66,r67 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_Mem_access_$du_dpdk_core_a.txt -e r8,r13,r66,r67 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_Mem_access_$du_worker_core_a.txt -e r8,r13,r66,r67 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Mem_access_$cu_worker_core_b.txt -e r8,r13,r66,r67 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_Inst_Spec_LD_ST_$cu_worker_core_a.txt -e r8,r1b,r70,r71 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_Inst_Spec_LD_ST_$cu_worker_core_a.txt -e r8,r1b,r70,r71 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_LD_ST_$du_dpdk_core_a.txt -e r8,r1b,r70,r71 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_LD_ST_$du_dpdk_core_a.txt -e r8,r1b,r70,r71 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_LD_ST_$du_dpdk_core_a.txt -e r8,r1b,r70,r71 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_LD_ST_$du_worker_core_a.txt -e r8,r1b,r70,r71 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_LD_ST_$cu_worker_core_b.txt -e r8,r1b,r70,r71 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_Inst_Spec_DP_ASE_$cu_worker_core_a.txt -e r8,r1b,r73,r74 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_Inst_Spec_DP_ASE_$cu_worker_core_a.txt -e r8,r1b,r73,r74 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_DP_ASE_$du_dpdk_core_a.txt -e r8,r1b,r73,r74 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_DP_ASE_$du_dpdk_core_a.txt -e r8,r1b,r73,r74 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_DP_ASE_$du_dpdk_core_a.txt -e r8,r1b,r73,r74 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_DP_ASE_$du_worker_core_a.txt -e r8,r1b,r73,r74 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_DP_ASE_$cu_worker_core_b.txt -e r8,r1b,r73,r74 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_Inst_Spec_VFP_Crypto_$cu_worker_core_a.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_Inst_Spec_VFP_Crypto_$cu_worker_core_a.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_VFP_Crypto_$du_dpdk_core_a.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_VFP_Crypto_$du_dpdk_core_a.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_VFP_Crypto_$du_dpdk_core_a.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_VFP_Crypto_$du_worker_core_a.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_VFP_Crypto_$cu_worker_core_b.txt -e r8,r1b,r75,r76,r77 -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec + +perf stat -A --output $1_Inst_Spec_BR_$cu_worker_core_a.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$cu_worker_core_a" -x "," sleep $sleep_sec +#perf stat -A --output $1_Inst_Spec_BR_$cu_worker_core_a.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$cu_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_BR_$du_dpdk_core_a.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$du_dpdk_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_BR_$du_dpdk_core_a.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$du_dpdk_core_b" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_BR_$du_dpdk_core_a.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$du_dpdk_core_c" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_BR_$du_worker_core_a.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$du_worker_core_a" -x "," sleep $sleep_sec +perf stat -A --output $1_Inst_Spec_BR_$cu_worker_core_b.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec +``` + diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md new file mode 100644 index 0000000000..155a5749f9 --- /dev/null +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md @@ -0,0 +1,14 @@ +--- +title: How to Measure Power Consumption on Arm 5G Server +weight: 5 + +### FIXED, DO NOT MODIFY +layout: learningpathall +--- + +## How to Measure Power Consumption on Arm 5G Server + +5G Workload Power Measurement +Measurement Method +Watts Per CPU +Watts Per Core From 4624679362b6a97e04b2e6ec63cccb296d575c4a Mon Sep 17 00:00:00 2001 From: westlaker Date: Thu, 8 Jun 2023 11:38:52 -0500 Subject: [PATCH 02/20] add more contents and modify some existing one --- .../5g-on-arm/how-to-1.md | 6 +- .../5g-on-arm/how-to-2.md | 164 ++++++++++++++++-- .../5g-on-arm/how-to-4.md | 10 +- 3 files changed, 164 insertions(+), 16 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index 4ccf280cd3..e64e316889 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -91,7 +91,9 @@ For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be We had issue when boot with USB Ubuntu from VGA or remote access, after adding "console=tty0" to the kernel boot argument, we are able to access via VGA - this server can have only two PCIe slots, so not much we can play with for 5G server. Probably ideal for 5GCN Core server. + This server can have only two PCIe slots, so not much we can play with for 5G server. Probably ideal for 5GCN Core server. + + Another noticeble thing is that HPE server supports 220V only, you can't use in 110V outlet. #### A few thoughts about Arm ServerReady @@ -103,7 +105,7 @@ For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be iii. and the top slots never work, wasting two precious locations -2. must make ServerReady pass PCIe card detection as required (not necessary to test its driver to work), but at least the PCIe slots on server should bind to PCIe standard in voltage spec. verify by "lspci -t -nn -v " +2. must make ServerReady pass PCIe card detection as required (not necessary to test its driver to work), but at least the PCIe slots should follow the PCIe standard in voltage spec. And it can be verified by commnad of "lspci -t -nn -v " 3. must require ODM/OEM who apply for ServerReady to have a support team ready for at least one-year period after passing certification, in case the customers run into any issue with their server during evaluation. diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index 927525cf19..9a9b68b7c4 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -44,17 +44,19 @@ To update the firmeware, take the example of Foxconn server: ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade altra_scp_signed_2.10.20220531.hpm force ``` +Please note that the update of the firmware via IPMI interface is hardware dependent, check your server ODM/OEM for the IPMI detail for updating its firmware. + ### BIOS Setting Typically SR-IOV enablement is required for Arm server to support 5G deployment in container environment. Through BIOS options, you can enable SR-IOV. ### PCIe Setting -This is really depending on the server ODM, for example for SuperMicro server PCIe slots need to be configured. +This is really depending on the server ODM, for example for SuperMicro server PCIe slots need to be re-configured for certain PCIe devices. Contact ODM/OEM to find out details. ### CPU Frequency Setting -Following script can be run on each boot to temporarily force the cores to run at max CPU frequency +Following script can be run to temporarily force the cores to run at max CPU frequency ```bash #!/bin/bash @@ -212,17 +214,18 @@ It's worth noting that disabling the audit system is generally not recommended, A low latency kernel is a version of the Linux operating system that has been optimized for real-time applications. The goal of a low latency kernel is to minimize the time it takes for the operating system to respond to events and processes, so that applications can run with minimal delay or interruption. -Low latency kernels are commonly used in applications that require real-time processing, such as audio and video production, gaming, and scientific computing. They are also used in environments where the time taken by the operating system to respond to events is critical, such as in financial trading systems and other high-frequency data processing systems. +Low latency kernels are commonly used in applications that require real-time processing, such as audio and video production, gaming, and scientific computing, of course in 5G space as well. They are also used in environments where the time taken by the operating system to respond to events is critical, such as in financial trading systems and other high-frequency data processing systems. To achieve low latency, low latency kernels typically implement a number of changes to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. They may also make use of specialized scheduling algorithms and memory management techniques to further optimize performance. -How to install Lowlatency kernel on Arm server +How to install Lowlatency kernel on Arm server: + ```bash sudo apt update sudo apt-cache policy 'linux-image-5.*-lowlatency' sudo apt install linux-headers-5.15.0-46-lowlatency linux-image-5.15.0-46-lowlatency -y ``` -then run this to verify the new low-latency kernel: +then run this follwoing script to verify the new low-latency kernel: awk -F\' '$1=="menuentry " {print i++ " : " $2} $1=="submenu " {print i++ " : " $2; j=0} $1=="\tmenuentry " {print "\t", j++ " : " $2}' /boot/grub/grub.cfg @@ -231,7 +234,8 @@ it will show: 0 : Ubuntu, with Linux 5.15.0-46-lowlatency 1 : Ubuntu, with Linux 5.15.0-46-lowlatency (recovery mode) -need to disable upgrade OS kernel: +We will also need to disable upgrade OS kernel so the kernel will stay same version as we desire: + ```bash sudo vi /etc/apt/apt.conf.d/20auto-upgrades, change 1 to 0: APT::Periodic::Update-Package-Lists "0"; @@ -242,7 +246,7 @@ APT::Periodic::Unattended-Upgrade "0"; An RT kernel is designed to provide a guaranteed minimum response time for certain system events and processes, even under heavy load conditions. This is achieved through a number of optimizations to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. Additionally, RT kernels may make use of specialized scheduling algorithms and memory management techniques to further optimize performance. -At this moment, RT kernel is not ready from apt repository, we have to rebuild the kernel to enable RT for Arm +At this moment, RT kernel is not ready from apt repository, we have to rebuild the kernel to enable RT for Arm. We will update as soob as it becomes widely available. ### SR-IOV @@ -351,13 +355,153 @@ This command displays the following sample output. ### PTP Setting In 5G world, the time synchronization is critical to synchronize all components in same timing cadence -#### GrandMaster based PTP +#### GrandMaster Hardware based PTP + requires a GrandMaster Qulsar with GPS capability -Also requires a PTP enabled switch, currently Arista switch seems to be able to PTP sync'ed all slave nodes like Nvidia/Keysight. +Also requires a PTP enabled switch, currently Arista switch seems to be able to have PTP sync'ed all slave nodes like Nvidia/Keysight. #### SW based PTP -setup ptp4l/phc2sys on Linux: + +How to setup ptp4l/phc2sys on Linux: + +Configure Slave node setting: + +```bash +$ cat /etc/5g-ptp.conf +[global] +verbose 1 +domainNumber 24 +slaveOnly 1 +priority1 128 +priority2 128 +use_syslog 0 +logging_level 6 +tx_timestamp_timeout 900 +hybrid_e2e 0 +dscp_event 46 +dscp_general 46 +#clock_type BC +boundary_clock_jbod 1 + +[enP1p3s0f0np0] +logAnnounceInterval -3 +announceReceiptTimeout 3 +logSyncInterval -4 +logMinDelayReqInterval -4 +delay_mechanism E2E +network_transport L2 +``` + +Setup Slave node PTP4L service: note that we need to assign a core for this task to run + +```bash +$ cat /lib/systemd/system/ptp4l.service +[Unit] +Description=Precision Time Protocol (PTP) service +Documentation=man:ptp4l + +[Service] +Type=simple +ExecStart=taskset -c 32 /usr/sbin/ptp4l -f /etc/5g-ptp.conf + +[Install] +WantedBy=multi-user.target +``` + +Setup Slave node PHC2SYS service: note the NIC interface used here is enP1p3s0f0np0, also assign a core to run it +```bash +$ cat /lib/systemd/system/phc2sys.service +[Unit] +Description=Synchronize system clock or PTP hardware clock (PHC) +Documentation=man:phc2sys +After=ntpdate.service +Requires=ptp4l.service +After=ptp4l.service + +[Service] +Type=simple +ExecStart=/bin/sh -c "taskset -c 31 /usr/sbin/phc2sys -s /dev/ptp$(ethtool -T enP1p3s0f0np0 | grep PTP | awk '{print $4}') -c CLOCK_REALTIME -n 24 -O 0 -R 256 -u 256" + +[Install] +WantedBy=multi-user.target +``` + +Configure Master node setting: +```bash +$ cat /etc/5g-ptp.conf +[global] +verbose 1 +domainNumber 24 +priority1 128 +priority2 128 +use_syslog 1 +logging_level 6 +tx_timestamp_timeout 30 +hybrid_e2e 0 +dscp_event 46 +dscp_general 46 +[ens1f0np0] +logAnnounceInterval -3 +announceReceiptTimeout 3 +logSyncInterval -4 +logMinDelayReqInterval -3 +delay_mechanism E2E +network_transport L2 +``` + +Setup Master node PTP4L service: note that we need to assign a core for this task to run +```bash +$ cat /lib/systemd/system/ptp4l.service +[Unit] +Description=Precision Time Protocol (PTP) service +Documentation=man:ptp4l + +[Service] +Type=simple +Restart=always +RestartSec=5s +ExecStart=taskset -c 32 /usr/sbin/ptp4l -f /etc/5g-ptp.conf + +[Install] +WantedBy=multi-user.target +``` + +Setup Master node PHC2SYS service: note the NIC interface used here is enP1p3s0f0np0, also assign a core to run it +```bash +$ cat /lib/systemd/system/phc2sys.service +[Unit] +Description=Synchronize system clock or PTP hardware clock (PHC) +Documentation=man:phc2sys +After=ntpdate.service +Requires=ptp4l.service +After=ptp4l.service + +[Service] +Type=simple +Restart=always +RestartSec=5s +ExecStart=/bin/sh -c "taskset -c 31 /usr/sbin/phc2sys -s /dev/ptp$(ethtool -T enP1p3s0f0np0 | grep PTP | awk '{print $4}') -c CLOCK_REALTIME -n 24 -O 0 -R 256 -u 256" + +[Install] +WantedBy=multi-user.target +``` + +Start PTP/PHC2SYS services: + +```bash +#PTP.service +sudo systemctl daemon-reload +sudo systemctl restart ptp4l.service +sudo systemctl enable ptp4l.service +sudo systemctl status ptp4l.service + +#phc2sys.service +sudo systemctl daemon-reload +sudo systemctl restart phc2sys.service +sudo systemctl enable phc2sys.service +sudo systemctl status phc2sys.service +``` Issues with synchronization of NIC port There was a problem with A100X NIC port enP1p3s0f0np0, every time to run phc2sys, the timedatectl will be set to the bogus ToD like 2093. work around: diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md index 155a5749f9..669a63aa99 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md @@ -8,7 +8,9 @@ layout: learningpathall ## How to Measure Power Consumption on Arm 5G Server -5G Workload Power Measurement -Measurement Method -Watts Per CPU -Watts Per Core +### 5G Workload Power Measurement + +#### Measurement Method + +##### Watts Per CPU +##### Watts Per Core From aa9587b33816fd8416c1589a9d7f7d894b3d841e Mon Sep 17 00:00:00 2001 From: westlaker Date: Wed, 21 Jun 2023 15:11:00 -0500 Subject: [PATCH 03/20] further editing... --- .../5g-on-arm/how-to-1.md | 68 ++++++------------- .../5g-on-arm/how-to-2.md | 29 +------- .../5g-on-arm/how-to-3.md | 4 -- .../5g-on-arm/how-to-4.md | 16 ----- 4 files changed, 23 insertions(+), 94 deletions(-) delete mode 100644 content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index e64e316889..d142de0f05 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -13,11 +13,11 @@ layout: learningpathall 1. 5G-in-one-box (software only) - 2. 5G with Genevisio L1 Accerlator + 2. 5G with Genevisio NXP L1 Accelerator 3. 5G with Nvidia A100X (A100 GPU + BlueField2 NIC) -#### We have evaluated variety of Arm server platforms from Foxconn, Gigabyte, SuperMicro and HPE +#### We have evaluated variety of Arm server platforms from Foxconn, Gigabyte, WIWYNN, SuperMicro and HPE 1. Gigabyte (Mt. Snow 1P) @@ -33,17 +33,17 @@ layout: learningpathall 1P vs 2P systems: -For evlaution purpose, 1P system should be sufficient to run any 5G software (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would add more overhead introduced by numa system. So for simplicity, we would recommend 1P system. +For evlaution purpose, 1P system should be sufficient to run any 5G software (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, it also adds more overhead introduced by numa system. So for simplicity, we would recommend 1P system to start with. 1U vs. 2U systems: -1U would be sufficient to run 5G CN Core software while 2U is definitely required for using additional HW such as PCIe card for L1 processing, e.g. the Nvidia A100X and Genvisio. +1U would be sufficient to run 5G CN Core software while 2U is definitely required for needing additional HW such as PCIe card for L1 processing, e.g. the Nvidia A100X and Genvisio. -#### Accommodate PCIe Accerlators +#### Accommodate PCIe Accelerators -Please note due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accerlators. +Please note due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accelerators we are going to use. -For full length PCIe cards, you need at least 2U server, however not every 2U server will support full length/full width PCIe devices. Also some PCIe full profile devices requirea little bit higher voltage to run properly, SuperMicro ARM server seems to accommondate Genevisio card better, the second Arm server we tested and it was working is FoxConn ARM server, but it is an 2P system, a bit overkill to start with. +For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also some PCIe full profile devices requirea little bit higher voltage to run properly, SuperMicro ARM server seems to accommondate Genevisio card better, the second Arm server we tested and it can support is the FoxConn ARM server, but it is an 2P system, a bit overkill to start with. For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one devices, we need to make sure its NIC ports face out. At this moment, only SuperMicro ARM server has designed its server with A100X support in mind. @@ -51,72 +51,44 @@ For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be 1. Gigabytes/WIWYNN servers - In genearal they are able to do what intends to do, supporting Nvidia A100X, but it was not designed for A100X for the NIC ports outfit. + In genearal they are able to do what intends to do, we have used Gigabyte servers to do CORE server, edge server or 5G network in one box, which is so far working well. - We have used Gigabyte servers to do CORE server, edge server or 5G network in one box (Altran), which is so far great + Its GPU version claims to support up to 3 GPUs, but we have tried that only one A100X can work, however it does not have proper orientation for A100X ethernet ports. - However certain Gigabytes/WIWYNN models have issue to support the requirement of supporting NXP Genevisio PCIe card, Intel QAT and two Intel NIC X710 cards due to its limited estate space - - the GPU version G242-P3X claims to support up to 3 GPUs, but we tried, it is only one A100X can work - - the GPU version does not work for Genevisio card + Its GPU version does not work for some other PCIe devices we have tested. 2. Foxconn Mt Collins 2P is a powerful 2P system to accommodate more than 4 PCIe slots, however - It can't accommodate A100X for having its NIC ports in right orientation - - It has three riser cages, but the right one (from its back panel view) is useless and in the both left and middle ones, there are only middle and bottom slots work, the top one slots never work + It can support A100X, but it can't have its NIC ports in right orientation. - the bottom ones can't fit a full length PCIe card as its wiring and other stuff is in conflict with the PCIe card on the bottom slot - - When inserting A100X or Genevisio card or Intel NIC 710 card, the server will generate tons of PCIe AER warning messages (even though it claims it is corrected, but not sure what extent of its impact) - - It never be able to detect Intel NIC 710 4-port card - - 2P server might not be necessary in our use case here. Additionally it introduces the overhead of numa system. + It seems to have some issues for different PCIe devices, so you need to make sure it can work for your hardware requirement. 3. SuperMicro server - SMC initially has PCIe power issues with A100X and GeneVisio cards. After put down extra power cables from SuperMicro (both the GPU power cable and the Y-shape cable to boost the 12V to its PCIe riser cards, both GeneVisio and A100X can be functional. - - It can detect Intel NIC 710 4-port, make it only Arm server being able to do that! - - Also it is tricky to play with SuperMicro to host various PCIe card - - It uses OpenBMC as its BMC, we have trouble to login into its BMC UI, however its motherboard has Password printed and use that password we can use ipmitool or its UI to operate/access the server + SMC seems to have its solutions for some of PCIe devices we would have issues with other servers. At this moment it is the server can support all of various PCIe devices we intend to use such as Nvidia 100X with proper orientation for its ethernet ports. 4. HPE server - Its BMC needs to enable IPMI i/f from its UI, however ipmi SOL is "not supported", but you can get serial console by... ssh-ing into the iLO and then using "vsp" (virtual serial port). There's also another one for physical, and both seem to work, but "vsp" is recommended by HP. - - We had issue when boot with USB Ubuntu from VGA or remote access, after adding "console=tty0" to the kernel boot argument, we are able to access via VGA - - This server can have only two PCIe slots, so not much we can play with for 5G server. Probably ideal for 5GCN Core server. + It is 1U server and can have only two PCIe slots, so pobably is ideal for 5GCN Core server. Another noticeble thing is that HPE server supports 220V only, you can't use in 110V outlet. #### A few thoughts about Arm ServerReady -1. current ServerReady is not strengthened enough to make sure the ODM/OEM have 100% ready for deployment in 5G space. For example, in the case of Foxconn, we were struggling to make it accommodating to support our 5G PoC requirement and it's quite fragile. - - i. We have to try all combination of PCIe cards with different PCIe slots to finalize only one combination work - - ii. the motherboard layout design was bad for accommodating full profile/full length PCIe card, especially for the slots in the bottom. - - iii. and the top slots never work, wasting two precious locations +1. Current ServerReady is not strengthened enough to make sure the ODM/OEM is 100% ready for deployment in 5G space. In order to qualify an Arm server for 5G Ready, we need to make sure that they can work with the required hardware for 5G. -2. must make ServerReady pass PCIe card detection as required (not necessary to test its driver to work), but at least the PCIe slots should follow the PCIe standard in voltage spec. And it can be verified by commnad of "lspci -t -nn -v " +2. The Arm server needs to pass PCIe device detection as minimal requirement. That is the lspci command should be able to list these PCIe devices without issues. -3. must require ODM/OEM who apply for ServerReady to have a support team ready for at least one-year period after passing certification, in case the customers run into any issue with their server during evaluation. +3. It is important to work with Arm server ODM/OEM and their firmware team for any issues rised during evaulation so any issues can be fedback to the ODM/OEM to be addressed in timely fashion. -4. ArmServerReady should also mandate standard IPMI interface must be supported because it is important to have integration of IPMI into automation testing +4. ArmServerReady should also mandate that standard IPMI interface must be supported because it is important standard. #### 5G ServerReady We like to develop our own methodology to qualify any server from ODM/OEM to be 5G ServerReady: - 1. Using existing PoC example to run the test to see if the server can pass (hopefully with automation) + 1. Using existing PoC example to run the qualification to see if the server can pass with possible automation - 2. Using Performance PoC example to test to see if the server can meet our performance goal (hopefully with automation) + 2. Using Performance PoC example to test to see if the server can meet our performance goal with possible automation 3. Developing General Testing Scripts to check the requirement for configuration and compliance for 5G ServerReady diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index 9a9b68b7c4..4f4ea8a3a5 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -10,9 +10,7 @@ layout: learningpathall ### Firmware Maintenance -It is important to have server's BIOS firmware updated, it is important to be in the loop of any ODM/OEM updating with its firmware. We need to have close relationship with ODM/OEM's FW team to address any issue we have encountered - -It is also important to have server's BMC firmware updated, it is important to be in the loop of any ODM/OEM updating with its FW. We need to have close relationship with ODM/OEM's FW team to address any issue we have encountered +The BIOS firmware would have impact on some of the functionalities and performance of the server. So it is important to have server's BIOS and BMC firmware updated, it is important to be in the loop of any ODM/OEM update with its firmware. We need to have close relationship with ODM/OEM's firmware team to address any issue we would have encountered. BMC should always support IPMI standard protocol which allows automation of controlling the servers @@ -52,7 +50,7 @@ Typically SR-IOV enablement is required for Arm server to support 5G deployment ### PCIe Setting -This is really depending on the server ODM, for example for SuperMicro server PCIe slots need to be re-configured for certain PCIe devices. Contact ODM/OEM to find out details. +This is really depending on the server ODM, contact ODM/OEM to find out details. ### CPU Frequency Setting @@ -99,7 +97,7 @@ sudo systemctl disable ondemand #### Disable freq scaling -set in the boot argument with cpufreq.off=1 +Set in the boot argument with cpufreq.off=1 cpufreq.off=1 is a kernel parameter in Linux that is used to disable CPU frequency scaling. CPU frequency scaling, also known as CPU throttling, is the process of adjusting the clock speed of the CPU to conserve energy and reduce heat generation. This is typically done by the operating system based on the load and performance requirements of the system. @@ -503,26 +501,5 @@ sudo systemctl enable phc2sys.service sudo systemctl status phc2sys.service ``` -Issues with synchronization of NIC port -There was a problem with A100X NIC port enP1p3s0f0np0, every time to run phc2sys, the timedatectl will be set to the bogus ToD like 2093. work around: - -changed /lib/systemd/system/phc2sys.service: - -ExecStart=/bin/sh -c "taskset -c 31 /usr/sbin/phc2sys -s CLOCK_REALTIME -c /dev/ptp$(ethtool -T enP1p3s0f0np0 | grep PTP | awk '{print $4}') -n 24 -O 0 -R 256 -u 256" - -then run: -```bash -sudo ntpdate us.pool.ntp.org -sudo hwclock -w -sudo systemctl daemon-reload -sudo systemctl restart phc2sys.service -sudo systemctl enable phc2sys.service -sudo systemctl status phc2sys.service -``` -then changed back the /lib/systemd/system/phc2sys.service - -it will start sync' up with switch. - -the A100X device somehow got messed up for its time clock, above steps are to update it with right ToD. diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md index 34e7030b56..6bfe578f1c 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -14,7 +14,6 @@ Please refer to early section of setting kernel boot arguments and the CPU scali ### DPDK Tuning - #### System level setting: Core isolation from Linux scheduler, Disabling interrupts, Huge page usage to minimize TLB misses are important before running DPDK PMDs. @@ -45,9 +44,6 @@ Refer testpmd application https://doc.dpdk.org/guides/testpmd_app_ug/run_app.htm Refer the same link above to run test pmd on Ampere. This will ensure that the Ethernet PMD is operating at optimal performance. -### Memory Tuning -### IO Tuning - ### Profiling/Tracing #### Using perf tools like : diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md deleted file mode 100644 index 669a63aa99..0000000000 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-4.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: How to Measure Power Consumption on Arm 5G Server -weight: 5 - -### FIXED, DO NOT MODIFY -layout: learningpathall ---- - -## How to Measure Power Consumption on Arm 5G Server - -### 5G Workload Power Measurement - -#### Measurement Method - -##### Watts Per CPU -##### Watts Per Core From 489ca9ff91e9f4971d143d7165a0e23464f3071e Mon Sep 17 00:00:00 2001 From: westlaker Date: Thu, 22 Jun 2023 11:07:21 -0500 Subject: [PATCH 04/20] further editing in review and next step. --- .../5g-on-arm/_index.md | 21 +++--- .../5g-on-arm/_next-steps.md | 33 ++++++--- .../5g-on-arm/_review.md | 74 +++++++++++++------ .../5g-on-arm/how-to-1.md | 22 +++--- .../5g-on-arm/how-to-3.md | 19 +++++ 5 files changed, 114 insertions(+), 55 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md index 4ad788f8cb..651bd7b32b 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md @@ -3,6 +3,7 @@ title: 5G On Arm Handbook description: This +learning_path_main_page: 'yes' minutes_to_complete: 10 who_is_this_for: This is an introductory topic for 5G software migrating to Arm. @@ -13,24 +14,22 @@ learning_objectives: prerequisites: - Arm Neoverse server - - 5G stacks + - 5G ORAN stack author_primary: 5G Solution Team/Arm Performance and Solution Engineering Team ### Tags skilllevels: Introductory -subjects: 5G ORAN stacks +subjects: 5G armips: - - Arm Neoverse IP -tools: - - Arm development toolset - - Virtualization software -softwares: - - Any ORAN stacks - - L1 with or without HW accerlators + - Neoverse +tools_software_languages: + - SVE + - GCC + - 5G ORAN + - Containers and Virtualization operatingsystems: - - Ubuntu 20.04+ - + - Linux ### FIXED, DO NOT MODIFY # ================================================================================ diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md index c4ae77cc92..865026f0bd 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md @@ -1,22 +1,31 @@ --- -next_step_guidance: PLACEHOLDER TEXT 1 +# ================================================================================ +# Edit +# ================================================================================ + +next_step_guidance: Please contact us for futher discussion for your need for 5G on Arm. -recommended_path: /learning-paths/PLACEHOLDER_CATEGORY/PLACEHOLDER_LEARNING_PATH/ +recommended_path: "/learning-paths/servers-and-cloud-computing/ran/" +# Link to the next learning path being recommended. + +# further_reading links to references related to this path. Can be: + # Manuals for a tool / software mentioned (type: documentation) + # Blog about related topics (type: blog) + # General online references (type: website) further_reading: - resource: - title: PLACEHOLDER MANUAL - link: PLACEHOLDER MANUAL LINK - type: documentation + title: 5G Infrastructure + link: https://www.arm.com/en/markets/5g/infrastructure + type: website - resource: - title: PLACEHOLDER BLOG - link: PLACEHOLDER BLOG LINK - type: blog + title: Arm RAN Acceleration Library Reference Guide + link: https://developer.arm.com/documentation/102249 + type: documentation - resource: - title: PLACEHOLDER GENERAL WEBSITE - link: PLACEHOLDER GENERAL WEBSITE LINK - type: website - + title: 5G RAN for Dummies + link: https://www.arm.com/resources/dummies-guide/5g-ran + type: documentation # ================================================================================ # FIXED, DO NOT MODIFY diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md index 9b1bfb96fe..435f97753f 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md @@ -1,38 +1,70 @@ --- +# ================================================================================ +# Edit +# ================================================================================ + +# Always 3 questions. Should try to test the reader's knowledge, and reinforce the key points you want them to remember. + # question: A one sentence question + # answers: The correct answers (from 2-4 answer options only). Should be surrounded by quotes. + # correct_answer: An integer indicating what answer is correct (index starts from 0) + # explanation: A short (1-3 sentence) explanation of why the correct answer is correct. Can add additional context if desired + review: - questions: question: > - PLACEHOLDER QUESTION 1? + Which server has better support of Nvidia A100X? answers: - - PLACEHOLDER ANSWER A - - PLACEHOLDER ANSWER B - correct_answer: 1 + - "HPE" + - "Foxconn" + - "Gigabyte" + - "WIWYNN" + - "Supermicro" + - "All the above" + correct_answer: 5 explanation: > - PLACEHOLDER EXPLANATION 1 - + The Supermicro server is designed to support A100X with correct orientation for its ethernet ports + - questions: + question: > + Which statement is corrected? + answers: + - "Regular Linux kernel should be enough for supporting 5G stack" + - "Low latency Linux kernel must be required for supporting 5G stack" + correct_answer: 2 + explanation: > + Low Latency Kernel minimize the time it takes for the operating system to respond to events and processes, it is essential for 5G stack. - questions: question: > - PLACEHOLDER QUESTION 2? + What is potential issue with a 2P server? answers: - - PLACEHOLDER ANSWER A - - PLACEHOLDER ANSWER B - - PLACEHOLDER ANSWER C - - PLACEHOLDER ANSWER D - correct_answer: 4 + - "PCIe devices sit on different node from the CPU" + - "Cross socket communication overhead" + - "Sometime can't put PCIe device and CPU on same node" + - "All the above" + correct_answer: 4 explanation: > - PLACEHOLDER EXPLANATION 2 - + Explain all potential issues related to multiple socket server - questions: question: > - PLACEHOLDER QUESTION 3? + What is isolcpus? answers: - - PLACEHOLDER ANSWER A - - PLACEHOLDER ANSWER B - - PLACEHOLDER ANSWER C - - PLACEHOLDER ANSWER D - correct_answer: 2 + - "Remove the core off access" + - "Isolate the core off limit" + - "Reserved for real-time or other special purpose tasks" + correct_answer: 3 explanation: > - PLACEHOLDER EXPLANATION 3 + Explain isolcpus setting is important for the dedicated tasks not interfered by the kernel + - questions: + question: > + Which way to affinitize your program to cores? + answers: + - "taskset" + - "numactl" + - "pthread_setaffinity_np" + - "All the above" + correct_answer: 4 + explanation: > + Explain the ways of how to assign your program to cpu/cores + diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index d142de0f05..afd4d6c3bc 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -9,7 +9,7 @@ layout: learningpathall ## How to Choose Right 5G Servers --- -#### We have done extensive 5G development on Arm servers with various HW configurations and SW components: +#### We have done extensive 5G development and testing on Arm servers with various HW configurations and SW components: 1. 5G-in-one-box (software only) @@ -17,7 +17,7 @@ layout: learningpathall 3. 5G with Nvidia A100X (A100 GPU + BlueField2 NIC) -#### We have evaluated variety of Arm server platforms from Foxconn, Gigabyte, WIWYNN, SuperMicro and HPE +#### We have evaluated a variety of Arm server platforms from Foxconn, Gigabyte, WIWYNN, Supermicro and HPE 1. Gigabyte (Mt. Snow 1P) @@ -25,7 +25,7 @@ layout: learningpathall 3. Foxconn (Mt. Collins 2P) - 4. SuperMicro (R12SPD 1P) + 4. Supermicro (R12SPD 1P) 5. HPE (ProLiant RL300 Gen11 (P59870-B21) 1P) @@ -33,19 +33,19 @@ layout: learningpathall 1P vs 2P systems: -For evlaution purpose, 1P system should be sufficient to run any 5G software (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, it also adds more overhead introduced by numa system. So for simplicity, we would recommend 1P system to start with. +For evaluation purpose, we recommend a 1P system which should be sufficient to run any 5G software stacks (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, we need to avoid socket communication, especially when some PCIe devices sit in a different node across from the CPU. -1U vs. 2U systems: +1U vs 2U systems: -1U would be sufficient to run 5G CN Core software while 2U is definitely required for needing additional HW such as PCIe card for L1 processing, e.g. the Nvidia A100X and Genvisio. +1U would be sufficient to run 5G CN Core software or pure 5G software stacks while 2U is definitely required for needing additional HW such as several PCIe cards or a full length PCIe card for L1 processing, for example, the Nvidia A100X and Genvisio. -#### Accommodate PCIe Accelerators +#### Accommodating PCIe Accelerators -Please note due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accelerators we are going to use. +Due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accelerators we are going to use. -For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also some PCIe full profile devices requirea little bit higher voltage to run properly, SuperMicro ARM server seems to accommondate Genevisio card better, the second Arm server we tested and it can support is the FoxConn ARM server, but it is an 2P system, a bit overkill to start with. +For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's ARM server seems to accommodate Genevisio card better. -For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one devices, we need to make sure its NIC ports face out. At this moment, only SuperMicro ARM server has designed its server with A100X support in mind. +For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one devices, we need to make sure its NIC ports face out. At this moment, only Supermicro ARM server has designed its server with A100X support in mind. #### Available ARM Servers @@ -63,7 +63,7 @@ For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be It seems to have some issues for different PCIe devices, so you need to make sure it can work for your hardware requirement. -3. SuperMicro server +3. Supermicro server SMC seems to have its solutions for some of PCIe devices we would have issues with other servers. At this moment it is the server can support all of various PCIe devices we intend to use such as Nvidia 100X with proper orientation for its ethernet ports. diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md index 6bfe578f1c..ea9dfbc7ae 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -44,6 +44,14 @@ Refer testpmd application https://doc.dpdk.org/guides/testpmd_app_ug/run_app.htm Refer the same link above to run test pmd on Ampere. This will ensure that the Ethernet PMD is operating at optimal performance. +### Allocate Cores to Different Tasks + +It is critical to be carefully alloacting the cores to various tasks for 5G stack. Make sure all of processes have their own cores to run on, not to step over each other. + +For multiple socket server, always use numactl to launch your program to associate with the cpu and the PCIe device your program will access on same node. + +Use taskset command to launch your program on specific cores. + ### Profiling/Tracing #### Using perf tools like : @@ -184,3 +192,14 @@ perf stat -A --output $1_Inst_Spec_BR_$du_worker_core_a.txt -e r8,r1b,r78,r79,r7 perf stat -A --output $1_Inst_Spec_BR_$cu_worker_core_b.txt -e r8,r1b,r78,r79,r7a -I $interval -C "$cu_worker_core_b" -x "," sleep $sleep_sec ``` +#### Take Advantage of Arm RAL and SVE/NEON + +Arm 5G RAN Acceleration Library (ArmRAL): +https://learn.arm.com/learning-paths/servers-and-cloud-computing/ran/ + +Port Code to Arm Scalable Vector Extension (SVE) +https://learn.arm.com/learning-paths/servers-and-cloud-computing/sve/ + +#### Using Other Arm Tools + +Refer this link https://developer.arm.com/Tools%20and%20Software/Streamline%20Performance%20Analyzer for Streamline Performance Analyzer From b034dbb29f805279865cbc8c721f679c4e5e7fc7 Mon Sep 17 00:00:00 2001 From: westlaker Date: Wed, 28 Jun 2023 09:44:50 -0500 Subject: [PATCH 05/20] some more removal --- .../5g-on-arm/how-to-1.md | 39 +------------------ 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index afd4d6c3bc..0c7e6de9eb 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -47,44 +47,9 @@ For full length PCIe cards, you need at least an 2U server, however not every 2U For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one devices, we need to make sure its NIC ports face out. At this moment, only Supermicro ARM server has designed its server with A100X support in mind. -#### Available ARM Servers - -1. Gigabytes/WIWYNN servers - - In genearal they are able to do what intends to do, we have used Gigabyte servers to do CORE server, edge server or 5G network in one box, which is so far working well. - - Its GPU version claims to support up to 3 GPUs, but we have tried that only one A100X can work, however it does not have proper orientation for A100X ethernet ports. - - Its GPU version does not work for some other PCIe devices we have tested. - -2. Foxconn Mt Collins 2P is a powerful 2P system to accommodate more than 4 PCIe slots, however - - It can support A100X, but it can't have its NIC ports in right orientation. - - It seems to have some issues for different PCIe devices, so you need to make sure it can work for your hardware requirement. - -3. Supermicro server - - SMC seems to have its solutions for some of PCIe devices we would have issues with other servers. At this moment it is the server can support all of various PCIe devices we intend to use such as Nvidia 100X with proper orientation for its ethernet ports. - -4. HPE server - - It is 1U server and can have only two PCIe slots, so pobably is ideal for 5GCN Core server. - - Another noticeble thing is that HPE server supports 220V only, you can't use in 110V outlet. - -#### A few thoughts about Arm ServerReady - -1. Current ServerReady is not strengthened enough to make sure the ODM/OEM is 100% ready for deployment in 5G space. In order to qualify an Arm server for 5G Ready, we need to make sure that they can work with the required hardware for 5G. - -2. The Arm server needs to pass PCIe device detection as minimal requirement. That is the lspci command should be able to list these PCIe devices without issues. - -3. It is important to work with Arm server ODM/OEM and their firmware team for any issues rised during evaulation so any issues can be fedback to the ODM/OEM to be addressed in timely fashion. - -4. ArmServerReady should also mandate that standard IPMI interface must be supported because it is important standard. - #### 5G ServerReady -We like to develop our own methodology to qualify any server from ODM/OEM to be 5G ServerReady: + +We like to develop a way to qualify any server from ODM/OEM to be 5G ServerReady: 1. Using existing PoC example to run the qualification to see if the server can pass with possible automation From dde40886f247bac83fb75fb16bdac97c05807e74 Mon Sep 17 00:00:00 2001 From: westlaker Date: Thu, 20 Jul 2023 10:52:31 -0500 Subject: [PATCH 06/20] updated --- .../5g-on-arm/_index.md | 8 ++++---- .../5g-on-arm/_review.md | 2 +- .../5g-on-arm/how-to-1.md | 20 +++++++++---------- .../5g-on-arm/how-to-2.md | 16 ++++----------- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md index 651bd7b32b..43462154cb 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md @@ -6,17 +6,17 @@ description: This learning_path_main_page: 'yes' minutes_to_complete: 10 -who_is_this_for: This is an introductory topic for 5G software migrating to Arm. +who_is_this_for: This is a startup guide for running 5G stack on Arm learning_objectives: - - Chooce the proper Arm server model to develop/deploy 5G stacks and 5GCN server - - Suggest the best practice and configuration for setting up Arm servers for 5G stacks + - How to choose the right Arm server for 5G stack - gNB (L1/CU/DU) and Core + - Tuning Guide for the selected servers for running 5G stack prerequisites: - Arm Neoverse server - 5G ORAN stack -author_primary: 5G Solution Team/Arm Performance and Solution Engineering Team +author_primary: Arm Performance and Solution Engineering Team ### Tags skilllevels: Introductory diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md index 435f97753f..7249d7bca2 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md @@ -31,7 +31,7 @@ review: - "Low latency Linux kernel must be required for supporting 5G stack" correct_answer: 2 explanation: > - Low Latency Kernel minimize the time it takes for the operating system to respond to events and processes, it is essential for 5G stack. + Low Latency Kernel minimize the time it takes for the operating system to respond to events and processes, it is essential for latency sensitive of processes in 5G stack. - questions: question: > What is potential issue with a 2P server? diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index 0c7e6de9eb..75dd2b3be8 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -13,9 +13,11 @@ layout: learningpathall 1. 5G-in-one-box (software only) - 2. 5G with Genevisio NXP L1 Accelerator + 2. 5G with Inline L1 Accelerator - 3. 5G with Nvidia A100X (A100 GPU + BlueField2 NIC) + 3. 5G with L1 GPU Offload + + 4. 5G with Lookaside Accelerator Offload #### We have evaluated a variety of Arm server platforms from Foxconn, Gigabyte, WIWYNN, Supermicro and HPE @@ -43,19 +45,17 @@ For evaluation purpose, we recommend a 1P system which should be sufficient to r Due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accelerators we are going to use. -For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's ARM server seems to accommodate Genevisio card better. +For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's ARM server seems to better accommodate some Inline L1 Accelerator cards. -For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one devices, we need to make sure its NIC ports face out. At this moment, only Supermicro ARM server has designed its server with A100X support in mind. +For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one device, we need to make sure its NIC ports face out. At this moment, only Supermicro ARM server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. -#### 5G ServerReady +#### 5G Ready on Arm -We like to develop a way to qualify any server from ODM/OEM to be 5G ServerReady: +This handbook attempt to provide the guide to make Arm servers ready for 5G development and deployment: - 1. Using existing PoC example to run the qualification to see if the server can pass with possible automation + 1. Using existing PoC example to run the qualification to see if the server can pass with automation - 2. Using Performance PoC example to test to see if the server can meet our performance goal with possible automation + 2. Using Performance PoC example to test to see if the server can meet our performance goal with automation - 3. Developing General Testing Scripts to check the requirement for configuration and compliance for 5G ServerReady - 4. Incorporate Testing equipment like Keysight and Viavi for 5G ServerReady tests diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index 4f4ea8a3a5..e17cfd8f11 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -10,9 +10,7 @@ layout: learningpathall ### Firmware Maintenance -The BIOS firmware would have impact on some of the functionalities and performance of the server. So it is important to have server's BIOS and BMC firmware updated, it is important to be in the loop of any ODM/OEM update with its firmware. We need to have close relationship with ODM/OEM's firmware team to address any issue we would have encountered. - -BMC should always support IPMI standard protocol which allows automation of controlling the servers +The BIOS firmware would have impact on some of the functionalities and performance of the server. So it is important to sign up for firmware updates from your server vendors ### How to update firmware @@ -87,7 +85,7 @@ return "ondemand" in the current setup. We may want to change the CPU governor from "ondemand" to "performance". -The following instructions show how we do it on x86. +The following instructions show how we do it: ```bash sudo apt install cpufrequtils @@ -95,14 +93,6 @@ echo 'GOVERNOR="performance"' | sudo tee /etc/default/cpufrequtils sudo systemctl disable ondemand ``` -#### Disable freq scaling - -Set in the boot argument with cpufreq.off=1 - -cpufreq.off=1 is a kernel parameter in Linux that is used to disable CPU frequency scaling. CPU frequency scaling, also known as CPU throttling, is the process of adjusting the clock speed of the CPU to conserve energy and reduce heat generation. This is typically done by the operating system based on the load and performance requirements of the system. - -By setting cpufreq.off=1, you are telling the Linux kernel not to use CPU frequency scaling. This means that the CPU will run at its maximum clock speed all the time, even if it is not needed. This can have performance and power consumption implications, as well as potential stability issues, so it's generally not recommended to disable CPU frequency scaling unless you have a specific use case that requires it. - ### Linux Requirements #### Boot Parameters @@ -110,6 +100,8 @@ By setting cpufreq.off=1, you are telling the Linux kernel not to use CPU freque ##### Hugepage setting default_hugepagesz=1G hugepagesz=1G hugepages=32 +The huge page setting is required for DPDK support which is commonly adopted in 5G networking. It reduces the TLB misses, improves performance by lowering memory overhead, and improves memory efficiency with larger and contiguois memory allocation + ##### isolcpus=cpuX-cpuY The setting "isolcpus" is a kernel command line parameter that is used to isolate specific CPUs in a multi-CPU system, such that they are not used for general purpose processing. Instead, they are reserved for real-time or other special purpose tasks that require dedicated and predictable processing resources. From 863ead8e668efbace34ebbdb8f0fb07b8546dac7 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 21 Jul 2023 08:47:47 -0400 Subject: [PATCH 07/20] Update _index.md --- .../servers-and-cloud-computing/5g-on-arm/_index.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md index 43462154cb..b4d6e3b69f 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md @@ -1,22 +1,20 @@ --- title: 5G On Arm Handbook -description: This - learning_path_main_page: 'yes' minutes_to_complete: 10 -who_is_this_for: This is a startup guide for running 5G stack on Arm +who_is_this_for: This is a introductory topic for software developers interested in running the 5G stack on Arm learning_objectives: - - How to choose the right Arm server for 5G stack - gNB (L1/CU/DU) and Core - - Tuning Guide for the selected servers for running 5G stack + - Choose the appropriate Arm server for running the 5G stack + - Tune the selected servers for running the 5G stack prerequisites: - - Arm Neoverse server + - Arm server - 5G ORAN stack -author_primary: Arm Performance and Solution Engineering Team +author_primary: Jici Gao ### Tags skilllevels: Introductory @@ -27,7 +25,6 @@ tools_software_languages: - SVE - GCC - 5G ORAN - - Containers and Virtualization operatingsystems: - Linux From a0329150ea4ea81e137d6ea79b1a5bf9901aeb98 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 21 Jul 2023 09:37:14 -0400 Subject: [PATCH 08/20] Update how-to-1.md --- .../5g-on-arm/how-to-1.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index 75dd2b3be8..a1e135f44d 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -1,15 +1,15 @@ --- -title: How to Choose Right 5G Servers +title: Choose the appropriate Arm server for running the 5G stack weight: 2 ### FIXED, DO NOT MODIFY layout: learningpathall --- -## How to Choose Right 5G Servers +## Choose the appropriate Arm 5G Servers --- -#### We have done extensive 5G development and testing on Arm servers with various HW configurations and SW components: +#### Extensive 5G development and testing on Arm servers has been done on the hardware configurations and software components listed below: 1. 5G-in-one-box (software only) @@ -19,7 +19,7 @@ layout: learningpathall 4. 5G with Lookaside Accelerator Offload -#### We have evaluated a variety of Arm server platforms from Foxconn, Gigabyte, WIWYNN, Supermicro and HPE +#### The Arm server platforms listed below have been evaluated to run the 5G stack on: 1. Gigabyte (Mt. Snow 1P) @@ -35,27 +35,27 @@ layout: learningpathall 1P vs 2P systems: -For evaluation purpose, we recommend a 1P system which should be sufficient to run any 5G software stacks (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, we need to avoid socket communication, especially when some PCIe devices sit in a different node across from the CPU. +For evaluation purposes, a 1P system is recommended which should be sufficient to run any 5G software stacks (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, socket communication should be avoided, especially when some PCIe devices sit in a different node across from the CPU. 1U vs 2U systems: -1U would be sufficient to run 5G CN Core software or pure 5G software stacks while 2U is definitely required for needing additional HW such as several PCIe cards or a full length PCIe card for L1 processing, for example, the Nvidia A100X and Genvisio. +1U is sufficient to run 5G CN Core software or pure 5G software stacks while 2U is required when you need additional hardware such as several PCIe cards or a full length PCIe card for L1 processing, for example, the Nvidia A100X and Genvisio. #### Accommodating PCIe Accelerators -Due to nature of PCIe devices, we need to consider carefully for picking up right Arm server to accommodating the PCIe Accelerators we are going to use. +Due to the nature of PCIe devices, you need to carefully select the right Arm server to accommodate the PCIe Accelerators you are going to use. -For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's ARM server seems to better accommodate some Inline L1 Accelerator cards. +For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's Arm server better accommodate some Inline L1 Accelerator cards. -For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one device, we need to make sure its NIC ports face out. At this moment, only Supermicro ARM server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. +For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one device, you need to make sure its NIC ports face out. At this moment, only Supermicro Arm server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. #### 5G Ready on Arm -This handbook attempt to provide the guide to make Arm servers ready for 5G development and deployment: +This learning path attempts to provide the guidance to make Arm servers ready for 5G development and deployment: 1. Using existing PoC example to run the qualification to see if the server can pass with automation - 2. Using Performance PoC example to test to see if the server can meet our performance goal with automation + 2. Using Performance PoC example to test to see if the server can meet performance goals with automation From 13f03dfd275ad62eac34ceed5e65b2518c7549cc Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 21 Jul 2023 10:08:54 -0400 Subject: [PATCH 09/20] Update how-to-2.md --- .../5g-on-arm/how-to-2.md | 97 +++++++++++-------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index e17cfd8f11..92df55ac18 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -1,5 +1,5 @@ --- -title: How to Setup and Configure 5G Servers +title: Setup and Configure 5G Arm Servers weight: 3 ### FIXED, DO NOT MODIFY @@ -10,7 +10,7 @@ layout: learningpathall ### Firmware Maintenance -The BIOS firmware would have impact on some of the functionalities and performance of the server. So it is important to sign up for firmware updates from your server vendors +The BIOS firmware impacts some of the functionality and performance of the server. So it is important to sign up for firmware updates from your server vendors. ### How to update firmware @@ -20,18 +20,18 @@ Most of Arm servers provide web UI for managing the firmware. Log into BMC IP via browser, - 1. check its current FW version by clicking its Firmware information + 1. Check the current FW version by clicking its `Firmware information`. - 2. go to its Firmware maintenance section to click Firmware update and follow up its interface to update BIOS/SCP and BMC software + 2. Go to the `Firmware maintenance` section and click on `Firmware update`. Follow te prompts to update BIOS/SCP and BMC software. #### Via ipmitool command line -The benefit of using ipmitool is for automation of controlling the servers remotely. +The benefit of using `ipmitool` is for automation of controlling the servers remotely. -To update the firmeware, take the example of Foxconn server: +To update the firmware, for example on the Foxconn server: -```bash +```console ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin raw 0x32 0x8f 0x3 0x1 ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade MtCollinsBmc0_43_4.hpm force ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin raw 0x32 0x8f 0x3 0x1 @@ -40,19 +40,21 @@ To update the firmeware, take the example of Foxconn server: ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin -z 8196 hpm upgrade altra_scp_signed_2.10.20220531.hpm force ``` -Please note that the update of the firmware via IPMI interface is hardware dependent, check your server ODM/OEM for the IPMI detail for updating its firmware. +{{% notice Note %}} +The update of the firmware via IPMI interface is hardware dependent, check your server ODM/OEM for the IPMI detail for updating its firmware. +{{% /notice %}} ### BIOS Setting -Typically SR-IOV enablement is required for Arm server to support 5G deployment in container environment. Through BIOS options, you can enable SR-IOV. +Typically SR-IOV enablement is required for Arm servers to support 5G deployment in a container environment. Through BIOS options, you can enable SR-IOV. ### PCIe Setting -This is really depending on the server ODM, contact ODM/OEM to find out details. +This setting is dependent on the server ODM, contact ODM/OEM to find out details. ### CPU Frequency Setting -Following script can be run to temporarily force the cores to run at max CPU frequency +The following script can be run to temporarily force the cores to run at max CPU frequency: ```bash #!/bin/bash @@ -77,15 +79,16 @@ done #### Change frequency scaling -There should be a way in the BIOS to disable frequency scaling. There's also a Linux option, described in the following. +Typically there is a way in the BIOS to disable frequency scaling. There's also a Linux option, described below: +```console cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor +``` +This command returns `ondemand` in the current setup. -return "ondemand" in the current setup. - -We may want to change the CPU governor from "ondemand" to "performance". +You may want to change the CPU governor from `ondemand` to `performance`. -The following instructions show how we do it: +The following instructions show you how to do it: ```bash sudo apt install cpufrequtils @@ -97,10 +100,12 @@ sudo systemctl disable ondemand #### Boot Parameters +The individual parameters are described below: + ##### Hugepage setting default_hugepagesz=1G hugepagesz=1G hugepages=32 -The huge page setting is required for DPDK support which is commonly adopted in 5G networking. It reduces the TLB misses, improves performance by lowering memory overhead, and improves memory efficiency with larger and contiguois memory allocation +The huge page setting is required for DPDK support which is commonly adopted in 5G networking. It reduces the TLB misses, improves performance by lowering memory overhead, and improves memory efficiency with larger and contiguous memory allocation. ##### isolcpus=cpuX-cpuY The setting "isolcpus" is a kernel command line parameter that is used to isolate specific CPUs in a multi-CPU system, such that they are not used for general purpose processing. Instead, they are reserved for real-time or other special purpose tasks that require dedicated and predictable processing resources. @@ -208,25 +213,28 @@ Low latency kernels are commonly used in applications that require real-time pro To achieve low latency, low latency kernels typically implement a number of changes to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. They may also make use of specialized scheduling algorithms and memory management techniques to further optimize performance. -How to install Lowlatency kernel on Arm server: +To install Low latency kernel on Arm server: -```bash +```console sudo apt update sudo apt-cache policy 'linux-image-5.*-lowlatency' sudo apt install linux-headers-5.15.0-46-lowlatency linux-image-5.15.0-46-lowlatency -y ``` -then run this follwoing script to verify the new low-latency kernel: +Then run this following script to verify the new low-latency kernel: +```console awk -F\' '$1=="menuentry " {print i++ " : " $2} $1=="submenu " {print i++ " : " $2; j=0} $1=="\tmenuentry " {print "\t", j++ " : " $2}' /boot/grub/grub.cfg +``` +The output from this command will look like: -it will show: - +```output 0 : Ubuntu, with Linux 5.15.0-46-lowlatency 1 : Ubuntu, with Linux 5.15.0-46-lowlatency (recovery mode) +``` -We will also need to disable upgrade OS kernel so the kernel will stay same version as we desire: +You will also need to disable upgrade OS kernel so the kernel will stay at the same version: -```bash +```console sudo vi /etc/apt/apt.conf.d/20auto-upgrades, change 1 to 0: APT::Periodic::Update-Package-Lists "0"; APT::Periodic::Unattended-Upgrade "0"; @@ -236,7 +244,7 @@ APT::Periodic::Unattended-Upgrade "0"; An RT kernel is designed to provide a guaranteed minimum response time for certain system events and processes, even under heavy load conditions. This is achieved through a number of optimizations to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. Additionally, RT kernels may make use of specialized scheduling algorithms and memory management techniques to further optimize performance. -At this moment, RT kernel is not ready from apt repository, we have to rebuild the kernel to enable RT for Arm. We will update as soob as it becomes widely available. +At this moment, RT kernel is not ready from apt repository, you will need to rebuild the kernel to enable RT for Arm. ### SR-IOV @@ -272,7 +280,7 @@ At this moment, RT kernel is not ready from apt repository, we have to rebuild t - Bare-Metal (SR-IOV VFs and Configuration map) -```bash +```console insertModules: - "i40e" - "vfio-pci" @@ -311,13 +319,13 @@ resourceConfig: SR-IOV Network Device Plugin supports running in a virtualized environment. However, not all device selectors are applicable as the VFs are passthrough to the VM without any association to their respective PF, hence any device selector that relies on the association between a VF and its PF will not work and therefore the pfNames and rootDevices extended selectors will not work in a virtual deployment. The common selector pciAddress can be used to select the virtual device. #### Configuring SR-IOV Resources -perform the following steps on the bare-metal host for enabling SR-IOV. +Perform the following steps on the bare-metal host for enabling SR-IOV. -Enable SR-IOV in BIOS settings. +- Enable SR-IOV in BIOS settings. -Execute the following command to append iommu.passthrough=1 option. GRUB_CMLINE_LINUX in /etc/default/grub +- Execute the following command to append iommu.passthrough=1 option. GRUB_CMLINE_LINUX in /etc/default/grub -Run the grub-mkconfig -o /boot/grub/grub.cfg script to update boot partitions grub configuration file. +- Run the grub-mkconfig -o /boot/grub/grub.cfg script to update boot partitions grub configuration file. Reboot Linux to reflect the changes. @@ -326,10 +334,14 @@ echo 2 > /sys/class/net//device/sriov_numvfs Execute the following command to check if the SR-IOV VFs interface names are displayed. -"lshw -c network -businfo" -NOTE: Execute the following command to remove the SR-IOV from the bare-metal host systems. +```console +lshw -c network -businfo +``` +Execute the following command to remove the SR-IOV from the bare-metal host systems. +```console echo 0 > /sys/class/net//device/sriov_numvfs +``` Type II - SR-IOV Configuration for Bare-metal Deployment Example of a SR-IOV deployment yaml file used for UPF @@ -338,8 +350,9 @@ Perform the following step to verify the SR-IOV CNI resource configuration. Execute the following command in the master node and confirm if allocatable resources were created on the worker node used for UPF deployment. +```console kubectl get node -o json | jq '.status.allocatable' - +``` This command displays the following sample output. ### PTP Setting @@ -347,9 +360,9 @@ In 5G world, the time synchronization is critical to synchronize all components #### GrandMaster Hardware based PTP -requires a GrandMaster Qulsar with GPS capability +This option requires a GrandMaster Qulsar with GPS capability -Also requires a PTP enabled switch, currently Arista switch seems to be able to have PTP sync'ed all slave nodes like Nvidia/Keysight. +Also requires a PTP enabled switch, currently Arista switch has PTP sync'ed to all slave nodes like Nvidia/Keysight. #### SW based PTP @@ -357,7 +370,7 @@ How to setup ptp4l/phc2sys on Linux: Configure Slave node setting: -```bash +```console $ cat /etc/5g-ptp.conf [global] verbose 1 @@ -385,7 +398,7 @@ network_transport L2 Setup Slave node PTP4L service: note that we need to assign a core for this task to run -```bash +```console $ cat /lib/systemd/system/ptp4l.service [Unit] Description=Precision Time Protocol (PTP) service @@ -400,7 +413,7 @@ WantedBy=multi-user.target ``` Setup Slave node PHC2SYS service: note the NIC interface used here is enP1p3s0f0np0, also assign a core to run it -```bash +```console $ cat /lib/systemd/system/phc2sys.service [Unit] Description=Synchronize system clock or PTP hardware clock (PHC) @@ -418,7 +431,7 @@ WantedBy=multi-user.target ``` Configure Master node setting: -```bash +```console $ cat /etc/5g-ptp.conf [global] verbose 1 @@ -441,7 +454,7 @@ network_transport L2 ``` Setup Master node PTP4L service: note that we need to assign a core for this task to run -```bash +```console $ cat /lib/systemd/system/ptp4l.service [Unit] Description=Precision Time Protocol (PTP) service @@ -458,7 +471,7 @@ WantedBy=multi-user.target ``` Setup Master node PHC2SYS service: note the NIC interface used here is enP1p3s0f0np0, also assign a core to run it -```bash +```console $ cat /lib/systemd/system/phc2sys.service [Unit] Description=Synchronize system clock or PTP hardware clock (PHC) @@ -479,7 +492,7 @@ WantedBy=multi-user.target Start PTP/PHC2SYS services: -```bash +```console #PTP.service sudo systemctl daemon-reload sudo systemctl restart ptp4l.service From 3ec387f5524bcbb174ae8fa9014fcdbc0d07c022 Mon Sep 17 00:00:00 2001 From: westlaker Date: Wed, 26 Jul 2023 12:24:05 -0500 Subject: [PATCH 10/20] added more Acronyms explaination. --- .../5g-on-arm/_index.md | 2 +- .../5g-on-arm/how-to-1.md | 10 ++--- .../5g-on-arm/how-to-2.md | 40 +++++++++---------- .../5g-on-arm/how-to-3.md | 2 +- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md index b4d6e3b69f..5ff383e503 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_index.md @@ -14,7 +14,7 @@ prerequisites: - Arm server - 5G ORAN stack -author_primary: Jici Gao +author_primary: Arm ### Tags skilllevels: Introductory diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index a1e135f44d..dfe31a9b05 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -33,13 +33,13 @@ layout: learningpathall #### Things to consider: -1P vs 2P systems: +1P (Single Processor) vs 2P (Dual Processors) systems: -For evaluation purposes, a 1P system is recommended which should be sufficient to run any 5G software stacks (L1/L2/L3) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, socket communication should be avoided, especially when some PCIe devices sit in a different node across from the CPU. +For evaluation purposes, a 1P system is recommended which should be sufficient to run any 5G software stacks (L1 physical layer/L2 datalink layer/L3 network layer) considering that the Arm CPU has at least 80 cores. 2P system would be great to run more processes on single box, however, socket communication should be avoided, especially when some PCIe devices sit in a different node across from the CPU. -1U vs 2U systems: +1U (One Rack Unit) vs 2U (Two Rack Units) systems: -1U is sufficient to run 5G CN Core software or pure 5G software stacks while 2U is required when you need additional hardware such as several PCIe cards or a full length PCIe card for L1 processing, for example, the Nvidia A100X and Genvisio. +1U is sufficient to run 5G CN (Core Network) software or pure 5G software stacks while 2U is required when you need additional hardware such as several PCIe cards or a full length PCIe card for L1 processing, for example, the Nvidia A100X and Genvisio. #### Accommodating PCIe Accelerators @@ -47,7 +47,7 @@ Due to the nature of PCIe devices, you need to carefully select the right Arm se For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's Arm server better accommodate some Inline L1 Accelerator cards. -For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC in one device, you need to make sure its NIC ports face out. At this moment, only Supermicro Arm server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. +For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC (Network Interface Card) in one device, you need to make sure its NIC ports face out. At this moment, only Supermicro Arm server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. #### 5G Ready on Arm diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index 92df55ac18..dc7541be1e 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -10,24 +10,24 @@ layout: learningpathall ### Firmware Maintenance -The BIOS firmware impacts some of the functionality and performance of the server. So it is important to sign up for firmware updates from your server vendors. +The BIOS (Basic Input/Output System) firmware impacts some of the functionality and performance of the server. So it is important to sign up for firmware updates from your server vendors. ### How to update firmware -#### Via Web UI +#### Via Web UI (User Interface) -Most of Arm servers provide web UI for managing the firmware. +Most of Arm servers provide web UI for managing the FW (firmware). -Log into BMC IP via browser, +Log into the BMC (Baseboard Management Controller) IP address via browser, 1. Check the current FW version by clicking its `Firmware information`. - 2. Go to the `Firmware maintenance` section and click on `Firmware update`. Follow te prompts to update BIOS/SCP and BMC software. + 2. Go to the `Firmware maintenance` section and click on `Firmware update`. Follow te prompts to update BIOSP and BMC software. #### Via ipmitool command line -The benefit of using `ipmitool` is for automation of controlling the servers remotely. +The benefit of using IPMI (Intelligent Platform Management Interface) tool `ipmitool` is for automation of controlling the servers remotely. To update the firmware, for example on the Foxconn server: @@ -41,12 +41,12 @@ To update the firmware, for example on the Foxconn server: ``` {{% notice Note %}} -The update of the firmware via IPMI interface is hardware dependent, check your server ODM/OEM for the IPMI detail for updating its firmware. +The update of the firmware via IPMI interface is hardware dependent, check your server ODM/OEM (Original Design/Equipment Manufacturer) for the IPMI detail for updating its firmware. {{% /notice %}} ### BIOS Setting -Typically SR-IOV enablement is required for Arm servers to support 5G deployment in a container environment. Through BIOS options, you can enable SR-IOV. +Typically SR-IOV (Single Root IO Virtualization) enablement is required for Arm servers to support 5G deployment in a container environment. Through BIOS options, you can enable SR-IOV. ### PCIe Setting @@ -105,7 +105,7 @@ The individual parameters are described below: ##### Hugepage setting default_hugepagesz=1G hugepagesz=1G hugepages=32 -The huge page setting is required for DPDK support which is commonly adopted in 5G networking. It reduces the TLB misses, improves performance by lowering memory overhead, and improves memory efficiency with larger and contiguous memory allocation. +The huge page setting is required for DPDK (Data Plane Development Kit) support which is commonly adopted in 5G networking. It reduces the TLB (Translation Lookaside Buffer) misses, improves performance by lowering memory overhead, and improves memory efficiency with larger and contiguous memory allocation. ##### isolcpus=cpuX-cpuY The setting "isolcpus" is a kernel command line parameter that is used to isolate specific CPUs in a multi-CPU system, such that they are not used for general purpose processing. Instead, they are reserved for real-time or other special purpose tasks that require dedicated and predictable processing resources. @@ -152,9 +152,9 @@ The "nosoftlockup" setting disables the software lockup detection mechanism. Whe It's important to carefully consider the impact of the "nosoftlockup" setting before using it in a production environment. Disabling the software lockup detection mechanism may result in the system hanging or becoming unresponsive if a software lockup occurs. This can cause data loss, corruption, or other serious problems. Additionally, it can make it more difficult to diagnose and resolve software lockup issues. As a result, it's generally recommended to leave the software lockup detection mechanism enabled, unless there is a specific reason to disable it. ##### iommu.passthrough=1 -The "iommu.passthrough=1" setting is used to enable IOMMU pass-through, which allows virtual machines to access physical devices without any virtualization of the I/O memory. +The "iommu.passthrough=1" setting is used to enable IOMMU (Input–Output Memory Management Unit) pass-through, which allows virtual machines to access physical devices without any virtualization of the I/O memory. -This setting is used in virtualization environments where it is important to have high performance I/O for certain applications or devices. For example, some graphics-intensive applications require direct access to the physical GPU, which can be achieved by enabling IOMMU pass-through. However, enabling IOMMU pass-through can also make the virtualization environment less secure, as it provides direct access to physical devices, bypassing any virtualization or security features. As a result, the setting is typically used with caution, and only in specific cases where the performance benefits outweigh the security risks. (from ChatGPT) +This setting is used in virtualization environments where it is important to have high performance I/O for certain applications or devices. For example, some graphics-intensive applications require direct access to the physical GPU, which can be achieved by enabling IOMMU pass-through. However, enabling IOMMU pass-through can also make the virtualization environment less secure, as it provides direct access to physical devices, bypassing any virtualization or security features. As a result, the setting is typically used with caution, and only in specific cases where the performance benefits outweigh the security risks. ##### cpufreq.off=1 The setting "cpufreq.off=1" is a kernel command line parameter that is used to disable the CPU frequency scaling feature in the Linux kernel. CPU frequency scaling, also known as CPU speed scaling, is a technique that allows the operating system to dynamically adjust the clock speed of the CPU based on the workload. This can help to conserve energy and reduce heat generation, while still providing sufficient performance for the current workload. @@ -178,7 +178,7 @@ modprobe.blacklist=cppc_cpufreq is a kernel parameter in Linux that blacklists t By blacklisting the cppc_cpufreq module, you are telling the Linux kernel not to load it at boot time. This means that the functionality provided by the module will not be available to the system. ##### tsc=reliable -The setting "tsc=reliable" is a kernel command line parameter that is used to set the time stamp counter (TSC) as a reliable source of time for the Linux kernel. The TSC is a CPU register that increments at a fixed rate based on the clock speed of the CPU. It is commonly used as a source of time for the operating system, as it provides a high-resolution and constant-rate timer that is usable even when other system timers are unavailable or unreliable. +The setting "tsc=reliable" is a kernel command line parameter that is used to set the Time Stamp Counter (TSC) as a reliable source of time for the Linux kernel. The TSC is a CPU register that increments at a fixed rate based on the clock speed of the CPU. It is commonly used as a source of time for the operating system, as it provides a high-resolution and constant-rate timer that is usable even when other system timers are unavailable or unreliable. The "tsc=reliable" setting is used to indicate that the TSC is a reliable source of time, and that it should be used as the primary source of time for the Linux kernel. This can be useful in some cases where the TSC is known to be reliable and accurate, as it can provide more consistent and accurate timing information for the operating system. @@ -240,7 +240,7 @@ APT::Periodic::Update-Package-Lists "0"; APT::Periodic::Unattended-Upgrade "0"; ``` -### RT Kernel +### RT (Rime-Time) Kernel An RT kernel is designed to provide a guaranteed minimum response time for certain system events and processes, even under heavy load conditions. This is achieved through a number of optimizations to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. Additionally, RT kernels may make use of specialized scheduling algorithms and memory management techniques to further optimize performance. @@ -248,7 +248,7 @@ At this moment, RT kernel is not ready from apt repository, you will need to reb ### SR-IOV -#### Supported SR-IOV NICs +#### Supported SR-IOV NICs (Network Interface Cards) Intel® Ethernet 800 Series (E810) @@ -272,11 +272,11 @@ At this moment, RT kernel is not ready from apt repository, you will need to reb #### Creating SR-IOV Resources - Creating SR-IOV virtual functions (VFs) + Creating SR-IOV Virtual Functions (VFs) Generation SR-IOV configuration maps for kubernetes - - VM based (K8s configuration map for corresponding worker node) + - VM (Virtual Machine) based (K8s configuration map for corresponding worker node) - Bare-Metal (SR-IOV VFs and Configuration map) @@ -316,7 +316,7 @@ resourceConfig: #### Configure Device Plugin extended selectors in virtual environments -SR-IOV Network Device Plugin supports running in a virtualized environment. However, not all device selectors are applicable as the VFs are passthrough to the VM without any association to their respective PF, hence any device selector that relies on the association between a VF and its PF will not work and therefore the pfNames and rootDevices extended selectors will not work in a virtual deployment. The common selector pciAddress can be used to select the virtual device. +SR-IOV Network Device Plugin supports running in a virtualized environment. However, not all device selectors are applicable as the VFs (Virtual Functions) are passthrough to the VM (Virtual Machine) without any association to their respective PF (Physical Function), hence any device selector that relies on the association between a VF and its PF will not work and therefore the pfNames and rootDevices extended selectors will not work in a virtual deployment. The common selector pciAddress can be used to select the virtual device. #### Configuring SR-IOV Resources Perform the following steps on the bare-metal host for enabling SR-IOV. @@ -343,10 +343,10 @@ Execute the following command to remove the SR-IOV from the bare-metal host syst echo 0 > /sys/class/net//device/sriov_numvfs ``` Type II - SR-IOV Configuration for Bare-metal Deployment -Example of a SR-IOV deployment yaml file used for UPF +Example of a SR-IOV deployment yaml file used for UPF (User Plane Function) ### SR-IOV CNI Resource Verification -Perform the following step to verify the SR-IOV CNI resource configuration. +Perform the following step to verify the SR-IOV CNI (Cloud Native Interface) resource configuration. Execute the following command in the master node and confirm if allocatable resources were created on the worker node used for UPF deployment. @@ -355,7 +355,7 @@ kubectl get node -o json | jq '.status.allocatable' ``` This command displays the following sample output. -### PTP Setting +### PTP (Precision Time Protocol) Setting In 5G world, the time synchronization is critical to synchronize all components in same timing cadence #### GrandMaster Hardware based PTP diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md index ea9dfbc7ae..c6891abc73 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -20,7 +20,7 @@ Core isolation from Linux scheduler, Disabling interrupts, Huge page usage to mi Make sure that each memory channel has at least one memory DIMM inserted with 8GB memory size. -Enable cache stashing on ampere to stash the packets coming through NIC to SLC cache. +Enable cache stashing on ampere to stash the packets coming through NIC to SLC cache (System Level Cache, aks Level 3 Cache). Cache stashing feature enable/disable on Ampere dynamically (No reboot required). slc_inst_s0 script can be found in attachment: slc_inst_s0 ```bash From a20c6986378d92d6d78d8ea1751e7a8c26be1ac0 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 09:22:22 -0400 Subject: [PATCH 11/20] Update _next-steps.md --- .../servers-and-cloud-computing/5g-on-arm/_next-steps.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md index 865026f0bd..a77f7bf657 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_next-steps.md @@ -3,7 +3,7 @@ # Edit # ================================================================================ -next_step_guidance: Please contact us for futher discussion for your need for 5G on Arm. +next_step_guidance: As a next step, we recommend the learning path about getting started with the Arm 5G RAN Acceleration Library. recommended_path: "/learning-paths/servers-and-cloud-computing/ran/" # Link to the next learning path being recommended. From 28436fe96447e90e643602819d4d6dd711208f5c Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 09:27:53 -0400 Subject: [PATCH 12/20] Update _review.md --- .../5g-on-arm/_review.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md index 7249d7bca2..92c291ca0d 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/_review.md @@ -25,24 +25,24 @@ review: The Supermicro server is designed to support A100X with correct orientation for its ethernet ports - questions: question: > - Which statement is corrected? + Which statement is correct? answers: - - "Regular Linux kernel should be enough for supporting 5G stack" - - "Low latency Linux kernel must be required for supporting 5G stack" + - "Regular Linux kernel is sufficient for supporting the 5G stack" + - "Low latency Linux kernel is required for supporting 5G stack" correct_answer: 2 explanation: > - Low Latency Kernel minimize the time it takes for the operating system to respond to events and processes, it is essential for latency sensitive of processes in 5G stack. + Low Latency Kernel minimizes the time it takes for the operating system to respond to events and processes, it is essential for the latency sensitive processes in the 5G stack. - questions: question: > - What is potential issue with a 2P server? + What is the potential issue with a 2P server? answers: - "PCIe devices sit on different node from the CPU" - "Cross socket communication overhead" - - "Sometime can't put PCIe device and CPU on same node" + - "Sometimes you can't put the PCIe device and CPU on same node" - "All the above" correct_answer: 4 explanation: > - Explain all potential issues related to multiple socket server + These are all potential issues related to multiple socket servers. - questions: question: > What is isolcpus? @@ -52,10 +52,10 @@ review: - "Reserved for real-time or other special purpose tasks" correct_answer: 3 explanation: > - Explain isolcpus setting is important for the dedicated tasks not interfered by the kernel + The isolcpus setting is important for dedicated tasks that are not interfered by the kernel - questions: question: > - Which way to affinitize your program to cores? + How do you affinitize your program to cores? answers: - "taskset" - "numactl" @@ -63,7 +63,7 @@ review: - "All the above" correct_answer: 4 explanation: > - Explain the ways of how to assign your program to cpu/cores + These are all the ways you can assign your program to cpu/cores From 32ca22250796d4be140a0dd133b32eae824c47d4 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 09:35:34 -0400 Subject: [PATCH 13/20] Update how-to-1.md --- .../servers-and-cloud-computing/5g-on-arm/how-to-1.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index dfe31a9b05..bb8fa286e8 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -8,6 +8,7 @@ layout: learningpathall ## Choose the appropriate Arm 5G Servers --- +It is essential to choose the appropriate Arm server to run the 5G stack on. In this section, you can learn about the different configurations of Arm servers the 5G stack has been developed and tested on. You can also learn about what needs to be considered when choosing the appropriate Arm server for this task. #### Extensive 5G development and testing on Arm servers has been done on the hardware configurations and software components listed below: @@ -45,7 +46,7 @@ For evaluation purposes, a 1P system is recommended which should be sufficient t Due to the nature of PCIe devices, you need to carefully select the right Arm server to accommodate the PCIe Accelerators you are going to use. -For full length PCIe cards, you need at least an 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's Arm server better accommodate some Inline L1 Accelerator cards. +For full length PCIe cards, you need at least a 2U server, however not every 2U server will support full length/full width PCIe devices. Also, some PCIe full profile devices require more power to run properly, for example, Supermicro's Arm server better accommodate some Inline L1 Accelerator cards. For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC (Network Interface Card) in one device, you need to make sure its NIC ports face out. At this moment, only Supermicro Arm server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. From 6ecd8aa905f26d4886e1fe93b6b5e8cf07aa12aa Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 10:03:25 -0400 Subject: [PATCH 14/20] Update how-to-2.md --- .../5g-on-arm/how-to-2.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index dc7541be1e..ce333c24e9 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -16,20 +16,20 @@ The BIOS (Basic Input/Output System) firmware impacts some of the functionality #### Via Web UI (User Interface) -Most of Arm servers provide web UI for managing the FW (firmware). +Most of Arm servers provide web UI for managing the firmware (FW). -Log into the BMC (Baseboard Management Controller) IP address via browser, +Log into the Baseboard Management Controller (BMC) IP address via your browser, 1. Check the current FW version by clicking its `Firmware information`. - 2. Go to the `Firmware maintenance` section and click on `Firmware update`. Follow te prompts to update BIOSP and BMC software. + 2. Go to the `Firmware maintenance` section and click on `Firmware update`. Follow the prompts to update BIOSP and BMC software. #### Via ipmitool command line -The benefit of using IPMI (Intelligent Platform Management Interface) tool `ipmitool` is for automation of controlling the servers remotely. +The benefit of using the Intelligent Platform Management Interface (IPMI) tool `ipmitool` is for automation of controlling the servers remotely. -To update the firmware, for example on the Foxconn server: + For example on the Foxconn server, run the following commands to update the firmware: ```console ipmitool -C 3 -I lanplus -H 10.118.45.98 -U admin -P admin raw 0x32 0x8f 0x3 0x1 @@ -46,17 +46,17 @@ The update of the firmware via IPMI interface is hardware dependent, check your ### BIOS Setting -Typically SR-IOV (Single Root IO Virtualization) enablement is required for Arm servers to support 5G deployment in a container environment. Through BIOS options, you can enable SR-IOV. +Typically, Single Root IO Virtualization (SR-IOV) enablement is required for Arm servers to support 5G deployment in a container environment. Through BIOS options, you can enable SR-IOV. ### PCIe Setting -This setting is dependent on the server ODM, contact ODM/OEM to find out details. +This setting is dependent on the server ODM, contact the ODM/OEM to find out details. ### CPU Frequency Setting The following script can be run to temporarily force the cores to run at max CPU frequency: -```bash +```console #!/bin/bash for ((i=0;i<`nproc`;i++)) @@ -88,9 +88,9 @@ This command returns `ondemand` in the current setup. You may want to change the CPU governor from `ondemand` to `performance`. -The following instructions show you how to do it: +The following commands show you how to do it: -```bash +```console sudo apt install cpufrequtils echo 'GOVERNOR="performance"' | sudo tee /etc/default/cpufrequtils sudo systemctl disable ondemand @@ -100,7 +100,7 @@ sudo systemctl disable ondemand #### Boot Parameters -The individual parameters are described below: +The individual Linux kernel command-line parameters that impact running the 5G stack are described below: ##### Hugepage setting default_hugepagesz=1G hugepagesz=1G hugepages=32 @@ -213,7 +213,7 @@ Low latency kernels are commonly used in applications that require real-time pro To achieve low latency, low latency kernels typically implement a number of changes to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. They may also make use of specialized scheduling algorithms and memory management techniques to further optimize performance. -To install Low latency kernel on Arm server: +To install a Llw latency kernel on Arm server, run the following commands: ```console sudo apt update @@ -240,7 +240,7 @@ APT::Periodic::Update-Package-Lists "0"; APT::Periodic::Unattended-Upgrade "0"; ``` -### RT (Rime-Time) Kernel +### Real-Time (RT) Kernel An RT kernel is designed to provide a guaranteed minimum response time for certain system events and processes, even under heavy load conditions. This is achieved through a number of optimizations to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. Additionally, RT kernels may make use of specialized scheduling algorithms and memory management techniques to further optimize performance. From b39feb5bcada1ac2e153e7be8e096dd16b59f586 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 10:09:32 -0400 Subject: [PATCH 15/20] Update how-to-3.md --- .../5g-on-arm/how-to-3.md | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md index c6891abc73..efe693942d 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -10,7 +10,7 @@ layout: learningpathall ### Kernel Tuning -Please refer to early section of setting kernel boot arguments and the CPU scaling. +For kernel tuning, please refer to the previous section on setting kernel boot arguments and CPU scaling. ### DPDK Tuning @@ -23,13 +23,14 @@ Make sure that each memory channel has at least one memory DIMM inserted with 8G Enable cache stashing on ampere to stash the packets coming through NIC to SLC cache (System Level Cache, aks Level 3 Cache). Cache stashing feature enable/disable on Ampere dynamically (No reboot required). slc_inst_s0 script can be found in attachment: slc_inst_s0 -```bash +```console ./slc_inst_s0 1 -- enable SLC installation for all root ports on Socket 0 ./slc_inst_s0 0 -- disable SLC installation for all root ports on Socket 0 ``` -Here is the grub settings from Ampere machine: -```bash +Here are the grub settings from Ampere machine: + +```console # cat /proc/cmdline BOOT_IMAGE=/boot/vmlinuz-5.15.0-46-lowlatency root=UUID=c0ef447a-8367-4d45-8991-47ece2fcb425 ro iommu.passthrough=1 default_hugepagesz=1G hugepagesz=1G hugepages=20 isolcpus=1-69 irqaffinity=0 rcu_nocbs=1-69 nohz_full=1-69 kpti=off nosoftlockup ``` @@ -46,7 +47,7 @@ Refer the same link above to run test pmd on Ampere. This will ensure that the E ### Allocate Cores to Different Tasks -It is critical to be carefully alloacting the cores to various tasks for 5G stack. Make sure all of processes have their own cores to run on, not to step over each other. +It is critical to carefully alloacte the cores to various tasks for 5G stack. Make sure all of processes have their own cores to run on, not to step over each other. For multiple socket server, always use numactl to launch your program to associate with the cpu and the PCIe device your program will access on same node. @@ -58,8 +59,9 @@ Use taskset command to launch your program on specific cores. - "perf record & report" can be used to identify bottleneck based on events on specific core - "perf stat" can be used to statistically measure the KPIs like IPC, Front End/Back End Stalls and L1/L2/LLC Cache misses. - - Perf script to run on Neoverse based system is attached here: -```bash + - Perf script to run on an Arm Neoverse based system is shown below: + +```console #!/bin/bash # This script must be run for every use case captured in SoW. This script captures the perf events on all the CU and DU DPDK and worker cores for 10 sec at 100 msec interval. if [ $# -eq 0 ]; then @@ -195,11 +197,10 @@ perf stat -A --output $1_Inst_Spec_BR_$cu_worker_core_b.txt -e r8,r1b,r78,r79,r7 #### Take Advantage of Arm RAL and SVE/NEON Arm 5G RAN Acceleration Library (ArmRAL): -https://learn.arm.com/learning-paths/servers-and-cloud-computing/ran/ +To learn more about getting started with the Arm 5G RAM Acceleration Library, refer to this [learning path] +(https://learn.arm.com/learning-paths/servers-and-cloud-computing/ran/) Port Code to Arm Scalable Vector Extension (SVE) -https://learn.arm.com/learning-paths/servers-and-cloud-computing/sve/ +To learn about porting your code to use Arm SVE, refer to this [learning path] (https://learn.arm.com/learning-paths/servers-and-cloud-computing/sve/) -#### Using Other Arm Tools -Refer this link https://developer.arm.com/Tools%20and%20Software/Streamline%20Performance%20Analyzer for Streamline Performance Analyzer From 86f846dc4af70e92f5cb579e7ff5a3b596088bf8 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 10:21:56 -0400 Subject: [PATCH 16/20] Update how-to-1.md --- .../servers-and-cloud-computing/5g-on-arm/how-to-1.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md index bb8fa286e8..8c49f9e121 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-1.md @@ -50,13 +50,7 @@ For full length PCIe cards, you need at least a 2U server, however not every 2U For PCIe cards taking up two PCIe slots like Nvidia A100X, not every 2U will be right choice. Because A100X combines GPU and Mellanox NIC (Network Interface Card) in one device, you need to make sure its NIC ports face out. At this moment, only Supermicro Arm server has designed its server with the full length PCIe cards such as Nvidia A100X converged card. -#### 5G Ready on Arm -This learning path attempts to provide the guidance to make Arm servers ready for 5G development and deployment: - - 1. Using existing PoC example to run the qualification to see if the server can pass with automation - - 2. Using Performance PoC example to test to see if the server can meet performance goals with automation From aa08f981b0fa45fb26c6e56133a561a4fc2f372f Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 11:04:02 -0400 Subject: [PATCH 17/20] Update how-to-2.md --- .../5g-on-arm/how-to-2.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index ce333c24e9..af4fa4bb1e 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -7,12 +7,14 @@ layout: learningpathall --- ## How to Setup and Configure 5G Servers +In this section you get an overview of the different settings used to configure your Arm server. ### Firmware Maintenance The BIOS (Basic Input/Output System) firmware impacts some of the functionality and performance of the server. So it is important to sign up for firmware updates from your server vendors. ### How to update firmware +Lets look at the different ways you can update the firmware on your server. #### Via Web UI (User Interface) @@ -213,7 +215,7 @@ Low latency kernels are commonly used in applications that require real-time pro To achieve low latency, low latency kernels typically implement a number of changes to the standard Linux kernel, such as reducing the frequency of interrupts, minimizing the number of context switches, and reducing the amount of time spent processing system calls. They may also make use of specialized scheduling algorithms and memory management techniques to further optimize performance. -To install a Llw latency kernel on Arm server, run the following commands: +To install a low latency kernel on Arm server, run the following commands: ```console sudo apt update @@ -274,7 +276,7 @@ At this moment, RT kernel is not ready from apt repository, you will need to reb Creating SR-IOV Virtual Functions (VFs) - Generation SR-IOV configuration maps for kubernetes + Generation of SR-IOV configuration maps for kubernetes - VM (Virtual Machine) based (K8s configuration map for corresponding worker node) - Bare-Metal (SR-IOV VFs and Configuration map) @@ -356,7 +358,7 @@ kubectl get node -o json | jq '.status.allocatable' This command displays the following sample output. ### PTP (Precision Time Protocol) Setting -In 5G world, the time synchronization is critical to synchronize all components in same timing cadence +In 5G world, the time synchronization is critical to synchronize all components in same timing cadence. #### GrandMaster Hardware based PTP @@ -368,7 +370,7 @@ Also requires a PTP enabled switch, currently Arista switch has PTP sync'ed to a How to setup ptp4l/phc2sys on Linux: -Configure Slave node setting: +Configure Slave node setting as shown below: ```console $ cat /etc/5g-ptp.conf @@ -396,7 +398,7 @@ delay_mechanism E2E network_transport L2 ``` -Setup Slave node PTP4L service: note that we need to assign a core for this task to run +Setup Slave node PTP4L service: note that you need to assign a core for this task to run ```console $ cat /lib/systemd/system/ptp4l.service @@ -413,6 +415,7 @@ WantedBy=multi-user.target ``` Setup Slave node PHC2SYS service: note the NIC interface used here is enP1p3s0f0np0, also assign a core to run it + ```console $ cat /lib/systemd/system/phc2sys.service [Unit] From 86134009662925fc578e7958ef084aae99cb9a28 Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 11:17:18 -0400 Subject: [PATCH 18/20] Update how-to-2.md --- .../servers-and-cloud-computing/5g-on-arm/how-to-2.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index af4fa4bb1e..77629b8a77 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -327,7 +327,7 @@ Perform the following steps on the bare-metal host for enabling SR-IOV. - Execute the following command to append iommu.passthrough=1 option. GRUB_CMLINE_LINUX in /etc/default/grub -- Run the grub-mkconfig -o /boot/grub/grub.cfg script to update boot partitions grub configuration file. +- Run the `grub-mkconfig -o /boot/grub/grub.cfg` script to update boot partitions grub configuration file. Reboot Linux to reflect the changes. @@ -370,7 +370,7 @@ Also requires a PTP enabled switch, currently Arista switch has PTP sync'ed to a How to setup ptp4l/phc2sys on Linux: -Configure Slave node setting as shown below: +Configure the slave node with the contents of the configuration file shown below: ```console $ cat /etc/5g-ptp.conf From 04723073dd3e533c706f8a6ee12560bc65c6f30c Mon Sep 17 00:00:00 2001 From: pareenaverma Date: Fri, 28 Jul 2023 11:23:01 -0400 Subject: [PATCH 19/20] Update how-to-3.md --- .../5g-on-arm/how-to-3.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md index efe693942d..58fd9cfd5d 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -39,19 +39,19 @@ Here are the grub settings from Ampere machine: DPDK Ethernet PMDs does the packet processing in a batch of 32 packets. Tuning the NIC Rx/Tx threshold through DPDK EAL config might provide performance boost. -Refer testpmd application https://doc.dpdk.org/guides/testpmd_app_ug/run_app.html and look for "–rxd" and "–txd" options to set the descriptor threshold. +Refer to `testpmd` application at https://doc.dpdk.org/guides/testpmd_app_ug/run_app.html and look for "–rxd" and "–txd" options to set the descriptor threshold. -#### Before running the 5G stack, check whether maximum no drop rate is achieved using DPDK test pmd application. +#### Before running the 5G stack, check whether maximum no drop rate is achieved using DPDK testpmd application. -Refer the same link above to run test pmd on Ampere. This will ensure that the Ethernet PMD is operating at optimal performance. +Refer to the same link above to run `testpmd` on Ampere. This will ensure that the Ethernet PMD is operating at optimal performance. ### Allocate Cores to Different Tasks -It is critical to carefully alloacte the cores to various tasks for 5G stack. Make sure all of processes have their own cores to run on, not to step over each other. +It is critical to carefully allocate the cores to various tasks for 5G stack. Make sure all of processes have their own cores to run on, not to step over each other. -For multiple socket server, always use numactl to launch your program to associate with the cpu and the PCIe device your program will access on same node. +For multiple socket server, always use `numactl` to launch your program to associate with the cpu and the PCIe device your program will access on same node. -Use taskset command to launch your program on specific cores. +Use `taskset` command to launch your program on specific cores. ### Profiling/Tracing @@ -59,7 +59,8 @@ Use taskset command to launch your program on specific cores. - "perf record & report" can be used to identify bottleneck based on events on specific core - "perf stat" can be used to statistically measure the KPIs like IPC, Front End/Back End Stalls and L1/L2/LLC Cache misses. - - Perf script to run on an Arm Neoverse based system is shown below: + +An example perf script that is run on an Arm Neoverse based server is shown below: ```console #!/bin/bash From 7e2eee0767e12aa2b079e41ab874d119c1e22e53 Mon Sep 17 00:00:00 2001 From: westlaker Date: Fri, 28 Jul 2023 12:54:28 -0500 Subject: [PATCH 20/20] more modification based on feedback from Pareena --- .../5g-on-arm/how-to-2.md | 419 +++++++++++++++++- .../5g-on-arm/how-to-3.md | 2 +- 2 files changed, 413 insertions(+), 8 deletions(-) diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md index 77629b8a77..7dc1f55fcc 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-2.md @@ -276,12 +276,12 @@ At this moment, RT kernel is not ready from apt repository, you will need to reb Creating SR-IOV Virtual Functions (VFs) - Generation of SR-IOV configuration maps for kubernetes + Generation of SR-IOV configuration maps for Kubernetes - VM (Virtual Machine) based (K8s configuration map for corresponding worker node) - Bare-Metal (SR-IOV VFs and Configuration map) - + Example of SR-IOV configuration map for Kubernetes: ```console insertModules: - "i40e" @@ -325,7 +325,7 @@ Perform the following steps on the bare-metal host for enabling SR-IOV. - Enable SR-IOV in BIOS settings. -- Execute the following command to append iommu.passthrough=1 option. GRUB_CMLINE_LINUX in /etc/default/grub +- Append iommu.passthrough=1 option to GRUB_CMLINE_LINUX line in /etc/default/grub - Run the `grub-mkconfig -o /boot/grub/grub.cfg` script to update boot partitions grub configuration file. @@ -334,7 +334,7 @@ Reboot Linux to reflect the changes. #### Type I - SR-IOV Configuration for VM Deployment echo 2 > /sys/class/net//device/sriov_numvfs -Execute the following command to check if the SR-IOV VFs interface names are displayed. +Execute the following command to check if the SR-IOV VFs interface names are displayed: ```console lshw -c network -businfo @@ -345,17 +345,422 @@ Execute the following command to remove the SR-IOV from the bare-metal host syst echo 0 > /sys/class/net//device/sriov_numvfs ``` Type II - SR-IOV Configuration for Bare-metal Deployment -Example of a SR-IOV deployment yaml file used for UPF (User Plane Function) +Example of a SR-IOV deployment yaml file used for UPF (User Plane Function): + +```console +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "1" + meta.helm.sh/release-name: upfnf + meta.helm.sh/release-namespace: radisys-upf1 + creationTimestamp: "2023-04-28T09:51:46Z" + generation: 1 + labels: + app: upf + app.kubernetes.io/managed-by: Helm + svc: upf + name: upf + namespace: radisys-upf1 + resourceVersion: "11234556" + uid: 1c98264a-93bf-4ac7-b807-5187779495dc +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: upf + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + annotations: + k8s.v1.cni.cncf.io/networks: '[ { "name": "upf1-ngu-interface", "interface": + "net1" }, { "name": "upf1-n6-interface", "interface": "net2" } ]' + myLivenessPath: /fgc-livez + myProbePort: "8090" + myReadinessPath: /fgc-readyz + creationTimestamp: null + labels: + app: upf + spec: + containers: + - command: + - /bin/bash + - -c + - sysctl -w net.ipv6.conf.all.disable_ipv6=0;sysctl -w net.ipv4.conf.all.arp_ignore=1;/root/bin/upf + env: + - name: INTF_NAME + value: eth0 + - name: LOG_OUTPUT + value: STDOUT + - name: MY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MY_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: MY_POD_LIVENESS_PATH + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['myLivenessPath'] + - name: MY_POD_READINESS_PATH + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['myReadinessPath'] + - name: MY_POD_PROBE_PORT + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['myProbePort'] + image: docker.io/library/upfsp:4.0.2-upfarm + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /fgc-livez + port: 8090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: upfsp + readinessProbe: + failureThreshold: 3 + httpGet: + path: /fgc-readyz + port: 8090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + hugepages-1Gi: 2Gi + memory: 1Gi + requests: + cpu: 500m + hugepages-1Gi: 2Gi + memory: 1Gi + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/podinfo + name: podinfo + - mountPath: /dev/shm + name: shm + - mountPath: /dev/hugepages + name: hugepage + - mountPath: /run/vpp + name: upf-memif + - mountPath: /root/config + name: oam-config-volume + - mountPath: /etc/fgc/upf + name: day0-config-volume + - mountPath: /var/local/core-dumps + name: core-path + - command: + - /bin/bash + - -c + - sysctl -w net.ipv6.conf.all.autoconf=0;sysctl -w net.ipv6.conf.default.autoconf=0;/opt/upf_fp/bin/upffpmgr + env: + - name: INTF_NAME + value: eth0 + - name: nguIntType + value: sriov + - name: n6IntType + value: sriov + - name: nguResourcePrefix + value: intel.com + - name: nguResourceName + value: sriov_netdevice_ngu_upf + - name: nguPciDeviceName + value: PCIDEVICE_INTEL_COM_SRIOV_NETDEVICE_NGU_UPF + - name: n6ResourcePrefix + value: intel.com + - name: n6ResourceName + value: sriov_netdevice_n6_upf + - name: n6PciDeviceName + value: PCIDEVICE_INTEL_COM_SRIOV_NETDEVICE_N6_UPF + - name: LOG_OUTPUT + value: STDOUT + - name: LD_LIBRARY_PATH + value: /opt/upf_fp/vpp/lib/:/lib/x86_64-linux-gnu + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/upf_fp/vpp/bin + image: docker.io/library/upffp:4.0.2-upfarm + imagePullPolicy: IfNotPresent + name: upffp + resources: + limits: + cpu: "4" + hugepages-1Gi: 3Gi + intel.com/sriov_netdevice_n6_upf: "1" + intel.com/sriov_netdevice_ngu_upf: "1" + memory: 1Gi + requests: + cpu: "4" + hugepages-1Gi: 3Gi + intel.com/sriov_netdevice_n6_upf: "1" + intel.com/sriov_netdevice_ngu_upf: "1" + memory: 1Gi + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /lib/modules + name: modules + - mountPath: /dev/vfio + name: dev + - mountPath: /sys/devices + name: devices + - mountPath: /sys/bus/pci + name: pci + - mountPath: /sys/module + name: sysmodule + - mountPath: /etc/podinfo + name: podinfo + - mountPath: /run/vpp + name: upf-memif + - mountPath: /dev/hugepages + name: hugepage + - mountPath: /dev/shm + name: shm + - mountPath: /opt/upf_fp/conf + name: config-volume + - mountPath: /opt/upf_fp/config + name: oam-config-volume + - mountPath: /var/local/core-dumps + name: core-path + - command: + - /bin/bash + - -c + - rsyslogd -n + env: + - name: INTF_NAME + value: eth0 + - name: LOG_OUTPUT + value: STDOUT + - name: MY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MY_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + image: localhost:5000/upfrsyslog:v1 + imagePullPolicy: IfNotPresent + name: upfrsyslog + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log/upfv3 + name: upfv3-syslog + - mountPath: /etc/rsyslog.d + name: rsyslog-config-volume + dnsPolicy: ClusterFirst + nodeName: node1 + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /lib/modules + type: "" + name: modules + - hostPath: + path: /dev/vfio + type: "" + name: dev + - hostPath: + path: /sys/devices + type: "" + name: devices + - hostPath: + path: /sys/bus/pci + type: "" + name: pci + - hostPath: + path: /sys/module + type: "" + name: sysmodule + - downwardAPI: + defaultMode: 420 + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: labels + - fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations + path: annotations + name: podinfo + - emptyDir: {} + name: upf-memif + - hostPath: + path: /dev/shm/ + type: "" + name: shm + - emptyDir: + medium: HugePages + name: hugepage + - configMap: + defaultMode: 420 + name: upf-radisys-upf1-configmap + name: config-volume + - configMap: + defaultMode: 420 + name: upf-radisys-upf1-oam-configmap + name: oam-config-volume + - configMap: + defaultMode: 420 + name: upf-radisys-upf1-day0-configmap + name: day0-config-volume + - configMap: + defaultMode: 420 + name: upf-radisys-upf1-rsyslog-configmap + name: rsyslog-config-volume + - hostPath: + path: /var/local/core-dumps + type: "" + name: core-path + - hostPath: + path: /var/log/upfv3 + type: "" + name: upfv3-syslog +status: + conditions: + - lastTransitionTime: "2023-04-28T09:51:46Z" + lastUpdateTime: "2023-04-28T09:51:46Z" + message: Deployment does not have minimum availability. + reason: MinimumReplicasUnavailable + status: "False" + type: Available + - lastTransitionTime: "2023-04-28T10:01:47Z" + lastUpdateTime: "2023-04-28T10:01:47Z" + message: ReplicaSet "upf-5bf69d8f7c" has timed out progressing. + reason: ProgressDeadlineExceeded + status: "False" + type: Progressing + observedGeneration: 1 + replicas: 1 + unavailableReplicas: 1 + updatedReplicas: 1 +``` + +```console +git clone https://github.com/k8snetworkplumbingwg/sriov-cni +kubectl apply -f sriov-cni/images/k8s-v1.16/sriov-cni-daemonset.yaml +``` + +sriov-cni-daemonset.yaml: +```console +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-sriov-cni-ds-arm64 + namespace: kube-system + labels: + tier: node + app: sriov-cni +spec: + selector: + matchLabels: + name: sriov-cni + template: + metadata: + labels: + name: sriov-cni + tier: node + app: sriov-cni + spec: + nodeSelector: + kubernetes.io/arch: arm64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: kube-sriov-cni-arm64 + image: localhost:5000/sriov-cni-arm64:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin +``` ### SR-IOV CNI Resource Verification Perform the following step to verify the SR-IOV CNI (Cloud Native Interface) resource configuration. -Execute the following command in the master node and confirm if allocatable resources were created on the worker node used for UPF deployment. +Execute the following command in the master node and confirm if allocatable resources were created on the worker node used for UPF deployment: ```console kubectl get node -o json | jq '.status.allocatable' ``` -This command displays the following sample output. +This command displays the following sample output: +```console +{ + "cpu": "31900m", + "ephemeral-storage": "189274027310", + "hugepages-1Gi": "10Gi", + "memory": "120754964Ki", + "pods": "110", + "arm.com/intel_sriov_netdevice_upf_n6": "1", + "arm.com/intel_sriov_netdevice_upf_ngu": "1" +} +``` ### PTP (Precision Time Protocol) Setting In 5G world, the time synchronization is critical to synchronize all components in same timing cadence. diff --git a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md index 58fd9cfd5d..32e3619f82 100644 --- a/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md +++ b/content/learning-paths/servers-and-cloud-computing/5g-on-arm/how-to-3.md @@ -22,7 +22,7 @@ Make sure that each memory channel has at least one memory DIMM inserted with 8G Enable cache stashing on ampere to stash the packets coming through NIC to SLC cache (System Level Cache, aks Level 3 Cache). -Cache stashing feature enable/disable on Ampere dynamically (No reboot required). slc_inst_s0 script can be found in attachment: slc_inst_s0 +Cache stashing feature enable/disable on Ampere dynamically (No reboot required). slc_inst_s0 is a program can be available upon request: ```console ./slc_inst_s0 1 -- enable SLC installation for all root ports on Socket 0 ./slc_inst_s0 0 -- disable SLC installation for all root ports on Socket 0