From 1bdb69dc0761e614abf5a163d3a6272ba15e0750 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Tue, 27 Jul 2021 11:36:56 -0500 Subject: [PATCH 01/54] Issue #6544 - Fixing broken `jetty.gzip.excludedMimeTypeList` property support + Adding GzipHandler tests + Adding Gzip module tests + Updating jetty-gzip.xml for includedMimeTypesList and excludedMimeTypesList behavior + Adding GzipHandler support for setIncludedMimeTypesList(String) and setExcludedMimeTypesList(String Signed-off-by: Joakim Erdfelt --- .../src/main/webapp/WEB-INF/web.xml | 5 + .../src/main/webapp/jetty.png | Bin 0 -> 5465 bytes .../src/main/webapp/jetty.webp | Bin 0 -> 3534 bytes .../src/main/config/etc/jetty-gzip.xml | 4 +- .../server/handler/gzip/GzipHandler.java | 22 +++ .../jetty/servlet/GzipHandlerTest.java | 103 ++++++++++-- .../tests/distribution/GzipModuleTests.java | 147 ++++++++++++++++++ 7 files changed, 265 insertions(+), 16 deletions(-) create mode 100644 demos/demo-simple-webapp/src/main/webapp/jetty.png create mode 100644 demos/demo-simple-webapp/src/main/webapp/jetty.webp create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java diff --git a/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml b/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml index da1263c1b8d7..1aebcaace5ce 100644 --- a/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml +++ b/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml @@ -6,4 +6,9 @@ Simple Web Application + + webp + image/webp + + diff --git a/demos/demo-simple-webapp/src/main/webapp/jetty.png b/demos/demo-simple-webapp/src/main/webapp/jetty.png new file mode 100644 index 0000000000000000000000000000000000000000..d579fffddfe176f020f7f74bdc663195fd39697f GIT binary patch literal 5465 zcmV-f6{hNmP)r1pz?>)F1TtVCnd}LfB{RKL zo%hGB>ZzHj?y1||!ASb|`SfRcs_NFMuDa)*d(QctTcWB|QAHKui2b84po%K?WK5siKM^s47SmRTM#@+!Kh%RNzpHCJH>Fs@uwyUB%uB5t#>!5834c zPpayMkTw&6qYIpJjjApU-J_}{e}M0J7p0b;w}2zckzM`leIG!8#|G(J>wyFNwfiEF zEz-sT&JL4VvIP;5ByjPBl*oUMuZi4lO^?fuHfDwa?+2bMRelwFB~Ag3dH=8k9~v>R z>ptF|-I8aSFv~EN&#*>UcjpVTY6dT88Y3LPSmlz6|VR(X11=PE~&rQO17&{{@V-kkJlYt*WI8ft^~MZLilw4+P3`6)`@77^TVqHx!axQXWJ^-UW<3x-m}l@o>+aUSSl3mZ7$m&;jE|y&6@< zTnlTfz|-b_ZBW&{RyjOe0dPfuV9<5z^VZ|qz~&DFQ$8^|$$m8hyZP_7Wm(hX0^Fmj zYYPc3)q+d{*e?}hMVB+!v3WeWBj%H=T&a}4N;LqbDo76C z)=e31-IN*Z*#1}#JmZYLClA^V{pZ&!7k!{fePAqKr4v?r|cpZz}q; zNUO3Tqvv7t&3Nj6!!PD%d$>~+qYzSXh&`?CMmfd;E8GV95|$O ziQm|_hHvZ}-PtY$*X%!pYxXaG%uYvXTGGMn#yEeQ8vR-P@3t(Tc%>U)TwwETpzi#! zNv^R7R(x!AH^1MWjRB*8V~53s)|m~FpXZjWGHz){|7-Cv9~g4U_!ME|=S>^Zm@$_d zR5b@6B1ZuKHm5NjI&EIo>F{VjhrSp}u^_X7SWD^okJEtKOUI|I9)v9>*2I`t6XVoT zNv?0p@a45VO$trTNhwP?_9Q&r;oxE)YJaNtSu750o(nL?`dm4HjhHFbgxQUDgI%Cs zED3HGk)gmBnv%lVV+KZ{-pVOI*^~v?3j8dvtElz3t}WD|FNjilikJVeRFL-koVs@2 z&^TA`M&$^?IBF?AGsc+curT32E4)ZrsmWRwcsj6oF2HPybG7z{?HA%Fj)&S^%TpZ% z123arynWP=~%CN>-3L7Gcqh z!m>cL>*8@WZ#&Qa>-!pov>hS$WGv{&A=q{5E?0TpJc}1pwH-i262Pp3hs0>I@IU&6 z?OVVulnwcShYKw7w~0s%Fux%#oIiG;_2_ot2POmFMc)kgWkCG0@in0lf*aa0zC(X) z#OR~l6wAMn2M&ob*5XDk?{w(#u55ThLbbaXG75KW&ak@Mr71>FYD!vJ!|T?kS(~<7=;Dl~B(p4JT(vIk2Q-S}nNEi(wRRD>B^?eXdfaEk zJ_4w3u^#>VD99PMq3muKo;Aw4LRC8gyUqm0er`;X5h1_r_RSeK8_RW@s&0o~hkpN> z7@rtD(4pUuQGU541ID4hC+aoY4yV(Bx-$4zv^2)C&@BM|^`)-h!{A%%(mXb`p1Bqe zw6(`2tMI3{a(%X+IVO4pE2nVPx(r>eoq~M6Il7D<*C&J* zIeXZ2%TV?+oerj-_cAL)BnDiT6u5Z&KSttT^9pG=8p?^=x^STCT+&W z>wSkl3TzdmB>y8Y&t#tE1AV+b=PSoz;9I~vh0B+8I#vqueD4(RM&J(M!P*$(rq|n@ z!&8p?fv-ns^2pG*jGAU4V_BEuE5}{HcZ(eQ8Q>z@!74D&EC_7&6eRk&w&#>p-7XmC zdIk`)c^+VX%?$%CW+`z&wb&As&aS-}7K|JcOp0aonFPpcjV+JJFKHP~6R1--xH zPpVq9mPJJN14bQf8P!?P8_am1m<%r|XVs})V3GL^>-y1G=8jGXlPre+r#l=xBbWC- z?1`(P*6a!aE-jP+Z_8)EuM2EG4;XgAxD<7b?B`~pVk&+WI~u?aU);t zaoOV8(T2EigvDsu?kE|BcAd8Uo zi(QW2QS#>&k%NF{+vroj>-;!Chok)Et;n+%Us>1tTqqIw05Id^kqIV;^szs`mGh}h z_p9nkuT2dwzdj~h5aQ7HxNwUZWIHK9X)lJzJ;Wy{&bB2e_Kp(FX^e-?4+~*vT=?VU zNQvq}@qCvHhGyR;BAz9*oSvA`{Vm|72!|UIq=kY62#V{A5$HtZU|@``&rm?Yy}F)~ z$`%w8DxLtGdDfbe82Lemd5{cvzPKrv!#EQd_u1xTXt3gcwq*QJ^iP7cny9e3D;4lR z{xLa-Z8>F~As0UHZEA@DM_WW(T6S7GHFlPmv+faCPoI{VOkB82Q3t?g5 zq*;GhcWTK0h7JtDF@LU_7|gHXFNg}a9{&y;^vTi5P%heU zw`XZJ7ka0vt}o@HtSm@P434z8k=t_0Y7^<8HU>vpjM$aLy(W7K@H>pNpR{%;?fj2& z={SS>AGI+%KWtsbWvz++EK*emKtx6ZlV{e)vEZHdxlV66olEdaAR@vAegoX0s;-Dk z0!}%sJ|60CzTE9{|Mnc1e941>T~`WlX(3vPAtk!>b?B`Ssh4I4QSV8X5s0G9)?%R{C~0N1yr{eG0U9wKraFe5^Z`hBXp zdcXk@5%tc@0c39JF?SBND9r`~?K$OTV+of>3=>*Az8f;? z$M+WxlE^A4r215NNYy(35jfXI%;-G%1H%*CeQ<5$_V;eha^7n_00)c6bHHf_)WrB$ z2&H{p#^qOr(*Adh)$vhEjk-0W-es>-l@#d!;9yGzt(S9ffNxth90&y-R#o5eJ+Lt@ zeyv3$ytK>pN3iEbL?ZBb4jPjb##xxm7kA}5@QPx`QDUcF*n2-01vaP zbZh~5@6b5IEDAsZIjI$x=~?Pa<5QtEBsaBX{A_zZ1MWNY5o**QQ`IE{4zNIE1+l%6 z8HJ?*f?`-)yAT3|I>i(@m4iotm4W|9+Ys^qHfQq>^z*=(BC?L&mHOPk-^VZt-Q1kC z8xTBh5x~VFaxSKJB}~Qy+h@_~C@Xg5@8<$j%n88MNwqPnVzqvL(i|u`pX30q0Dl;h z6h0F|Y2T4kZrhyCfCm?KGuC798uhZQ2SiX1Z*8HYCR^UfWu1=CeF0#ig#}sK>5vde z2npcu$PN2Jfwarvz-D@H*3f#3$P&P*qmq1eAD1Pa4sWF0Ase%LkgHy=RlnydJw{dr zNQL;CMe)g|B&)hzmUTJ2k#=i0^^@VExaRj@8-ZKe>5vpihTPBhmFIt)m9iu#b~x%= z%N3X$M#C4!r3mXDKirt1(ujUn_E|RJCnLyA`|Nb2CX0r$Z7`GuIO_2cZpt4 z5CguwUvU{rIvwV`&}Ef92RLR}W$q{B{STj@IBj}EJhXEc!jeu0)8}~>xTrBMe4#m< z(*DDZ`3(3E1G_E*VwX(_cjyB(>SbFGi1z$S@;?G2%^v{PMWv@!5CQJg^1$ZDGYX43 zqEjL)Wi16B-|1+OAXyn1o}=B`Q+{->6l}>Vuk_>ziZg+jZQv#_a83f6&TUSHTEm}j z&iE9id^Q|$nn|MZYrfCUI)#%!0{`){L%bvLqGq+y2RE67I)_RKMtVirUX)4Fy^kRJ4ED+w96k)dAVCoZ%Q(|G0tK2 zF_L2EEZd+h6o=Kt7?ToGC0tWLme132y)~vD zzjI6W@K?HB&KsLzT3w7WNh#@fzur6OJOC`-nAL0E`$89|HzhfKc!F`M7$Xv5MFt%n z7pugtH;wcOinkw76T?w_GT@+CwafAA$cX`kUYp^qxZa=AegV)Bp+^1ts+t~nL}`#> zA$<`!3;4zL6NfP0;uQzyws!M#6P`O#RhN0^Iv=Zq3x7} zkBm%m@1%iU;h`NlPBP0w{a#g11@QQDYa8Oy^!kj30cE(|E*!YLgUwmp3GC?j&b9z`ekoVLR2l0`4Q8l{^_Z0JET7 zDa);d%2L(Mz)2!988`{pzYG~UOeNz(A~F^;jk_Q7;+L72b)`nrciJpK127C24&)cl znp7K$tQ=sHZ<)Te>>tNOlx$XE%mT*)jaL+iLS5SyN=iH%Xo^sy{^Lq22NbX%B616G z;gs4KCyq#1t$MyKr!+s)0cN?+_gCyn_i~8H<-k{{z!I$o4~cQ#lsevR;a6U>KFwF( zNCTXys&|wk*Q>*M@|a->9yw%SozJ2Uhq*6y0sKu>|HW$)2i7EoiECywGNGT(;wVgh zwv*S>ZW@@Vs!^7Ljf_Ih#YBL|rqyHfJ_{&FimRdwh)5G~^@NnfZ5h!Gabc+U8JQ4{ z8y4p<%W6!3C&Rg1wm1@CdR@$C=k~!Z8$%a$YXT-7Q6Fbnr^B*-ZJz6Ld=A^~l~xWY ztRN}Cihb)NP*WX+7G6a(v;gsQnp1o&r@`idln6&n4aVwjm+!1k`@TmSI7d~pgEW7~ zp^$>4Ma(Ok{Wq;cFLO2jQKiQ<%jvH)1w{^UQPp}>6j>m%G2^B_w5s47+zN_O1P5?c z^WV2-m9=S?etEx#Fwb;NMTGy8p-`CjZ0uH`bAbCW?`hbRUeNiDfQWnv^Jc7u-7IPw z@LJ&Wh_@KSUWo;AB(!#cJf P00000NkvXXu0mjf*FjSZ literal 0 HcmV?d00001 diff --git a/demos/demo-simple-webapp/src/main/webapp/jetty.webp b/demos/demo-simple-webapp/src/main/webapp/jetty.webp new file mode 100644 index 0000000000000000000000000000000000000000..2d1bfea3ef792ce55a4c1f518947a93d29468bd7 GIT binary patch literal 3534 zcmV;<4KeakNk&G-4FCXFMM6+kP&il$0000G0002T001}u06|PpNL2*@00E#yZQJq2 zdXlEAyBK7TZQHhO+qP}nwr$(C?MbE>)kxlh=KbIAPtqREK}1Xd{`s1xrPdZn;vMB% z#%`X$62E9fd_%xDh}fMo8C`GpHjs${RkA5LhFAno=< zT#6{t9_{UVCpDA-IugGaN-dIu&QOn}*|}yUP49Arh`$0Do^#O|lw>C9FlU%02{Let zB;VokJ2lh=8WO(+N^6NqJHu7U5%6~MEKdiVJNCfpe0*twq%Uj39nI zkU55XVo-EG4tfZfLL`HhNo)xKFAEp)Kd^%Mwoq#djf0i}MM+xxf#d;z?!+$uQiO9K zO^S+99NZchC<#kqlPm*_Z@8#*z;xpKLh(SoZq}J4nqTm{@2zUDUMg!PCuuLR=}Foej`vHx0I*7?O%!DS?s4|qz;NQnLAAyC zQa2Li<64O_Fh3+afvhC0y;1TNs80Mr!0S5~Y*v+-oZcVvB)^Q@GRaps&Q8+4KnIE4 z+)XkUIH#c$(4F{sP~Fb?&fuoxIQWN>-ArduAz+QIWviTT zs&|xRG?&OG8NjTU{Ph4{NLE4NJ`J^jX2h?9@~6&MxFSi*b2Uj?lzAmN4J0LL`;3cD zYCYc3h#WTV5rBJ~Er&9*-Ra(9j3m;Vn<5D{IZ95_2|xo;mKP+fgLTBW2C5OiAIj^H z(~rP$$qn%5CFkL|k*F##M`FoD(m`n=xeq4boQC4+rt1_lQ9PkE@Jh1Mz%Y-GdN!Iyu!De7B@y&G@BHFB9cVLyWzMEX;Vs(;UC4L@|CX##Wp~HU3w@&)y z1sW?W*xF^LO%z3Pk!gUr#P@)XKP0=&L3;f%0LHV5{uoV{6-DY-`;18AdcYjaS-f<} zM$!SOW7p{f0QlWja}r4OP|f}kdfCnX%IrCTvBZynjvYlsfZk#H^o@li2ka{P0!V&D zNmum$tIt#1RJI@R_UC>lRF`6Bge1fSz+7+6v6HHgbu>8uvJF{!@TB%*9ZiUxbJ)^@ zC$%5#o+f3=jNMlqzJBxO&0EjDhA3Et^h*wE-SbyN37`-0GogM7iO<0u5C22Dw}-GS z@fQG(4_rh_9gfH*yQ5?=q<g$fYB6W+X1~+&` z5Yrs)*chzW@vLx%oi)YmD-HKNI9Ofa?VZyr!%gOC(Y#r+rcD|*YFIZl-1dKo6(;iY z@iH0kZ(aaaP&gn;2mk=^CjgxRD#!pi06vjCnMtLiA|WdioH(!&32Xq=iGluS`2nrn z(my1Q&|^pE|% z(?9Ngg@0@PfxmS}uwA=^FpQ--}`3;;M#wFRx0O2t6MT zAvj|;tv9DjM;oGnK(R@H-nj74`+65!q5&Y)}ERWEM0bqsmWb9A>{>D&R z0m)c+-$-Gp%}ZPX$LHLS)R+I@mHYCVLHdby%li_-+^l42x}FVYasEA7^KYqUM9bOr zPjv2&tE#LTyxiKbcA4Lix&*uV#^!127t9&qJ}e(}mo7IMRUYXq={;+v`CQ$g)=(BN zi7NW*M@X~vck~)MABVt-56s?A-NQ`YRk3^KCJH zLspg~k-uar;D!Jo;UVNmF}wqfGvw^`&rP|HWp+zO``GL$=T8$0FMfyR9S$Kfy_N^;&lJh`3-JTs9U z5rIjgp3HWQ`tD{Xfef$LjUok{>0k7Q)aT)kdO&!_q7C6gQUL$;MaFxnh0fm;e=BR3 zm`#gg24+iHi5km(um=Te4g18ZC{%5ty%wHoMV?;aIj{X_5Xf&KS&CmU&Le)VKTZ;x zQ@Qd&DmCm1vEEdmqq-u(%=6BCfC)v(IoOZk=uP^s!boObrnanwVzJ8~?9ZAdE0D42 zK|ejyk;$Ql3Nf~gxrDk*fUe8_A;%s*Ws6gDQlj<5tXi%XhKq^Si5{IDOgb!4Cz+P# zL!&p0x~mQUu@&}3bsnj>w5I<_arnO+wDvwjme3yJdCNMuRoya`R{li98lQ=^$IOw3+OH1eXbS1A zZ$7{__z%Fdt9lSEngCJHw_9>%auuSS6o)q$^yg#<-kFN2Zx>y&_%I$h8!dgxr#&Jr?mSTYE}?(Ai0|+5HMwF_Ci3FI|E&=n}3?~d4~`WRt|SBqaq)Wi_$Q%kjxSg4U!SkPAk*tAlgs%YMP zhd>pM6Y_=S9PrY+xg$Xc(f0u!0f_^kPx}IUz{7GHo!k1L;l0CH?%==u+*Bn03iyk1 zDe!Dhg&NbjwmOCms{{yd#{5PfGwV!^kPn}9%~mdCz3h3Gy$_=}ZJv5pR&!LyDIk6g&U14;mQTD;q7)@l0k z37VYtRrI;Oc^v0T32nr1Dq{cqKy}xTA2QY`<$vHxr9yhDATL`8j9>NIHc1>r?>i+> z1LcAVau^#lK`eOwl;>h{cb za@=w)*M;yo{YOEWva`a<^oY>4zGA>k_|eHktkbi6#4J3Q4vxe2jtA;YvEUAYvl*P{ zs#U{NtYD*_yRwxzi)xY0&_>Vzzz?I40GNLK{+}*R^~J{%3&QdZNJ?N(I7(tyRJDsh zX$W#H0=nMtIC6q(QMa3u!pTnkv5N)MMTSq%+Z?u)@2f3J?Sxes*7rG3;2&tfT6n25vz0lMu1eUQ}60B~joI8?yN7)bXR+-nc7$ zA26NoBkN;SC9kbwwUVS1yl9rNL=CfC_)^s}B*b&O=AU=)7P5((8O(KF2ykBly;@Qi z8SptNGoJ^t5LQr*{`-Ma-8Z~AOL8C6tOe$I=UwT4q^E3J9QeTzpjSD1`{cNjZ}DVB zXQ*bj94FU~dZQEcS-^nTHLMqv#1B#f+42;Ru-Ho;_KFNC#^5HT00MaAFxVFePRswK I1ONa40I<~Z`v3p{ literal 0 HcmV?d00001 diff --git a/jetty-server/src/main/config/etc/jetty-gzip.xml b/jetty-server/src/main/config/etc/jetty-gzip.xml index 7659b2a195d0..933f2ef4893e 100644 --- a/jetty-server/src/main/config/etc/jetty-gzip.xml +++ b/jetty-server/src/main/config/etc/jetty-gzip.xml @@ -18,8 +18,8 @@ - - + + diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHandler.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHandler.java index 9df6291acf4d..1a2cfd921284 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHandler.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHandler.java @@ -768,6 +768,17 @@ public void setExcludedMimeTypes(String... types) _mimeTypes.exclude(types); } + /** + * Set the excluded filter list of MIME types (replacing any previously set) + * + * @param csvTypes The list of mime types to exclude (without charset or other parameters), CSV format + * @see #setIncludedMimeTypesList(String) + */ + public void setExcludedMimeTypesList(String csvTypes) + { + setExcludedMimeTypes(StringUtil.csvSplit(csvTypes)); + } + /** * Set the excluded filter list of Path specs (replacing any previously set) * @@ -819,6 +830,17 @@ public void setIncludedMimeTypes(String... types) _mimeTypes.include(types); } + /** + * Set the included filter list of MIME types (replacing any previously set) + * + * @param csvTypes The list of mime types to include (without charset or other parameters), CSV format + * @see #setExcludedMimeTypesList(String) + */ + public void setIncludedMimeTypesList(String csvTypes) + { + setIncludedMimeTypes(StringUtil.csvSplit(csvTypes)); + } + /** * Set the included filter list of Path specs (replacing any previously set) * diff --git a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java index 43d3f031f854..e27d87ab6974 100644 --- a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java +++ b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java @@ -61,7 +61,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; -@SuppressWarnings("serial") public class GzipHandlerTest { private static final String __content = @@ -88,6 +87,8 @@ public class GzipHandlerTest private Server _server; private LocalConnector _connector; + private GzipHandler gzipHandler; + private ServletContextHandler context; @BeforeEach public void init() throws Exception @@ -96,25 +97,25 @@ public void init() throws Exception _connector = new LocalConnector(_server); _server.addConnector(_connector); - GzipHandler gzipHandler = new GzipHandler(); + gzipHandler = new GzipHandler(); gzipHandler.setMinGzipSize(16); gzipHandler.setInflateBufferSize(4096); - ServletContextHandler context = new ServletContextHandler(gzipHandler, "/ctx"); - ServletHandler servlets = context.getServletHandler(); + context = new ServletContextHandler(gzipHandler, "/ctx"); _server.setHandler(gzipHandler); gzipHandler.setHandler(context); - servlets.addServletWithMapping(MicroServlet.class, "/micro"); - servlets.addServletWithMapping(MicroChunkedServlet.class, "/microchunked"); - servlets.addServletWithMapping(TestServlet.class, "/content"); - servlets.addServletWithMapping(ForwardServlet.class, "/forward"); - servlets.addServletWithMapping(IncludeServlet.class, "/include"); - servlets.addServletWithMapping(EchoServlet.class, "/echo/*"); - servlets.addServletWithMapping(DumpServlet.class, "/dump/*"); - servlets.addServletWithMapping(AsyncServlet.class, "/async/*"); - servlets.addServletWithMapping(BufferServlet.class, "/buffer/*"); - servlets.addFilterWithMapping(CheckFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST)); + context.addServlet(MicroServlet.class, "/micro"); + context.addServlet(MicroChunkedServlet.class, "/microchunked"); + context.addServlet(TestServlet.class, "/content"); + context.addServlet(MimeTypeContentServlet.class, "/mimetypes/*"); + context.addServlet(ForwardServlet.class, "/forward"); + context.addServlet(IncludeServlet.class, "/include"); + context.addServlet(EchoServlet.class, "/echo/*"); + context.addServlet(DumpServlet.class, "/dump/*"); + context.addServlet(AsyncServlet.class, "/async/*"); + context.addServlet(BufferServlet.class, "/buffer/*"); + context.addFilter(CheckFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST)); _server.start(); } @@ -147,6 +148,34 @@ protected void doGet(HttpServletRequest req, HttpServletResponse response) throw } } + public static class MimeTypeContentServlet extends HttpServlet + { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException + { + String pathInfo = req.getPathInfo(); + resp.setContentType(getContentTypeFromRequest(pathInfo, req)); + resp.getWriter().println("This is content for " + pathInfo); + } + + private String getContentTypeFromRequest(String filename, HttpServletRequest req) + { + String defaultContentType = "application/octet-stream"; + if (req.getParameter("type") != null) + defaultContentType = req.getParameter("type"); + + ServletContextHandler servletContextHandler = ServletContextHandler.getServletContextHandler(getServletContext()); + if (servletContextHandler == null) + return defaultContentType; + String contentType = servletContextHandler.getMimeTypes().getMimeByExtension(filename); + if (contentType != null) + { + return contentType; + } + return defaultContentType; + } + } + public static class TestServlet extends HttpServlet { @Override @@ -797,6 +826,52 @@ public void testGzipBomb() throws Exception assertThat(response.getContentBytes().length, is(512 * 1024)); } + @Test + public void testGzipExcludeNewMimeType() throws Exception + { + // setting all excluded mime-types to a mimetype new mime-type + // Note: this mime-type does not exist in MimeTypes object. + gzipHandler.setExcludedMimeTypes("image/webfoo"); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + // Request something that is not present on MimeTypes and is also + // excluded by GzipHandler configuration + request.setMethod("GET"); + request.setURI("/ctx/mimetypes/foo.webfoo?type=image/webfoo"); + request.setVersion("HTTP/1.1"); + request.setHeader("Host", "tester"); + request.setHeader("Accept", "*/*"); + request.setHeader("Accept-Encoding", "gzip"); // allow compressed responses + request.setHeader("Connection", "close"); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + assertThat(response.getStatus(), is(200)); + assertThat("Should not be compressed with gzip", response.get("Content-Encoding"), nullValue()); + assertThat(response.get("ETag"), nullValue()); + assertThat(response.get("Vary"), nullValue()); + + // Request something that is present on MimeTypes and is also compressible + // by the GzipHandler configuration + request.setMethod("GET"); + request.setURI("/ctx/mimetypes/zed.txt"); + request.setVersion("HTTP/1.1"); + request.setHeader("Host", "tester"); + request.setHeader("Accept", "*/*"); + request.setHeader("Accept-Encoding", "gzip"); // allow compressed responses + request.setHeader("Connection", "close"); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + assertThat(response.getStatus(), is(200)); + assertThat(response.get("Content-Encoding"), containsString("gzip")); + assertThat(response.get("ETag"), nullValue()); + assertThat(response.get("Vary"), is("Accept-Encoding")); + } + public static class CheckFilter implements Filter { @Override diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java new file mode 100644 index 000000000000..64cd664d2b13 --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java @@ -0,0 +1,147 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution; + +import java.io.File; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpStatus; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class GzipModuleTests extends AbstractJettyHomeTest +{ + @Test + public void testGzipDefault() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + int httpsPort = distribution.freePort(); + assertThat("httpPort != httpsPort", httpPort, is(not(httpsPort))); + + String[] argsConfig = { + "--add-modules=gzip", + "--add-modules=deploy,webapp,http" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(5, TimeUnit.SECONDS)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort, + "jetty.httpConfig.port=" + httpsPort, + "jetty.ssl.port=" + httpsPort + }; + + File war = distribution.resolveArtifact("org.eclipse.jetty.demos:demo-simple-webapp:war:" + jettyVersion); + distribution.installWarFile(war, "demo"); + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/index.html"); + assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat("Ensure that gzip is working", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), containsString("gzip")); + } + } + } + + @Test + public void testGzipExcludeMimeType() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + int httpsPort = distribution.freePort(); + assertThat("httpPort != httpsPort", httpPort, is(not(httpsPort))); + + String[] argsConfig = { + "--add-modules=gzip", + "--add-modules=deploy,webapp,http" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(5, TimeUnit.SECONDS)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort, + "jetty.httpConfig.port=" + httpsPort, + "jetty.ssl.port=" + httpsPort, + "jetty.gzip.excludedMimeTypeList=image/webp" + }; + + File war = distribution.resolveArtifact("org.eclipse.jetty.demos:demo-simple-webapp:war:" + jettyVersion); + distribution.installWarFile(war, "demo"); + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/jetty.webp"); + assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat("Ensure that gzip exclusion worked", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); + } + } + } + + private static class ResponseDetails implements Supplier + { + private final ContentResponse response; + + public ResponseDetails(ContentResponse response) + { + this.response = response; + } + + @Override + public String get() + { + StringBuilder ret = new StringBuilder(); + ret.append(response.toString()).append(System.lineSeparator()); + ret.append(response.getHeaders().toString()).append(System.lineSeparator()); + ret.append(response.getContentAsString()).append(System.lineSeparator()); + return ret.toString(); + } + } +} From 61bcc6068e494943686585973fb14a9d3db5fd7f Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Thu, 29 Jul 2021 15:07:43 +1000 Subject: [PATCH 02/54] Update VERSION.txt with CVE-2021-34429. Signed-off-by: Lachlan Roberts --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 405b0ed27831..372baea126a7 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -9,7 +9,7 @@ jetty-10.0.6 - 29 June 2021 + 6410 Ensure Jetty IO uses SocketAddress instead of InetSocketAddress + 6418 Bad and/or missing Require-Capability for osgi.serviceloader + 6425 Update to asm 9.1 - + 6447 Deprecate support for UTF16 encoding in URIs + + 6447 Deprecate support for UTF16 encoding in URIs (Resolves CVE-2021-34429) + 6451 Request#getServletPath() returns null for ROOT mapping + 6464 Wrong files/lib definitions in certain *-capture.mod files? + 6473 Improve alias checking in PathResource From 9e047ab41219ad659d30feb2e88fe2c99f5635c8 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Mon, 26 Jul 2021 11:03:18 +0200 Subject: [PATCH 03/54] #6541 improve testTake perf Signed-off-by: Ludovic Orban --- .../jetty/util/BlockingArrayQueueTest.java | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java index c3bc28a204b4..cd6149e2d65e 100644 --- a/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java @@ -13,11 +13,14 @@ package org.eclipse.jetty.util; +import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.ListIterator; +import java.util.Objects; import java.util.Random; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; @@ -27,9 +30,11 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIfSystemProperty; +import static org.eclipse.jetty.util.BlockingArrayQueueTest.Await.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -161,12 +166,12 @@ public void testGrow() throws Exception } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testTake() throws Exception { final String[] data = new String[4]; final BlockingArrayQueue queue = new BlockingArrayQueue<>(); + CyclicBarrier barrier = new CyclicBarrier(2); Thread thread = new Thread() { @@ -177,7 +182,7 @@ public void run() { data[0] = queue.take(); data[1] = queue.take(); - Thread.sleep(1000); + barrier.await(5, TimeUnit.SECONDS); // Wait until the main thread already called offer(). data[2] = queue.take(); data[3] = queue.poll(100, TimeUnit.MILLISECONDS); } @@ -191,17 +196,19 @@ public void run() thread.start(); - Thread.sleep(1000); + // Wait until the spawned thread is blocked in queue.take(). + await().atMost(5, TimeUnit.SECONDS).until(() -> thread.getState() == Thread.State.WAITING); queue.offer("zero"); queue.offer("one"); queue.offer("two"); + barrier.await(5, TimeUnit.SECONDS); // Notify the spawned thread that offer() was called. thread.join(); assertEquals("zero", data[0]); assertEquals("one", data[1]); assertEquals("two", data[2]); - assertEquals(null, data[3]); + assertNull(data[3]); } @Test @@ -525,4 +532,35 @@ public void testDrainTo() throws Exception assertThat(queue.size(), Matchers.is(0)); assertThat(queue, Matchers.empty()); } + + static class Await + { + private Duration duration; + + public static Await await() + { + return new Await(); + } + + public Await atMost(long time, TimeUnit unit) + { + duration = Duration.ofMillis(unit.toMillis(time)); + return this; + } + + public void until(Callable condition) throws Exception + { + Objects.requireNonNull(duration); + long start = System.nanoTime(); + + while (true) + { + if (condition.call()) + return; + if (duration.minus(Duration.ofNanos(System.nanoTime() - start)).isNegative()) + throw new AssertionError("Duration expired"); + Thread.sleep(10); + } + } + } } From d8a890f71ee66b3c0932155ba742f696c44ae434 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Mon, 26 Jul 2021 11:30:31 +0200 Subject: [PATCH 04/54] #6541 improve testConcurrentAccess perf Signed-off-by: Ludovic Orban --- .../jetty/util/BlockingArrayQueueTest.java | 66 +++++++++---------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java index cd6149e2d65e..eac290032191 100644 --- a/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java @@ -15,20 +15,20 @@ import java.time.Duration; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.ListIterator; import java.util.Objects; -import java.util.Random; +import java.util.Set; import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import static org.eclipse.jetty.util.BlockingArrayQueueTest.Await.await; import static org.hamcrest.MatcherAssert.assertThat; @@ -212,21 +212,20 @@ public void run() } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testConcurrentAccess() throws Exception { - final int THREADS = 50; + final int THREADS = 32; final int LOOPS = 1000; - final BlockingArrayQueue queue = new BlockingArrayQueue<>(1 + THREADS * LOOPS); + BlockingArrayQueue queue = new BlockingArrayQueue<>(1 + THREADS * LOOPS); - final ConcurrentLinkedQueue produced = new ConcurrentLinkedQueue<>(); - final ConcurrentLinkedQueue consumed = new ConcurrentLinkedQueue<>(); + Set produced = ConcurrentHashMap.newKeySet(); + Set consumed = ConcurrentHashMap.newKeySet(); - final AtomicBoolean running = new AtomicBoolean(true); + AtomicBoolean consumersRunning = new AtomicBoolean(true); // start consumers - final CyclicBarrier barrier0 = new CyclicBarrier(THREADS + 1); + CyclicBarrier consumersBarrier = new CyclicBarrier(THREADS + 1); for (int i = 0; i < THREADS; i++) { new Thread() @@ -234,20 +233,18 @@ public void testConcurrentAccess() throws Exception @Override public void run() { - final Random random = new Random(); - setPriority(getPriority() - 1); try { - while (running.get()) + while (consumersRunning.get()) { - int r = 1 + random.nextInt(10); + int r = 1 + ThreadLocalRandom.current().nextInt(10); if (r % 2 == 0) { Integer msg = queue.poll(); if (msg == null) { - Thread.sleep(1 + random.nextInt(10)); + Thread.sleep(ThreadLocalRandom.current().nextInt(2)); continue; } consumed.add(msg); @@ -268,7 +265,7 @@ public void run() { try { - barrier0.await(); + consumersBarrier.await(); } catch (Exception e) { @@ -280,7 +277,7 @@ public void run() } // start producers - final CyclicBarrier barrier1 = new CyclicBarrier(THREADS + 1); + CyclicBarrier producersBarrier = new CyclicBarrier(THREADS + 1); for (int i = 0; i < THREADS; i++) { final int id = i; @@ -289,16 +286,15 @@ public void run() @Override public void run() { - final Random random = new Random(); try { for (int j = 0; j < LOOPS; j++) { - Integer msg = random.nextInt(); + Integer msg = ThreadLocalRandom.current().nextInt(); produced.add(msg); if (!queue.offer(msg)) throw new Exception(id + " FULL! " + queue.size()); - Thread.sleep(1 + random.nextInt(10)); + Thread.sleep(ThreadLocalRandom.current().nextInt(2)); } } catch (Exception e) @@ -309,7 +305,7 @@ public void run() { try { - barrier1.await(); + producersBarrier.await(); } catch (Exception e) { @@ -320,22 +316,22 @@ public void run() }.start(); } - barrier1.await(); - int size = queue.size(); - int last = size - 1; - while (size > 0 && size != last) + producersBarrier.await(); + + AtomicInteger last = new AtomicInteger(queue.size() - 1); + await().atMost(5, TimeUnit.SECONDS).until(() -> { - last = size; - Thread.sleep(500); - size = queue.size(); - } - running.set(false); - barrier0.await(); + int size = queue.size(); + if (size == 0 && last.get() == size) + return true; + last.set(size); + return false; + }); - HashSet prodSet = new HashSet<>(produced); - HashSet consSet = new HashSet<>(consumed); + consumersRunning.set(false); + consumersBarrier.await(); - assertEquals(prodSet, consSet); + assertEquals(produced, consumed); } @Test From 6201d3b1072b05d9e683377f5741f467a44089fa Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 30 Jul 2021 09:08:36 +1000 Subject: [PATCH 05/54] Non blocking ReservedThreadExecutor (#6535) (#6559) A call to offer must never block, nor even yield, since to do so give an opportunity for the allocated CPU core to change, defeating the whole purpose of the class. There is also some reasonable level of diagnostic warnings if a reserved thread misses too many offers consecutively, based on tracking the state of the reserved thread. Remove the stack data structure entirely. ReservedThreads all poll the same SynchronousQueue and tryExecute does a non blocking offer. Added test for busy shrinking Remember last time we hit zero reserved threads Co-authored-by: Simone Bordet --- .../util/thread/ReservedThreadExecutor.java | 341 ++++++++++-------- .../thread/ReservedThreadExecutorTest.java | 43 ++- 2 files changed, 221 insertions(+), 163 deletions(-) diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/thread/ReservedThreadExecutor.java b/jetty-util/src/main/java/org/eclipse/jetty/util/thread/ReservedThreadExecutor.java index c1510eaff40b..81d41d0c7a2a 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/thread/ReservedThreadExecutor.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/thread/ReservedThreadExecutor.java @@ -13,34 +13,45 @@ package org.eclipse.jetty.util.thread; -import java.util.concurrent.ConcurrentLinkedDeque; +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.eclipse.jetty.util.AtomicBiInteger; import org.eclipse.jetty.util.ProcessorUtils; import org.eclipse.jetty.util.annotation.ManagedAttribute; import org.eclipse.jetty.util.annotation.ManagedObject; import org.eclipse.jetty.util.component.AbstractLifeCycle; +import org.eclipse.jetty.util.component.Dumpable; +import org.eclipse.jetty.util.component.DumpableCollection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.eclipse.jetty.util.AtomicBiInteger.getHi; +import static org.eclipse.jetty.util.AtomicBiInteger.getLo; + + /** - * An Executor using preallocated/reserved Threads from a wrapped Executor. + * An Executor using pre-allocated/reserved Threads from a wrapped Executor. *

Calls to {@link #execute(Runnable)} on a {@link ReservedThreadExecutor} will either succeed * with a Thread immediately being assigned the Runnable task, or fail if no Thread is * available. - *

Threads are reserved lazily, with a new reserved thread being allocated from a - * wrapped {@link Executor} when an execution fails. If the {@link #setIdleTimeout(long, TimeUnit)} - * is set to non zero (default 1 minute), then the reserved thread pool will shrink by 1 thread - * whenever it has been idle for that period. + *

Threads are reserved lazily, with a new reserved threads being allocated from the + * {@link Executor} passed to the constructor. Whenever 1 or more reserved threads have been + * idle for more than {@link #getIdleTimeoutMs()} then one reserved thread will return to + * the executor. */ @ManagedObject("A pool for reserved threads") -public class ReservedThreadExecutor extends AbstractLifeCycle implements TryExecutor +public class ReservedThreadExecutor extends AbstractLifeCycle implements TryExecutor, Dumpable { private static final Logger LOG = LoggerFactory.getLogger(ReservedThreadExecutor.class); + private static final long DEFAULT_IDLE_TIMEOUT = TimeUnit.MINUTES.toNanos(1); private static final Runnable STOP = new Runnable() { @Override @@ -57,13 +68,13 @@ public String toString() private final Executor _executor; private final int _capacity; - private final ConcurrentLinkedDeque _stack; - private final AtomicInteger _size = new AtomicInteger(); - private final AtomicInteger _pending = new AtomicInteger(); + private final Set _threads = ConcurrentHashMap.newKeySet(); + private final SynchronousQueue _queue = new SynchronousQueue<>(false); + private final AtomicBiInteger _count = new AtomicBiInteger(); // hi=pending; lo=size; + private final AtomicLong _lastEmptyTime = new AtomicLong(System.nanoTime()); private ThreadPoolBudget.Lease _lease; - private long _idleTime = 1L; - private TimeUnit _idleTimeUnit = TimeUnit.MINUTES; + private long _idleTimeNanos = DEFAULT_IDLE_TIMEOUT; /** * @param executor The executor to use to obtain threads @@ -75,7 +86,6 @@ public ReservedThreadExecutor(Executor executor, int capacity) { _executor = executor; _capacity = reservedThreads(executor, capacity); - _stack = new ConcurrentLinkedDeque<>(); if (LOG.isDebugEnabled()) LOG.debug("{}", this); } @@ -121,42 +131,39 @@ public int getCapacity() @ManagedAttribute(value = "available reserved threads", readonly = true) public int getAvailable() { - return _stack.size(); + return _count.getLo(); } @ManagedAttribute(value = "pending reserved threads", readonly = true) public int getPending() { - return _pending.get(); + return _count.getHi(); } - @ManagedAttribute(value = "idletimeout in MS", readonly = true) + @ManagedAttribute(value = "idle timeout in ms", readonly = true) public long getIdleTimeoutMs() { - if (_idleTimeUnit == null) - return 0; - return _idleTimeUnit.toMillis(_idleTime); + return NANOSECONDS.toMillis(_idleTimeNanos); } /** * Set the idle timeout for shrinking the reserved thread pool * - * @param idleTime Time to wait before shrinking, or 0 for no timeout. + * @param idleTime Time to wait before shrinking, or 0 for default timeout. * @param idleTimeUnit Time units for idle timeout */ public void setIdleTimeout(long idleTime, TimeUnit idleTimeUnit) { if (isRunning()) throw new IllegalStateException(); - _idleTime = idleTime; - _idleTimeUnit = idleTimeUnit; + _idleTimeNanos = (idleTime <= 0 || idleTimeUnit == null) ? DEFAULT_IDLE_TIMEOUT : idleTimeUnit.toNanos(idleTime); } @Override public void doStart() throws Exception { _lease = ThreadPoolBudget.leaseFrom(getExecutor(), this, _capacity); - _size.set(0); + _count.set(0, 0); super.doStart(); } @@ -168,26 +175,22 @@ public void doStop() throws Exception super.doStop(); - while (true) + // Offer STOP task to all waiting reserved threads. + for (int i = _count.getAndSetLo(-1); i-- > 0;) { - int size = _size.get(); - // If no reserved threads left try setting size to -1 to - // atomically prevent other threads adding themselves to stack. - if (size == 0 && _size.compareAndSet(size, -1)) - break; - - ReservedThread thread = _stack.pollFirst(); - if (thread == null) - { - // Reserved thread must have incremented size but not yet added itself to queue. - // We will spin until it is added. - Thread.onSpinWait(); - continue; - } - - _size.decrementAndGet(); - thread.stop(); + // yield to wait for any reserved threads that have incremented the size but not yet polled + Thread.yield(); + _queue.offer(STOP); } + // Interrupt any reserved thread missed the offer so it doesn't wait too long. + for (ReservedThread reserved : _threads) + { + Thread thread = reserved._thread; + if (thread != null) + thread.interrupt(); + } + _threads.clear(); + _count.getAndSetHi(0); } @Override @@ -207,52 +210,61 @@ public boolean tryExecute(Runnable task) { if (LOG.isDebugEnabled()) LOG.debug("{} tryExecute {}", this, task); - if (task == null) return false; - ReservedThread thread = _stack.pollFirst(); - if (thread == null) - { - if (task != STOP) - startReservedThread(); - return false; - } + // Offer will only succeed if there is a reserved thread waiting + boolean offered = _queue.offer(task); - int size = _size.decrementAndGet(); - if (!thread.offer(task)) - return false; + // If the offer succeeded we need to reduce the size, unless it is set to -1 in the meantime + int size = _count.getLo(); + while (offered && size > 0 && !_count.compareAndSetLo(size, --size)) + size = _count.getLo(); + // If size is 0 and we are not stopping, start a new reserved thread if (size == 0 && task != STOP) startReservedThread(); - return true; + return offered; } private void startReservedThread() { - try + while (true) { - while (true) + long count = _count.get(); + int pending = getHi(count); + int size = getLo(count); + if (size < 0 || pending + size >= _capacity) + return; + if (size == 0) + _lastEmptyTime.set(System.nanoTime()); + if (!_count.compareAndSet(count, pending + 1, size)) + continue; + + if (LOG.isDebugEnabled()) + LOG.debug("{} startReservedThread p={}", this, pending + 1); + try { - // Not atomic, but there is a re-check in ReservedThread.run(). - int pending = _pending.get(); - int size = _size.get(); - if (pending + size >= _capacity) - return; - if (_pending.compareAndSet(pending, pending + 1)) - { - if (LOG.isDebugEnabled()) - LOG.debug("{} startReservedThread p={}", this, pending + 1); - _executor.execute(new ReservedThread()); - return; - } + ReservedThread thread = new ReservedThread(); + _threads.add(thread); + _executor.execute(thread); } + catch (Throwable e) + { + _count.add(-1, 0); + if (LOG.isDebugEnabled()) + LOG.debug("ignored", e); + } + return; } - catch (RejectedExecutionException e) - { - LOG.trace("IGNORED", e); - } + } + + @Override + public void dump(Appendable out, String indent) throws IOException + { + Dumpable.dumpObjects(out, indent, this, + new DumpableCollection("reserved", _threads)); } @Override @@ -261,136 +273,149 @@ public String toString() return String.format("%s@%x{s=%d/%d,p=%d}", getClass().getSimpleName(), hashCode(), - _size.get(), + _count.getLo(), _capacity, - _pending.get()); + _count.getHi()); } - private class ReservedThread implements Runnable + private enum State { - private final SynchronousQueue _task = new SynchronousQueue<>(); - private boolean _starting = true; - - public boolean offer(Runnable task) - { - if (LOG.isDebugEnabled()) - LOG.debug("{} offer {}", this, task); - - try - { - _task.put(task); - return true; - } - catch (Throwable e) - { - LOG.trace("IGNORED", e); - _size.getAndIncrement(); - _stack.offerFirst(this); - return false; - } - } + PENDING, + RESERVED, + RUNNING, + IDLE, + STOPPED + } - public void stop() - { - offer(STOP); - } + private class ReservedThread implements Runnable + { + // The state and thread are kept only for dumping + private volatile State _state = State.PENDING; + private volatile Thread _thread; private Runnable reservedWait() { if (LOG.isDebugEnabled()) - LOG.debug("{} waiting", this); + LOG.debug("{} waiting {}", this, ReservedThreadExecutor.this); - while (true) + // Keep waiting until stopped, tasked or idle + while (_count.getLo() >= 0) { try { - Runnable task = _idleTime <= 0 ? _task.take() : _task.poll(_idleTime, _idleTimeUnit); + // Always poll at some period as safety to ensure we don't poll forever. + Runnable task = _queue.poll(_idleTimeNanos, NANOSECONDS); if (LOG.isDebugEnabled()) - LOG.debug("{} task={}", this, task); + LOG.debug("{} task={} {}", this, task, ReservedThreadExecutor.this); if (task != null) return task; - if (_stack.remove(this)) + // we have idled out + int size = _count.getLo(); + // decrement size if we have not also been stopped. + while (size > 0) { - if (LOG.isDebugEnabled()) - LOG.debug("{} IDLE", this); - _size.decrementAndGet(); - return STOP; + if (_count.compareAndSetLo(size, --size)) + break; + size = _count.getLo(); } + _state = size >= 0 ? State.IDLE : State.STOPPED; + return STOP; + } catch (InterruptedException e) { - LOG.trace("IGNORED", e); + if (LOG.isDebugEnabled()) + LOG.debug("ignored", e); } } + _state = State.STOPPED; + return STOP; } @Override public void run() { - while (isRunning()) + _thread = Thread.currentThread(); + try { - // test and increment size BEFORE decrementing pending, - // so that we don't have a race starting new pending. - int size = _size.get(); + while (true) + { + long count = _count.get(); - // Are we stopped? - if (size < 0) - return; + // reduce pending if this thread was pending + int pending = getHi(count) - (_state == State.PENDING ? 1 : 0); + int size = getLo(count); - // Are we surplus to capacity? - if (size >= _capacity) - { - if (LOG.isDebugEnabled()) - LOG.debug("{} size {} > capacity {}", this, size, _capacity); - if (_starting) - _pending.decrementAndGet(); - return; - } + State next; + if (size < 0 || size >= _capacity) + { + // The executor has stopped or this thread is excess to capacity + next = State.STOPPED; + } + else + { + long now = System.nanoTime(); + long lastEmpty = _lastEmptyTime.get(); + if (size > 0 && _idleTimeNanos < (now - lastEmpty) && _lastEmptyTime.compareAndSet(lastEmpty, now)) + { + // it has been too long since we hit zero reserved threads, so are "busy" idle + next = State.IDLE; + } + else + { + // We will become a reserved thread if we can update the count below. + next = State.RESERVED; + size++; + } + } - // If we cannot update size then recalculate - if (!_size.compareAndSet(size, size + 1)) - continue; + // Update count for pending and size + if (!_count.compareAndSet(count, pending, size)) + continue; - if (_starting) - { if (LOG.isDebugEnabled()) - LOG.debug("{} started", this); - _pending.decrementAndGet(); - _starting = false; - } + LOG.debug("{} was={} next={} size={}+{} capacity={}", this, _state, next, pending, size, _capacity); + _state = next; + if (next != State.RESERVED) + break; - // Insert ourselves in the stack. Size is already incremented, but - // that only effects the decision to keep other threads reserved. - _stack.offerFirst(this); + // We are reserved whilst we are waiting for an offered _task. + Runnable task = reservedWait(); - // Once added to the stack, we must always wait for a job on the _task Queue - // and never return early, else we may leave a thread blocked offering a _task. - Runnable task = reservedWait(); + // Is the task the STOP poison pill? + if (task == STOP) + break; - if (task == STOP) - // return on STOP poison pill - break; - - // Run the task - try - { - task.run(); - } - catch (Throwable e) - { - LOG.warn("Unable to run task", e); + // Run the task + try + { + _state = State.RUNNING; + task.run(); + } + catch (Throwable e) + { + LOG.warn("Unable to run task", e); + } } } - - if (LOG.isDebugEnabled()) - LOG.debug("{} Exited", this); + finally + { + if (LOG.isDebugEnabled()) + LOG.debug("{} exited {}", this, ReservedThreadExecutor.this); + _threads.remove(this); + _thread = null; + } } @Override public String toString() { - return String.format("%s@%x", ReservedThreadExecutor.this, hashCode()); + return String.format("%s@%x{%s,thread=%s}", + getClass().getSimpleName(), + hashCode(), + _state, + _thread); } } -} +} \ No newline at end of file diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/thread/ReservedThreadExecutorTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/thread/ReservedThreadExecutorTest.java index 5f86fa2b1be1..e3a08b876bf3 100644 --- a/jetty-util/src/test/java/org/eclipse/jetty/util/thread/ReservedThreadExecutorTest.java +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/thread/ReservedThreadExecutorTest.java @@ -22,10 +22,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -180,6 +180,35 @@ public void testShrink() throws Exception assertThat(_reservedExecutor.getAvailable(), is(0)); } + @Test + public void testBusyShrink() throws Exception + { + final long IDLE = 1000; + + _reservedExecutor.stop(); + _reservedExecutor.setIdleTimeout(IDLE, TimeUnit.MILLISECONDS); + _reservedExecutor.start(); + assertThat(_reservedExecutor.getAvailable(), is(0)); + + assertThat(_reservedExecutor.tryExecute(NOOP), is(false)); + assertThat(_reservedExecutor.tryExecute(NOOP), is(false)); + + _executor.startThread(); + _executor.startThread(); + + waitForAvailable(2); + + int available = _reservedExecutor.getAvailable(); + assertThat(available, is(2)); + + for (int i = 10; i-- > 0;) + { + assertThat(_reservedExecutor.tryExecute(NOOP), is(true)); + Thread.sleep(200); + } + assertThat(_reservedExecutor.getAvailable(), is(1)); + } + @Test public void testReservedIdleTimeoutWithOneReservedThread() throws Exception { @@ -261,7 +290,6 @@ public void run() } } - @Disabled @Test public void stressTest() throws Exception { @@ -271,9 +299,9 @@ public void stressTest() throws Exception reserved.setIdleTimeout(0, null); reserved.start(); - final int LOOPS = 1000000; + final int LOOPS = 200000; final AtomicInteger executions = new AtomicInteger(LOOPS); - final CountDownLatch executed = new CountDownLatch(executions.get()); + final CountDownLatch executed = new CountDownLatch(LOOPS); final AtomicInteger usedReserved = new AtomicInteger(0); final AtomicInteger usedPool = new AtomicInteger(0); @@ -322,10 +350,15 @@ public void run() assertTrue(executed.await(60, TimeUnit.SECONDS)); + // ensure tryExecute is still working + while (!reserved.tryExecute(() -> {})) + Thread.yield(); + reserved.stop(); pool.stop(); + assertThat(usedReserved.get(), greaterThan(0)); assertThat(usedReserved.get() + usedPool.get(), is(LOOPS)); - System.err.printf("reserved=%d pool=%d total=%d%n", usedReserved.get(), usedPool.get(), LOOPS); + // System.err.printf("reserved=%d pool=%d total=%d%n", usedReserved.get(), usedPool.get(), LOOPS); } } From 72fc018cc56adc8971900a449de7e3331fbfc650 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 30 Jul 2021 07:25:03 -0500 Subject: [PATCH 06/54] Issue #6544 - Updates based on review Signed-off-by: Joakim Erdfelt --- .../src/main/webapp/WEB-INF/web.xml | 4 +- .../src/main/webapp/jetty.icon | Bin 0 -> 6586 bytes .../tests/distribution/GzipModuleTests.java | 60 ++++++++++++++---- 3 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 demos/demo-simple-webapp/src/main/webapp/jetty.icon diff --git a/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml b/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml index 1aebcaace5ce..8ce8b679a5a0 100644 --- a/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml +++ b/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml @@ -7,8 +7,8 @@ Simple Web Application - webp - image/webp + icon + image/vnd.microsoft.icon diff --git a/demos/demo-simple-webapp/src/main/webapp/jetty.icon b/demos/demo-simple-webapp/src/main/webapp/jetty.icon new file mode 100644 index 0000000000000000000000000000000000000000..54e2e6104332729142765d92f2a8168ca4a84891 GIT binary patch literal 6586 zcmZQzU}Rur5D;Jh(h3Y2EDQ``3=9kk3QRygBZC1W1H(R`n1KP5&&XYLHY5)R|{v06fC&|E|0i=OaKoN*IkYZo}nu8AhkAl$<7!83D8v^L%0jYdYIimo? z4nPbluRx#yOhO1IAO*r801RXh9#8;r$-t4I@G=161R#co7bMt#B9O4c3@>ol0mE&S zh9n;Pg#$QJMu`Cn0Z@Kv0ND=ABe49!zyT6~fWcP23~1ptYWZjgjE2By2#kinXb6nZ z5P-D{M(NQI7!85Z5Eu=C(GVC7fsqpeodJICyj)UTKp*pZdbk9E2M%B(2PK?SPXi^4 zJzX3_GVZ-SwGcR{z{8~Qzdp)8BDTS&szZUJ2^%`}-r)14hnGTf=DLGI($m$?Wt~$( F697q|!wUca literal 0 HcmV?d00001 diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java index 64cd664d2b13..af1755687e38 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java @@ -25,7 +25,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -44,8 +43,6 @@ public void testGzipDefault() throws Exception .build(); int httpPort = distribution.freePort(); - int httpsPort = distribution.freePort(); - assertThat("httpPort != httpsPort", httpPort, is(not(httpsPort))); String[] argsConfig = { "--add-modules=gzip", @@ -59,8 +56,7 @@ public void testGzipDefault() throws Exception String[] argsStart = { "jetty.http.port=" + httpPort, - "jetty.httpConfig.port=" + httpsPort, - "jetty.ssl.port=" + httpsPort + "jetty.httpConfig.port=" + httpPort }; File war = distribution.resolveArtifact("org.eclipse.jetty.demos:demo-simple-webapp:war:" + jettyVersion); @@ -79,7 +75,7 @@ public void testGzipDefault() throws Exception } @Test - public void testGzipExcludeMimeType() throws Exception + public void testGzipDefaultExcludedMimeType() throws Exception { Path jettyBase = newTestJettyBaseDirectory(); String jettyVersion = System.getProperty("jettyVersion"); @@ -90,8 +86,6 @@ public void testGzipExcludeMimeType() throws Exception .build(); int httpPort = distribution.freePort(); - int httpsPort = distribution.freePort(); - assertThat("httpPort != httpsPort", httpPort, is(not(httpsPort))); String[] argsConfig = { "--add-modules=gzip", @@ -105,9 +99,7 @@ public void testGzipExcludeMimeType() throws Exception String[] argsStart = { "jetty.http.port=" + httpPort, - "jetty.httpConfig.port=" + httpsPort, - "jetty.ssl.port=" + httpsPort, - "jetty.gzip.excludedMimeTypeList=image/webp" + "jetty.httpConfig.port=" + httpPort }; File war = distribution.resolveArtifact("org.eclipse.jetty.demos:demo-simple-webapp:war:" + jettyVersion); @@ -120,11 +112,57 @@ public void testGzipExcludeMimeType() throws Exception startHttpClient(); ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/jetty.webp"); assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat("Correct Content-Type", response.getHeaders().get(HttpHeader.CONTENT_TYPE), containsString("image/webp")); assertThat("Ensure that gzip exclusion worked", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); } } } + @Test + public void testGzipAddWebappSpecificExcludeMimeType() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + + String[] argsConfig = { + "--add-modules=gzip", + "--add-modules=deploy,webapp,http" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(5, TimeUnit.SECONDS)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort, + "jetty.httpConfig.port=" + httpPort, + "jetty.gzip.excludedMimeTypeList=image/vnd.microsoft.icon" + }; + + File war = distribution.resolveArtifact("org.eclipse.jetty.demos:demo-simple-webapp:war:" + jettyVersion); + distribution.installWarFile(war, "demo"); + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/jetty.icon"); + assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat("Ensure that gzip exclusion worked", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); + assertThat("Correct Content-Type", response.getHeaders().get(HttpHeader.CONTENT_TYPE), containsString("image/vnd.microsoft.icon")); + } + } + } + private static class ResponseDetails implements Supplier { private final ContentResponse response; From 386836055562cdde7dfb8a7af4fdc33eeb5dc46c Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 30 Jul 2021 07:29:59 -0500 Subject: [PATCH 07/54] Issue #6544 - More updates based on review and merge Signed-off-by: Joakim Erdfelt --- .../tests/distribution/GzipModuleTests.java | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java index af1755687e38..040e79e9d6ae 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/GzipModuleTests.java @@ -16,7 +16,6 @@ import java.io.File; import java.nio.file.Path; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.http.HttpHeader; @@ -68,8 +67,9 @@ public void testGzipDefault() throws Exception startHttpClient(); ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/index.html"); - assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); - assertThat("Ensure that gzip is working", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), containsString("gzip")); + String responseDetails = toResponseDetails(response); + assertEquals(HttpStatus.OK_200, response.getStatus(), responseDetails); + assertThat(responseDetails, response.getHeaders().get(HttpHeader.CONTENT_ENCODING), containsString("gzip")); } } } @@ -111,9 +111,10 @@ public void testGzipDefaultExcludedMimeType() throws Exception startHttpClient(); ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/jetty.webp"); - assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); - assertThat("Correct Content-Type", response.getHeaders().get(HttpHeader.CONTENT_TYPE), containsString("image/webp")); - assertThat("Ensure that gzip exclusion worked", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); + String responseDetails = toResponseDetails(response); + assertEquals(HttpStatus.OK_200, response.getStatus(), responseDetails); + assertThat(responseDetails, response.getHeaders().get(HttpHeader.CONTENT_TYPE), containsString("image/webp")); + assertThat(responseDetails, response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); } } } @@ -156,30 +157,20 @@ public void testGzipAddWebappSpecificExcludeMimeType() throws Exception startHttpClient(); ContentResponse response = client.GET("http://localhost:" + httpPort + "/demo/jetty.icon"); - assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); - assertThat("Ensure that gzip exclusion worked", response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); - assertThat("Correct Content-Type", response.getHeaders().get(HttpHeader.CONTENT_TYPE), containsString("image/vnd.microsoft.icon")); + String responseDetails = toResponseDetails(response); + assertEquals(HttpStatus.OK_200, response.getStatus(), responseDetails); + assertThat(responseDetails, response.getHeaders().get(HttpHeader.CONTENT_ENCODING), not(containsString("gzip"))); + assertThat(responseDetails, response.getHeaders().get(HttpHeader.CONTENT_TYPE), containsString("image/vnd.microsoft.icon")); } } } - private static class ResponseDetails implements Supplier + private static String toResponseDetails(ContentResponse response) { - private final ContentResponse response; - - public ResponseDetails(ContentResponse response) - { - this.response = response; - } - - @Override - public String get() - { - StringBuilder ret = new StringBuilder(); - ret.append(response.toString()).append(System.lineSeparator()); - ret.append(response.getHeaders().toString()).append(System.lineSeparator()); - ret.append(response.getContentAsString()).append(System.lineSeparator()); - return ret.toString(); - } + StringBuilder ret = new StringBuilder(); + ret.append(response.toString()).append(System.lineSeparator()); + ret.append(response.getHeaders().toString()).append(System.lineSeparator()); + ret.append(response.getContentAsString()).append(System.lineSeparator()); + return ret.toString(); } } From d5da842ea8c914f15a5880b2c14ba28daa6755b2 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 30 Jul 2021 07:32:19 -0500 Subject: [PATCH 08/54] Issue #6544 - Remove braces Signed-off-by: Joakim Erdfelt --- .../test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java index e27d87ab6974..c6470583c79c 100644 --- a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java +++ b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java @@ -163,15 +163,12 @@ private String getContentTypeFromRequest(String filename, HttpServletRequest req String defaultContentType = "application/octet-stream"; if (req.getParameter("type") != null) defaultContentType = req.getParameter("type"); - ServletContextHandler servletContextHandler = ServletContextHandler.getServletContextHandler(getServletContext()); if (servletContextHandler == null) return defaultContentType; String contentType = servletContextHandler.getMimeTypes().getMimeByExtension(filename); if (contentType != null) - { return contentType; - } return defaultContentType; } } From dc48a4b4521f069d50e19426407a6769ce516d5b Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 30 Jul 2021 08:56:21 -0500 Subject: [PATCH 09/54] Issue #6544 - Adding comment about mime-type in web.xml Signed-off-by: Joakim Erdfelt --- demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml b/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml index 8ce8b679a5a0..79d16dcd8500 100644 --- a/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml +++ b/demos/demo-simple-webapp/src/main/webapp/WEB-INF/web.xml @@ -6,6 +6,7 @@ Simple Web Application + icon image/vnd.microsoft.icon From 90a72b079826e65e11f638641918d98651103025 Mon Sep 17 00:00:00 2001 From: Jan Bartel Date: Mon, 2 Aug 2021 10:04:51 +1000 Subject: [PATCH 10/54] Issue #6556 Ensure context classloader set when operating on memcache. (#6557) * Issue #6556 Ensure context classloader set when operating on memcache. Signed-off-by: Jan Bartel --- .../session/MemcachedSessionDataMap.java | 46 ++++++++++++++- .../session/AbstractSessionDataStore.java | 59 +++++-------------- .../org/eclipse/jetty/util/FuturePromise.java | 24 ++++++++ 3 files changed, 81 insertions(+), 48 deletions(-) diff --git a/jetty-memcached/jetty-memcached-sessions/src/main/java/org/eclipse/jetty/memcached/session/MemcachedSessionDataMap.java b/jetty-memcached/jetty-memcached-sessions/src/main/java/org/eclipse/jetty/memcached/session/MemcachedSessionDataMap.java index 531da8eaf1b3..0d503bc5dba6 100644 --- a/jetty-memcached/jetty-memcached-sessions/src/main/java/org/eclipse/jetty/memcached/session/MemcachedSessionDataMap.java +++ b/jetty-memcached/jetty-memcached-sessions/src/main/java/org/eclipse/jetty/memcached/session/MemcachedSessionDataMap.java @@ -25,6 +25,7 @@ import org.eclipse.jetty.server.session.SessionData; import org.eclipse.jetty.server.session.SessionDataMap; import org.eclipse.jetty.util.ClassLoadingObjectInputStream; +import org.eclipse.jetty.util.FuturePromise; import org.eclipse.jetty.util.annotation.ManagedAttribute; import org.eclipse.jetty.util.annotation.ManagedObject; import org.eclipse.jetty.util.component.AbstractLifeCycle; @@ -43,6 +44,7 @@ public class MemcachedSessionDataMap extends AbstractLifeCycle implements Sessio protected int _expirySec = 0; protected boolean _heartbeats = true; protected XMemcachedClientBuilder _builder; + protected SessionContext _context; /** * SessionDataTranscoder @@ -140,8 +142,12 @@ public void setHeartbeats(boolean heartbeats) @Override public void initialize(SessionContext context) { + if (isStarted()) + throw new IllegalStateException("Context set after MemcachedSessionDataMap started"); + try { + _context = context; _builder.setTranscoder(new SessionDataTranscoder()); _client = _builder.build(); _client.setEnableHeartBeat(isHeartbeats()); @@ -155,14 +161,48 @@ public void initialize(SessionContext context) @Override public SessionData load(String id) throws Exception { - SessionData data = _client.get(id); - return data; + if (!isStarted()) + throw new IllegalStateException("Not started"); + + final FuturePromise result = new FuturePromise<>(); + + Runnable r = () -> + { + try + { + result.succeeded(_client.get(id)); + } + catch (Exception e) + { + result.failed(e); + } + }; + + _context.run(r); + return result.getOrThrow(); } @Override public void store(String id, SessionData data) throws Exception { - _client.set(id, _expirySec, data); + if (!isStarted()) + throw new IllegalStateException("Not started"); + + final FuturePromise result = new FuturePromise<>(); + Runnable r = () -> + { + try + { + _client.set(id, _expirySec, data); + result.succeeded(null); + } + catch (Exception e) + { + result.failed(e); + } + }; + _context.run(r); + result.getOrThrow(); } @Override diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/session/AbstractSessionDataStore.java b/jetty-server/src/main/java/org/eclipse/jetty/server/session/AbstractSessionDataStore.java index 49ee9690f160..524bccb5d881 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/session/AbstractSessionDataStore.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/session/AbstractSessionDataStore.java @@ -15,8 +15,10 @@ import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.eclipse.jetty.util.FuturePromise; import org.eclipse.jetty.util.annotation.ManagedAttribute; import org.eclipse.jetty.util.annotation.ManagedObject; import org.eclipse.jetty.util.component.ContainerLifeCycle; @@ -40,41 +42,6 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem protected long _lastOrphanSweepTime = 0; //last time in ms that we deleted orphaned sessions protected int _savePeriodSec = DEFAULT_SAVE_PERIOD_SEC; //time in sec between saves - /** - * Small utility class to allow us to - * return a result and an Exception - * from invocation of Runnables. - * - * @param the type of the result. - */ - private class Result - { - private V _result; - private Exception _exception; - - public void setResult(V result) - { - _result = result; - } - - public void setException(Exception exception) - { - _exception = exception; - } - - private void throwIfException() throws Exception - { - if (_exception != null) - throw _exception; - } - - public V getOrThrow() throws Exception - { - throwIfException(); - return _result; - } - } - /** * Check if a session for the given id exists. * @@ -171,21 +138,22 @@ public SessionData load(String id) throws Exception if (!isStarted()) throw new IllegalStateException("Not started"); - final Result result = new Result<>(); - + final FuturePromise result = new FuturePromise<>(); + Runnable r = () -> { try { - result.setResult(doLoad(id)); + result.succeeded(doLoad(id)); } catch (Exception e) { - result.setException(e); + result.failed(e); } }; _context.run(r); + return result.getOrThrow(); } @@ -214,7 +182,7 @@ public void store(String id, SessionData data) throws Exception //set the last saved time to now data.setLastSaved(System.currentTimeMillis()); - final Result result = new Result<>(); + final FuturePromise result = new FuturePromise<>(); Runnable r = () -> { try @@ -222,32 +190,33 @@ public void store(String id, SessionData data) throws Exception //call the specific store method, passing in previous save time doStore(id, data, lastSave); data.clean(); //unset all dirty flags + result.succeeded(null); } catch (Exception e) { //reset last save time if save failed data.setLastSaved(lastSave); - result.setException(e); + result.failed(e); } }; _context.run(r); - result.throwIfException(); + result.getOrThrow(); } } @Override public boolean exists(String id) throws Exception { - Result result = new Result<>(); + FuturePromise result = new FuturePromise<>(); Runnable r = () -> { try { - result.setResult(doExists(id)); + result.succeeded(doExists(id)); } catch (Exception e) { - result.setException(e); + result.failed(e); } }; diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/FuturePromise.java b/jetty-util/src/main/java/org/eclipse/jetty/util/FuturePromise.java index ad8ab65aa207..deae5c669c0a 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/FuturePromise.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/FuturePromise.java @@ -118,6 +118,30 @@ public C get() throws InterruptedException, ExecutionException throw (CancellationException)new CancellationException().initCause(_cause); throw new ExecutionException(_cause); } + + /** + * Return the result if completed successfully + * or in the case of failure, throw the + * Exception/Error, or an ExecutionException wrapping + * the cause if it is neither an Exception or Error. + * + * @return the computed result + * @throws Exception if the cause is an Exception or Error, + * otherwise an ExecutionException wrapping the cause + */ + public C getOrThrow() throws Exception + { + _latch.await(); + + if (_cause == COMPLETED) + return _result; + if (_cause instanceof Exception) + throw (Exception)_cause; + if (_cause instanceof Error) + throw (Error)_cause; + + throw new ExecutionException(_cause); + } @Override public C get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException From 4e3e99c5c5dfb7f2b6f7884c4c97cc071bb73d07 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Mon, 2 Aug 2021 18:16:34 +1000 Subject: [PATCH 11/54] Fix #6565 Deploy Symlinked applications (#6567) Fix #6565 Deploy Symlinked applications by treating extracting context name (which becomes the default context path) from the base resource and then following aliases, so that base resource will not be an alias. Added warning in ContextHandler if the base resource is an alias that we may not support this in future releases. Signed-off-by: Greg Wilkins --- .../deploy/providers/WebAppProvider.java | 99 ++++++++++++------- .../deploy/providers/WebAppProviderTest.java | 6 +- .../jetty/server/handler/ContextHandler.java | 8 +- 3 files changed, 72 insertions(+), 41 deletions(-) diff --git a/jetty-deploy/src/main/java/org/eclipse/jetty/deploy/providers/WebAppProvider.java b/jetty-deploy/src/main/java/org/eclipse/jetty/deploy/providers/WebAppProvider.java index 24c6958c6674..b531ce1d1ea1 100644 --- a/jetty-deploy/src/main/java/org/eclipse/jetty/deploy/providers/WebAppProvider.java +++ b/jetty-deploy/src/main/java/org/eclipse/jetty/deploy/providers/WebAppProvider.java @@ -22,6 +22,7 @@ import org.eclipse.jetty.deploy.util.FileID; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.util.StringUtil; import org.eclipse.jetty.util.URIUtil; import org.eclipse.jetty.util.annotation.ManagedAttribute; import org.eclipse.jetty.util.annotation.ManagedObject; @@ -265,8 +266,18 @@ public ContextHandler createContextHandler(final App app) throws Exception if (!resource.exists()) throw new IllegalStateException("App resource does not exist " + resource); - String context = file.getName(); + final String contextName = file.getName(); + // Resource aliases (after getting name) to ensure baseResource is not an alias + if (resource.isAlias()) + { + file = new File(resource.getAlias()).toPath().toRealPath().toFile(); + resource = Resource.newResource(file); + if (!resource.exists()) + throw new IllegalStateException("App resource does not exist " + resource); + } + + // Handle a context XML file if (resource.exists() && FileID.isXmlFile(file)) { XmlConfiguration xmlc = new XmlConfiguration(resource) @@ -276,11 +287,15 @@ public void initializeDefaults(Object context) { super.initializeDefaults(context); + // If the XML created object is a ContextHandler + if (context instanceof ContextHandler) + // Initialize the context path prior to running context XML + initializeContextPath((ContextHandler)context, contextName, true); + + // If it is a webapp if (context instanceof WebAppContext) - { - WebAppContext webapp = (WebAppContext)context; - initializeWebAppContextDefaults(webapp); - } + // initialize other defaults prior to running context XML + initializeWebAppContextDefaults((WebAppContext)context); } }; @@ -290,54 +305,62 @@ public void initializeDefaults(Object context) xmlc.getProperties().putAll(getConfigurationManager().getProperties()); return (ContextHandler)xmlc.configure(); } - else if (file.isDirectory()) - { - // must be a directory - } - else if (FileID.isWebArchiveFile(file)) - { - // Context Path is the same as the archive. - context = context.substring(0, context.length() - 4); - } - else + // Otherwise it must be a directory or an archive + else if (!file.isDirectory() && !FileID.isWebArchiveFile(file)) { throw new IllegalStateException("unable to create ContextHandler for " + app); } - // Ensure "/" is Not Trailing in context paths. - if (context.endsWith("/") && context.length() > 0) - { - context = context.substring(0, context.length() - 1); - } - - // Start building the webapplication + // Build the web application WebAppContext webAppContext = new WebAppContext(); - webAppContext.setDisplayName(context); + webAppContext.setWar(file.getAbsolutePath()); + initializeContextPath(webAppContext, contextName, !file.isDirectory()); + initializeWebAppContextDefaults(webAppContext); + + return webAppContext; + } + + protected void initializeContextPath(ContextHandler context, String contextName, boolean stripExtension) + { + String contextPath = contextName; + + // Strip any 3 char extension from non directories + if (stripExtension && contextPath.length() > 4 && contextPath.charAt(contextPath.length() - 4) == '.') + contextPath = contextPath.substring(0, contextPath.length() - 4); + + // Ensure "/" is Not Trailing in context paths. + if (contextPath.endsWith("/") && contextPath.length() > 1) + contextPath = contextPath.substring(0, contextPath.length() - 1); // special case of archive (or dir) named "root" is / context - if (context.equalsIgnoreCase("root")) + if (contextPath.equalsIgnoreCase("root")) { - context = URIUtil.SLASH; + contextPath = URIUtil.SLASH; } - else if (context.toLowerCase(Locale.ENGLISH).startsWith("root-")) + // handle root with virtual host form + else if (StringUtil.startsWithIgnoreCase(contextPath, "root-")) { - int dash = context.toLowerCase(Locale.ENGLISH).indexOf('-'); - String virtual = context.substring(dash + 1); - webAppContext.setVirtualHosts(new String[]{virtual}); - context = URIUtil.SLASH; + int dash = contextPath.indexOf('-'); + String virtual = contextPath.substring(dash + 1); + context.setVirtualHosts(virtual.split(",")); + contextPath = URIUtil.SLASH; } // Ensure "/" is Prepended to all context paths. - if (context.charAt(0) != '/') + if (contextPath.charAt(0) != '/') + contextPath = "/" + contextPath; + + // Set the display name and context Path + context.setDisplayName(contextName); + if (context instanceof WebAppContext) { - context = "/" + context; + WebAppContext webAppContext = (WebAppContext)context; + webAppContext.setDefaultContextPath(contextPath); + } + else + { + context.setContextPath(contextPath); } - - webAppContext.setDefaultContextPath(context); - webAppContext.setWar(file.getAbsolutePath()); - initializeWebAppContextDefaults(webAppContext); - - return webAppContext; } @Override diff --git a/jetty-deploy/src/test/java/org/eclipse/jetty/deploy/providers/WebAppProviderTest.java b/jetty-deploy/src/test/java/org/eclipse/jetty/deploy/providers/WebAppProviderTest.java index 90a687b9663f..e0ec2a8e8c4b 100644 --- a/jetty-deploy/src/test/java/org/eclipse/jetty/deploy/providers/WebAppProviderTest.java +++ b/jetty-deploy/src/test/java/org/eclipse/jetty/deploy/providers/WebAppProviderTest.java @@ -25,6 +25,7 @@ import org.eclipse.jetty.deploy.test.XmlConfiguredJetty; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.toolchain.test.MavenTestingUtils; import org.eclipse.jetty.toolchain.test.jupiter.WorkDir; @@ -124,9 +125,12 @@ public void testStartupSymlinkContext() // Check Server for expected Handlers jetty.assertWebAppContextsExists("/bar", "/foo", "/bob"); + // Check that baseResources are not aliases + jetty.getServer().getContainedBeans(ContextHandler.class).forEach(h -> assertFalse(h.getBaseResource().isAlias())); + // Test for expected work/temp directory behaviour File workDir = jetty.getJettyDir("workish"); - assertTrue(hasJettyGeneratedPath(workDir, "bar_war"), "Should have generated directory in work directory: " + workDir); + assertTrue(hasJettyGeneratedPath(workDir, "_war-_bar"), "Should have generated directory in work directory: " + workDir); } @Test diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/ContextHandler.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/ContextHandler.java index 9e4735621c3a..414ff29d510a 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/ContextHandler.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/ContextHandler.java @@ -853,11 +853,15 @@ public void setLogger(Logger logger) @Override protected void doStart() throws Exception { - _availability.set(Availability.STARTING); - if (_contextPath == null) throw new IllegalStateException("Null contextPath"); + if (getBaseResource() != null && getBaseResource().isAlias()) + LOG.warn("BaseResource {} is aliased to {} in {}. May not be supported in future releases.", + getBaseResource(), getBaseResource().getAlias(), this); + + _availability.set(Availability.STARTING); + if (_logger == null) _logger = LoggerFactory.getLogger(ContextHandler.class.getName() + getLogNameSuffix()); From bbabaee8cc1a1017b20816a675d8024412301fcc Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 4 Aug 2021 10:05:50 +1000 Subject: [PATCH 12/54] Issue #6566 - add executor to WebSocketComponents & use for Dispatched Messages Signed-off-by: Lachlan Roberts --- .../core/client/WebSocketCoreClient.java | 9 ++----- .../client/internal/HttpClientProvider.java | 7 +----- .../websocket/core/WebSocketComponents.java | 16 +++++++++++++ .../messages/DispatchedMessageSink.java | 7 ++++-- .../server/WebSocketServerComponents.java | 13 +++++++--- .../JavaxWebSocketServerContainer.java | 24 +------------------ 6 files changed, 35 insertions(+), 41 deletions(-) diff --git a/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/WebSocketCoreClient.java b/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/WebSocketCoreClient.java index c8db7f1d82f3..950231f0fa17 100644 --- a/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/WebSocketCoreClient.java +++ b/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/WebSocketCoreClient.java @@ -40,13 +40,6 @@ public class WebSocketCoreClient extends ContainerLifeCycle private final WebSocketComponents components; private ClassLoader classLoader; - // TODO: Things to consider for inclusion in this class (or removal if they can be set elsewhere, like HttpClient) - // - AsyncWrite Idle Timeout - // - Bind Address - // - SslContextFactory setup - // - Connect Timeout - // - Cookie Store - public WebSocketCoreClient() { this(null, new WebSocketComponents()); @@ -61,6 +54,8 @@ public WebSocketCoreClient(HttpClient httpClient, WebSocketComponents webSocketC { if (httpClient == null) httpClient = Objects.requireNonNull(HttpClientProvider.get()); + if (httpClient.getExecutor() == null) + httpClient.setExecutor(webSocketComponents.getExecutor()); this.classLoader = Thread.currentThread().getContextClassLoader(); this.httpClient = httpClient; diff --git a/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/internal/HttpClientProvider.java b/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/internal/HttpClientProvider.java index f6b855e8525f..d1ea884064ed 100644 --- a/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/internal/HttpClientProvider.java +++ b/jetty-websocket/websocket-core-client/src/main/java/org/eclipse/jetty/websocket/core/client/internal/HttpClientProvider.java @@ -14,7 +14,6 @@ package org.eclipse.jetty.websocket.core.client.internal; import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.util.thread.QueuedThreadPool; public interface HttpClientProvider { @@ -30,11 +29,7 @@ static HttpClient get() private static HttpClient newDefaultHttpClient() { - HttpClient client = new HttpClient(); - QueuedThreadPool threadPool = new QueuedThreadPool(); - threadPool.setName("WebSocketClient@" + client.hashCode()); - client.setExecutor(threadPool); - return client; + return new HttpClient(); } default HttpClient newHttpClient() diff --git a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java index f9c6e86e742e..76168da640a0 100644 --- a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java +++ b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java @@ -13,6 +13,7 @@ package org.eclipse.jetty.websocket.core; +import java.util.concurrent.Executor; import java.util.zip.Deflater; import org.eclipse.jetty.io.ByteBufferPool; @@ -22,6 +23,7 @@ import org.eclipse.jetty.util.compression.CompressionPool; import org.eclipse.jetty.util.compression.DeflaterPool; import org.eclipse.jetty.util.compression.InflaterPool; +import org.eclipse.jetty.util.thread.QueuedThreadPool; /** * A collection of components which are the resources needed for websockets such as @@ -31,6 +33,7 @@ public class WebSocketComponents extends ContainerLifeCycle { private final DecoratedObjectFactory objectFactory; private final WebSocketExtensionRegistry extensionRegistry; + private final Executor executor; private final ByteBufferPool bufferPool; private final InflaterPool inflaterPool; private final DeflaterPool deflaterPool; @@ -42,18 +45,26 @@ public WebSocketComponents() public WebSocketComponents(WebSocketExtensionRegistry extensionRegistry, DecoratedObjectFactory objectFactory, ByteBufferPool bufferPool, InflaterPool inflaterPool, DeflaterPool deflaterPool) + { + this (extensionRegistry, objectFactory, bufferPool, inflaterPool, deflaterPool, null); + } + + public WebSocketComponents(WebSocketExtensionRegistry extensionRegistry, DecoratedObjectFactory objectFactory, + ByteBufferPool bufferPool, InflaterPool inflaterPool, DeflaterPool deflaterPool, Executor executor) { this.extensionRegistry = (extensionRegistry == null) ? new WebSocketExtensionRegistry() : extensionRegistry; this.objectFactory = (objectFactory == null) ? new DecoratedObjectFactory() : objectFactory; this.bufferPool = (bufferPool == null) ? new MappedByteBufferPool() : bufferPool; this.inflaterPool = (inflaterPool == null) ? new InflaterPool(CompressionPool.DEFAULT_CAPACITY, true) : inflaterPool; this.deflaterPool = (deflaterPool == null) ? new DeflaterPool(CompressionPool.DEFAULT_CAPACITY, Deflater.DEFAULT_COMPRESSION, true) : deflaterPool; + this.executor = (executor == null) ? new QueuedThreadPool() : executor; addBean(inflaterPool); addBean(deflaterPool); addBean(bufferPool); addBean(extensionRegistry); addBean(objectFactory); + addBean(executor); } public ByteBufferPool getBufferPool() @@ -61,6 +72,11 @@ public ByteBufferPool getBufferPool() return bufferPool; } + public Executor getExecutor() + { + return executor; + } + public WebSocketExtensionRegistry getExtensionRegistry() { return extensionRegistry; diff --git a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/internal/messages/DispatchedMessageSink.java b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/internal/messages/DispatchedMessageSink.java index 875fab0096bd..57d31e0d3f67 100644 --- a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/internal/messages/DispatchedMessageSink.java +++ b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/internal/messages/DispatchedMessageSink.java @@ -16,6 +16,7 @@ import java.io.Closeable; import java.lang.invoke.MethodHandle; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.IO; @@ -95,10 +96,12 @@ public abstract class DispatchedMessageSink extends AbstractMessageSink { private CompletableFuture dispatchComplete; private MessageSink typeSink; + private final Executor executor; public DispatchedMessageSink(CoreSession session, MethodHandle methodHandle) { super(session, methodHandle); + executor = session.getWebSocketComponents().getExecutor(); } public abstract MessageSink newSink(Frame frame); @@ -112,7 +115,7 @@ public void accept(Frame frame, final Callback callback) // Dispatch to end user function (will likely start with blocking for data/accept). // If the MessageSink can be closed do this after invoking and before completing the CompletableFuture. - new Thread(() -> + executor.execute(() -> { try { @@ -129,7 +132,7 @@ public void accept(Frame frame, final Callback callback) dispatchComplete.completeExceptionally(throwable); } - }).start(); + }); } Callback frameCallback = callback; diff --git a/jetty-websocket/websocket-core-server/src/main/java/org/eclipse/jetty/websocket/core/server/WebSocketServerComponents.java b/jetty-websocket/websocket-core-server/src/main/java/org/eclipse/jetty/websocket/core/server/WebSocketServerComponents.java index 946882acda1b..d5cd19562f2e 100644 --- a/jetty-websocket/websocket-core-server/src/main/java/org/eclipse/jetty/websocket/core/server/WebSocketServerComponents.java +++ b/jetty-websocket/websocket-core-server/src/main/java/org/eclipse/jetty/websocket/core/server/WebSocketServerComponents.java @@ -14,6 +14,7 @@ package org.eclipse.jetty.websocket.core.server; import java.util.Objects; +import java.util.concurrent.Executor; import javax.servlet.ServletContext; import org.eclipse.jetty.io.ByteBufferPool; @@ -40,9 +41,9 @@ public class WebSocketServerComponents extends WebSocketComponents public static final String WEBSOCKET_DEFLATER_POOL_ATTRIBUTE = "jetty.websocket.deflater"; public static final String WEBSOCKET_BUFFER_POOL_ATTRIBUTE = "jetty.websocket.bufferPool"; - WebSocketServerComponents(InflaterPool inflaterPool, DeflaterPool deflaterPool, ByteBufferPool bufferPool, DecoratedObjectFactory objectFactory) + WebSocketServerComponents(InflaterPool inflaterPool, DeflaterPool deflaterPool, ByteBufferPool bufferPool, DecoratedObjectFactory objectFactory, Executor executor) { - super(null, objectFactory, bufferPool, inflaterPool, deflaterPool); + super(null, objectFactory, bufferPool, inflaterPool, deflaterPool, executor); } /** @@ -79,8 +80,12 @@ public static WebSocketComponents ensureWebSocketComponents(Server server, Servl if (bufferPool == null) bufferPool = server.getBean(ByteBufferPool.class); + Executor executor = (Executor)servletContext.getAttribute("org.eclipse.jetty.server.Executor"); + if (executor == null) + executor = server.getThreadPool(); + DecoratedObjectFactory objectFactory = (DecoratedObjectFactory)servletContext.getAttribute(DecoratedObjectFactory.ATTR); - WebSocketComponents serverComponents = new WebSocketServerComponents(inflaterPool, deflaterPool, bufferPool, objectFactory); + WebSocketComponents serverComponents = new WebSocketServerComponents(inflaterPool, deflaterPool, bufferPool, objectFactory, executor); if (objectFactory != null) serverComponents.unmanage(objectFactory); @@ -92,6 +97,8 @@ public static WebSocketComponents ensureWebSocketComponents(Server server, Servl serverComponents.unmanage(deflaterPool); if (server.contains(bufferPool)) serverComponents.unmanage(bufferPool); + if (executor != null) + serverComponents.unmanage(executor); // Stop the WebSocketComponents when the ContextHandler stops. ContextHandler contextHandler = Objects.requireNonNull(ContextHandler.getContextHandler(servletContext)); diff --git a/jetty-websocket/websocket-javax-server/src/main/java/org/eclipse/jetty/websocket/javax/server/internal/JavaxWebSocketServerContainer.java b/jetty-websocket/websocket-javax-server/src/main/java/org/eclipse/jetty/websocket/javax/server/internal/JavaxWebSocketServerContainer.java index ac488c842da8..6683b931244d 100644 --- a/jetty-websocket/websocket-javax-server/src/main/java/org/eclipse/jetty/websocket/javax/server/internal/JavaxWebSocketServerContainer.java +++ b/jetty-websocket/websocket-javax-server/src/main/java/org/eclipse/jetty/websocket/javax/server/internal/JavaxWebSocketServerContainer.java @@ -73,12 +73,7 @@ public static JavaxWebSocketServerContainer ensureContainer(ServletContext servl if (httpClient == null) httpClient = (HttpClient)contextHandler.getServer().getAttribute(JavaxWebSocketServletContainerInitializer.HTTPCLIENT_ATTRIBUTE); - Executor executor = httpClient == null ? null : httpClient.getExecutor(); - if (executor == null) - executor = (Executor)servletContext.getAttribute("org.eclipse.jetty.server.Executor"); - if (executor == null) - executor = contextHandler.getServer().getThreadPool(); - + Executor executor = wsComponents.getExecutor(); if (httpClient != null && httpClient.getExecutor() == null) httpClient.setExecutor(executor); @@ -123,23 +118,6 @@ public void lifeCycleStopping(LifeCycle event) private List> deferredEndpointClasses; private List deferredEndpointConfigs; - /** - * Main entry point for {@link JavaxWebSocketServletContainerInitializer}. - * - * @param webSocketMappings the {@link WebSocketMappings} that this container belongs to - */ - public JavaxWebSocketServerContainer(WebSocketMappings webSocketMappings) - { - this(webSocketMappings, new WebSocketComponents()); - } - - public JavaxWebSocketServerContainer(WebSocketMappings webSocketMappings, WebSocketComponents components) - { - super(components); - this.webSocketMappings = webSocketMappings; - this.frameHandlerFactory = new JavaxWebSocketServerFrameHandlerFactory(this); - } - /** * Main entry point for {@link JavaxWebSocketServletContainerInitializer}. * From 14c09e3c989d0126f3f3352133cc878ca5e80a54 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 4 Aug 2021 13:28:10 +1000 Subject: [PATCH 13/54] Issue #6566 - fix WebSocketComponents LifeCycle issue Signed-off-by: Lachlan Roberts --- .../websocket/core/WebSocketComponents.java | 48 +++++++++---------- .../JettyWebSocketFrameHandlerTest.java | 38 +++++++++------ 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java index 76168da640a0..5579b7ad5a02 100644 --- a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java +++ b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java @@ -31,12 +31,12 @@ */ public class WebSocketComponents extends ContainerLifeCycle { - private final DecoratedObjectFactory objectFactory; - private final WebSocketExtensionRegistry extensionRegistry; - private final Executor executor; - private final ByteBufferPool bufferPool; - private final InflaterPool inflaterPool; - private final DeflaterPool deflaterPool; + private final DecoratedObjectFactory _objectFactory; + private final WebSocketExtensionRegistry _extensionRegistry; + private final Executor _executor; + private final ByteBufferPool _bufferPool; + private final InflaterPool _inflaterPool; + private final DeflaterPool _deflaterPool; public WebSocketComponents() { @@ -52,48 +52,48 @@ public WebSocketComponents(WebSocketExtensionRegistry extensionRegistry, Decorat public WebSocketComponents(WebSocketExtensionRegistry extensionRegistry, DecoratedObjectFactory objectFactory, ByteBufferPool bufferPool, InflaterPool inflaterPool, DeflaterPool deflaterPool, Executor executor) { - this.extensionRegistry = (extensionRegistry == null) ? new WebSocketExtensionRegistry() : extensionRegistry; - this.objectFactory = (objectFactory == null) ? new DecoratedObjectFactory() : objectFactory; - this.bufferPool = (bufferPool == null) ? new MappedByteBufferPool() : bufferPool; - this.inflaterPool = (inflaterPool == null) ? new InflaterPool(CompressionPool.DEFAULT_CAPACITY, true) : inflaterPool; - this.deflaterPool = (deflaterPool == null) ? new DeflaterPool(CompressionPool.DEFAULT_CAPACITY, Deflater.DEFAULT_COMPRESSION, true) : deflaterPool; - this.executor = (executor == null) ? new QueuedThreadPool() : executor; + _extensionRegistry = (extensionRegistry == null) ? new WebSocketExtensionRegistry() : extensionRegistry; + _objectFactory = (objectFactory == null) ? new DecoratedObjectFactory() : objectFactory; + _bufferPool = (bufferPool == null) ? new MappedByteBufferPool() : bufferPool; + _inflaterPool = (inflaterPool == null) ? new InflaterPool(CompressionPool.DEFAULT_CAPACITY, true) : inflaterPool; + _deflaterPool = (deflaterPool == null) ? new DeflaterPool(CompressionPool.DEFAULT_CAPACITY, Deflater.DEFAULT_COMPRESSION, true) : deflaterPool; + _executor = (executor == null) ? new QueuedThreadPool() : executor; - addBean(inflaterPool); - addBean(deflaterPool); - addBean(bufferPool); - addBean(extensionRegistry); - addBean(objectFactory); - addBean(executor); + addBean(_inflaterPool); + addBean(_deflaterPool); + addBean(_bufferPool); + addBean(_extensionRegistry); + addBean(_objectFactory); + addBean(_executor); } public ByteBufferPool getBufferPool() { - return bufferPool; + return _bufferPool; } public Executor getExecutor() { - return executor; + return _executor; } public WebSocketExtensionRegistry getExtensionRegistry() { - return extensionRegistry; + return _extensionRegistry; } public DecoratedObjectFactory getObjectFactory() { - return objectFactory; + return _objectFactory; } public InflaterPool getInflaterPool() { - return inflaterPool; + return _inflaterPool; } public DeflaterPool getDeflaterPool() { - return deflaterPool; + return _deflaterPool; } } diff --git a/jetty-websocket/websocket-jetty-common/src/test/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandlerTest.java b/jetty-websocket/websocket-jetty-common/src/test/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandlerTest.java index 1fd9853c81e6..668c63ab63f7 100644 --- a/jetty-websocket/websocket-jetty-common/src/test/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandlerTest.java +++ b/jetty-websocket/websocket-jetty-common/src/test/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandlerTest.java @@ -22,6 +22,7 @@ import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.websocket.api.Session; import org.eclipse.jetty.websocket.api.StatusCode; import org.eclipse.jetty.websocket.api.WebSocketConnectionListener; @@ -46,6 +47,10 @@ public class JettyWebSocketFrameHandlerTest { private static DummyContainer container; + private final WebSocketComponents components; + private final JettyWebSocketFrameHandlerFactory endpointFactory; + private final CoreSession coreSession; + @BeforeAll public static void startContainer() throws Exception { @@ -59,22 +64,27 @@ public static void stopContainer() throws Exception container.stop(); } - private final WebSocketComponents components = new WebSocketComponents(); - private final JettyWebSocketFrameHandlerFactory endpointFactory = new JettyWebSocketFrameHandlerFactory(container, components); - private final CoreSession coreSession = new CoreSession.Empty() + public JettyWebSocketFrameHandlerTest() { - @Override - public Behavior getBehavior() - { - return Behavior.CLIENT; - } - - @Override - public WebSocketComponents getWebSocketComponents() + components = new WebSocketComponents(); + endpointFactory = new JettyWebSocketFrameHandlerFactory(container, components); + coreSession = new CoreSession.Empty() { - return components; - } - }; + @Override + public Behavior getBehavior() + { + return Behavior.CLIENT; + } + + @Override + public WebSocketComponents getWebSocketComponents() + { + return components; + } + }; + + LifeCycle.start(components); + } private JettyWebSocketFrameHandler newLocalFrameHandler(Object wsEndpoint) { From de8cbcf588bb512743973d0b18b6c90c5ca46812 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 4 Aug 2021 15:11:35 +1000 Subject: [PATCH 14/54] Fix broken tests using WebSocketComponents Signed-off-by: Lachlan Roberts --- .../javax/common/AbstractSessionTest.java | 14 +++++++++- .../client/AbstractClientSessionTest.java | 13 +++++++++- ...etFrameHandlerOnMessageTextStreamTest.java | 26 ++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractSessionTest.java b/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractSessionTest.java index e9ae179af5c7..30fa05edad77 100644 --- a/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractSessionTest.java +++ b/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractSessionTest.java @@ -18,6 +18,7 @@ import javax.websocket.Session; import org.eclipse.jetty.websocket.core.CoreSession; +import org.eclipse.jetty.websocket.core.WebSocketComponents; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -25,16 +26,26 @@ public abstract class AbstractSessionTest { protected static JavaxWebSocketSession session; protected static JavaxWebSocketContainer container; + protected static WebSocketComponents components; @BeforeAll public static void initSession() throws Exception { container = new DummyContainer(); container.start(); + components = new WebSocketComponents(); + components.start(); Object websocketPojo = new DummyEndpoint(); UpgradeRequest upgradeRequest = new UpgradeRequestAdapter(); JavaxWebSocketFrameHandler frameHandler = container.newFrameHandler(websocketPojo, upgradeRequest); - CoreSession coreSession = new CoreSession.Empty(); + CoreSession coreSession = new CoreSession.Empty() + { + @Override + public WebSocketComponents getWebSocketComponents() + { + return components; + } + }; session = new JavaxWebSocketSession(container, coreSession, frameHandler, container.getFrameHandlerFactory() .newDefaultEndpointConfig(websocketPojo.getClass())); } @@ -42,6 +53,7 @@ public static void initSession() throws Exception @AfterAll public static void stopContainer() throws Exception { + components.stop(); container.stop(); } diff --git a/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/client/AbstractClientSessionTest.java b/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/client/AbstractClientSessionTest.java index a5bbde19c45d..135c4b08a0ab 100644 --- a/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/client/AbstractClientSessionTest.java +++ b/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/client/AbstractClientSessionTest.java @@ -14,6 +14,7 @@ package org.eclipse.jetty.websocket.javax.tests.client; import org.eclipse.jetty.websocket.core.CoreSession; +import org.eclipse.jetty.websocket.core.WebSocketComponents; import org.eclipse.jetty.websocket.javax.client.internal.BasicClientEndpointConfig; import org.eclipse.jetty.websocket.javax.client.internal.JavaxWebSocketClientContainer; import org.eclipse.jetty.websocket.javax.common.JavaxWebSocketContainer; @@ -29,16 +30,26 @@ public abstract class AbstractClientSessionTest { protected static JavaxWebSocketSession session; protected static JavaxWebSocketContainer container; + protected static WebSocketComponents components; @BeforeAll public static void initSession() throws Exception { container = new JavaxWebSocketClientContainer(); container.start(); + components = new WebSocketComponents(); + components.start(); Object websocketPojo = new DummyEndpoint(); UpgradeRequest upgradeRequest = new UpgradeRequestAdapter(); JavaxWebSocketFrameHandler frameHandler = container.newFrameHandler(websocketPojo, upgradeRequest); - CoreSession coreSession = new CoreSession.Empty(); + CoreSession coreSession = new CoreSession.Empty() + { + @Override + public WebSocketComponents getWebSocketComponents() + { + return components; + } + }; session = new JavaxWebSocketSession(container, coreSession, frameHandler, new BasicClientEndpointConfig()); } diff --git a/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/server/JavaxWebSocketFrameHandlerOnMessageTextStreamTest.java b/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/server/JavaxWebSocketFrameHandlerOnMessageTextStreamTest.java index d60015606606..6eda07ca6e30 100644 --- a/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/server/JavaxWebSocketFrameHandlerOnMessageTextStreamTest.java +++ b/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/server/JavaxWebSocketFrameHandlerOnMessageTextStreamTest.java @@ -27,10 +27,13 @@ import org.eclipse.jetty.websocket.core.CoreSession; import org.eclipse.jetty.websocket.core.Frame; import org.eclipse.jetty.websocket.core.OpCode; +import org.eclipse.jetty.websocket.core.WebSocketComponents; import org.eclipse.jetty.websocket.javax.common.JavaxWebSocketFrameHandler; import org.eclipse.jetty.websocket.javax.common.UpgradeRequest; import org.eclipse.jetty.websocket.javax.common.UpgradeRequestAdapter; import org.eclipse.jetty.websocket.javax.tests.WSEventTracker; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import static org.hamcrest.MatcherAssert.assertThat; @@ -38,6 +41,20 @@ public class JavaxWebSocketFrameHandlerOnMessageTextStreamTest extends AbstractJavaxWebSocketServerFrameHandlerTest { + private static final WebSocketComponents components = new WebSocketComponents(); + + @BeforeAll + public static void beforeAll() throws Exception + { + components.start(); + } + + @AfterAll + public static void afterAll() throws Exception + { + components.stop(); + } + @SuppressWarnings("Duplicates") private T performOnMessageInvocation(T socket, Consumer func) throws Exception { @@ -46,7 +63,14 @@ private T performOnMessageInvocation(T socket, Consum // Establish endpoint function JavaxWebSocketFrameHandler frameHandler = container.newFrameHandler(socket, request); - frameHandler.onOpen(new CoreSession.Empty(), Callback.NOOP); + frameHandler.onOpen(new CoreSession.Empty() + { + @Override + public WebSocketComponents getWebSocketComponents() + { + return components; + } + }, Callback.NOOP); func.accept(frameHandler); return socket; } From 316c46675e7e64ba23a83eace7f4d4c913866cae Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 4 Aug 2021 16:19:34 +1000 Subject: [PATCH 15/54] Fix WebSocketComponents LifeCycle in AbstractJavaxWebSocketFrameHandlerTest Signed-off-by: Lachlan Roberts --- .../javax/common/AbstractJavaxWebSocketFrameHandlerTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractJavaxWebSocketFrameHandlerTest.java b/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractJavaxWebSocketFrameHandlerTest.java index b9b2de4c6083..f4a6321b0b46 100644 --- a/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractJavaxWebSocketFrameHandlerTest.java +++ b/jetty-websocket/websocket-javax-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractJavaxWebSocketFrameHandlerTest.java @@ -28,17 +28,21 @@ public abstract class AbstractJavaxWebSocketFrameHandlerTest { protected static DummyContainer container; + private static WebSocketComponents components; @BeforeAll public static void initContainer() throws Exception { container = new DummyContainer(); container.start(); + components = new WebSocketComponents(); + components.start(); } @AfterAll public static void stopContainer() throws Exception { + components.stop(); container.stop(); } @@ -48,7 +52,6 @@ public static void stopContainer() throws Exception protected EndpointConfig endpointConfig; protected CoreSession coreSession = new CoreSession.Empty() { - private final WebSocketComponents components = new WebSocketComponents(); @Override public WebSocketComponents getWebSocketComponents() From edec52893b081581abee4e03e763e137db24713b Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 4 Aug 2021 16:49:15 +1000 Subject: [PATCH 16/54] Add name on WebSocketComponents default threadpool Signed-off-by: Lachlan Roberts --- .../jetty/websocket/core/WebSocketComponents.java | 12 +++++++++++- .../jetty/websocket/client/WebSocketClient.java | 4 ---- .../jetty/websocket/client/HttpClientInitTest.java | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java index 5579b7ad5a02..22353c83b825 100644 --- a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java +++ b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/WebSocketComponents.java @@ -57,7 +57,17 @@ public WebSocketComponents(WebSocketExtensionRegistry extensionRegistry, Decorat _bufferPool = (bufferPool == null) ? new MappedByteBufferPool() : bufferPool; _inflaterPool = (inflaterPool == null) ? new InflaterPool(CompressionPool.DEFAULT_CAPACITY, true) : inflaterPool; _deflaterPool = (deflaterPool == null) ? new DeflaterPool(CompressionPool.DEFAULT_CAPACITY, Deflater.DEFAULT_COMPRESSION, true) : deflaterPool; - _executor = (executor == null) ? new QueuedThreadPool() : executor; + + if (executor == null) + { + QueuedThreadPool threadPool = new QueuedThreadPool(); + threadPool.setName("WebSocket@" + hashCode()); + _executor = threadPool; + } + else + { + _executor = executor; + } addBean(_inflaterPool); addBean(_deflaterPool); diff --git a/jetty-websocket/websocket-jetty-client/src/main/java/org/eclipse/jetty/websocket/client/WebSocketClient.java b/jetty-websocket/websocket-jetty-client/src/main/java/org/eclipse/jetty/websocket/client/WebSocketClient.java index 5e184a3f4887..b4ff5563d228 100644 --- a/jetty-websocket/websocket-jetty-client/src/main/java/org/eclipse/jetty/websocket/client/WebSocketClient.java +++ b/jetty-websocket/websocket-jetty-client/src/main/java/org/eclipse/jetty/websocket/client/WebSocketClient.java @@ -85,10 +85,6 @@ public WebSocketClient(HttpClient httpClient) { coreClient = new WebSocketCoreClient(httpClient, components); addManaged(coreClient); - - if (httpClient == null) - coreClient.getHttpClient().setName("Jetty-WebSocketClient@" + hashCode()); - frameHandlerFactory = new JettyWebSocketFrameHandlerFactory(this, components); sessionListeners.add(sessionTracker); addBean(sessionTracker); diff --git a/jetty-websocket/websocket-jetty-client/src/test/java/org/eclipse/jetty/websocket/client/HttpClientInitTest.java b/jetty-websocket/websocket-jetty-client/src/test/java/org/eclipse/jetty/websocket/client/HttpClientInitTest.java index 4a652c1450e5..0159b3d44f82 100644 --- a/jetty-websocket/websocket-jetty-client/src/test/java/org/eclipse/jetty/websocket/client/HttpClientInitTest.java +++ b/jetty-websocket/websocket-jetty-client/src/test/java/org/eclipse/jetty/websocket/client/HttpClientInitTest.java @@ -41,7 +41,7 @@ public void testDefaultInit() throws Exception assertThat("Executor exists", executor, notNullValue()); assertThat("Executor instanceof", executor, instanceOf(QueuedThreadPool.class)); QueuedThreadPool threadPool = (QueuedThreadPool)executor; - assertThat("QueuedThreadPool.name", threadPool.getName(), startsWith("WebSocketClient@")); + assertThat("QueuedThreadPool.name", threadPool.getName(), startsWith("WebSocket@")); } finally { From 266d8f0dca4edc0ea323f55f8f2b46f9640160f0 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Tue, 3 Aug 2021 17:09:28 +0200 Subject: [PATCH 17/54] Fixes #6558 - Allow configuring return type in JSON array parsing. Introduced `arrayConverter` in both JSON and AsyncJSON.Factory. Signed-off-by: Simone Bordet (cherry picked from commit 342396c7ee33a4fa51e3d6b1facc5337942dc354) --- .../eclipse/jetty/util/ajax/AsyncJSON.java | 37 ++++++++++++++- .../org/eclipse/jetty/util/ajax/JSON.java | 45 +++++++++++++++--- .../jetty/util/ajax/AsyncJSONTest.java | 47 +++++++++++++++++++ 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/AsyncJSON.java b/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/AsyncJSON.java index a93de93d5016..b19842084b02 100644 --- a/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/AsyncJSON.java +++ b/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/AsyncJSON.java @@ -19,7 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import org.eclipse.jetty.util.BufferUtil; import org.eclipse.jetty.util.Index; @@ -62,6 +64,8 @@ * *

Class {@code com.acme.Person} must either implement {@link Convertible}, * or be mapped with a {@link Convertor} via {@link Factory#putConvertor(String, Convertor)}.

+ *

JSON arrays are by default represented with a {@code List}, but the + * Java representation can be customized via {@link Factory#setArrayConverter(Function)}.

*/ public class AsyncJSON { @@ -75,8 +79,31 @@ public static class Factory { private Index.Mutable cache; private Map convertors; + private Function, Object> arrayConverter = list -> list; private boolean detailedParseException; + /** + * @return the function to customize the Java representation of JSON arrays + * @see #setArrayConverter(Function) + */ + public Function, Object> getArrayConverter() + { + return arrayConverter; + } + + /** + *

Sets the function to convert JSON arrays from their default Java + * representation, a {@code List}, to another Java data structure + * such as an {@code Object[]}.

+ * + * @param arrayConverter the function to customize the Java representation of JSON arrays + * @see #getArrayConverter() + */ + public void setArrayConverter(Function, Object> arrayConverter) + { + this.arrayConverter = Objects.requireNonNull(arrayConverter); + } + /** * @return whether a parse failure should report the whole JSON string or just the last chunk */ @@ -870,9 +897,10 @@ private boolean parseArray(ByteBuffer buffer) case ']': { buffer.get(); - Object array = stack.peek().value; + @SuppressWarnings("unchecked") + List array = (List)stack.peek().value; stack.pop(); - stack.peek().value(array); + stack.peek().value(convertArray(array)); return true; } case ',': @@ -1067,6 +1095,11 @@ private boolean parseObjectFieldValue(ByteBuffer buffer) return true; } + private Object convertArray(List array) + { + return factory.getArrayConverter().apply(array); + } + private Object convertObject(Map object) { Object result = convertObject("x-class", object); diff --git a/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/JSON.java b/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/JSON.java index 6407c8f3d9b8..719745c27369 100644 --- a/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/JSON.java +++ b/jetty-util-ajax/src/main/java/org/eclipse/jetty/util/ajax/JSON.java @@ -19,11 +19,14 @@ import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import org.eclipse.jetty.util.Loader; import org.eclipse.jetty.util.TypeUtil; @@ -81,6 +84,7 @@ public class JSON private final Map _convertors = new ConcurrentHashMap<>(); private int _stringBufferSize = 1024; + private Function, Object> _arrayConverter = List::toArray; /** * @return the initial stringBuffer size to use when creating JSON strings @@ -461,7 +465,9 @@ protected Map newMap() * * @param size the size of the array * @return a new array representing the JSON array + * @deprecated use {@link #setArrayConverter(Function)} instead. */ + @Deprecated protected Object[] newArray(int size) { return new Object[size]; @@ -601,6 +607,28 @@ public Convertor getConvertorFor(String name) return _convertors.get(name); } + /** + * @return the function to customize the Java representation of JSON arrays + * @see #setArrayConverter(Function) + */ + public Function, Object> getArrayConverter() + { + return _arrayConverter; + } + + /** + *

Sets the function to convert JSON arrays from their default Java + * representation, a {@code List}, to another Java data structure + * such as an {@code Object[]}.

+ * + * @param arrayConverter the function to customize the Java representation of JSON arrays + * @see #getArrayConverter() + */ + public void setArrayConverter(Function, Object> arrayConverter) + { + _arrayConverter = Objects.requireNonNull(arrayConverter); + } + /** *

Parses the given JSON source into an object.

*

Although the JSON specification does not allow comments (of any kind) @@ -928,14 +956,16 @@ protected Object parseArray(Source source) switch (size) { case 0: - return newArray(0); + list = Collections.emptyList(); + break; case 1: - Object array = newArray(1); - Array.set(array, 0, item); - return array; + list = Collections.singletonList(item); + break; default: - return list.toArray(newArray(list.size())); + break; } + return getArrayConverter().apply(list); + case ',': if (comma) throw new IllegalStateException(); @@ -970,6 +1000,7 @@ else if (list == null) item = null; } } + break; } } @@ -1199,7 +1230,7 @@ public Number parseNumber(Source source) break doubleLoop; } } - return Double.parseDouble(buffer.toString()); + return Double.valueOf(buffer.toString()); } protected void seekTo(char seek, Source source) @@ -1585,7 +1616,7 @@ public interface Generator */ public static class Literal implements Generator { - private String _json; + private final String _json; /** * Constructs a literal JSON instance. diff --git a/jetty-util-ajax/src/test/java/org/eclipse/jetty/util/ajax/AsyncJSONTest.java b/jetty-util-ajax/src/test/java/org/eclipse/jetty/util/ajax/AsyncJSONTest.java index b9a2f82ac5d9..2c8c8ebbdccf 100644 --- a/jetty-util-ajax/src/test/java/org/eclipse/jetty/util/ajax/AsyncJSONTest.java +++ b/jetty-util-ajax/src/test/java/org/eclipse/jetty/util/ajax/AsyncJSONTest.java @@ -19,15 +19,19 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -520,4 +524,47 @@ public void testEncodedCaching() assertSame(foo, item); } } + + @Test + public void testArrayConverter() + { + // Test root arrays. + testArrayConverter("[1]", Function.identity()); + + // Test non-root arrays. + testArrayConverter("{\"array\": [1]}", object -> + { + @SuppressWarnings("unchecked") + Map map = (Map)object; + return map.get("array"); + }); + } + + private void testArrayConverter(String json, Function extractor) + { + AsyncJSON.Factory factory = new AsyncJSON.Factory(); + AsyncJSON async = factory.newAsyncJSON(); + JSON sync = new JSON(); + + async.parse(UTF_8.encode(json)); + Object result = extractor.apply(async.complete()); + // AsyncJSON historically defaults to list. + assertThat(result, Matchers.instanceOf(List.class)); + // JSON historically defaults to array. + result = extractor.apply(sync.parse(new JSON.StringSource(json))); + assertNotNull(result); + assertTrue(result.getClass().isArray(), json + " -> " + result); + + // Configure AsyncJSON to return arrays. + factory.setArrayConverter(List::toArray); + async.parse(UTF_8.encode(json)); + result = extractor.apply(async.complete()); + assertNotNull(result); + assertTrue(result.getClass().isArray(), json + " -> " + result); + + // Configure JSON to return lists. + sync.setArrayConverter(list -> list); + result = extractor.apply(sync.parse(new JSON.StringSource(json))); + assertThat(result, Matchers.instanceOf(List.class)); + } } From b0140dae059a9293c39fed839b16f2670aeb1009 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 5 Aug 2021 09:13:12 +1000 Subject: [PATCH 18/54] Fix #6562 last written bytebuffer (#6563) (#6579) Fixes #6562 the last written bytebuffer calculation. Also fixed an associated issue with unnecessary flush of an empty when last calculation already signalled last. --- .../org/eclipse/jetty/server/HttpOutput.java | 16 +- .../eclipse/jetty/server/HttpOutputTest.java | 194 +++++++++++++++++- 2 files changed, 199 insertions(+), 11 deletions(-) diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java index 65a182b24fd5..f8df5ea27a26 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java @@ -847,10 +847,12 @@ public void write(byte[] b, int off, int len) throws IOException // Blocking write try { + boolean complete = false; // flush any content from the aggregate if (BufferUtil.hasContent(_aggregate)) { - channelWrite(_aggregate, last && len == 0); + complete = last && len == 0; + channelWrite(_aggregate, complete); // should we fill aggregate again from the buffer? if (len > 0 && !last && len <= _commitSize && len <= maximizeAggregateSpace()) @@ -880,7 +882,7 @@ public void write(byte[] b, int off, int len) throws IOException } channelWrite(view, last); } - else if (last) + else if (last && !complete) { channelWrite(BufferUtil.EMPTY_BUFFER, true); } @@ -907,7 +909,7 @@ public void write(ByteBuffer buffer) throws IOException { checkWritable(); long written = _written + len; - last = _channel.getResponse().isAllContentWritten(_written); + last = _channel.getResponse().isAllContentWritten(written); flush = last || len > 0 || BufferUtil.hasContent(_aggregate); if (last && _state == State.OPEN) @@ -951,13 +953,17 @@ public void write(ByteBuffer buffer) throws IOException { // Blocking write // flush any content from the aggregate + boolean complete = false; if (BufferUtil.hasContent(_aggregate)) - channelWrite(_aggregate, last && len == 0); + { + complete = last && len == 0; + channelWrite(_aggregate, complete); + } // write any remaining content in the buffer directly if (len > 0) channelWrite(buffer, last); - else if (last) + else if (last && !complete) channelWrite(BufferUtil.EMPTY_BUFFER, true); onWriteComplete(last, null); diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java index d9d9e5452cae..af67693b8e5c 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java @@ -22,6 +22,8 @@ import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import javax.servlet.AsyncContext; import javax.servlet.ServletException; @@ -36,6 +38,7 @@ import org.eclipse.jetty.server.handler.HotSwapHandler; import org.eclipse.jetty.util.BufferUtil; import org.eclipse.jetty.util.Callback; +import org.eclipse.jetty.util.FuturePromise; import org.eclipse.jetty.util.resource.Resource; import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; @@ -45,6 +48,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -355,6 +359,7 @@ public void testWriteByteKnown() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length")); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -369,6 +374,7 @@ public void testWriteSmallKnown() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length")); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -383,6 +389,7 @@ public void testWriteMedKnown() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length")); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -397,6 +404,7 @@ public void testWriteLargeKnown() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length")); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -414,6 +422,7 @@ public void testWriteHugeKnown() throws Exception String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length")); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -428,6 +437,7 @@ public void testWriteBufferSmall() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -442,6 +452,7 @@ public void testWriteBufferMed() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -456,6 +467,52 @@ public void testWriteBufferLarge() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); + } + + @Test + public void testWriteBufferSmallKnown() throws Exception + { + final Resource big = Resource.newClassPathResource("simple/big.txt"); + _handler._writeLengthIfKnown = true; + _handler._content = BufferUtil.toBuffer(big, false); + _handler._byteBuffer = BufferUtil.allocate(8); + + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(response, containsString("Content-Length")); + assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); + } + + @Test + public void testWriteBufferMedKnown() throws Exception + { + final Resource big = Resource.newClassPathResource("simple/big.txt"); + _handler._writeLengthIfKnown = true; + _handler._content = BufferUtil.toBuffer(big, false); + _handler._byteBuffer = BufferUtil.allocate(4000); + + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(response, containsString("Content-Length")); + assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); + } + + @Test + public void testWriteBufferLargeKnown() throws Exception + { + final Resource big = Resource.newClassPathResource("simple/big.txt"); + _handler._writeLengthIfKnown = true; + _handler._content = BufferUtil.toBuffer(big, false); + _handler._byteBuffer = BufferUtil.allocate(8192); + + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(response, containsString("Content-Length")); + assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -471,6 +528,7 @@ public void testAsyncWriteByte() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -486,6 +544,7 @@ public void testAsyncWriteSmall() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -501,6 +560,7 @@ public void testAsyncWriteMed() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -516,12 +576,13 @@ public void testAsyncWriteLarge() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test public void testAsyncWriteHuge() throws Exception { - _handler._writeLengthIfKnown = true; + _handler._writeLengthIfKnown = false; _handler._content = BufferUtil.allocate(4 * 1024 * 1024); _handler._content.limit(_handler._content.capacity()); for (int i = _handler._content.capacity(); i-- > 0; ) @@ -533,7 +594,8 @@ public void testAsyncWriteHuge() throws Exception String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); assertThat(response, containsString("HTTP/1.1 200 OK")); - assertThat(response, containsString("Content-Length")); + assertThat(response, Matchers.not(containsString("Content-Length"))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -549,6 +611,7 @@ public void testAsyncWriteBufferSmall() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -564,6 +627,7 @@ public void testAsyncWriteBufferMed() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -579,6 +643,7 @@ public void testAsyncWriteBufferLarge() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -595,6 +660,7 @@ public void testAsyncWriteBufferLargeDirect() assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, Matchers.not(containsString("Content-Length"))); assertThat(response, endsWith(toUTF8String(big))); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(false)); } @Test @@ -629,6 +695,7 @@ public void testAsyncWriteSimpleKnown() throws Exception assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length: 11")); assertThat(response, containsString("simple text")); + assertThat(_handler._closedAfterWrite.get(10, TimeUnit.SECONDS), is(true)); } @Test @@ -686,6 +753,116 @@ public void setNext(Interceptor interceptor) assertThat(response, containsString("400\tTHIS IS A BIGGER FILE")); } + @Test + public void testEmptyArray() throws Exception + { + FuturePromise committed = new FuturePromise<>(); + AbstractHandler handler = new AbstractHandler() + { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + baseRequest.setHandled(true); + response.setStatus(200); + try + { + response.getOutputStream().write(new byte[0]); + committed.succeeded(response.isCommitted()); + } + catch (Throwable t) + { + committed.failed(t); + } + } + }; + + _swap.setHandler(handler); + handler.start(); + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(committed.get(10, TimeUnit.SECONDS), is(false)); + } + + @Test + public void testEmptyArrayKnown() throws Exception + { + FuturePromise committed = new FuturePromise<>(); + AbstractHandler handler = new AbstractHandler() + { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + baseRequest.setHandled(true); + response.setStatus(200); + response.setContentLength(0); + try + { + response.getOutputStream().write(new byte[0]); + committed.succeeded(response.isCommitted()); + } + catch (Throwable t) + { + committed.failed(t); + } + } + }; + + _swap.setHandler(handler); + handler.start(); + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(response, containsString("Content-Length: 0")); + assertThat(committed.get(10, TimeUnit.SECONDS), is(true)); + } + + @Test + public void testEmptyBuffer() throws Exception + { + AtomicBoolean committed = new AtomicBoolean(); + AbstractHandler handler = new AbstractHandler() + { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + baseRequest.setHandled(true); + response.setStatus(200); + ((HttpOutput)response.getOutputStream()).write(ByteBuffer.wrap(new byte[0])); + committed.set(response.isCommitted()); + } + }; + + _swap.setHandler(handler); + handler.start(); + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(committed.get(), is(false)); + } + + @Test + public void testEmptyBufferKnown() throws Exception + { + AtomicBoolean committed = new AtomicBoolean(); + AbstractHandler handler = new AbstractHandler() + { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + baseRequest.setHandled(true); + response.setStatus(200); + response.setContentLength(0); + ((HttpOutput)response.getOutputStream()).write(ByteBuffer.wrap(new byte[0])); + committed.set(response.isCommitted()); + } + }; + + _swap.setHandler(handler); + handler.start(); + String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); + assertThat(response, containsString("HTTP/1.1 200 OK")); + assertThat(response, containsString("Content-Length: 0")); + assertThat(committed.get(), is(true)); + } + @Test public void testAggregation() throws Exception { @@ -851,7 +1028,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques aggregated += data.length; } - // write data that will not be aggregated + // write data that will not be aggregated because it is too large data = new byte[bufferSize + 1]; Arrays.fill(data, (byte)(fill++)); expected.write(data); @@ -1025,6 +1202,7 @@ static class ContentHandler extends AbstractHandler ReadableByteChannel _contentChannel; ByteBuffer _content; ChainedInterceptor _interceptor; + final FuturePromise _closedAfterWrite = new FuturePromise<>(); @Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException @@ -1045,6 +1223,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques { out.sendContent(_contentInputStream); _contentInputStream = null; + _closedAfterWrite.succeeded(out.isClosed()); return; } @@ -1052,6 +1231,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques { out.sendContent(_contentChannel); _contentChannel = null; + _closedAfterWrite.succeeded(out.isClosed()); return; } @@ -1078,6 +1258,7 @@ public void onWritePossible() throws IOException len = _arrayBuffer.length; if (len == 0) { + _closedAfterWrite.succeeded(out.isClosed()); async.complete(); break; } @@ -1088,7 +1269,6 @@ public void onWritePossible() throws IOException else out.write(_arrayBuffer, 0, len); } - // assertFalse(out.isReady()); } @Override @@ -1113,7 +1293,7 @@ public void onError(Throwable t) else out.write(_arrayBuffer, 0, len); } - + _closedAfterWrite.succeeded(out.isClosed()); return; } @@ -1137,6 +1317,7 @@ public void onWritePossible() throws IOException assertTrue(out.isReady()); if (BufferUtil.isEmpty(_content)) { + _closedAfterWrite.succeeded(out.isClosed()); async.complete(); break; } @@ -1167,7 +1348,7 @@ public void onError(Throwable t) BufferUtil.flipToFlush(_byteBuffer, 0); out.write(_byteBuffer); } - + _closedAfterWrite.succeeded(out.isClosed()); return; } @@ -1178,6 +1359,7 @@ public void onError(Throwable t) else out.sendContent(_content); _content = null; + _closedAfterWrite.succeeded(out.isClosed()); return; } } From 49a08450c2f03e4ed4a3c81866274735ba7c97ee Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Thu, 5 Aug 2021 10:04:37 +0200 Subject: [PATCH 19/54] Fixes #6043 - Reimplement UnixSocket support based on Java 16. (#6522) Fixes #6043 - Reimplement UnixSocket support based on Java 16. * Introduced new module "jetty-server-unixdomain". It uses reflection to access the Java 16 Unix-Domain classes to keep compatibility with the other modules and the build. * Added Jetty module with only HTTP/1.1 support for now (requires review of the modules to reuse them with various connectors). * Updated documentation to mention UnixDomainServerConnector. * Updated client libraries to support Unix-Domain. * Updated PROXY protocol implementation to support Unix-Domain. * Replaced unix.socket.tmp with better named jetty.unixdomain.dir property. Defaulted jetty.unixdomain.dir property to system property user.home under Windows. Simplified code that runs Unix-Domain tests. Signed-off-by: Simone Bordet --- Jenkinsfile | 2 +- Jenkinsfile-autobahn | 2 +- documentation/jetty-documentation/pom.xml | 5 + .../deploy/deploy-virtual-hosts.adoc | 6 +- .../modules/modules-custom.adoc | 2 +- .../asciidoc/programming-guide/arch-io.adoc | 14 +- .../server/http/server-http-connector.adoc | 41 ++- .../server/server-io-arch.adoc | 56 ++- .../docs/programming/server/ServerDocs.java | 19 + .../server/http/HTTPServerDocs.java | 63 +++- .../client/AbstractHttpClientTransport.java | 2 +- .../org/eclipse/jetty/client/HttpClient.java | 17 +- .../dynamic/HttpClientTransportDynamic.java | 4 + .../http/HttpClientTransportOverHTTP.java | 3 + .../http/HttpClientTransportOverFCGI.java | 4 + jetty-fcgi/fcgi-server/pom.xml | 6 + .../server/proxy/FastCGIProxyServlet.java | 30 +- .../server/proxy/FastCGIProxyServletTest.java | 104 ++++-- .../test/resources/jetty-logging.properties | 2 +- jetty-home/pom.xml | 7 + .../org/eclipse/jetty/io/ClientConnector.java | 136 +++++++- .../jetty/io/SocketChannelEndPoint.java | 8 +- .../jetty/server/ProxyConnectionFactory.java | 86 ++++- jetty-unixdomain-server/pom.xml | 43 +++ .../main/config/etc/jetty-unixdomain-http.xml | 34 ++ .../main/config/modules/unixdomain-http.mod | 36 ++ .../src/main/java/module-info.java | 20 ++ .../server/UnixDomainServerConnector.java | 326 ++++++++++++++++++ .../unixdomain/server/UnixDomainTest.java | 272 +++++++++++++++ .../test/resources/jetty-logging.properties | 2 + .../jetty/unixsocket/UnixSocketTest.java | 21 +- pom.xml | 16 +- .../tests/distribution/DistributionTests.java | 59 +++- .../jetty/http/client/TransportScenario.java | 24 +- 34 files changed, 1303 insertions(+), 169 deletions(-) create mode 100644 jetty-unixdomain-server/pom.xml create mode 100644 jetty-unixdomain-server/src/main/config/etc/jetty-unixdomain-http.xml create mode 100644 jetty-unixdomain-server/src/main/config/modules/unixdomain-http.mod create mode 100644 jetty-unixdomain-server/src/main/java/module-info.java create mode 100644 jetty-unixdomain-server/src/main/java/org/eclipse/jetty/unixdomain/server/UnixDomainServerConnector.java create mode 100644 jetty-unixdomain-server/src/test/java/org/eclipse/jetty/unixdomain/server/UnixDomainTest.java create mode 100644 jetty-unixdomain-server/src/test/resources/jetty-logging.properties diff --git a/Jenkinsfile b/Jenkinsfile index 7d5820e08803..bc747e8b6cc2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,7 @@ def mavenBuild(jdk, cmdline, mvnName) { "MAVEN_OPTS=-Xms2g -Xmx4g -Djava.awt.headless=true"]) { configFileProvider( [configFile(fileId: 'oss-settings.xml', variable: 'GLOBAL_MVN_SETTINGS')]) { - sh "mvn --no-transfer-progress -s $GLOBAL_MVN_SETTINGS -Dmaven.repo.local=.repository -Pci -V -B -e -Djetty.testtracker.log=true $cmdline -Dunix.socket.tmp=/tmp/unixsocket" + sh "mvn --no-transfer-progress -s $GLOBAL_MVN_SETTINGS -Dmaven.repo.local=.repository -Pci -V -B -e -Djetty.testtracker.log=true $cmdline" } } } diff --git a/Jenkinsfile-autobahn b/Jenkinsfile-autobahn index 0944e587368d..0e92efd217d6 100644 --- a/Jenkinsfile-autobahn +++ b/Jenkinsfile-autobahn @@ -80,7 +80,7 @@ def mavenBuild(jdk, cmdline, mvnName, junitPublishDisabled) { mavenOpts: mavenOpts, mavenLocalRepo: localRepo) { // Some common Maven command line + provided command line - sh "mvn -Pci -V -B -e -fae -Dmaven.test.failure.ignore=true -Djetty.testtracker.log=true $cmdline -Dunix.socket.tmp=" + env.JENKINS_HOME + sh "mvn -Pci -V -B -e -fae -Dmaven.test.failure.ignore=true -Djetty.testtracker.log=true $cmdline" } } diff --git a/documentation/jetty-documentation/pom.xml b/documentation/jetty-documentation/pom.xml index 01d3db94affd..48dd43ee6834 100644 --- a/documentation/jetty-documentation/pom.xml +++ b/documentation/jetty-documentation/pom.xml @@ -201,6 +201,11 @@ http2-http-client-transport ${project.version} + + org.eclipse.jetty + jetty-unixdomain-server + ${project.version} + org.eclipse.jetty jetty-slf4j-impl diff --git a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/deploy/deploy-virtual-hosts.adoc b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/deploy/deploy-virtual-hosts.adoc index 7e63f26756ad..9412f3da1f93 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/deploy/deploy-virtual-hosts.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/deploy/deploy-virtual-hosts.adoc @@ -41,8 +41,8 @@ A wildcard domain name which will match only one level of arbitrary subdomains. An IP address may be set as a virtual host to indicate that a web application should handle requests received on the network interface with that IP address for protocols that do not indicate a host name such as HTTP/0.9 or HTTP/1.0. `@ConnectorName`:: -A Jetty `ServerConnector` name to indicate that a web application should handle requests received on the `ServerConnector` with that name, and therefore received on a specific IP port. -A `ServerConnector` name can be set via link:{javadoc-url}/org/eclipse/jetty/server/AbstractConnector.html#setName(java.lang.String)[]. +A Jetty server `Connector` name to indicate that a web application should handle requests received on the server `Connector` with that name, and therefore received on a specific socket address (either an IP port for `ServerConnector`, or a Unix-Domain path for `UnixDomainServerConnector`). +A server `Connector` name can be set via link:{javadoc-url}/org/eclipse/jetty/server/AbstractConnector.html#setName(java.lang.String)[]. `www.√integral.com`:: Non-ASCII and https://en.wikipedia.org/wiki/Internationalized_domain_name[IDN] domain names can be set as virtual hosts using https://en.wikipedia.org/wiki/Punycode[Puny Code] equivalents that may be obtained from a https://www.punycoder.com/[Punycode/IDN converters]. @@ -134,7 +134,7 @@ To achieve this, you simply use the same context path of `/` for each of your we [[og-deploy-virtual-hosts-port]] ===== Different Port, Different Web Application -Sometimes it is required to serve different web applications from different IP ports, and therefore from different ``ServerConnector``s. +Sometimes it is required to serve different web applications from different socket addresses (either different IP ports, or different Unix-Domain paths), and therefore from different server ``Connector``s. For example, you want requests to `+http://localhost:8080/+` to be served by one web application, but requests to `+http://localhost:9090/+` to be served by another web application. diff --git a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules-custom.adoc b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules-custom.adoc index 826570e42e16..15d23edc5d2d 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules-custom.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules-custom.adoc @@ -131,7 +131,7 @@ In the cases where you need to enhance Jetty with a custom functionality, you ca For example, let's assume that you need to add a custom auditing component that integrates with the auditing tools used by your company. This custom auditing component should measure the HTTP request processing times and record them (how they are recorded is irrelevant here -- could be in a local log file or sent via network to an external service). -The Jetty libraries already provide a way to measure HTTP request processing times via xref:{prog-guide}#pg-server-http-channel-events[`HttpChannel` events]: you write a custom component that implements the `HttpChannel.Listener` interface and add it as a bean to the `ServerConnector` that receives the HTTP requests. +The Jetty libraries already provide a way to measure HTTP request processing times via xref:{prog-guide}#pg-server-http-channel-events[`HttpChannel` events]: you write a custom component that implements the `HttpChannel.Listener` interface and add it as a bean to the server `Connector` that receives the HTTP requests. The steps to create a Jetty module are similar to those necessary to xref:og-modules-custom-modify[modify an existing module]: diff --git a/documentation/jetty-documentation/src/main/asciidoc/programming-guide/arch-io.adoc b/documentation/jetty-documentation/src/main/asciidoc/programming-guide/arch-io.adoc index 5b2b8a6a995c..23f4de0d797d 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/programming-guide/arch-io.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/programming-guide/arch-io.adoc @@ -26,7 +26,7 @@ Each `ManagedSelector` wraps an instance of `java.nio.channels.Selector` that in NOTE: TODO: add image -`SocketChannel` instances can be created by network clients when connecting to a server and by a network server when accepting connections from network clients. +`SocketChannel` instances can be created by clients when connecting to a server and by a server when accepting connections from clients. In both cases the `SocketChannel` instance is passed to `SelectorManager` (which passes it to `ManagedSelector` and eventually to `java.nio.channels.Selector`) to be registered for use within Jetty. It is possible for an application to create the `SocketChannel` instances outside Jetty, even perform some initial network traffic also outside Jetty (for example for authentication purposes), and then pass the `SocketChannel` instance to `SelectorManager` for use within Jetty. @@ -50,7 +50,7 @@ include::{doc_code}/org/eclipse/jetty/docs/programming/SelectorManagerDocs.java[ ``SocketChannel``s that are passed to `SelectorManager` are wrapped into two related components: an link:{javadoc-url}/org/eclipse/jetty/io/EndPoint.html[`EndPoint`] and a link:{javadoc-url}/org/eclipse/jetty/io/Connection.html[`Connection`]. -`EndPoint` is the Jetty abstraction for a `SocketChannel`: you can read bytes from an `EndPoint` via `EndPoint.fill(ByteBuffer)`, you can write bytes to an `EndPoint` via `EndPoint.flush(ByteBuffer...)` and `EndPoint.write(Callback, ByteBuffer...)`, you can close an `EndPoint` via `EndPoint.close()`, etc. +`EndPoint` is the Jetty abstraction for a `SocketChannel`: you can read bytes from an `EndPoint` via `EndPoint.fill(ByteBuffer)`, you can write bytes to an `EndPoint` via `EndPoint.flush(ByteBuffer\...)` and `EndPoint.write(Callback, ByteBuffer\...)`, you can close an `EndPoint` via `EndPoint.close()`, etc. `Connection` is the Jetty abstraction that is responsible to read bytes from the `EndPoint` and to deserialize the read bytes into objects. For example, an HTTP/1.1 server-side `Connection` implementation is responsible to deserialize HTTP/1.1 request bytes into an HTTP request object. @@ -63,9 +63,9 @@ The writing side for a specific protocol _may_ be implemented in the `Connection While there is primarily just one implementation of `EndPoint`,link:{javadoc-url}/org/eclipse/jetty/io/SocketChannelEndPoint.html[`SocketChannelEndPoint`] (used both on the client-side and on the server-side), there are many implementations of `Connection`, typically two for each protocol (one for the client-side and one for the server-side). The `EndPoint` and `Connection` pairs can be chained, for example in case of encrypted communication using the TLS protocol. -There is an `EndPoint` and `Connection` TLS pair where the `EndPoint` reads the encrypted bytes from the network and the `Connection` decrypts them; next in the chain there is an `EndPoint` and `Connection` pair where the `EndPoint` "reads" decrypted bytes (provided by the previous `Connection`) and the `Connection` deserializes them into specific protocol objects (for example HTTP/2 frame objects). +There is an `EndPoint` and `Connection` TLS pair where the `EndPoint` reads the encrypted bytes from the socket and the `Connection` decrypts them; next in the chain there is an `EndPoint` and `Connection` pair where the `EndPoint` "reads" decrypted bytes (provided by the previous `Connection`) and the `Connection` deserializes them into specific protocol objects (for example HTTP/2 frame objects). -Certain protocols, such as WebSocket, start the communication with the server using one protocol (e.g. HTTP/1.1), but then change the communication to use another protocol (e.g. WebSocket). +Certain protocols, such as WebSocket, start the communication with the server using one protocol (for example, HTTP/1.1), but then change the communication to use another protocol (for example, WebSocket). `EndPoint` supports changing the `Connection` object on-the-fly via `EndPoint.upgrade(Connection)`. This allows to use the HTTP/1.1 `Connection` during the initial communication and later to replace it with a WebSocket `Connection`. @@ -75,9 +75,9 @@ NOTE: TODO: add a section on `UpgradeFrom` and `UpgradeTo`? Creating `Connection` instances is performed on the server-side by link:{javadoc-url}/org/eclipse/jetty/server/ConnectionFactory.html[`ConnectionFactory`]s and on the client-side by link:{javadoc-url}/org/eclipse/jetty/io/ClientConnectionFactory.html[`ClientConnectionFactory`]s -On the server-side, the component that aggregates a `SelectorManager` with a set of ``ConnectionFactory``s is link:{javadoc-url}/org/eclipse/jetty/server/ServerConnector.html[`ServerConnector`]s, see xref:pg-server-io-arch[]. +On the server-side, the component that aggregates a `SelectorManager` with a set of ``ConnectionFactory``s is link:{javadoc-url}/org/eclipse/jetty/server/ServerConnector.html[`ServerConnector`]s for TCP/IP sockets, and link:{JDURL}/org/eclipse/jetty/unixdomain/server/UnixDomainServerConnector.html[`UnixDomainServerConnector`] for Unix-Domain sockets (see the xref:pg-server-io-arch[server-side architecture section] for more information). -On the client-side, the components that aggregates a `SelectorManager` with a set of ``ClientConnectionFactory``s are link:{javadoc-url}/org/eclipse/jetty/client/HttpClientTransport.html[`HttpClientTransport`] subclasses, see xref:pg-client-io-arch[]. +On the client-side, the components that aggregates a `SelectorManager` with a set of ``ClientConnectionFactory``s are link:{javadoc-url}/org/eclipse/jetty/client/HttpClientTransport.html[`HttpClientTransport`] subclasses (see the xref:pg-client-io-arch[client-side architecture section] for more information). [[pg-arch-io-endpoint]] ==== Jetty I/O: `EndPoint` @@ -86,7 +86,7 @@ The Jetty I/O library use Java NIO to handle I/O, so that I/O is non-blocking. At the Java NIO level, in order to be notified when a `SocketChannel` has data to be read, the `SelectionKey.OP_READ` flag must be set. -In the Jetty I/O library, you can call `EndPoint.fillInterested(Callback)` to declare interest in the "read" (or "fill") event, and the `Callback` parameter is the object that is notified when such an event occurs. +In the Jetty I/O library, you can call `EndPoint.fillInterested(Callback)` to declare interest in the "read" (also called "fill") event, and the `Callback` parameter is the object that is notified when such an event occurs. At the Java NIO level, a `SocketChannel` is always writable, unless it becomes TCP congested. In order to be notified when a `SocketChannel` uncongests and it is therefore writable again, the `SelectionKey.OP_WRITE` flag must be set. diff --git a/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/http/server-http-connector.adoc b/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/http/server-http-connector.adoc index a736a9ddfd7f..767c94a82d3b 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/http/server-http-connector.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/http/server-http-connector.adoc @@ -16,20 +16,31 @@ A `Connector` is the component that handles incoming requests from clients, and works in conjunction with `ConnectionFactory` instances. -The primary implementation is `org.eclipse.jetty.server.ServerConnector`. -`ServerConnector` uses a `java.nio.channels.ServerSocketChannel` to listen to a TCP port and to accept TCP connections. +The available implementations are: -Since `ServerConnector` wraps a `ServerSocketChannel`, it can be configured in a similar way, for example the port to listen to, the network address to bind to, etc.: +* `org.eclipse.jetty.server.ServerConnector`, for TCP/IP sockets. +* `org.eclipse.jetty.unixdomain.server.UnixDomainServerConnector` for Unix-Domain sockets. + +Both use a `java.nio.channels.ServerSocketChannel` to listen to a socket address and to accept socket connections. + +Since `ServerConnector` wraps a `ServerSocketChannel`, it can be configured in a similar way, for example the IP port to listen to, the IP address to bind to, etc.: [source,java,indent=0] ---- include::../../{doc_code}/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java[tags=configureConnector] ---- -The _acceptors_ are threads (typically only one) that compete to accept TCP connections on the listening port. +Likewise, `UnixDomainServerConnector` also wraps a `ServerSocketChannel` and can be configured with the Unix-Domain path to listen to: + +[source,java,indent=0] +---- +include::../../{doc_code}/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java[tags=configureConnectorUnix] +---- + +The _acceptors_ are threads (typically only one) that compete to accept socket connections. When a connection is accepted, `ServerConnector` wraps the accepted `SocketChannel` and passes it to the xref:pg-arch-io-selector-manager[`SelectorManager`]. Therefore, there is a little moment where the acceptor thread is not accepting new connections because it is busy wrapping the just accepted connection to pass it to the `SelectorManager`. -Connections that are ready to be accepted but are not accepted yet are queued in a bounded queue (at the OS level) whose capacity can be configured with the `ServerConnector.acceptQueueSize` parameter. +Connections that are ready to be accepted but are not accepted yet are queued in a bounded queue (at the OS level) whose capacity can be configured with the `acceptQueueSize` parameter. If your application must withstand a very high rate of connections opened, configuring more than one acceptor thread may be beneficial: when one acceptor thread accepts one connection, another acceptor thread can take over accepting connections. @@ -42,7 +53,7 @@ In this case a single selector may be able to manage many sockets because chance On the contrary, web messaging applications tend to send many small messages at a very high frequency so that sockets are rarely idle. In this case a single selector may be able to manage less sockets because chances are that many of them will be active at the same time. -It is possible to configure more than one `ServerConnector`, each listening on a different port: +It is possible to configure more than one `ServerConnector` (each listening on a different port), or more than one `UnixDomainServerConnector` (each listening on a different path), or ``ServerConnector``s and ``UnixDomainServerConnector``s, for example: [source,java,indent=0] ---- @@ -52,9 +63,9 @@ include::../../{doc_code}/org/eclipse/jetty/docs/programming/server/http/HTTPSer [[pg-server-http-connector-protocol]] ===== Configuring Protocols -For each accepted TCP connection, `ServerConnector` asks a `ConnectionFactory` to create a `Connection` object that handles the network traffic on that TCP connection, parsing and generating bytes for a specific protocol (see xref:pg-arch-io[this section] for more details about `Connection` objects). +For each accepted socket connection, the server `Connector` asks a `ConnectionFactory` to create a `Connection` object that handles the traffic on that socket connection, parsing and generating bytes for a specific protocol (see xref:pg-arch-io[this section] for more details about `Connection` objects). -A `ServerConnector` can be configured with one or more ``ConnectionFactory``s. +A server `Connector` can be configured with one or more ``ConnectionFactory``s. If no `ConnectionFactory` is specified then `HttpConnectionFactory` is implicitly configured. [[pg-server-http-connector-protocol-http11]] @@ -87,7 +98,7 @@ By using those ports, a client had _prior knowledge_ that the server would speak HTTP/2 was designed to be a smooth transition from HTTP/1.1 for users and as such the HTTP ports were not changed. However the HTTP/2 protocol is, on the wire, a binary protocol, completely different from HTTP/1.1. -Therefore, with HTTP/2, clients that connect to port `80` may speak either HTTP/1.1 or HTTP/2, and the server must figure out which version of the HTTP protocol the client is speaking. +Therefore, with HTTP/2, clients that connect to port `80` (or to a specific Unix-Domain path) may speak either HTTP/1.1 or HTTP/2, and the server must figure out which version of the HTTP protocol the client is speaking. Jetty can support both HTTP/1.1 and HTTP/2 on the same clear-text port by configuring both the HTTP/1.1 and the HTTP/2 ``ConnectionFactory``s: @@ -128,7 +139,7 @@ Note also that the default protocol set in the ALPN ``ConnectionFactory``, which It is often the case that Jetty receives connections from a load balancer configured to distribute the load among many Jetty backend servers. -From the Jetty point of view, all the connections arrive from the load balancer, rather than the real clients, but is possible to configure the load balancer to forward the real client IP address and port to the backend Jetty server using the link:https://www.haproxy.org/download/2.1/doc/proxy-protocol.txt[PROXY protocol]. +From the Jetty point of view, all the connections arrive from the load balancer, rather than the real clients, but is possible to configure the load balancer to forward the real client IP address and IP port to the backend Jetty server using the link:https://www.haproxy.org/download/2.1/doc/proxy-protocol.txt[PROXY protocol]. NOTE: The PROXY protocol is widely supported by load balancers such as link:http://cbonte.github.io/haproxy-dconv/2.2/configuration.html#5.2-send-proxy[HAProxy] (via its `send-proxy` directive), link:https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol[Nginx](via its `proxy_protocol on` directive) and others. @@ -144,3 +155,13 @@ Note also how the PROXY `ConnectionFactory` needs to know its _next_ protocol (i Each `ConnectionFactory` is asked to create a `Connection` object for each accepted TCP connection; the `Connection` objects will be chained together to handle the bytes, each for its own protocol. Therefore the `ProxyConnection` will handle the PROXY protocol bytes and `HttpConnection` will handle the HTTP/1.1 bytes producing a request object and response object that will be processed by ``Handler``s. + +The load balancer may be configured to communicate with Jetty backend servers via Unix-Domain sockets (requires Java 16 or later). +For example: + +[source,java,indent=0] +---- +include::../../{doc_code}/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java[tags=proxyHTTPUnix] +---- + +Note that the only difference when using Unix-Domain sockets is instantiating `UnixDomainServerConnector` instead of `ServerConnector` and configuring the Unix-Domain path instead of the IP port. diff --git a/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/server-io-arch.adoc b/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/server-io-arch.adoc index 3e98026afcc3..ab6c9b8bd010 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/server-io-arch.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/programming-guide/server/server-io-arch.adoc @@ -12,23 +12,30 @@ // [[pg-server-io-arch]] -=== Server Libraries I/O Architecture +=== Server I/O Architecture The Jetty server libraries provide the basic components and APIs to implement a network server. They build on the common xref:pg-arch-io[Jetty I/O Architecture] and provide server specific concepts. -The central I/O server-side component is `org.eclipse.jetty.server.ServerConnector`. +The Jetty server libraries provide I/O support for TCP/IP sockets (for both IPv4 and IPv6) and, when using Java 16 or later, for Unix-Domain sockets. + +Support for Unix-Domain sockets is interesting when Jetty is deployed behind a proxy or a load-balancer: it is possible to configure the proxy or load balancer to communicate with Jetty via Unix-Domain sockets, rather than via the loopback network interface. + +The central I/O server-side component are `org.eclipse.jetty.server.ServerConnector`, that handles the TCP/IP socket traffic, and `org.eclipse.jetty.unixdomain.server.UnixDomainServerConnector`, that handles the Unix-Domain socket traffic. + +`ServerConnector` and `UnixDomainServerConnector` are very similar, and while in the following sections `ServerConnector` is used, the same concepts apply to `UnixDomainServerConnector`, unless otherwise noted. + A `ServerConnector` manages a list of ``ConnectionFactory``s, that indicate what protocols the connector is able to speak. [[pg-server-io-arch-connection-factory]] ==== Creating Connections with `ConnectionFactory` -Recall from the xref:pg-arch-io-connection[`Connection` section] of the Jetty I/O architecture that `Connection` instances are responsible for parsing bytes read from a TCP connection and generating bytes to write to that TCP connection. +Recall from the xref:pg-arch-io-connection[`Connection` section] of the Jetty I/O architecture that `Connection` instances are responsible for parsing bytes read from a socket and generating bytes to write to that socket. On the server-side, a `ConnectionFactory` creates `Connection` instances that know how to parse and generate bytes for the specific protocol they support -- it can be either HTTP/1.1, or TLS, or FastCGI, or the link:https://www.haproxy.org/download/2.1/doc/proxy-protocol.txt[PROXY protocol]. -For example, this is how clear-text HTTP/1.1 is configured: +For example, this is how clear-text HTTP/1.1 is configured for TCP/IP sockets: [source,java,indent=0] ---- @@ -36,7 +43,24 @@ include::../{doc_code}/org/eclipse/jetty/docs/programming/server/ServerDocs.java ---- With this configuration, the `ServerConnector` will listen on port `8080`. -When a new TCP connection is established, `ServerConnector` delegates to the `ConnectionFactory` the creation of the `Connection` instance for that TCP connection, that is linked to the corresponding `EndPoint`: + +Similarly, this is how clear-text HTTP/1.1 is configured for Unix-Domain sockets: + +[source,java,indent=0] +---- +include::../{doc_code}/org/eclipse/jetty/docs/programming/server/ServerDocs.java[tags=httpUnix] +---- + +With this configuration, the `UnixDomainServerConnector` will listen on file `/tmp/jetty.sock`. + +[NOTE] +==== +`ServerConnector` and `UnixDomainServerConnector` only differ by how they are configured -- for `ServerConnector` you specify the IP port it listens to, for `UnixDomainServerConnector` you specify the Unix-Domain path it listens to. + +Both configure ``ConnectionFactory``s in exactly the same way. +==== + +When a new socket connection is established, `ServerConnector` delegates to the `ConnectionFactory` the creation of the `Connection` instance for that socket connection, that is linked to the corresponding `EndPoint`: [plantuml] ---- @@ -53,12 +77,12 @@ scale 1.5 circle network circle application -network - SocketEndPoint -SocketEndPoint - HttpConnection +network - SocketChannelEndPoint +SocketChannelEndPoint - HttpConnection HttpConnection - application ---- -For every TCP connection there will be an `EndPoint` + `Connection` pair. +For every socket connection there will be an `EndPoint` + `Connection` pair. [[pg-server-io-arch-connection-factory-wrapping]] ==== Wrapping a `ConnectionFactory` @@ -72,7 +96,7 @@ include::../{doc_code}/org/eclipse/jetty/docs/programming/server/ServerDocs.java ---- With this configuration, the `ServerConnector` will listen on port `8443`. -When a new TCP connection is established, the first `ConnectionFactory` configured in `ServerConnector` is invoked to create a `Connection`. +When a new socket connection is established, the first `ConnectionFactory` configured in `ServerConnector` is invoked to create a `Connection`. In the example above, `SslConnectionFactory` creates a `SslConnection` and then asks to its wrapped `ConnectionFactory` (in the example, `HttpConnectionFactory`) to create the wrapped `Connection` (an `HttpConnection`) and will then link the two ``Connection``s together, in this way: [plantuml] @@ -90,16 +114,16 @@ scale 1.5 circle network circle application -network - SocketEndPoint -SocketEndPoint - SslConnection +network - SocketChannelEndPoint +SocketChannelEndPoint - SslConnection SslConnection -- DecryptedEndPoint DecryptedEndPoint - HttpConnection HttpConnection - application ---- -Bytes read by the `SocketEndPoint` will be interpreted as TLS bytes by the `SslConnection`, then decrypted and made available to the `DecryptedEndPoint` (a component part of `SslConnection`), which will then provide them to `HttpConnection`. +Bytes read by the `SocketChannelEndPoint` will be interpreted as TLS bytes by the `SslConnection`, then decrypted and made available to the `DecryptedEndPoint` (a component part of `SslConnection`), which will then provide them to `HttpConnection`. -The application writes bytes through the `HttpConnection` to the `DecryptedEndPoint`, which will encrypt them through the `SslConnection` and write the encrypted bytes to the `SocketEndPoint`. +The application writes bytes through the `HttpConnection` to the `DecryptedEndPoint`, which will encrypt them through the `SslConnection` and write the encrypted bytes to the `SocketChannelEndPoint`. [[pg-server-io-arch-connection-factory-detecting]] ==== Choosing `ConnectionFactory` via Bytes Detection @@ -124,18 +148,18 @@ With this configuration, the detector will delegate to `SslConnectionFactory` to <2> Creates the `ServerConnector` with `DetectorConnectionFactory` as the first `ConnectionFactory`, and `HttpConnectionFactory` as the next `ConnectionFactory` to invoke if the detection fails. In the example above `ServerConnector` will listen on port 8181. -When a new TCP connection is established, `DetectorConnectionFactory` is invoked to create a `Connection`, because it is the first `ConnectionFactory` specified in the `ServerConnector` list. +When a new socket connection is established, `DetectorConnectionFactory` is invoked to create a `Connection`, because it is the first `ConnectionFactory` specified in the `ServerConnector` list. `DetectorConnectionFactory` reads the initial bytes and asks to its detecting ``ConnectionFactory``s if they recognize the bytes. In the example above, the detecting ``ConnectionFactory`` is `SslConnectionFactory` which will therefore detect whether the initial bytes are TLS bytes. If one of the detecting ``ConnectionFactory``s recognizes the bytes, it creates a `Connection`; otherwise `DetectorConnectionFactory` will try the next `ConnectionFactory` after itself in the `ServerConnector` list. In the example above, the next `ConnectionFactory` after `DetectorConnectionFactory` is `HttpConnectionFactory`. -The final result is that when new TCP connection is established, the initial bytes are examined: if they are TLS bytes, a `SslConnectionFactory` will create a `SslConnection` that wraps an `HttpConnection` as explained xref:pg-server-io-arch-connection-factory-wrapping[here], therefore supporting `https`; otherwise they are not TLS bytes and an `HttpConnection` is created, therefore supporting `http`. +The final result is that when new socket connection is established, the initial bytes are examined: if they are TLS bytes, a `SslConnectionFactory` will create a `SslConnection` that wraps an `HttpConnection` as explained xref:pg-server-io-arch-connection-factory-wrapping[here], therefore supporting `https`; otherwise they are not TLS bytes and an `HttpConnection` is created, therefore supporting `http`. [[pg-server-io-arch-connection-factory-custom]] ==== Writing a Custom `ConnectionFactory` -This section explains how to use the Jetty server-side libraries to write a generic network server able to parse and generate any protocol based on TCP. +This section explains how to use the Jetty server-side libraries to write a generic network server able to parse and generate any protocol.. Let's suppose that we want to write a custom protocol that is based on JSON but has the same semantic as HTTP; let's call this custom protocol `JSONHTTP`, so that a request would look like this: diff --git a/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/ServerDocs.java b/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/ServerDocs.java index 5c063daf3428..6829c6d19719 100644 --- a/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/ServerDocs.java +++ b/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/ServerDocs.java @@ -14,6 +14,7 @@ package org.eclipse.jetty.docs.programming.server; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.Map; import java.util.concurrent.Executor; @@ -31,6 +32,7 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.unixdomain.server.UnixDomainServerConnector; import org.eclipse.jetty.util.BufferUtil; import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.IteratingCallback; @@ -57,6 +59,23 @@ public void http() throws Exception // end::http[] } + public void httpUnix() throws Exception + { + // tag::httpUnix[] + // Create the HTTP/1.1 ConnectionFactory. + HttpConnectionFactory http = new HttpConnectionFactory(); + + Server server = new Server(); + + // Create the connector with the ConnectionFactory. + UnixDomainServerConnector connector = new UnixDomainServerConnector(server, http); + connector.setUnixDomainPath(Path.of("/tmp/jetty.sock")); + + server.addConnector(connector); + server.start(); + // end::httpUnix[] + } + public void tlsHttp() throws Exception { // tag::tlsHttp[] diff --git a/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java b/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java index 103d24402cfd..6aaa1e43a519 100644 --- a/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java +++ b/documentation/jetty-documentation/src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java @@ -14,6 +14,7 @@ package org.eclipse.jetty.docs.programming.server.http; import java.io.IOException; +import java.nio.file.Path; import java.util.EnumSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -62,6 +63,7 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.servlets.CrossOriginFilter; +import org.eclipse.jetty.unixdomain.server.UnixDomainServerConnector; import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.resource.ResourceCollection; @@ -163,14 +165,15 @@ public void configureConnector() throws Exception int selectors = 1; // Create a ServerConnector instance. - ServerConnector connector = new ServerConnector(server, 1, 1, new HttpConnectionFactory()); + ServerConnector connector = new ServerConnector(server, acceptors, selectors, new HttpConnectionFactory()); - // Configure TCP parameters. + // Configure TCP/IP parameters. - // The TCP port to listen to. + // The port to listen to. connector.setPort(8080); - // The TCP address to bind to. + // The address to bind to. connector.setHost("127.0.0.1"); + // The TCP accept queue size. connector.setAcceptQueueSize(128); @@ -179,6 +182,33 @@ public void configureConnector() throws Exception // end::configureConnector[] } + public void configureConnectorUnix() throws Exception + { + // tag::configureConnectorUnix[] + Server server = new Server(); + + // The number of acceptor threads. + int acceptors = 1; + + // The number of selectors. + int selectors = 1; + + // Create a ServerConnector instance. + UnixDomainServerConnector connector = new UnixDomainServerConnector(server, acceptors, selectors, new HttpConnectionFactory()); + + // Configure Unix-Domain parameters. + + // The Unix-Domain path to listen to. + connector.setUnixDomainPath(Path.of("/tmp/jetty.sock")); + + // The TCP accept queue size. + connector.setAcceptQueueSize(128); + + server.addConnector(connector); + server.start(); + // end::configureConnectorUnix[] + } + public void configureConnectors() throws Exception { // tag::configureConnectors[] @@ -248,6 +278,31 @@ public void proxyHTTP() throws Exception // end::proxyHTTP[] } + public void proxyHTTPUnix() throws Exception + { + // tag::proxyHTTPUnix[] + Server server = new Server(); + + // The HTTP configuration object. + HttpConfiguration httpConfig = new HttpConfiguration(); + // Configure the HTTP support, for example: + httpConfig.setSendServerVersion(false); + + // The ConnectionFactory for HTTP/1.1. + HttpConnectionFactory http11 = new HttpConnectionFactory(httpConfig); + + // The ConnectionFactory for the PROXY protocol. + ProxyConnectionFactory proxy = new ProxyConnectionFactory(http11.getProtocol()); + + // Create the ServerConnector. + UnixDomainServerConnector connector = new UnixDomainServerConnector(server, proxy, http11); + connector.setUnixDomainPath(Path.of("/tmp/jetty.sock")); + + server.addConnector(connector); + server.start(); + // end::proxyHTTPUnix[] + } + public void tlsHttp11() throws Exception { // tag::tlsHttp11[] diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java b/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java index 4fd7f4c72e27..2ed04c31a020 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/AbstractHttpClientTransport.java @@ -25,7 +25,7 @@ @ManagedObject public abstract class AbstractHttpClientTransport extends ContainerLifeCycle implements HttpClientTransport { - protected static final Logger LOG = LoggerFactory.getLogger(HttpClientTransport.class); + private static final Logger LOG = LoggerFactory.getLogger(HttpClientTransport.class); private HttpClient client; private ConnectionPool.Factory factory; diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java index 2e89a292a4d4..76caef9bb3d1 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java @@ -161,7 +161,7 @@ public HttpClient(HttpClientTransport transport) { this.transport = Objects.requireNonNull(transport); addBean(transport); - this.connector = ((AbstractHttpClientTransport)transport).getBean(ClientConnector.class); + this.connector = ((AbstractHttpClientTransport)transport).getContainedBeans(ClientConnector.class).stream().findFirst().orElseThrow(); addBean(handlers); addBean(decoderFactories); } @@ -553,24 +553,25 @@ public List getDestinations() return new ArrayList<>(destinations.values()); } - protected void send(final HttpRequest request, List listeners) + protected void send(HttpRequest request, List listeners) { HttpDestination destination = (HttpDestination)resolveDestination(request); destination.send(request, listeners); } - protected void newConnection(final HttpDestination destination, final Promise promise) + protected void newConnection(HttpDestination destination, Promise promise) { + // Multiple threads may access the map, especially with DEBUG logging enabled. + Map context = new ConcurrentHashMap<>(); + context.put(ClientConnectionFactory.CLIENT_CONTEXT_KEY, HttpClient.this); + context.put(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY, destination); + Origin.Address address = destination.getConnectAddress(); resolver.resolve(address.getHost(), address.getPort(), new Promise<>() { @Override public void succeeded(List socketAddresses) { - // Multiple threads may access the map, especially with DEBUG logging enabled. - Map context = new ConcurrentHashMap<>(); - context.put(ClientConnectionFactory.CLIENT_CONTEXT_KEY, HttpClient.this); - context.put(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY, destination); connect(socketAddresses, 0, context); } @@ -1229,7 +1230,7 @@ public boolean containsAll(Collection c) @Override public Iterator iterator() { - final Iterator iterator = set.iterator(); + Iterator iterator = set.iterator(); return new Iterator<>() { @Override diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java b/jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java index c45f3664e65a..f52eff81a7d2 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java @@ -41,6 +41,8 @@ import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.io.Connection; import org.eclipse.jetty.io.EndPoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** *

A {@link HttpClientTransport} that can dynamically switch among different application protocols.

@@ -79,6 +81,8 @@ */ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTransport { + private static final Logger LOG = LoggerFactory.getLogger(HttpClientTransportDynamic.class); + private final List factoryInfos; private final List protocols; diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientTransportOverHTTP.java b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientTransportOverHTTP.java index 4064a3a0b1c5..0eb54ef3502e 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientTransportOverHTTP.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientTransportOverHTTP.java @@ -29,11 +29,14 @@ import org.eclipse.jetty.util.ProcessorUtils; import org.eclipse.jetty.util.annotation.ManagedAttribute; import org.eclipse.jetty.util.annotation.ManagedObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @ManagedObject("The HTTP/1.1 client transport") public class HttpClientTransportOverHTTP extends AbstractConnectorHttpClientTransport { public static final Origin.Protocol HTTP11 = new Origin.Protocol(List.of("http/1.1"), false); + private static final Logger LOG = LoggerFactory.getLogger(HttpClientTransportOverHTTP.class); private final ClientConnectionFactory factory = new HttpClientConnectionFactory(); private int headerCacheSize = 1024; diff --git a/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpClientTransportOverFCGI.java b/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpClientTransportOverFCGI.java index a33fa967e10d..039f813ed319 100644 --- a/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpClientTransportOverFCGI.java +++ b/jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpClientTransportOverFCGI.java @@ -34,10 +34,14 @@ import org.eclipse.jetty.util.Promise; import org.eclipse.jetty.util.annotation.ManagedAttribute; import org.eclipse.jetty.util.annotation.ManagedObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @ManagedObject("The FastCGI/1.0 client transport") public class HttpClientTransportOverFCGI extends AbstractConnectorHttpClientTransport { + private static final Logger LOG = LoggerFactory.getLogger(HttpClientTransportOverFCGI.class); + private final String scriptRoot; public HttpClientTransportOverFCGI(String scriptRoot) diff --git a/jetty-fcgi/fcgi-server/pom.xml b/jetty-fcgi/fcgi-server/pom.xml index bed9361cede4..0a2226e4b81d 100644 --- a/jetty-fcgi/fcgi-server/pom.xml +++ b/jetty-fcgi/fcgi-server/pom.xml @@ -69,5 +69,11 @@ ${project.version} test + + org.eclipse.jetty + jetty-unixdomain-server + ${project.version} + test + diff --git a/jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServlet.java b/jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServlet.java index a543c22df221..e22b81c70d62 100644 --- a/jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServlet.java +++ b/jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServlet.java @@ -14,6 +14,7 @@ package org.eclipse.jetty.fcgi.server.proxy; import java.net.URI; +import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Set; @@ -36,6 +37,7 @@ import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpScheme; +import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.proxy.AsyncProxyServlet; import org.eclipse.jetty.util.ProcessorUtils; @@ -62,6 +64,8 @@ * to force the FastCGI {@code HTTPS} parameter to the value {@code on} *
  • {@code fastCGI.envNames}, optional, a comma separated list of environment variable * names read via {@link System#getenv(String)} that are forwarded as FastCGI parameters.
    • + *
  • {@code unixDomainPath}, optional, that specifies the Unix-Domain path the FastCGI + * server listens to.
    • * * * @see TryFilesFilter @@ -122,11 +126,23 @@ protected HttpClient newHttpClient() String scriptRoot = config.getInitParameter(SCRIPT_ROOT_INIT_PARAM); if (scriptRoot == null) throw new IllegalArgumentException("Mandatory parameter '" + SCRIPT_ROOT_INIT_PARAM + "' not configured"); - int selectors = Math.max(1, ProcessorUtils.availableProcessors() / 2); - String value = config.getInitParameter("selectors"); - if (value != null) - selectors = Integer.parseInt(value); - return new HttpClient(new ProxyHttpClientTransportOverFCGI(selectors, scriptRoot)); + + ClientConnector connector; + String unixDomainPath = config.getInitParameter("unixDomainPath"); + if (unixDomainPath != null) + { + connector = ClientConnector.forUnixDomain(Path.of(unixDomainPath)); + } + else + { + int selectors = Math.max(1, ProcessorUtils.availableProcessors() / 2); + String value = config.getInitParameter("selectors"); + if (value != null) + selectors = Integer.parseInt(value); + connector = new ClientConnector(); + connector.setSelectors(selectors); + } + return new HttpClient(new ProxyHttpClientTransportOverFCGI(connector, scriptRoot)); } @Override @@ -261,9 +277,9 @@ protected void customizeFastCGIHeaders(Request proxyRequest, HttpFields.Mutable private class ProxyHttpClientTransportOverFCGI extends HttpClientTransportOverFCGI { - private ProxyHttpClientTransportOverFCGI(int selectors, String scriptRoot) + private ProxyHttpClientTransportOverFCGI(ClientConnector connector, String scriptRoot) { - super(selectors, scriptRoot); + super(connector, scriptRoot); } @Override diff --git a/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServletTest.java b/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServletTest.java index 6df325b47312..1498be1b98fc 100644 --- a/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServletTest.java +++ b/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServletTest.java @@ -14,9 +14,12 @@ package org.eclipse.jetty.fcgi.server.proxy; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -29,18 +32,22 @@ import org.eclipse.jetty.fcgi.FCGI; import org.eclipse.jetty.fcgi.server.ServerFCGIConnectionFactory; import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.unixdomain.server.UnixDomainServerConnector; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -49,19 +56,13 @@ public class FastCGIProxyServletTest { - public static Stream factories() - { - return Stream.of( - true, // send status 200 - false // don't send status 200 - ).map(Arguments::of); - } - + private final Map fcgiParams = new HashMap<>(); private Server server; private ServerConnector httpConnector; - private ServerConnector fcgiConnector; + private Connector fcgiConnector; private ServletContextHandler context; private HttpClient client; + private Path unixDomainPath; public void prepare(boolean sendStatus200, HttpServlet servlet) throws Exception { @@ -71,19 +72,32 @@ public void prepare(boolean sendStatus200, HttpServlet servlet) throws Exception httpConnector = new ServerConnector(server); server.addConnector(httpConnector); - fcgiConnector = new ServerConnector(server, new ServerFCGIConnectionFactory(new HttpConfiguration(), sendStatus200)); + ServerFCGIConnectionFactory fcgi = new ServerFCGIConnectionFactory(new HttpConfiguration(), sendStatus200); + if (unixDomainPath == null) + { + fcgiConnector = new ServerConnector(server, fcgi); + } + else + { + UnixDomainServerConnector connector = new UnixDomainServerConnector(server, fcgi); + connector.setUnixDomainPath(unixDomainPath); + fcgiConnector = connector; + } server.addConnector(fcgiConnector); - final String contextPath = "/"; + String contextPath = "/"; context = new ServletContextHandler(server, contextPath); - final String servletPath = "/script"; + String servletPath = "/script"; FastCGIProxyServlet fcgiServlet = new FastCGIProxyServlet() { @Override protected String rewriteTarget(HttpServletRequest request) { - return "http://localhost:" + fcgiConnector.getLocalPort() + servletPath + request.getServletPath(); + String uri = "http://localhost"; + if (unixDomainPath == null) + uri += ":" + ((ServerConnector)fcgiConnector).getLocalPort(); + return uri + servletPath + request.getServletPath(); } }; ServletHolder fcgiServletHolder = new ServletHolder(fcgiServlet); @@ -91,6 +105,7 @@ protected String rewriteTarget(HttpServletRequest request) fcgiServletHolder.setInitParameter(FastCGIProxyServlet.SCRIPT_ROOT_INIT_PARAM, "/scriptRoot"); fcgiServletHolder.setInitParameter("proxyTo", "http://localhost"); fcgiServletHolder.setInitParameter(FastCGIProxyServlet.SCRIPT_PATTERN_INIT_PARAM, "(.+?\\.php)"); + fcgiParams.forEach(fcgiServletHolder::setInitParameter); context.addServlet(fcgiServletHolder, "*.php"); context.addServlet(new ServletHolder(servlet), servletPath + "/*"); @@ -111,36 +126,36 @@ public void dispose() throws Exception } @ParameterizedTest(name = "[{index}] sendStatus200={0}") - @MethodSource("factories") + @ValueSource(booleans = {true, false}) public void testGETWithSmallResponseContent(boolean sendStatus200) throws Exception { testGETWithResponseContent(sendStatus200, 1024, 0); } @ParameterizedTest(name = "[{index}] sendStatus200={0}") - @MethodSource("factories") + @ValueSource(booleans = {true, false}) public void testGETWithLargeResponseContent(boolean sendStatus200) throws Exception { testGETWithResponseContent(sendStatus200, 16 * 1024 * 1024, 0); } @ParameterizedTest(name = "[{index}] sendStatus200={0}") - @MethodSource("factories") + @ValueSource(booleans = {true, false}) public void testGETWithLargeResponseContentWithSlowClient(boolean sendStatus200) throws Exception { testGETWithResponseContent(sendStatus200, 16 * 1024 * 1024, 1); } - private void testGETWithResponseContent(boolean sendStatus200, int length, final long delay) throws Exception + private void testGETWithResponseContent(boolean sendStatus200, int length, long delay) throws Exception { - final byte[] data = new byte[length]; + byte[] data = new byte[length]; new Random().nextBytes(data); - final String path = "/foo/index.php"; + String path = "/foo/index.php"; prepare(sendStatus200, new HttpServlet() { @Override - protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { assertTrue(request.getRequestURI().endsWith(path)); response.setContentLength(data.length); @@ -173,16 +188,20 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t } @ParameterizedTest(name = "[{index}] sendStatus200={0}") - @MethodSource("factories") + @ValueSource(booleans = {true, false}) public void testURIRewrite(boolean sendStatus200) throws Exception { String originalPath = "/original/index.php"; String originalQuery = "foo=bar"; String remotePath = "/remote/index.php"; + String pathAttribute = "_path_attribute"; + String queryAttribute = "_query_attribute"; + fcgiParams.put(FastCGIProxyServlet.ORIGINAL_URI_ATTRIBUTE_INIT_PARAM, pathAttribute); + fcgiParams.put(FastCGIProxyServlet.ORIGINAL_QUERY_ATTRIBUTE_INIT_PARAM, queryAttribute); prepare(sendStatus200, new HttpServlet() { @Override - protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + protected void service(HttpServletRequest request, HttpServletResponse response) { assertThat((String)request.getAttribute(FCGI.Headers.REQUEST_URI), Matchers.startsWith(originalPath)); assertEquals(originalQuery, request.getAttribute(FCGI.Headers.QUERY_STRING)); @@ -190,11 +209,6 @@ protected void service(HttpServletRequest request, HttpServletResponse response) } }); context.stop(); - String pathAttribute = "_path_attribute"; - String queryAttribute = "_query_attribute"; - ServletHolder fcgi = context.getServletHandler().getServlet("fcgi"); - fcgi.setInitParameter(FastCGIProxyServlet.ORIGINAL_URI_ATTRIBUTE_INIT_PARAM, pathAttribute); - fcgi.setInitParameter(FastCGIProxyServlet.ORIGINAL_QUERY_ATTRIBUTE_INIT_PARAM, queryAttribute); context.insertHandler(new HandlerWrapper() { @Override @@ -216,4 +230,34 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, assertEquals(HttpStatus.OK_200, response.getStatus()); } + + @Test + @EnabledForJreRange(min = JRE.JAVA_16) + public void testUnixDomain() throws Exception + { + int maxUnixDomainPathLength = 108; + Path path = Files.createTempFile("unix", ".sock"); + if (path.normalize().toAbsolutePath().toString().length() > maxUnixDomainPathLength) + path = Files.createTempFile(Path.of("/tmp"), "unix", ".sock"); + assertTrue(Files.deleteIfExists(path)); + unixDomainPath = path; + fcgiParams.put("unixDomainPath", path.toString()); + byte[] content = new byte[512]; + new Random().nextBytes(content); + prepare(true, new HttpServlet() + { + @Override + protected void service(HttpServletRequest request, HttpServletResponse response) throws IOException + { + response.getOutputStream().write(content); + } + }); + + ContentResponse response = client.newRequest("localhost", httpConnector.getLocalPort()) + .path("/index.php") + .send(); + + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertArrayEquals(content, response.getContent()); + } } diff --git a/jetty-fcgi/fcgi-server/src/test/resources/jetty-logging.properties b/jetty-fcgi/fcgi-server/src/test/resources/jetty-logging.properties index 4e7406f1b548..9d915cb3b003 100644 --- a/jetty-fcgi/fcgi-server/src/test/resources/jetty-logging.properties +++ b/jetty-fcgi/fcgi-server/src/test/resources/jetty-logging.properties @@ -1,3 +1,3 @@ -# Jetty Logging using jetty-slf4j-impl +#org.eclipse.jetty.LEVEL=DEBUG #org.eclipse.jetty.client.LEVEL=DEBUG #org.eclipse.jetty.fcgi.LEVEL=DEBUG diff --git a/jetty-home/pom.xml b/jetty-home/pom.xml index d0daeeb66574..a7ee899e87b0 100644 --- a/jetty-home/pom.xml +++ b/jetty-home/pom.xml @@ -534,6 +534,7 @@ + @@ -648,6 +649,12 @@ jetty-proxy ${project.version} + + org.eclipse.jetty + jetty-unixdomain-server + ${project.version} + true + org.eclipse.jetty jetty-unixsocket-server diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java b/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java index f2d5ebb4ce8b..f0a4c987e17f 100644 --- a/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java +++ b/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java @@ -14,18 +14,24 @@ package org.eclipse.jetty.io; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ProtocolFamily; import java.net.SocketAddress; import java.net.SocketException; +import java.net.StandardProtocolFamily; import java.net.StandardSocketOptions; import java.nio.channels.SelectableChannel; import java.nio.channels.SelectionKey; import java.nio.channels.SocketChannel; +import java.nio.file.Path; import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.Executor; import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.JavaVersion; import org.eclipse.jetty.util.Promise; import org.eclipse.jetty.util.component.ContainerLifeCycle; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -43,6 +49,12 @@ public class ClientConnector extends ContainerLifeCycle public static final String CONNECTION_PROMISE_CONTEXT_KEY = CLIENT_CONNECTOR_CONTEXT_KEY + ".connectionPromise"; private static final Logger LOG = LoggerFactory.getLogger(ClientConnector.class); + public static ClientConnector forUnixDomain(Path path) + { + return new ClientConnector(SocketChannelWithAddress.Factory.forUnixDomain(path)); + } + + private final SocketChannelWithAddress.Factory factory; private Executor executor; private Scheduler scheduler; private ByteBufferPool byteBufferPool; @@ -55,6 +67,16 @@ public class ClientConnector extends ContainerLifeCycle private SocketAddress bindAddress; private boolean reuseAddress = true; + public ClientConnector() + { + this((address, context) -> new SocketChannelWithAddress(SocketChannel.open(), address)); + } + + private ClientConnector(SocketChannelWithAddress.Factory factory) + { + this.factory = Objects.requireNonNull(factory); + } + public Executor getExecutor() { return executor; @@ -221,20 +243,16 @@ public void connect(SocketAddress address, Map context) context.put(ClientConnector.CLIENT_CONNECTOR_CONTEXT_KEY, this); context.putIfAbsent(REMOTE_SOCKET_ADDRESS_CONTEXT_KEY, address); - channel = SocketChannel.open(); + SocketChannelWithAddress channelWithAddress = factory.newSocketChannelWithAddress(address, context); + channel = channelWithAddress.getSocketChannel(); + address = channelWithAddress.getSocketAddress(); SocketAddress bindAddress = getBindAddress(); if (bindAddress != null) - { - boolean reuseAddress = getReuseAddress(); - if (LOG.isDebugEnabled()) - LOG.debug("Binding to {} to connect to {}{}", bindAddress, address, (reuseAddress ? " reusing address" : "")); - channel.setOption(StandardSocketOptions.SO_REUSEADDR, reuseAddress); - channel.bind(bindAddress); - } + bind(channel, bindAddress); configure(channel); boolean connected = true; - boolean blocking = isConnectBlocking(); + boolean blocking = isConnectBlocking() && address instanceof InetSocketAddress; if (LOG.isDebugEnabled()) LOG.debug("Connecting {} to {}", blocking ? "blocking" : "non-blocking", address); if (blocking) @@ -288,9 +306,34 @@ public void accept(SocketChannel channel, Map context) } } + private void bind(SocketChannel channel, SocketAddress bindAddress) + { + try + { + boolean reuseAddress = getReuseAddress(); + if (LOG.isDebugEnabled()) + LOG.debug("Binding to {} reusing address {}", bindAddress, reuseAddress); + channel.setOption(StandardSocketOptions.SO_REUSEADDR, reuseAddress); + channel.bind(bindAddress); + } + catch (Throwable x) + { + if (LOG.isDebugEnabled()) + LOG.debug("Could not bind {}", channel); + } + } + protected void configure(SocketChannel channel) throws IOException { - channel.socket().setTcpNoDelay(true); + try + { + channel.setOption(StandardSocketOptions.TCP_NODELAY, true); + } + catch (Throwable x) + { + if (LOG.isDebugEnabled()) + LOG.debug("Could not configure {}", channel); + } } protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey) @@ -351,4 +394,77 @@ protected void connectionFailed(SelectableChannel channel, Throwable failure, Ob connectFailed(failure, context); } } + + /** + *

    A pair/record holding a {@link SocketChannel} and a {@link SocketAddress} to connect to.

    + */ + private static class SocketChannelWithAddress + { + private final SocketChannel channel; + private final SocketAddress address; + + private SocketChannelWithAddress(SocketChannel channel, SocketAddress address) + { + this.channel = channel; + this.address = address; + } + + private SocketChannel getSocketChannel() + { + return channel; + } + + private SocketAddress getSocketAddress() + { + return address; + } + + /** + *

    A factory for {@link SocketChannelWithAddress} instances.

    + */ + private interface Factory + { + private static Factory forUnixDomain(Path path) + { + return (address, context) -> + { + try + { + ProtocolFamily family = Enum.valueOf(StandardProtocolFamily.class, "UNIX"); + SocketChannel socketChannel = (SocketChannel)SocketChannel.class.getMethod("open", ProtocolFamily.class).invoke(null, family); + Class addressClass = Class.forName("java.net.UnixDomainSocketAddress"); + SocketAddress socketAddress = (SocketAddress)addressClass.getMethod("of", Path.class).invoke(null, path); + return new SocketChannelWithAddress(socketChannel, socketAddress); + } + catch (Throwable x) + { + String message = "Unix-Domain SocketChannels are available starting from Java 16, your Java version is: " + JavaVersion.VERSION; + throw new UnsupportedOperationException(message, x); + } + }; + } + + /** + *

    Creates a new {@link SocketChannel} to connect to a {@link SocketAddress} + * derived from the input socket address.

    + *

    The input socket address represents the destination socket address to + * connect to, as it is typically specified by a URI authority, for example + * {@code localhost:8080} if the URI is {@code http://localhost:8080/path}.

    + *

    However, the returned socket address may be different as the implementation + * may use a Unix-Domain socket address to physically connect to the virtual + * destination socket address given as input.

    + *

    The return type is a pair/record holding the socket channel and the + * socket address, with the socket channel not yet connected. + * The implementation of this methods must not call + * {@link SocketChannel#connect(SocketAddress)}, as this is done later, + * after configuring the socket, by the {@link ClientConnector} implementation.

    + * + * @param address the destination socket address, typically specified in a URI + * @param context the context to create the new socket channel + * @return a new {@link SocketChannel} with an associated {@link SocketAddress} to connect to + * @throws IOException if the socket channel or the socket address cannot be created + */ + public SocketChannelWithAddress newSocketChannelWithAddress(SocketAddress address, Map context) throws IOException; + } + } } diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java b/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java index 1258424ddbb6..2ff61f4ae84c 100644 --- a/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java +++ b/jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java @@ -165,8 +165,10 @@ public SocketAddress getLocalSocketAddress() { return _channel.getLocalAddress(); } - catch (IOException x) + catch (Throwable x) { + if (LOG.isDebugEnabled()) + LOG.debug("Could not retrieve local socket address", x); return null; } } @@ -178,8 +180,10 @@ public SocketAddress getRemoteSocketAddress() { return _channel.getRemoteAddress(); } - catch (IOException e) + catch (Throwable x) { + if (LOG.isDebugEnabled()) + LOG.debug("Could not retrieve remote socket address", x); return null; } } diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ProxyConnectionFactory.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ProxyConnectionFactory.java index b84e2fad1566..1bfe3e65e4a7 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/ProxyConnectionFactory.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ProxyConnectionFactory.java @@ -585,30 +585,43 @@ private void parseBodyAndUpgrade() throws IOException { byte[] addr = new byte[4]; _buffer.get(addr); - InetAddress src = Inet4Address.getByAddress(addr); + InetAddress srcAddr = Inet4Address.getByAddress(addr); _buffer.get(addr); - InetAddress dst = Inet4Address.getByAddress(addr); - int sp = _buffer.getChar(); - int dp = _buffer.getChar(); - local = new InetSocketAddress(dst, dp); - remote = new InetSocketAddress(src, sp); + InetAddress dstAddr = Inet4Address.getByAddress(addr); + int srcPort = _buffer.getChar(); + int dstPort = _buffer.getChar(); + local = new InetSocketAddress(dstAddr, dstPort); + remote = new InetSocketAddress(srcAddr, srcPort); break; } case INET6: { byte[] addr = new byte[16]; _buffer.get(addr); - InetAddress src = Inet6Address.getByAddress(addr); + InetAddress srcAddr = Inet6Address.getByAddress(addr); _buffer.get(addr); - InetAddress dst = Inet6Address.getByAddress(addr); - int sp = _buffer.getChar(); - int dp = _buffer.getChar(); - local = new InetSocketAddress(dst, dp); - remote = new InetSocketAddress(src, sp); + InetAddress dstAddr = Inet6Address.getByAddress(addr); + int srcPort = _buffer.getChar(); + int dstPort = _buffer.getChar(); + local = new InetSocketAddress(dstAddr, dstPort); + remote = new InetSocketAddress(srcAddr, srcPort); + break; + } + case UNIX: + { + byte[] addr = new byte[108]; + _buffer.get(addr); + String src = UnixDomain.toPath(addr); + _buffer.get(addr); + String dst = UnixDomain.toPath(addr); + local = UnixDomain.newSocketAddress(dst); + remote = UnixDomain.newSocketAddress(src); break; } default: - throw new IllegalStateException(); + { + throw new IllegalStateException("Unsupported family " + _family); + } } proxyEndPoint = new ProxyEndPoint(endPoint, local, remote); @@ -706,7 +719,7 @@ private void parseHeader() throws IOException } Transport transport; - switch (0xF & transportAndFamily) + switch (transportAndFamily & 0xF) { case 0: transport = Transport.UNSPEC; @@ -723,7 +736,7 @@ private void parseHeader() throws IOException _length = _buffer.getChar(); - if (!_local && (_family == Family.UNSPEC || _family == Family.UNIX || transport != Transport.STREAM)) + if (!_local && (_family == Family.UNSPEC || transport != Transport.STREAM)) throw new IOException(String.format("Proxy v2 unsupported PROXY mode 0x%x,0x%x", versionAndCommand, transportAndFamily)); if (_length > getMaxProxyHeader()) @@ -957,4 +970,47 @@ public void write(Callback callback, ByteBuffer... buffers) throws WritePendingE _endPoint.write(callback, buffers); } } + + private static class UnixDomain + { + private static final Class unixDomainSocketAddress = probe(); + + private static Class probe() + { + try + { + return ClassLoader.getPlatformClassLoader().loadClass("java.net.UnixDomainSocketAddress"); + } + catch (Throwable ignored) + { + return null; + } + } + + private static SocketAddress newSocketAddress(String path) + { + try + { + if (unixDomainSocketAddress != null) + return (SocketAddress)unixDomainSocketAddress.getMethod("of", String.class).invoke(null, path); + return null; + } + catch (Throwable ignored) + { + return null; + } + } + + private static String toPath(byte[] bytes) + { + // Unix-Domain paths are zero-terminated. + int i = 0; + while (i < bytes.length) + { + if (bytes[i++] == 0) + break; + } + return new String(bytes, 0, i, StandardCharsets.US_ASCII).trim(); + } + } } diff --git a/jetty-unixdomain-server/pom.xml b/jetty-unixdomain-server/pom.xml new file mode 100644 index 000000000000..b86729a3e704 --- /dev/null +++ b/jetty-unixdomain-server/pom.xml @@ -0,0 +1,43 @@ + + + + org.eclipse.jetty + jetty-project + 10.0.7-SNAPSHOT + + + 4.0.0 + jetty-unixdomain-server + Jetty :: Unix-Domain Sockets :: Server + Jetty Unix-Domain Sockets Server + + + ${project.groupId}.unixdomain.server + org.eclipse.jetty.unixdomain.* + + + + + org.eclipse.jetty + jetty-server + ${project.version} + + + org.slf4j + slf4j-api + + + + org.eclipse.jetty + jetty-client + ${project.version} + test + + + org.eclipse.jetty + jetty-slf4j-impl + test + + + + diff --git a/jetty-unixdomain-server/src/main/config/etc/jetty-unixdomain-http.xml b/jetty-unixdomain-server/src/main/config/etc/jetty-unixdomain-http.xml new file mode 100644 index 000000000000..d59074567bea --- /dev/null +++ b/jetty-unixdomain-server/src/main/config/etc/jetty-unixdomain-http.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + /jetty.sock + + + + + + + + + + + diff --git a/jetty-unixdomain-server/src/main/config/modules/unixdomain-http.mod b/jetty-unixdomain-server/src/main/config/modules/unixdomain-http.mod new file mode 100644 index 000000000000..689b226e904f --- /dev/null +++ b/jetty-unixdomain-server/src/main/config/modules/unixdomain-http.mod @@ -0,0 +1,36 @@ +[description] +Enables support for clear-text HTTP/1.1 over Java 16 Unix-Domain server sockets. + +[tag] +connector +unixdomain + +[depends] +server + +[lib] +lib/jetty-unixdomain-server-*.jar + +[xml] +etc/jetty-unixdomain-http.xml + +[ini-template] +# tag::documentation[] +## The number of acceptors (-1 picks a default value based on number of cores). +# jetty.unixdomain.acceptors=1 + +## The number of selectors (-1 picks a default value based on number of cores). +# jetty.unixdomain.selectors=-1 + +## The Unix-Domain path the ServerSocketChannel listens to. +# jetty.unixdomain.path=/tmp/jetty.sock + +## The ServerSocketChannel accept queue backlog (0 picks the platform default). +# jetty.unixdomain.acceptQueueSize=0 + +## The SO_RCVBUF option for accepted SocketChannels (0 picks the platform default). +# jetty.unixdomain.acceptedReceiveBufferSize=0 + +## The SO_SNDBUF option for accepted SocketChannels (0 picks the platform default). +# jetty.unixdomain.acceptedSendBufferSize=0 +# end::documentation[] diff --git a/jetty-unixdomain-server/src/main/java/module-info.java b/jetty-unixdomain-server/src/main/java/module-info.java new file mode 100644 index 000000000000..97065cabeadf --- /dev/null +++ b/jetty-unixdomain-server/src/main/java/module-info.java @@ -0,0 +1,20 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +module org.eclipse.jetty.unixdomain.server +{ + exports org.eclipse.jetty.unixdomain.server; + + requires transitive org.eclipse.jetty.server; + requires org.slf4j; +} diff --git a/jetty-unixdomain-server/src/main/java/org/eclipse/jetty/unixdomain/server/UnixDomainServerConnector.java b/jetty-unixdomain-server/src/main/java/org/eclipse/jetty/unixdomain/server/UnixDomainServerConnector.java new file mode 100644 index 000000000000..9270ca3460e7 --- /dev/null +++ b/jetty-unixdomain-server/src/main/java/org/eclipse/jetty/unixdomain/server/UnixDomainServerConnector.java @@ -0,0 +1,326 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.unixdomain.server; + +import java.io.Closeable; +import java.io.IOException; +import java.net.ProtocolFamily; +import java.net.SocketAddress; +import java.net.StandardProtocolFamily; +import java.net.StandardSocketOptions; +import java.nio.channels.Channel; +import java.nio.channels.SelectableChannel; +import java.nio.channels.SelectionKey; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jetty.io.ByteBufferPool; +import org.eclipse.jetty.io.Connection; +import org.eclipse.jetty.io.EndPoint; +import org.eclipse.jetty.io.ManagedSelector; +import org.eclipse.jetty.io.SelectorManager; +import org.eclipse.jetty.io.SocketChannelEndPoint; +import org.eclipse.jetty.server.AbstractConnector; +import org.eclipse.jetty.server.ConnectionFactory; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.JavaVersion; +import org.eclipse.jetty.util.annotation.ManagedAttribute; +import org.eclipse.jetty.util.annotation.ManagedObject; +import org.eclipse.jetty.util.thread.Scheduler; + +/** + *

    A {@link Connector} implementation for Unix-Domain server socket channels.

    + *

    UnixDomainServerConnector "listens" to a {@link #setUnixDomainPath(Path) Unix-Domain path} + * and behaves {@link ServerConnector} with respect to acceptors, selectors and connection + * factories.

    + *

    Important: the unix-domain path must be less than 108 bytes. + * This limit is set by the way Unix-Domain sockets work at the OS level.

    + */ +@ManagedObject +public class UnixDomainServerConnector extends AbstractConnector +{ + private final AtomicReference acceptor = new AtomicReference<>(); + private final SelectorManager selectorManager; + private ServerSocketChannel serverChannel; + private Path unixDomainPath; + private boolean inheritChannel; + private int acceptQueueSize; + private int acceptedReceiveBufferSize; + private int acceptedSendBufferSize; + + public UnixDomainServerConnector(Server server, ConnectionFactory... factories) + { + this(server, null, null, null, -1, -1, factories); + } + + public UnixDomainServerConnector(Server server, int acceptors, int selectors, ConnectionFactory... factories) + { + this(server, null, null, null, acceptors, selectors, factories); + } + + public UnixDomainServerConnector(Server server, Executor executor, Scheduler scheduler, ByteBufferPool pool, int acceptors, int selectors, ConnectionFactory... factories) + { + super(server, executor, scheduler, pool, acceptors, factories.length > 0 ? factories : new ConnectionFactory[]{new HttpConnectionFactory()}); + selectorManager = newSelectorManager(getExecutor(), getScheduler(), selectors); + addBean(selectorManager, true); + } + + protected SelectorManager newSelectorManager(Executor executor, Scheduler scheduler, int selectors) + { + return new UnixDomainSelectorManager(executor, scheduler, selectors); + } + + @ManagedAttribute("The Unix-Domain path this connector listens to") + public Path getUnixDomainPath() + { + return unixDomainPath; + } + + public void setUnixDomainPath(Path unixDomainPath) + { + this.unixDomainPath = unixDomainPath; + } + + @ManagedAttribute("Whether this connector uses a server channel inherited from the JVM") + public boolean isInheritChannel() + { + return inheritChannel; + } + + public void setInheritChannel(boolean inheritChannel) + { + this.inheritChannel = inheritChannel; + } + + @ManagedAttribute("The accept queue size (backlog) for the server socket") + public int getAcceptQueueSize() + { + return acceptQueueSize; + } + + public void setAcceptQueueSize(int acceptQueueSize) + { + this.acceptQueueSize = acceptQueueSize; + } + + @ManagedAttribute("The SO_RCVBUF option for accepted sockets") + public int getAcceptedReceiveBufferSize() + { + return acceptedReceiveBufferSize; + } + + public void setAcceptedReceiveBufferSize(int acceptedReceiveBufferSize) + { + this.acceptedReceiveBufferSize = acceptedReceiveBufferSize; + } + + @ManagedAttribute("The SO_SNDBUF option for accepted sockets") + public int getAcceptedSendBufferSize() + { + return acceptedSendBufferSize; + } + + public void setAcceptedSendBufferSize(int acceptedSendBufferSize) + { + this.acceptedSendBufferSize = acceptedSendBufferSize; + } + + @Override + protected void doStart() throws Exception + { + getBeans(SelectorManager.SelectorManagerListener.class).forEach(selectorManager::addEventListener); + serverChannel = open(); + addBean(serverChannel); + super.doStart(); + } + + @Override + protected void doStop() throws Exception + { + super.doStop(); + removeBean(serverChannel); + close(); + getBeans(SelectorManager.SelectorManagerListener.class).forEach(selectorManager::removeEventListener); + } + + @Override + protected void accept(int acceptorID) throws IOException + { + ServerSocketChannel serverChannel = this.serverChannel; + if (serverChannel != null) + { + SocketChannel channel = serverChannel.accept(); + accepted(channel); + } + } + + private void accepted(SocketChannel channel) throws IOException + { + channel.configureBlocking(false); + configure(channel); + selectorManager.accept(channel); + } + + protected void configure(SocketChannel channel) throws IOException + { + // Unix-Domain does not support TCP_NODELAY. + // Unix-Domain does not support SO_REUSEADDR. + int rcvBufSize = getAcceptedReceiveBufferSize(); + if (rcvBufSize > 0) + channel.setOption(StandardSocketOptions.SO_RCVBUF, rcvBufSize); + int sndBufSize = getAcceptedSendBufferSize(); + if (sndBufSize > 0) + channel.setOption(StandardSocketOptions.SO_SNDBUF, sndBufSize); + } + + @Override + public Object getTransport() + { + return serverChannel; + } + + private ServerSocketChannel open() throws IOException + { + ServerSocketChannel serverChannel = openServerSocketChannel(); + if (getAcceptors() == 0) + { + serverChannel.configureBlocking(false); + acceptor.set(selectorManager.acceptor(serverChannel)); + } + return serverChannel; + } + + private void close() throws IOException + { + ServerSocketChannel serverChannel = this.serverChannel; + this.serverChannel = null; + IO.close(serverChannel); + Files.deleteIfExists(getUnixDomainPath()); + } + + private ServerSocketChannel openServerSocketChannel() throws IOException + { + ServerSocketChannel serverChannel = null; + if (isInheritChannel()) + { + Channel channel = System.inheritedChannel(); + if (channel instanceof ServerSocketChannel) + serverChannel = (ServerSocketChannel)channel; + else + LOG.warn("Unable to use System.inheritedChannel() {}. Trying a new Unix-Domain ServerSocketChannel at {}", channel, getUnixDomainPath()); + } + if (serverChannel == null) + serverChannel = bindServerSocketChannel(); + return serverChannel; + } + + private ServerSocketChannel bindServerSocketChannel() + { + try + { + ProtocolFamily family = Enum.valueOf(StandardProtocolFamily.class, "UNIX"); + Class channelClass = Class.forName("java.nio.channels.ServerSocketChannel"); + ServerSocketChannel serverChannel = (ServerSocketChannel)channelClass.getMethod("open", ProtocolFamily.class).invoke(null, family); + // Unix-Domain does not support SO_REUSEADDR. + Class addressClass = Class.forName("java.net.UnixDomainSocketAddress"); + SocketAddress socketAddress = (SocketAddress)addressClass.getMethod("of", Path.class).invoke(null, getUnixDomainPath()); + serverChannel.bind(socketAddress, getAcceptQueueSize()); + return serverChannel; + } + catch (Throwable x) + { + String message = "Unix-Domain SocketChannels are available starting from Java 16, your Java version is: " + JavaVersion.VERSION; + throw new UnsupportedOperationException(message, x); + } + } + + @Override + public void setAccepting(boolean accepting) + { + super.setAccepting(accepting); + if (getAcceptors() == 0) + return; + if (accepting) + { + if (acceptor.get() == null) + { + Closeable acceptor = selectorManager.acceptor(serverChannel); + if (!this.acceptor.compareAndSet(null, acceptor)) + IO.close(acceptor); + } + } + else + { + Closeable acceptor = this.acceptor.get(); + if (acceptor != null && this.acceptor.compareAndSet(acceptor, null)) + IO.close(acceptor); + } + } + + @Override + public String toString() + { + return String.format("%s@%h[%s]", getClass().getSimpleName(), hashCode(), getUnixDomainPath()); + } + + protected class UnixDomainSelectorManager extends SelectorManager + { + public UnixDomainSelectorManager(Executor executor, Scheduler scheduler, int selectors) + { + super(executor, scheduler, selectors); + } + + @Override + protected void accepted(SelectableChannel channel) throws IOException + { + UnixDomainServerConnector.this.accepted((SocketChannel)channel); + } + + @Override + protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey) + { + SocketChannelEndPoint endPoint = new SocketChannelEndPoint((SocketChannel)channel, selector, selectionKey, getScheduler()); + endPoint.setIdleTimeout(getIdleTimeout()); + return endPoint; + } + + @Override + public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment) + { + return getDefaultConnectionFactory().newConnection(UnixDomainServerConnector.this, endpoint); + } + + @Override + protected void endPointOpened(EndPoint endpoint) + { + super.endPointOpened(endpoint); + onEndPointOpened(endpoint); + } + + @Override + protected void endPointClosed(EndPoint endpoint) + { + onEndPointClosed(endpoint); + super.endPointClosed(endpoint); + } + } +} diff --git a/jetty-unixdomain-server/src/test/java/org/eclipse/jetty/unixdomain/server/UnixDomainTest.java b/jetty-unixdomain-server/src/test/java/org/eclipse/jetty/unixdomain/server/UnixDomainTest.java new file mode 100644 index 000000000000..0d17d609f2c4 --- /dev/null +++ b/jetty-unixdomain-server/src/test/java/org/eclipse/jetty/unixdomain/server/UnixDomainTest.java @@ -0,0 +1,272 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.unixdomain.server; + +import java.net.SocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.HttpProxy; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.http.HttpURI; +import org.eclipse.jetty.io.ClientConnector; +import org.eclipse.jetty.io.EndPoint; +import org.eclipse.jetty.server.ConnectionFactory; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.ProxyConnectionFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.eclipse.jetty.toolchain.test.FS; +import org.eclipse.jetty.util.component.LifeCycle; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +import static org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1; +import static org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@EnabledForJreRange(min = JRE.JAVA_16) +public class UnixDomainTest +{ + private static final Class unixDomainSocketAddressClass = probe(); + + private static Class probe() + { + try + { + return ClassLoader.getPlatformClassLoader().loadClass("java.net.UnixDomainSocketAddress"); + } + catch (Throwable x) + { + return null; + } + } + + private ConnectionFactory[] factories = new ConnectionFactory[]{new HttpConnectionFactory()}; + private Server server; + private Path unixDomainPath; + + @BeforeEach + public void prepare() + { + Assumptions.assumeTrue(unixDomainSocketAddressClass != null); + } + + private void start(Handler handler) throws Exception + { + server = new Server(); + UnixDomainServerConnector connector = new UnixDomainServerConnector(server, factories); + String dir = System.getProperty("jetty.unixdomain.dir"); + assertNotNull(dir); + unixDomainPath = Files.createTempFile(Path.of(dir), "unix_", ".sock"); + assertTrue(unixDomainPath.toAbsolutePath().toString().length() < 108, "Unix-Domain path too long"); + Files.delete(unixDomainPath); + connector.setUnixDomainPath(unixDomainPath); + server.addConnector(connector); + server.setHandler(handler); + server.start(); + } + + @AfterEach + public void dispose() + { + LifeCycle.stop(server); + } + + @Test + public void testHTTPOverUnixDomain() throws Exception + { + String uri = "http://localhost:1234/path"; + start(new AbstractHandler() + { + @Override + public void handle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) + { + jettyRequest.setHandled(true); + + // Verify the URI is preserved. + assertEquals(uri, request.getRequestURL().toString()); + + EndPoint endPoint = jettyRequest.getHttpChannel().getEndPoint(); + + // Verify the SocketAddresses. + SocketAddress local = endPoint.getLocalSocketAddress(); + assertThat(local, Matchers.instanceOf(unixDomainSocketAddressClass)); + SocketAddress remote = endPoint.getRemoteSocketAddress(); + assertThat(remote, Matchers.instanceOf(unixDomainSocketAddressClass)); + + // Verify that other address methods don't throw. + local = assertDoesNotThrow(endPoint::getLocalAddress); + assertNull(local); + remote = assertDoesNotThrow(endPoint::getRemoteAddress); + assertNull(remote); + + assertDoesNotThrow(endPoint::toString); + } + }); + + ClientConnector clientConnector = ClientConnector.forUnixDomain(unixDomainPath); + HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + httpClient.start(); + try + { + ContentResponse response = httpClient.newRequest(uri) + .timeout(5, TimeUnit.SECONDS) + .send(); + + assertEquals(HttpStatus.OK_200, response.getStatus()); + } + finally + { + httpClient.stop(); + } + } + + @Test + public void testHTTPOverUnixDomainWithHTTPProxy() throws Exception + { + int fakeProxyPort = 4567; + int fakeServerPort = 5678; + start(new AbstractHandler() + { + @Override + public void handle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) + { + jettyRequest.setHandled(true); + // Proxied requests must have an absolute URI. + HttpURI uri = jettyRequest.getMetaData().getURI(); + assertNotNull(uri.getScheme()); + assertEquals(fakeServerPort, uri.getPort()); + } + }); + + ClientConnector clientConnector = ClientConnector.forUnixDomain(unixDomainPath); + + HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + httpClient.getProxyConfiguration().getProxies().add(new HttpProxy("localhost", fakeProxyPort)); + httpClient.start(); + try + { + ContentResponse response = httpClient.newRequest("localhost", fakeServerPort) + .timeout(5, TimeUnit.SECONDS) + .send(); + + assertEquals(HttpStatus.OK_200, response.getStatus()); + } + finally + { + httpClient.stop(); + } + } + + @Test + public void testHTTPOverUnixDomainWithProxyProtocol() throws Exception + { + String srcAddr = "/proxySrcAddr"; + String dstAddr = "/proxyDstAddr"; + factories = new ConnectionFactory[]{new ProxyConnectionFactory(), new HttpConnectionFactory()}; + start(new AbstractHandler() + { + @Override + public void handle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) + { + jettyRequest.setHandled(true); + EndPoint endPoint = jettyRequest.getHttpChannel().getEndPoint(); + assertThat(endPoint, Matchers.instanceOf(ProxyConnectionFactory.ProxyEndPoint.class)); + assertThat(endPoint.getLocalSocketAddress(), Matchers.instanceOf(unixDomainSocketAddressClass)); + assertThat(endPoint.getRemoteSocketAddress(), Matchers.instanceOf(unixDomainSocketAddressClass)); + if ("/v1".equals(target)) + { + // As PROXYv1 does not support UNIX, the wrapped EndPoint data is used. + Path localPath = toUnixDomainPath(endPoint.getLocalSocketAddress()); + assertThat(localPath, Matchers.equalTo(unixDomainPath)); + } + else if ("/v2".equals(target)) + { + assertThat(toUnixDomainPath(endPoint.getLocalSocketAddress()).toString(), Matchers.equalTo(FS.separators(dstAddr))); + assertThat(toUnixDomainPath(endPoint.getRemoteSocketAddress()).toString(), Matchers.equalTo(FS.separators(srcAddr))); + } + else + { + Assertions.fail("Invalid PROXY protocol version " + target); + } + } + }); + + // Java 11+ portable way to implement SocketChannelWithAddress.Factory. + ClientConnector clientConnector = ClientConnector.forUnixDomain(unixDomainPath); + + HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + httpClient.start(); + try + { + // Try PROXYv1 with the PROXY information retrieved from the EndPoint. + // PROXYv1 does not support the UNIX family. + ContentResponse response1 = httpClient.newRequest("localhost", 0) + .path("/v1") + .tag(new V1.Tag()) + .timeout(5, TimeUnit.SECONDS) + .send(); + + assertEquals(HttpStatus.OK_200, response1.getStatus()); + + // Try PROXYv2 with explicit PROXY information. + var tag = new V2.Tag(V2.Tag.Command.PROXY, V2.Tag.Family.UNIX, V2.Tag.Protocol.STREAM, srcAddr, 0, dstAddr, 0, null); + ContentResponse response2 = httpClient.newRequest("localhost", 0) + .path("/v2") + .tag(tag) + .timeout(5, TimeUnit.SECONDS) + .send(); + + assertEquals(HttpStatus.OK_200, response2.getStatus()); + } + finally + { + httpClient.stop(); + } + } + + private static Path toUnixDomainPath(SocketAddress address) + { + try + { + Assertions.assertNotNull(unixDomainSocketAddressClass); + return (Path)unixDomainSocketAddressClass.getMethod("getPath").invoke(address); + } + catch (Throwable x) + { + Assertions.fail(x); + throw new AssertionError(); + } + } +} diff --git a/jetty-unixdomain-server/src/test/resources/jetty-logging.properties b/jetty-unixdomain-server/src/test/resources/jetty-logging.properties new file mode 100644 index 000000000000..bc2cf0effd0d --- /dev/null +++ b/jetty-unixdomain-server/src/test/resources/jetty-logging.properties @@ -0,0 +1,2 @@ +#org.eclipse.jetty.LEVEL=DEBUG +#org.eclipse.jetty.unixdomain.LEVEL=DEBUG diff --git a/jetty-unixsocket/jetty-unixsocket-client/src/test/java/org/eclipse/jetty/unixsocket/UnixSocketTest.java b/jetty-unixsocket/jetty-unixsocket-client/src/test/java/org/eclipse/jetty/unixsocket/UnixSocketTest.java index 3c3d981a9b4b..9ceb0ca6b5e3 100644 --- a/jetty-unixsocket/jetty-unixsocket-client/src/test/java/org/eclipse/jetty/unixsocket/UnixSocketTest.java +++ b/jetty-unixsocket/jetty-unixsocket-client/src/test/java/org/eclipse/jetty/unixsocket/UnixSocketTest.java @@ -18,7 +18,6 @@ import java.net.ConnectException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Date; import java.util.concurrent.ExecutionException; import javax.servlet.http.HttpServletRequest; @@ -32,7 +31,6 @@ import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.unixsocket.client.HttpClientTransportOverUnixSockets; import org.eclipse.jetty.unixsocket.server.UnixSocketConnector; -import org.eclipse.jetty.util.StringUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; @@ -45,9 +43,9 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.junit.jupiter.api.condition.OS.LINUX; import static org.junit.jupiter.api.condition.OS.MAC; @@ -65,18 +63,11 @@ public void before() throws Exception { server = null; httpClient = null; - String unixSocketTmp = System.getProperty("unix.socket.tmp"); - if (StringUtil.isNotBlank(unixSocketTmp)) - sockFile = Files.createTempFile(Paths.get(unixSocketTmp), "unix", ".sock"); - else - sockFile = Files.createTempFile("unix", ".sock"); - if (sockFile.toAbsolutePath().toString().length() > UnixSocketConnector.MAX_UNIX_SOCKET_PATH_LENGTH) - { - Path tmp = Paths.get("/tmp"); - assumeTrue(Files.exists(tmp) && Files.isDirectory(tmp)); - sockFile = Files.createTempFile(tmp, "unix", ".sock"); - } - assertTrue(Files.deleteIfExists(sockFile), "temp sock file cannot be deleted"); + String dir = System.getProperty("jetty.unixdomain.dir"); + assertNotNull(dir); + sockFile = Files.createTempFile(Path.of(dir), "unix_", ".sock"); + assertTrue(sockFile.toAbsolutePath().toString().length() < UnixSocketConnector.MAX_UNIX_SOCKET_PATH_LENGTH, "Unix-Domain path too long"); + Files.delete(sockFile); } @AfterEach diff --git a/pom.xml b/pom.xml index 7eb35f3bee1a..317cfa956c03 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,7 @@ 1995 + /tmp 11 11 11 @@ -44,7 +45,6 @@ 3.4.2.Final 1.0.6 1.10.9 - org.slf4j;version="[1.7,3.0)", org.slf4j.event;version="[1.7,3.0)", org.slf4j.helpers;version="[1.7,3.0)", org.slf4j.spi;version="[1.7,3.0)" @@ -150,6 +150,7 @@ jetty-bom documentation jetty-keystore + jetty-unixdomain-server @@ -684,7 +685,7 @@ alphabetical ${project.build.directory} - ${unix.socket.tmp} + ${jetty.unixdomain.dir} true ${jetty.testtracker.log} @@ -1243,6 +1244,17 @@ + + unix-domain-windows + + + Windows + + + + ${user.home} + + errorprone diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java index 7de00f6112c2..089d539fe79b 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java @@ -29,6 +29,7 @@ import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic; import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.http2.client.HTTP2Client; @@ -38,7 +39,6 @@ import org.eclipse.jetty.unixsocket.server.UnixSocketConnector; import org.eclipse.jetty.util.BlockingArrayQueue; import org.eclipse.jetty.util.IO; -import org.eclipse.jetty.util.StringUtil; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.websocket.api.Session; import org.eclipse.jetty.websocket.api.StatusCode; @@ -48,6 +48,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnJre; import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.EnabledForJreRange; import org.junit.jupiter.api.condition.JRE; import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.params.ParameterizedTest; @@ -61,8 +62,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; public class DistributionTests extends AbstractJettyHomeTest { @@ -277,20 +278,11 @@ private void testSimpleWebAppWithJSPOverHTTP2(boolean ssl) throws Exception @DisabledOnOs(OS.WINDOWS) // jnr not supported on windows public void testUnixSocket() throws Exception { - Path tmpSockFile; - String unixSocketTmp = System.getProperty("unix.socket.tmp"); - if (StringUtil.isNotBlank(unixSocketTmp)) - tmpSockFile = Files.createTempFile(Paths.get(unixSocketTmp), "unix", ".sock"); - else - tmpSockFile = Files.createTempFile("unix", ".sock"); - if (tmpSockFile.toAbsolutePath().toString().length() > UnixSocketConnector.MAX_UNIX_SOCKET_PATH_LENGTH) - { - Path tmp = Paths.get("/tmp"); - assumeTrue(Files.exists(tmp) && Files.isDirectory(tmp)); - tmpSockFile = Files.createTempFile(tmp, "unix", ".sock"); - } - Path sockFile = tmpSockFile; - assertTrue(Files.deleteIfExists(sockFile), "temp sock file cannot be deleted"); + String dir = System.getProperty("jetty.unixdomain.dir"); + assertNotNull(dir); + Path sockFile = Files.createTempFile(Path.of(dir), "unix_", ".sock"); + assertTrue(sockFile.toAbsolutePath().toString().length() < UnixSocketConnector.MAX_UNIX_SOCKET_PATH_LENGTH, "Unix-Domain path too long"); + Files.delete(sockFile); String jettyVersion = System.getProperty("jettyVersion"); JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() @@ -311,7 +303,7 @@ public void testUnixSocket() throws Exception File war = distribution.resolveArtifact("org.eclipse.jetty.demos:demo-jsp-webapp:war:" + jettyVersion); distribution.installWarFile(war, "test"); - try (JettyHomeTester.Run run2 = distribution.start("jetty.unixsocket.path=" + sockFile.toString())) + try (JettyHomeTester.Run run2 = distribution.start("jetty.unixsocket.path=" + sockFile)) { assertTrue(run2.awaitConsoleLogsFor("Started Server@", 10, TimeUnit.SECONDS)); @@ -906,4 +898,37 @@ public void testDefaultLoggingProviderNotActiveWhenExplicitProviderIsPresent() t assertFalse(Files.exists(jettyBase.resolve("resources/jetty-logging.properties"))); } } + + @Test + @EnabledForJreRange(min = JRE.JAVA_16) + public void testUnixDomain() throws Exception + { + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + try (JettyHomeTester.Run run1 = distribution.start("--add-modules=unixdomain-http")) + { + assertTrue(run1.awaitFor(10, TimeUnit.SECONDS)); + assertEquals(0, run1.getExitValue()); + + int maxUnixDomainPathLength = 108; + Path path = Files.createTempFile("unix", ".sock"); + if (path.normalize().toAbsolutePath().toString().length() > maxUnixDomainPathLength) + path = Files.createTempFile(Path.of("/tmp"), "unix", ".sock"); + assertTrue(Files.deleteIfExists(path)); + try (JettyHomeTester.Run run2 = distribution.start("jetty.unixdomain.path=" + path)) + { + assertTrue(run2.awaitConsoleLogsFor("Started Server@", 10, TimeUnit.SECONDS)); + + ClientConnector connector = ClientConnector.forUnixDomain(path); + client = new HttpClient(new HttpClientTransportDynamic(connector)); + client.start(); + ContentResponse response = client.GET("http://localhost/path"); + assertEquals(HttpStatus.NOT_FOUND_404, response.getStatus()); + } + } + } } diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/TransportScenario.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/TransportScenario.java index a0d49548f065..4f5b0f3990a9 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/TransportScenario.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/TransportScenario.java @@ -17,7 +17,6 @@ import java.lang.management.ManagementFactory; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -52,12 +51,10 @@ import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; -import org.eclipse.jetty.toolchain.test.MavenTestingUtils; import org.eclipse.jetty.unixsocket.client.HttpClientTransportOverUnixSockets; import org.eclipse.jetty.unixsocket.server.UnixSocketConnector; import org.eclipse.jetty.util.BlockingArrayQueue; import org.eclipse.jetty.util.SocketAddressResolver; -import org.eclipse.jetty.util.StringUtil; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.junit.jupiter.api.Assumptions; @@ -65,7 +62,8 @@ import org.slf4j.LoggerFactory; import static org.eclipse.jetty.http.client.Transport.UNIX_SOCKET; -import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TransportScenario { @@ -86,20 +84,10 @@ public TransportScenario(final Transport transport) throws IOException { this.transport = transport; - Path unixSocketTmp; - String tmpProp = System.getProperty("unix.socket.tmp"); - if (StringUtil.isBlank(tmpProp)) - unixSocketTmp = MavenTestingUtils.getTargetPath(); - else - unixSocketTmp = Paths.get(tmpProp); - sockFile = Files.createTempFile(unixSocketTmp, "unix", ".sock"); - if (sockFile.toAbsolutePath().toString().length() > UnixSocketConnector.MAX_UNIX_SOCKET_PATH_LENGTH) - { - Files.delete(sockFile); - Path tmp = Paths.get("/tmp"); - assumeTrue(Files.exists(tmp) && Files.isDirectory(tmp)); - sockFile = Files.createTempFile(tmp, "unix", ".sock"); - } + String dir = System.getProperty("jetty.unixdomain.dir"); + assertNotNull(dir); + sockFile = Files.createTempFile(Path.of(dir), "unix_", ".sock"); + assertTrue(sockFile.toAbsolutePath().toString().length() < UnixSocketConnector.MAX_UNIX_SOCKET_PATH_LENGTH, "Unix-Domain path too long"); Files.delete(sockFile); // Disable UNIX_SOCKET due to jnr/jnr-unixsocket#69. From c7f0c5a4ca782b235cfe908618de243d02e0116b Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Fri, 6 Aug 2021 12:07:09 +1000 Subject: [PATCH 20/54] Issue #6373 testing modules, starting with session storage (#6529) * Issue #6373 testing modules, starting with session storage Signed-off-by: Olivier Lamy --- .../etc/sessions/gcloud/session-store.xml | 4 +- .../modules/session-store-gcloud.mod | 2 + .../session/GCloudSessionDataStore.java | 42 +++- .../GCloudSessionDataStoreFactory.java | 24 +++ .../src/main/resources/modules/hawtio.mod | 5 +- .../src/main/resources/modules/jminix.mod | 3 +- .../sessions/infinispan/infinispan-remote.xml | 2 +- .../session-store-infinispan-remote.mod | 1 - pom.xml | 8 +- tests/test-distribution/pom.xml | 19 ++ .../tests/distribution/JettyHomeTester.java | 17 ++ .../distribution/AbstractJettyHomeTest.java | 21 ++ .../tests/distribution/DemoModulesTests.java | 20 -- .../distribution/ThirdPartyModulesTests.java | 181 ++++++++++++++++++ .../AbstractSessionDistributionTests.java | 136 +++++++++++++ .../session/FileSessionDistributionTests.java | 55 ++++++ ...eSessionWithMemcacheDistributionTests.java | 89 +++++++++ .../GCloudSessionDistributionTests.java | 111 +++++++++++ .../HazelcastSessionDistributionTests.java | 124 +++++------- .../InfinispanSessionDistributionTests.java | 116 +++++++++++ .../session/JDBCSessionDistributionTests.java | 107 +++++++++++ .../MongodbSessionDistributionTests.java | 79 ++++++++ tests/test-loginservice/pom.xml | 2 +- .../test-sessions/test-jdbc-sessions/pom.xml | 2 - 24 files changed, 1064 insertions(+), 106 deletions(-) create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/ThirdPartyModulesTests.java create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/AbstractSessionDistributionTests.java create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionDistributionTests.java create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionWithMemcacheDistributionTests.java create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/GCloudSessionDistributionTests.java rename tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/{ => session}/HazelcastSessionDistributionTests.java (65%) create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/InfinispanSessionDistributionTests.java create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/JDBCSessionDistributionTests.java create mode 100644 tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/MongodbSessionDistributionTests.java diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/etc/sessions/gcloud/session-store.xml b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/etc/sessions/gcloud/session-store.xml index 43d7fd0fe03f..070c76aa25ab 100644 --- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/etc/sessions/gcloud/session-store.xml +++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/etc/sessions/gcloud/session-store.xml @@ -14,7 +14,9 @@ - + + + diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/modules/session-store-gcloud.mod b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/modules/session-store-gcloud.mod index 9ee2a31f749d..90856809b16f 100644 --- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/modules/session-store-gcloud.mod +++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/config-template/modules/session-store-gcloud.mod @@ -33,6 +33,8 @@ etc/sessions/gcloud/session-store.xml #jetty.session.gcloud.maxRetries=5 #jetty.session.gcloud.backoffMs=1000 #jetty.session.gcloud.namespace= +#jetty.session.gcloud.host= +#jetty.session.gcloud.projectId= #jetty.session.gcloud.model.kind=GCloudSession #jetty.session.gcloud.model.id=id #jetty.session.gcloud.model.contextPath=contextPath diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java index d120ff593ac1..fd5fb1a87160 100644 --- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java +++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java @@ -67,6 +67,8 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore protected EntityDataModel _model; protected boolean _modelProvided; private String _namespace = DEFAULT_NAMESPACE; + private String _host; + private String _projectId; /** * EntityDataModel @@ -455,15 +457,49 @@ public int getMaxRetries() return _maxRetries; } + public void setHost(String host) + { + _host = host; + } + + @ManagedAttribute(value = "gcloud host", readonly = true) + public String getHost() + { + return _host; + } + + public void setProjectId(String projectId) + { + _projectId = projectId; + } + + @ManagedAttribute(value = "gcloud project Id", readonly = true) + public String getProjectId() + { + return _projectId; + } + @Override protected void doStart() throws Exception { if (!_dsProvided) { - if (!StringUtil.isBlank(getNamespace())) - _datastore = DatastoreOptions.newBuilder().setNamespace(getNamespace()).build().getService(); + boolean blankCustomnamespace = StringUtil.isBlank(getNamespace()); + boolean blankCustomHost = StringUtil.isBlank(getHost()); + boolean blankCustomProjectId = StringUtil.isBlank(getProjectId()); + if (blankCustomnamespace && blankCustomHost && blankCustomProjectId) + _datastore = DatastoreOptions.getDefaultInstance().getService(); else - _datastore = DatastoreOptions.getDefaultInstance().getService(); + { + DatastoreOptions.Builder builder = DatastoreOptions.newBuilder(); + if (!blankCustomnamespace) + builder.setNamespace(getNamespace()); + if (!blankCustomHost) + builder.setHost(getHost()); + if (!blankCustomProjectId) + builder.setProjectId(getProjectId()); + _datastore = builder.build().getService(); + } } if (_model == null) diff --git a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStoreFactory.java b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStoreFactory.java index 716e18f91028..6875a334a4ec 100644 --- a/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStoreFactory.java +++ b/jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStoreFactory.java @@ -26,6 +26,8 @@ public class GCloudSessionDataStoreFactory extends AbstractSessionDataStoreFacto private int _maxRetries = GCloudSessionDataStore.DEFAULT_MAX_RETRIES; private int _backoffMs = GCloudSessionDataStore.DEFAULT_BACKOFF_MS; private GCloudSessionDataStore.EntityDataModel _model; + private String _host; + private String _projectId; public GCloudSessionDataStore.EntityDataModel getEntityDataModel() { @@ -73,6 +75,26 @@ public void setNamespace(String namespace) _namespace = namespace; } + public void setHost(String host) + { + _host = host; + } + + public String getHost() + { + return _host; + } + + public void setProjectId(String projectId) + { + _projectId = projectId; + } + + public String getProjectId() + { + return _projectId; + } + @Override public SessionDataStore getSessionDataStore(SessionHandler handler) throws Exception { @@ -83,6 +105,8 @@ public SessionDataStore getSessionDataStore(SessionHandler handler) throws Excep ds.setNamespace(getNamespace()); ds.setSavePeriodSec(getSavePeriodSec()); ds.setEntityDataModel(getEntityDataModel()); + ds.setHost(getHost()); + ds.setProjectId(getProjectId()); return ds; } } diff --git a/jetty-home/src/main/resources/modules/hawtio.mod b/jetty-home/src/main/resources/modules/hawtio.mod index 2181a65259fe..050c6265449a 100644 --- a/jetty-home/src/main/resources/modules/hawtio.mod +++ b/jetty-home/src/main/resources/modules/hawtio.mod @@ -17,7 +17,7 @@ etc/hawtio.xml [files] etc/hawtio/ lib/hawtio/ -maven://io.hawt/hawtio-default/1.4.16|lib/hawtio/hawtio.war +maven://io.hawt/hawtio-default/${hawtio.version}/war|lib/hawtio/hawtio.war basehome:modules/hawtio/hawtio.xml|etc/hawtio.xml [license] @@ -26,6 +26,9 @@ http://hawt.io/ http://github.com/hawtio/hawtio http://www.apache.org/licenses/LICENSE-2.0.html +[ini] +hawtio.version?=2.13.5 + [ini-template] ## Hawt.io configuration -Dhawtio.authenticationEnabled?=false diff --git a/jetty-home/src/main/resources/modules/jminix.mod b/jetty-home/src/main/resources/modules/jminix.mod index a52f9d5e50d8..c3d6f7f71bbb 100644 --- a/jetty-home/src/main/resources/modules/jminix.mod +++ b/jetty-home/src/main/resources/modules/jminix.mod @@ -9,8 +9,7 @@ Deploys the Jminix JMX Console within the server. [depend] stats jmx -jcl-api -jcl-impl +commons-logging [xml] etc/jminix.xml diff --git a/jetty-infinispan/infinispan-remote/src/main/config-template/etc/sessions/infinispan/infinispan-remote.xml b/jetty-infinispan/infinispan-remote/src/main/config-template/etc/sessions/infinispan/infinispan-remote.xml index 7fa442548a04..ebce2c692761 100644 --- a/jetty-infinispan/infinispan-remote/src/main/config-template/etc/sessions/infinispan/infinispan-remote.xml +++ b/jetty-infinispan/infinispan-remote/src/main/config-template/etc/sessions/infinispan/infinispan-remote.xml @@ -6,7 +6,7 @@ - + diff --git a/jetty-infinispan/infinispan-remote/src/main/config-template/modules/session-store-infinispan-remote.mod b/jetty-infinispan/infinispan-remote/src/main/config-template/modules/session-store-infinispan-remote.mod index e627553096a5..cfa1a0354951 100644 --- a/jetty-infinispan/infinispan-remote/src/main/config-template/modules/session-store-infinispan-remote.mod +++ b/jetty-infinispan/infinispan-remote/src/main/config-template/modules/session-store-infinispan-remote.mod @@ -29,4 +29,3 @@ http://www.apache.org/licenses/LICENSE-2.0.html #jetty.session.infinispan.idleTimeout.seconds=0 #jetty.session.gracePeriod.seconds=3600 #jetty.session.savePeriod.seconds=0 - diff --git a/pom.xml b/pom.xml index 317cfa956c03..30de5fe3b537 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ false 5.5 2.2 + 10.3.6 @@ -79,7 +80,7 @@ false 0 1.15.1 - 2.7.0 + 2.7.0 @@ -1137,6 +1138,11 @@ pom import     + + org.mariadb.jdbc + mariadb-java-client + ${mariadb.version} + net.java.dev.jna jna diff --git a/tests/test-distribution/pom.xml b/tests/test-distribution/pom.xml index 107ba0f26e52..304f6da8356f 100644 --- a/tests/test-distribution/pom.xml +++ b/tests/test-distribution/pom.xml @@ -12,6 +12,7 @@ ${project.groupId}.tests.distribution -1 + 10 @@ -180,6 +181,21 @@ testcontainers test + + org.testcontainers + mariadb + test + + + org.testcontainers + gcloud + test + + + org.mariadb.jdbc + mariadb-java-client + test +     @@ -192,7 +208,10 @@ ${settings.localRepository} ${project.version} ${hazelcast.version} + ${mariadb.docker.version} $(distribution.debug.port} + ${home.start.timeout} + ${mariadb.version} diff --git a/tests/test-distribution/src/main/java/org/eclipse/jetty/tests/distribution/JettyHomeTester.java b/tests/test-distribution/src/main/java/org/eclipse/jetty/tests/distribution/JettyHomeTester.java index ebbb5b43d0e9..16ce3a2fbabd 100644 --- a/tests/test-distribution/src/main/java/org/eclipse/jetty/tests/distribution/JettyHomeTester.java +++ b/tests/test-distribution/src/main/java/org/eclipse/jetty/tests/distribution/JettyHomeTester.java @@ -180,6 +180,7 @@ public JettyHomeTester.Run start(List args) throws Exception ProcessBuilder pbCmd = new ProcessBuilder(commands); pbCmd.directory(jettyBaseDir); + pbCmd.environment().putAll(config.env); Process process = pbCmd.start(); return new Run(config, process); @@ -393,6 +394,7 @@ public static class Config private String jettyVersion; private String mavenLocalRepository = System.getProperty("mavenRepoPath", System.getProperty("user.home") + "/.m2/repository"); private List jvmArgs = new ArrayList<>(); + private Map env = new HashMap<>(); public Path getJettyBase() { @@ -419,6 +421,11 @@ public List getJVMArgs() return Collections.unmodifiableList(jvmArgs); } + public Map getEnv() + { + return Collections.unmodifiableMap(env); + } + @Override public String toString() { @@ -765,6 +772,16 @@ public Builder jvmArgs(List jvmArgs) return this; } + /** + * @param env the env to add + * @return this Builder + */ + public Builder env(Map env) + { + config.env = env; + return this; + } + /** * @return an empty instance of Builder */ diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/AbstractJettyHomeTest.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/AbstractJettyHomeTest.java index 37468835f650..7f33ce80b3b6 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/AbstractJettyHomeTest.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/AbstractJettyHomeTest.java @@ -19,6 +19,7 @@ import java.util.function.Supplier; import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP; import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.toolchain.test.FS; @@ -68,4 +69,24 @@ public void dispose() throws Exception if (client != null) client.stop(); } + + protected class ResponseDetails implements Supplier + { + private final ContentResponse response; + + public ResponseDetails(ContentResponse response) + { + this.response = response; + } + + @Override + public String get() + { + StringBuilder ret = new StringBuilder(); + ret.append(response.toString()).append(System.lineSeparator()); + ret.append(response.getHeaders().toString()).append(System.lineSeparator()); + ret.append(response.getContentAsString()).append(System.lineSeparator()); + return ret.toString(); + } + } } diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DemoModulesTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DemoModulesTests.java index ec5a06826bb2..d1e0cde25015 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DemoModulesTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DemoModulesTests.java @@ -296,24 +296,4 @@ public void testSessionDump() throws Exception } } } - - private class ResponseDetails implements Supplier - { - private final ContentResponse response; - - public ResponseDetails(ContentResponse response) - { - this.response = response; - } - - @Override - public String get() - { - StringBuilder ret = new StringBuilder(); - ret.append(response.toString()).append(System.lineSeparator()); - ret.append(response.getHeaders().toString()).append(System.lineSeparator()); - ret.append(response.getContentAsString()).append(System.lineSeparator()); - return ret.toString(); - } - } } diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/ThirdPartyModulesTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/ThirdPartyModulesTests.java new file mode 100644 index 000000000000..5c92ac5017e1 --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/ThirdPartyModulesTests.java @@ -0,0 +1,181 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution; + +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.http.HttpStatus; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ThirdPartyModulesTests extends AbstractJettyHomeTest +{ + @Test + public void testHawtio() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + + String[] argsConfig = { + "--approve-all-licenses", + "--add-modules=hawtio,http" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(2, TimeUnit.MINUTES)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort + }; + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + httpPort + "/hawtio"); + assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat(response.getContentAsString(), containsString("Hawtio")); + } + } + } + + @Test + public void testJAMon() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + + String[] argsConfig = { + "--approve-all-licenses", + "--add-modules=jamon,http" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(2, TimeUnit.MINUTES)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort + }; + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + httpPort + "/jamon"); + assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat(response.getContentAsString(), containsString("JAMon")); + } + } + } + + @Test + public void testjminix() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + + String[] argsConfig = { + "--approve-all-licenses", + "--add-modules=jminix,http,logging-jcl-capture" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(2, TimeUnit.MINUTES)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort + }; + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + } + } + } + + @Test + public void testjolokia() throws Exception + { + Path jettyBase = newTestJettyBaseDirectory(); + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .jettyBase(jettyBase) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + int httpPort = distribution.freePort(); + + String[] argsConfig = { + "--approve-all-licenses", + "--add-modules=jolokia,http" + }; + + try (JettyHomeTester.Run runConfig = distribution.start(argsConfig)) + { + assertTrue(runConfig.awaitFor(2, TimeUnit.MINUTES)); + assertEquals(0, runConfig.getExitValue()); + + String[] argsStart = { + "jetty.http.port=" + httpPort + }; + + try (JettyHomeTester.Run runStart = distribution.start(argsStart)) + { + assertTrue(runStart.awaitConsoleLogsFor("Started Server@", 20, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + httpPort + "/jolokia"); + assertEquals(HttpStatus.OK_200, response.getStatus(), new ResponseDetails(response)); + assertThat(response.getContentAsString(), containsString("\"agentType\":\"servlet\"")); + } + } + } + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/AbstractSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/AbstractSessionDistributionTests.java new file mode 100644 index 000000000000..c35171cd5fab --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/AbstractSessionDistributionTests.java @@ -0,0 +1,136 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.io.BufferedWriter; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.tests.distribution.AbstractJettyHomeTest; +import org.eclipse.jetty.tests.distribution.JettyHomeTester; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public abstract class AbstractSessionDistributionTests extends AbstractJettyHomeTest +{ + + private String jettyVersion = System.getProperty("jettyVersion"); + + protected JettyHomeTester jettyHomeTester; + + private static final int START_TIMEOUT = Integer.getInteger("home.start.timeout", 10); + + @BeforeEach + public void prepareJettyHomeTester() throws Exception + { + + jettyHomeTester = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .env(env()) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + } + + @Test + public void stopRestartWebappTestSessionContentSaved() throws Exception + { + startExternalSessionStorage(); + + List<String> args = new ArrayList<>(Arrays.asList( + "--create-startd", + "--approve-all-licenses", + "--add-module=resources,server,http,webapp,deploy,jmx,servlet,servlets," + getFirstStartExtraModules() + )); + args.addAll(getFirstStartExtraArgs()); + String[] argsStart = args.toArray(new String[0]); + + try (JettyHomeTester.Run run1 = jettyHomeTester.start(argsStart)) + { + assertTrue(run1.awaitFor(5, TimeUnit.SECONDS)); + assertEquals(0, run1.getExitValue()); + + File war = jettyHomeTester.resolveArtifact("org.eclipse.jetty.tests:test-simple-session-webapp:war:" + jettyVersion); + jettyHomeTester.installWarFile(war, "test"); + + int port = jettyHomeTester.freePort(); + args = new ArrayList<>(Collections.singletonList("jetty.http.port=" + port)); + args.addAll(getSecondStartExtraArgs()); + argsStart = args.toArray(new String[0]); + + try (JettyHomeTester.Run run2 = jettyHomeTester.start(argsStart)) + { + assertTrue(run2.awaitConsoleLogsFor("Started Server@", START_TIMEOUT, TimeUnit.SECONDS)); + + startHttpClient(); + ContentResponse response = client.GET("http://localhost:" + port + "/test/session?action=CREATE"); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getContentAsString(), containsString("SESSION CREATED")); + + response = client.GET("http://localhost:" + port + "/test/session?action=READ"); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getContentAsString(), containsString("SESSION READ CHOCOLATE THE BEST:FRENCH")); + } + + Path logFile = jettyHomeTester.getJettyBase().resolve("resources").resolve("jetty-logging.properties"); + Files.deleteIfExists(logFile); + try (BufferedWriter writer = Files.newBufferedWriter(logFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE)) + { + writer.write("org.eclipse.jetty.server.session.LEVEL=DEBUG"); + } + + try (JettyHomeTester.Run run2 = jettyHomeTester.start(argsStart)) + { + assertTrue(run2.awaitConsoleLogsFor("Started Server@", START_TIMEOUT, TimeUnit.SECONDS)); + + ContentResponse response = client.GET("http://localhost:" + port + "/test/session?action=READ"); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getContentAsString(), containsString("SESSION READ CHOCOLATE THE BEST:FRENCH")); + } + } + + stopExternalSessionStorage(); + } + + public Map<String, String> env() + { + return Collections.emptyMap(); + } + + public abstract List<String> getFirstStartExtraArgs(); + + public abstract String getFirstStartExtraModules(); + + public abstract List<String> getSecondStartExtraArgs(); + + public abstract void startExternalSessionStorage() throws Exception; + + public abstract void stopExternalSessionStorage() throws Exception; + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionDistributionTests.java new file mode 100644 index 000000000000..025bee15b696 --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionDistributionTests.java @@ -0,0 +1,55 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.util.Collections; +import java.util.List; + +/** + * + */ +public class FileSessionDistributionTests extends AbstractSessionDistributionTests +{ + + @Override + public void startExternalSessionStorage() throws Exception + { + // no op + } + + @Override + public void stopExternalSessionStorage() throws Exception + { + // no op + } + + @Override + public List<String> getFirstStartExtraArgs() + { + return Collections.emptyList(); + } + + @Override + public String getFirstStartExtraModules() + { + return "session-store-file"; + } + + @Override + public List<String> getSecondStartExtraArgs() + { + return Collections.emptyList(); + } + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionWithMemcacheDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionWithMemcacheDistributionTests.java new file mode 100644 index 000000000000..70973d964158 --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/FileSessionWithMemcacheDistributionTests.java @@ -0,0 +1,89 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +/** + * + */ +public class FileSessionWithMemcacheDistributionTests extends AbstractSessionDistributionTests +{ + + private static final Logger LOGGER = LoggerFactory.getLogger(FileSessionWithMemcacheDistributionTests.class); + private static final Logger MEMCACHED_LOG = LoggerFactory.getLogger("org.eclipse.jetty.tests.distribution.session.memcached"); + + private GenericContainer memcached; + + private String host; + private int port; + + @Override + @BeforeEach + public void prepareJettyHomeTester() throws Exception + { + memcached = + new GenericContainer("memcached:" + System.getProperty("memcached.docker.version", "1.6.6")) + .withLogConsumer(new Slf4jLogConsumer(MEMCACHED_LOG)); + memcached.start(); + this.host = memcached.getContainerIpAddress(); + this.port = memcached.getMappedPort(11211); + super.prepareJettyHomeTester(); + } + + @Override + public void startExternalSessionStorage() throws Exception + { + // no op + } + + @Override + public Map<String, String> env() + { + return Map.of("MEMCACHE_PORT_11211_TCP_ADDR", host, "MEMCACHE_PORT_11211_TCP_PORT", Integer.toString(port)); + } + + @Override + public void stopExternalSessionStorage() throws Exception + { + memcached.stop(); + } + + @Override + public List<String> getFirstStartExtraArgs() + { + return Collections.singletonList("session-data-cache=xmemcached"); + } + + @Override + public String getFirstStartExtraModules() + { + return "session-store-file,session-store-cache"; + } + + @Override + public List<String> getSecondStartExtraArgs() + { + return Collections.singletonList("session-data-cache=xmemcached"); + } + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/GCloudSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/GCloudSessionDistributionTests.java new file mode 100644 index 000000000000..a7c4b1117a7f --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/GCloudSessionDistributionTests.java @@ -0,0 +1,111 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.net.InetAddress; +import java.net.URL; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.DatastoreEmulatorContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * + */ +public class GCloudSessionDistributionTests extends AbstractSessionDistributionTests +{ + + private static final Logger LOGGER = LoggerFactory.getLogger(GCloudSessionDistributionTests.class); + private static final Logger GCLOUD_LOG = LoggerFactory.getLogger("org.eclipse.jetty.tests.distribution.session.gcloudLogs"); + + public DatastoreEmulatorContainer emulator = + new CustomDatastoreEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:316.0.0-emulators")) + .withLogConsumer(new Slf4jLogConsumer(GCLOUD_LOG)); + + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk"); + + private static final String CMD = "gcloud beta emulators datastore start --project test-project --host-port 0.0.0.0:8081 --consistency=1.0"; + private static final int HTTP_PORT = 8081; + + String host; + + public static class CustomDatastoreEmulatorContainer extends DatastoreEmulatorContainer + { + public CustomDatastoreEmulatorContainer(DockerImageName dockerImageName) + { + super(dockerImageName); + + dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); + + withExposedPorts(HTTP_PORT); + setWaitStrategy(Wait.forHttp("/").forStatusCode(200)); + withCommand("/bin/sh", "-c", CMD); + } + } + + @Override + public void startExternalSessionStorage() throws Exception + { + emulator.start(); + + //work out if we're running locally or not: if not local, then the host passed to + //DatastoreOptions must be prefixed with a scheme + String endPoint = emulator.getEmulatorEndpoint(); + InetAddress hostAddr = InetAddress.getByName(new URL("http://" + endPoint).getHost()); + LOGGER.info("endPoint: {} ,hostAddr.isAnyLocalAddress(): {},hostAddr.isLoopbackAddress(): {}", + endPoint, + hostAddr.isAnyLocalAddress(), + hostAddr.isLoopbackAddress()); + if (hostAddr.isAnyLocalAddress() || hostAddr.isLoopbackAddress()) + host = endPoint; + else + host = "http://" + endPoint; + } + + @Override + public void stopExternalSessionStorage() throws Exception + { + emulator.stop(); + } + + @Override + public List<String> getFirstStartExtraArgs() + { + return Arrays.asList( + "jetty.session.gcloud.host=" + host, + "jetty.session.gcloud.projectId=foobar" + ); + } + + @Override + public String getFirstStartExtraModules() + { + return "session-store-gcloud"; + } + + @Override + public List<String> getSecondStartExtraArgs() + { + return Arrays.asList( + "jetty.session.gcloud.host=" + host, + "jetty.session.gcloud.projectId=foobar" + ); + } + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/HazelcastSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/HazelcastSessionDistributionTests.java similarity index 65% rename from tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/HazelcastSessionDistributionTests.java rename to tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/HazelcastSessionDistributionTests.java index 6fe937dd4743..338391e766f5 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/HazelcastSessionDistributionTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/HazelcastSessionDistributionTests.java @@ -11,7 +11,7 @@ // ======================================================================== // -package org.eclipse.jetty.tests.distribution; +package org.eclipse.jetty.tests.distribution.session; import java.io.File; import java.io.OutputStream; @@ -21,6 +21,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,8 +29,8 @@ import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.tests.distribution.JettyHomeTester; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.BindMode; @@ -42,90 +43,67 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class HazelcastSessionDistributionTests extends AbstractJettyHomeTest +/** + * This simulate the onlyClient option which means the JVM running Jetty is only an Hazelcast client and not part + * of the cluster + */ +public class HazelcastSessionDistributionTests extends AbstractSessionDistributionTests { - private static final Logger HAZELCAST_LOG = LoggerFactory.getLogger("org.eclipse.jetty.tests.distribution.HazelcastLogs"); + private static final Logger HAZELCAST_LOG = LoggerFactory.getLogger("org.eclipse.jetty.tests.distribution.session.HazelcastLogs"); private static final Logger LOGGER = LoggerFactory.getLogger(HazelcastSessionDistributionTests.class); + private GenericContainer hazelcast = new GenericContainer("hazelcast/hazelcast:" + System.getProperty("hazelcast.version", "4.1")) + .withExposedPorts(5701) + .waitingFor(Wait.forListeningPort()) + .withLogConsumer(new Slf4jLogConsumer(HAZELCAST_LOG)); - /** - * This simulate the onlyClient option which means the JVM running Jetty is only an Hazelcast client and not part - * of the cluster - */ - @Test - public void testHazelcastRemoteOnlyClient() throws Exception - { - try (GenericContainer hazelcast = - new GenericContainer("hazelcast/hazelcast:" + System.getProperty("hazelcast.version", "4.1")) - .withExposedPorts(5701) - .waitingFor(Wait.forListeningPort()) - .withLogConsumer(new Slf4jLogConsumer(HAZELCAST_LOG))) - { - hazelcast.start(); - String hazelcastHost = hazelcast.getContainerIpAddress(); - int hazelcastPort = hazelcast.getMappedPort(5701); + private Path hazelcastJettyPath; - LOGGER.info("hazelcast started on {}:{}", hazelcastHost, hazelcastPort); - - Map<String, String> tokenValues = new HashMap<>(); - tokenValues.put("hazelcast_ip", hazelcastHost); - tokenValues.put("hazelcast_port", Integer.toString(hazelcastPort)); - Path hazelcastJettyPath = Paths.get("target/hazelcast-client.xml"); - transformFileWithHostAndPort(Paths.get("src/test/resources/hazelcast-client.xml"), - hazelcastJettyPath, - tokenValues); - - String jettyVersion = System.getProperty("jettyVersion"); - JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() - .jettyVersion(jettyVersion) - .mavenLocalRepository(System.getProperty("mavenRepoPath")) - .build(); - - String[] args1 = { - "--create-startd", - "--approve-all-licenses", - "--add-to-start=resources,server,http,webapp,deploy,jmx,servlet,servlets,session-store-hazelcast-remote" - }; - try (JettyHomeTester.Run run1 = distribution.start(args1)) - { - assertTrue(run1.awaitFor(5, TimeUnit.SECONDS)); - assertEquals(0, run1.getExitValue()); + @Override + public void startExternalSessionStorage() throws Exception + { + hazelcast.start(); - File war = distribution.resolveArtifact("org.eclipse.jetty.tests:test-simple-session-webapp:war:" + jettyVersion); - distribution.installWarFile(war, "test"); + String hazelcastHost = hazelcast.getContainerIpAddress(); + int hazelcastPort = hazelcast.getMappedPort(5701); - int port = distribution.freePort(); - String[] argsStart = { - "jetty.http.port=" + port, - "jetty.session.hazelcast.configurationLocation=" + hazelcastJettyPath.toAbsolutePath(), - "jetty.session.hazelcast.onlyClient=true" - }; - try (JettyHomeTester.Run run2 = distribution.start(argsStart)) - { - assertTrue(run2.awaitConsoleLogsFor("Started Server@", 10, TimeUnit.SECONDS)); + LOGGER.info("hazelcast started on {}:{}", hazelcastHost, hazelcastPort); - startHttpClient(); - ContentResponse response = client.GET("http://localhost:" + port + "/test/session?action=CREATE"); - assertEquals(HttpStatus.OK_200, response.getStatus()); - assertThat(response.getContentAsString(), containsString("SESSION CREATED")); + Map<String, String> tokenValues = new HashMap<>(); + tokenValues.put("hazelcast_ip", hazelcastHost); + tokenValues.put("hazelcast_port", Integer.toString(hazelcastPort)); + this.hazelcastJettyPath = Paths.get("target/hazelcast-client.xml"); + transformFileWithHostAndPort(Paths.get("src/test/resources/hazelcast-client.xml"), + hazelcastJettyPath, + tokenValues); + } - response = client.GET("http://localhost:" + port + "/test/session?action=READ"); - assertEquals(HttpStatus.OK_200, response.getStatus()); - assertThat(response.getContentAsString(), containsString("SESSION READ CHOCOLATE THE BEST:FRENCH")); - } + @Override + public void stopExternalSessionStorage() throws Exception + { + hazelcast.stop(); + } - try (JettyHomeTester.Run run2 = distribution.start(argsStart)) - { - assertTrue(run2.awaitConsoleLogsFor("Started Server@", 10, TimeUnit.SECONDS)); + @Override + public List<String> getFirstStartExtraArgs() + { + return Collections.emptyList(); + } - ContentResponse response = client.GET("http://localhost:" + port + "/test/session?action=READ"); - assertEquals(HttpStatus.OK_200, response.getStatus()); - assertThat(response.getContentAsString(), containsString("SESSION READ CHOCOLATE THE BEST:FRENCH")); - } - } + @Override + public String getFirstStartExtraModules() + { + return "session-store-hazelcast-remote"; + } - } + @Override + public List<String> getSecondStartExtraArgs() + { + return Arrays.asList( + "jetty.session.hazelcast.configurationLocation=" + hazelcastJettyPath.toAbsolutePath(), + "jetty.session.hazelcast.onlyClient=true" + ); } @Disabled("not working see https://github.com/hazelcast/hazelcast/issues/18508") diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/InfinispanSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/InfinispanSessionDistributionTests.java new file mode 100644 index 000000000000..917c181b972a --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/InfinispanSessionDistributionTests.java @@ -0,0 +1,116 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.eclipse.jetty.util.IO; +import org.infinispan.client.hotrod.RemoteCacheManager; +import org.infinispan.client.hotrod.configuration.Configuration; +import org.infinispan.client.hotrod.configuration.ConfigurationBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; + +/** + * + */ +public class InfinispanSessionDistributionTests extends AbstractSessionDistributionTests +{ + + private static final Logger LOGGER = LoggerFactory.getLogger(InfinispanSessionDistributionTests.class); + private static final Logger INFINISPAN_LOG = LoggerFactory.getLogger("org.eclipse.jetty.tests.distribution.session.infinispan"); + + private GenericContainer infinispan; + + private String host; + + @Override + public void startExternalSessionStorage() throws Exception + { + String infinispanVersion = System.getProperty("infinispan.docker.image.version", "9.4.8.Final"); + infinispan = + new GenericContainer(System.getProperty("infinispan.docker.image.name", "jboss/infinispan-server") + + ":" + infinispanVersion) + //.withEnv("APP_USER", "theuser") + //.withEnv("APP_PASS", "foobar") + .withEnv("MGMT_USER", "admin") + .withEnv("MGMT_PASS", "admin") + .withCommand("standalone") + .waitingFor(new LogMessageWaitStrategy() + .withRegEx(".*Infinispan Server.*started in.*\\s")) + .withExposedPorts(4712, 4713, 8088, 8089, 8443, 9990, 9993, 11211, 11222, 11223, 11224) + .withLogConsumer(new Slf4jLogConsumer(INFINISPAN_LOG)); + infinispan.start(); + String host = infinispan.getContainerIpAddress(); + int port = infinispan.getMappedPort(11222); + + Path resourcesDirectory = Path.of(jettyHomeTester.getJettyBase().toString(), "resources/"); + if (Files.exists(resourcesDirectory)) + { + IO.delete(resourcesDirectory.toFile()); + } + Files.createDirectories(resourcesDirectory); + Properties properties = new Properties(); + properties.put("infinispan.client.hotrod.server_list", host + ":" + port); + //properties.put("jetty.session.infinispan.clientIntelligence", "BASIC"); + + Path hotrod = Path.of(resourcesDirectory.toString(), "hotrod-client.properties"); + Files.deleteIfExists(hotrod); + Files.createFile(hotrod); + try (Writer writer = Files.newBufferedWriter(hotrod)) + { + properties.store(writer, null); + } + + Configuration configuration = new ConfigurationBuilder().withProperties(properties) + .addServer().host(host).port(port).build(); + + RemoteCacheManager remoteCacheManager = new RemoteCacheManager(configuration); + remoteCacheManager.administration().getOrCreateCache("sessions", (String)null); + + } + + @Override + public void stopExternalSessionStorage() throws Exception + { + infinispan.stop(); + } + + @Override + public List<String> getFirstStartExtraArgs() + { + return Arrays.asList(); + } + + @Override + public String getFirstStartExtraModules() + { + return "session-store-infinispan-remote"; + } + + @Override + public List<String> getSecondStartExtraArgs() + { + return Arrays.asList(); + } + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/JDBCSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/JDBCSessionDistributionTests.java new file mode 100644 index 000000000000..580a34317c39 --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/JDBCSessionDistributionTests.java @@ -0,0 +1,107 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +/** + * + */ +public class JDBCSessionDistributionTests extends AbstractSessionDistributionTests +{ + + private static final Logger LOGGER = LoggerFactory.getLogger(JDBCSessionDistributionTests.class); + + private static final String MARIA_DB_USER = "beer"; + private static final String MARIA_DB_PASSWORD = "pacific_ale"; + private String jdbcUrl; + private String driverClassName; + + private MariaDBContainer mariaDBContainer = new MariaDBContainer("mariadb:" + System.getProperty("mariadb.docker.version", "10.3.6")) + .withUsername(MARIA_DB_USER) + .withPassword(MARIA_DB_PASSWORD) + .withDatabaseName("sessions"); + + @Override + public void startExternalSessionStorage() throws Exception + { + mariaDBContainer.start(); + jdbcUrl = mariaDBContainer.getJdbcUrl() + "?user=" + MARIA_DB_USER + + "&password=" + MARIA_DB_PASSWORD; + driverClassName = mariaDBContainer.getDriverClassName(); + + // prepare mariadb driver mod file + String mariaDBVersion = System.getProperty("mariadb.version"); + StringBuilder modFileContent = new StringBuilder(); + modFileContent.append("[lib]").append(System.lineSeparator()); + modFileContent.append("lib/mariadb-java-client-" + mariaDBVersion + ".jar").append(System.lineSeparator()); + modFileContent.append("[files]").append(System.lineSeparator()); + modFileContent.append("maven://org.mariadb.jdbc/mariadb-java-client/" + mariaDBVersion + + "|lib/mariadb-java-client-" + mariaDBVersion + ".jar") + .append(System.lineSeparator()); + + Path modulesDirectory = Path.of(jettyHomeTester.getJettyBase().toString(), "modules"); + if (Files.notExists(modulesDirectory)) + { + Files.createDirectories(modulesDirectory); + } + Path mariaDbModPath = Path.of(modulesDirectory.toString(), "mariadb-driver.mod"); + Files.deleteIfExists(mariaDbModPath); + Files.createFile(mariaDbModPath); + LOGGER.info("create file modfile: {} with content {} ", mariaDbModPath, modFileContent); + Files.writeString(mariaDbModPath, modFileContent); + } + + @Override + public void stopExternalSessionStorage() throws Exception + { + mariaDBContainer.stop(); + } + + @Override + public List<String> getFirstStartExtraArgs() + { + return Arrays.asList( + "jetty.session.jdbc.driverUrl=" + jdbcUrl, + "db-connection-type=driver", + "jetty.session.jdbc.driverClass=" + driverClassName + ); + } + + @Override + public String getFirstStartExtraModules() + { + return "session-store-jdbc,mariadb-driver"; + } + + @Override + public List<String> getSecondStartExtraArgs() + { + return Arrays.asList( + "jetty.session.jdbc.driverUrl=" + jdbcUrl, + "db-connection-type=driver", + "jetty.session.jdbc.driverClass=" + driverClassName + ); + } + +} diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/MongodbSessionDistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/MongodbSessionDistributionTests.java new file mode 100644 index 000000000000..80558f61d3cb --- /dev/null +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/session/MongodbSessionDistributionTests.java @@ -0,0 +1,79 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.tests.distribution.session; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; + +/** + * + */ +public class MongodbSessionDistributionTests extends AbstractSessionDistributionTests +{ + private static final Logger LOGGER = LoggerFactory.getLogger(MongodbSessionDistributionTests.class); + + private static final Logger MONGO_LOG = LoggerFactory.getLogger("org.eclipse.jetty.tests.distribution.session.mongo"); + + final String imageName = "mongo:" + System.getProperty("mongo.docker.version", "2.2.7"); + final GenericContainer mongoDBContainer = + new GenericContainer(imageName) + .withLogConsumer(new Slf4jLogConsumer(MONGO_LOG)) + .waitingFor(new LogMessageWaitStrategy() + .withRegEx(".*waiting for connections.*")); + private String host; + private int port; + + @Override + public void startExternalSessionStorage() throws Exception + { + mongoDBContainer.start(); + host = mongoDBContainer.getHost(); + port = mongoDBContainer.getMappedPort(27017); + } + + @Override + public void stopExternalSessionStorage() throws Exception + { + mongoDBContainer.stop(); + } + + @Override + public List<String> getFirstStartExtraArgs() + { + return Collections.emptyList(); + } + + @Override + public String getFirstStartExtraModules() + { + return "session-store-mongo"; + } + + @Override + public List<String> getSecondStartExtraArgs() + { + return Arrays.asList( + "jetty.session.mongo.host=" + host, + "jetty.session.mongo.port=" + port + ); + } + +} diff --git a/tests/test-loginservice/pom.xml b/tests/test-loginservice/pom.xml index 959214680b54..a53fd881972f 100644 --- a/tests/test-loginservice/pom.xml +++ b/tests/test-loginservice/pom.xml @@ -65,7 +65,7 @@ <dependency> <groupId>org.mariadb.jdbc</groupId> <artifactId>mariadb-java-client</artifactId> - <version>${maria.version}</version> + <version>${mariadb.version}</version> <scope>test</scope> </dependency> <dependency> diff --git a/tests/test-sessions/test-jdbc-sessions/pom.xml b/tests/test-sessions/test-jdbc-sessions/pom.xml index a38fab2cf2df..cb114b0ef0be 100644 --- a/tests/test-sessions/test-jdbc-sessions/pom.xml +++ b/tests/test-sessions/test-jdbc-sessions/pom.xml @@ -10,7 +10,6 @@ <name>Jetty Tests :: Sessions :: JDBC</name> <properties> <bundle-symbolic-name>${project.groupId}.sessions.jdbc</bundle-symbolic-name> - <mariadb.docker.version>10.3.6</mariadb.docker.version> </properties> <build> <plugins> @@ -91,7 +90,6 @@ <dependency> <groupId>org.mariadb.jdbc</groupId> <artifactId>mariadb-java-client</artifactId> - <version>${maria.version}</version> <scope>test</scope> </dependency> </dependencies> From cc8ece5d566ea3bf0cf9316a4082198f2abc9c7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Aug 2021 08:02:32 +0000 Subject: [PATCH 21/54] Bump ant from 1.10.9 to 1.10.11 Bumps ant from 1.10.9 to 1.10.11. --- updated-dependencies: - dependency-name: org.apache.ant:ant dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 30de5fe3b537..3048dc8de693 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ <weld.version>3.1.8.Final</weld.version> <jboss.logging.version>3.4.2.Final</jboss.logging.version> <jetty.perf-helper.version>1.0.6</jetty.perf-helper.version> - <ant.version>1.10.9</ant.version> + <ant.version>1.10.11</ant.version> <!-- OSGI import-package --> <osgi.slf4j.import.packages>org.slf4j;version="[1.7,3.0)", org.slf4j.event;version="[1.7,3.0)", org.slf4j.helpers;version="[1.7,3.0)", org.slf4j.spi;version="[1.7,3.0)"</osgi.slf4j.import.packages> From 10ef07b9e7fa35b71200b02cb07eb6fd767e91e7 Mon Sep 17 00:00:00 2001 From: Simone Bordet <simone.bordet@gmail.com> Date: Fri, 6 Aug 2021 13:01:57 +0200 Subject: [PATCH 22/54] Fixes #6369 - Increment default jetty.http2.rateControl.maxEventsPerSecond Incremented rateControl.maxEventsPerSecond to 50. Fixed mistakes in the http2c module. Signed-off-by: Simone Bordet <simone.bordet@gmail.com> (cherry picked from commit 784293aa6d3e8c0bee77264258ac64ac48a4836f) --- .../http2-server/src/main/config/etc/jetty-http2.xml | 2 +- .../http2-server/src/main/config/etc/jetty-http2c.xml | 6 +++--- .../http2-server/src/main/config/modules/http2.mod | 2 +- .../http2-server/src/main/config/modules/http2c.mod | 8 ++++---- .../server/AbstractHTTP2ServerConnectionFactory.java | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/jetty-http2/http2-server/src/main/config/etc/jetty-http2.xml b/jetty-http2/http2-server/src/main/config/etc/jetty-http2.xml index 3110c911d9d9..78904fd4bba3 100644 --- a/jetty-http2/http2-server/src/main/config/etc/jetty-http2.xml +++ b/jetty-http2/http2-server/src/main/config/etc/jetty-http2.xml @@ -12,7 +12,7 @@ <Set name="maxSettingsKeys"><Property name="jetty.http2.maxSettingsKeys" default="64"/></Set> <Set name="rateControlFactory"> <New class="org.eclipse.jetty.http2.parser.WindowRateControl$Factory"> - <Arg type="int"><Property name="jetty.http2.rateControl.maxEventsPerSecond" default="20"/></Arg> + <Arg type="int"><Property name="jetty.http2.rateControl.maxEventsPerSecond" default="50"/></Arg> </New> </Set> </New> diff --git a/jetty-http2/http2-server/src/main/config/etc/jetty-http2c.xml b/jetty-http2/http2-server/src/main/config/etc/jetty-http2c.xml index 86b10ada984f..a13cf3970f68 100644 --- a/jetty-http2/http2-server/src/main/config/etc/jetty-http2c.xml +++ b/jetty-http2/http2-server/src/main/config/etc/jetty-http2c.xml @@ -8,11 +8,11 @@ <Arg name="config"><Ref refid="httpConfig"/></Arg> <Set name="maxConcurrentStreams" property="jetty.http2c.maxConcurrentStreams"/> <Set name="initialStreamRecvWindow" property="jetty.http2c.initialStreamRecvWindow"/> - <Set name="initialSessionRecvWindow" property="jetty.http2.initialSessionRecvWindow"/> - <Set name="maxSettingsKeys" property="jetty.http2.maxSettingsKeys"/> + <Set name="initialSessionRecvWindow" property="jetty.http2c.initialSessionRecvWindow"/> + <Set name="maxSettingsKeys" property="jetty.http2c.maxSettingsKeys"/> <Set name="rateControlFactory"> <New class="org.eclipse.jetty.http2.parser.WindowRateControl$Factory"> - <Arg type="int"><Property name="jetty.http2.rateControl.maxEventsPerSecond" default="20"/></Arg> + <Arg type="int"><Property name="jetty.http2c.rateControl.maxEventsPerSecond" default="50"/></Arg> </New> </Set> </New> diff --git a/jetty-http2/http2-server/src/main/config/modules/http2.mod b/jetty-http2/http2-server/src/main/config/modules/http2.mod index f1742041b519..2447ef364ec9 100644 --- a/jetty-http2/http2-server/src/main/config/modules/http2.mod +++ b/jetty-http2/http2-server/src/main/config/modules/http2.mod @@ -33,5 +33,5 @@ etc/jetty-http2.xml ## Specifies the maximum number of bad frames and pings per second, ## after which a session is closed to avoid denial of service attacks. -# jetty.http2.rateControl.maxEventsPerSecond=20 +# jetty.http2.rateControl.maxEventsPerSecond=50 # end::documentation[] diff --git a/jetty-http2/http2-server/src/main/config/modules/http2c.mod b/jetty-http2/http2-server/src/main/config/modules/http2c.mod index f1a6fc4f55af..eb40a13ed66c 100644 --- a/jetty-http2/http2-server/src/main/config/modules/http2c.mod +++ b/jetty-http2/http2-server/src/main/config/modules/http2c.mod @@ -20,16 +20,16 @@ etc/jetty-http2c.xml ## Specifies the maximum number of concurrent requests per session. # jetty.http2c.maxConcurrentStreams=128 - ## Specifies the initial stream receive window (client to server) in bytes. +## Specifies the initial stream receive window (client to server) in bytes. # jetty.http2c.initialStreamRecvWindow=524288 ## Specifies the initial session receive window (client to server) in bytes. -# jetty.http2.initialSessionRecvWindow=1232896 +# jetty.http2c.initialSessionRecvWindow=1232896 ## Specifies the maximum number of keys in all SETTINGS frames received by a session. -# jetty.http2.maxSettingsKeys=64 +# jetty.http2c.maxSettingsKeys=64 ## Specifies the maximum number of bad frames and pings per second, ## after which a session is closed to avoid denial of service attacks. -# jetty.http2.rateControl.maxEventsPerSecond=20 +# jetty.http2c.rateControl.maxEventsPerSecond=50 # end::documentation[] diff --git a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java index ff386f198fa5..1436e1188317 100644 --- a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java +++ b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java @@ -61,7 +61,7 @@ public abstract class AbstractHTTP2ServerConnectionFactory extends AbstractConne private int maxFrameLength = Frame.DEFAULT_MAX_LENGTH; private int maxSettingsKeys = SettingsFrame.DEFAULT_MAX_KEYS; private boolean connectProtocolEnabled = true; - private RateControl.Factory rateControlFactory = new WindowRateControl.Factory(20); + private RateControl.Factory rateControlFactory = new WindowRateControl.Factory(50); private FlowControlStrategy.Factory flowControlStrategyFactory = () -> new BufferingFlowControlStrategy(0.5F); private long streamIdleTimeout; private boolean useInputDirectByteBuffers; From 646707b85ca265cf70a9642fa89b57b4eef849b5 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts <lachlan@webtide.com> Date: Thu, 12 Aug 2021 15:50:25 +1000 Subject: [PATCH 23/54] Issue #6602 - do not invoke SessionListener onOpen if session has been closed in OnOpen Signed-off-by: Lachlan Roberts <lachlan@webtide.com> --- .../jetty/websocket/core/CoreSession.java | 2 +- .../common/JavaxWebSocketFrameHandler.java | 7 +- .../javax/tests/CloseInOnOpenTest.java | 97 +++++++++++++++++++ .../common/JettyWebSocketFrameHandler.java | 5 +- .../websocket/tests/CloseInOnOpenTest.java | 95 ++++++++++++++++++ 5 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/CloseInOnOpenTest.java create mode 100644 jetty-websocket/websocket-jetty-tests/src/test/java/org/eclipse/jetty/websocket/tests/CloseInOnOpenTest.java diff --git a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/CoreSession.java b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/CoreSession.java index 7c6bd9bb4ec0..bbaf05c4e5a7 100644 --- a/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/CoreSession.java +++ b/jetty-websocket/websocket-core-common/src/main/java/org/eclipse/jetty/websocket/core/CoreSession.java @@ -256,7 +256,7 @@ public SocketAddress getRemoteAddress() @Override public boolean isOutputOpen() { - return false; + return true; } @Override diff --git a/jetty-websocket/websocket-javax-common/src/main/java/org/eclipse/jetty/websocket/javax/common/JavaxWebSocketFrameHandler.java b/jetty-websocket/websocket-javax-common/src/main/java/org/eclipse/jetty/websocket/javax/common/JavaxWebSocketFrameHandler.java index 99cfd8bb2e6f..030312178243 100644 --- a/jetty-websocket/websocket-javax-common/src/main/java/org/eclipse/jetty/websocket/javax/common/JavaxWebSocketFrameHandler.java +++ b/jetty-websocket/websocket-javax-common/src/main/java/org/eclipse/jetty/websocket/javax/common/JavaxWebSocketFrameHandler.java @@ -133,6 +133,9 @@ public void onOpen(CoreSession coreSession, Callback callback) // Rewire EndpointConfig to call CoreSession setters if Jetty specific properties are set. endpointConfig = getWrappedEndpointConfig(); session = new JavaxWebSocketSession(container, coreSession, this, endpointConfig); + if (!session.isOpen()) + throw new IllegalStateException("Session is not open"); + openHandle = InvokerUtils.bindTo(openHandle, session, endpointConfig); closeHandle = InvokerUtils.bindTo(closeHandle, session); errorHandle = InvokerUtils.bindTo(errorHandle, session); @@ -171,7 +174,9 @@ public void onOpen(CoreSession coreSession, Callback callback) if (openHandle != null) openHandle.invoke(); - container.notifySessionListeners((listener) -> listener.onJavaxWebSocketSessionOpened(session)); + if (session.isOpen()) + container.notifySessionListeners((listener) -> listener.onJavaxWebSocketSessionOpened(session)); + callback.succeeded(); } catch (Throwable cause) diff --git a/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/CloseInOnOpenTest.java b/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/CloseInOnOpenTest.java new file mode 100644 index 000000000000..859a8568e5e6 --- /dev/null +++ b/jetty-websocket/websocket-javax-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/CloseInOnOpenTest.java @@ -0,0 +1,97 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.websocket.javax.tests; + +import java.net.URI; +import java.util.concurrent.TimeUnit; +import javax.websocket.CloseReason; +import javax.websocket.OnOpen; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.websocket.javax.client.internal.JavaxWebSocketClientContainer; +import org.eclipse.jetty.websocket.javax.server.config.JavaxWebSocketServletContainerInitializer; +import org.eclipse.jetty.websocket.javax.server.internal.JavaxWebSocketServerContainer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CloseInOnOpenTest +{ + private Server server; + private ServerConnector connector; + private JavaxWebSocketServerContainer serverContainer; + private JavaxWebSocketClientContainer client; + + @BeforeEach + public void beforeEach() throws Exception + { + server = new Server(); + + connector = new ServerConnector(server); + server.addConnector(connector); + + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + server.setHandler(context); + + JavaxWebSocketServletContainerInitializer.configure(context, (servletContext, wsContainer) -> + wsContainer.addEndpoint(ClosingListener.class)); + server.start(); + + serverContainer = JavaxWebSocketServerContainer.getContainer(context.getServletContext()); + assertNotNull(serverContainer); + + client = new JavaxWebSocketClientContainer(); + client.start(); + } + + @AfterEach + public void afterEach() throws Exception + { + client.stop(); + server.stop(); + } + + @Test + public void testCloseInOnWebSocketConnect() throws Exception + { + URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws"); + EventSocket clientEndpoint = new EventSocket(); + + client.connectToServer(clientEndpoint, uri); + assertTrue(clientEndpoint.closeLatch.await(5, TimeUnit.SECONDS)); + assertThat(clientEndpoint.closeReason.getCloseCode(), is(CloseReason.CloseCodes.VIOLATED_POLICY)); + + assertThat(serverContainer.getOpenSessions().size(), is(0)); + } + + @ServerEndpoint("/ws") + public static class ClosingListener + { + @OnOpen + public void onWebSocketConnect(Session session) throws Exception + { + session.close(new CloseReason(CloseReason.CloseCodes.VIOLATED_POLICY, "I am a WS that closes immediately")); + } + } +} diff --git a/jetty-websocket/websocket-jetty-common/src/main/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandler.java b/jetty-websocket/websocket-jetty-common/src/main/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandler.java index d3235ad0ed73..992c9042dd2a 100644 --- a/jetty-websocket/websocket-jetty-common/src/main/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandler.java +++ b/jetty-websocket/websocket-jetty-common/src/main/java/org/eclipse/jetty/websocket/common/JettyWebSocketFrameHandler.java @@ -151,6 +151,8 @@ public void onOpen(CoreSession coreSession, Callback callback) { customizer.customize(coreSession); session = new WebSocketSession(container, coreSession, this); + if (!session.isOpen()) + throw new IllegalStateException("Session is not open"); frameHandle = InvokerUtils.bindTo(frameHandle, session); openHandle = InvokerUtils.bindTo(openHandle, session); @@ -172,7 +174,8 @@ public void onOpen(CoreSession coreSession, Callback callback) if (openHandle != null) openHandle.invoke(); - container.notifySessionListeners((listener) -> listener.onWebSocketSessionOpened(session)); + if (session.isOpen()) + container.notifySessionListeners((listener) -> listener.onWebSocketSessionOpened(session)); callback.succeeded(); demand(); diff --git a/jetty-websocket/websocket-jetty-tests/src/test/java/org/eclipse/jetty/websocket/tests/CloseInOnOpenTest.java b/jetty-websocket/websocket-jetty-tests/src/test/java/org/eclipse/jetty/websocket/tests/CloseInOnOpenTest.java new file mode 100644 index 000000000000..903c51c5ed6d --- /dev/null +++ b/jetty-websocket/websocket-jetty-tests/src/test/java/org/eclipse/jetty/websocket/tests/CloseInOnOpenTest.java @@ -0,0 +1,95 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.websocket.tests; + +import java.net.URI; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.StatusCode; +import org.eclipse.jetty.websocket.api.WebSocketConnectionListener; +import org.eclipse.jetty.websocket.client.WebSocketClient; +import org.eclipse.jetty.websocket.server.JettyWebSocketServerContainer; +import org.eclipse.jetty.websocket.server.config.JettyWebSocketServletContainerInitializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CloseInOnOpenTest +{ + private Server server; + private ServerConnector connector; + private JettyWebSocketServerContainer serverContainer; + private WebSocketClient client; + + @BeforeEach + public void beforeEach() throws Exception + { + server = new Server(); + + connector = new ServerConnector(server); + server.addConnector(connector); + + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + server.setHandler(context); + + JettyWebSocketServletContainerInitializer.configure(context, (servletContext, wsContainer) -> + wsContainer.addMapping("/ws", (req, resp) -> new ClosingListener())); + server.start(); + + serverContainer = JettyWebSocketServerContainer.getContainer(context.getServletContext()); + assertNotNull(serverContainer); + + client = new WebSocketClient(); + client.start(); + } + + @AfterEach + public void afterEach() throws Exception + { + client.stop(); + server.stop(); + } + + @Test + public void testCloseInOnWebSocketConnect() throws Exception + { + URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws"); + EventSocket clientEndpoint = new EventSocket(); + + client.connect(clientEndpoint, uri).get(5, TimeUnit.SECONDS); + assertTrue(clientEndpoint.closeLatch.await(5, TimeUnit.SECONDS)); + assertThat(clientEndpoint.closeCode, is(StatusCode.POLICY_VIOLATION)); + + assertThat(serverContainer.getOpenSessions().size(), is(0)); + } + + public static class ClosingListener implements WebSocketConnectionListener + { + @Override + public void onWebSocketConnect(Session session) + { + session.close(StatusCode.POLICY_VIOLATION, "I am a WS that closes immediately"); + } + } +} From 26d144d708fbec4122869df2482048fdd521406b Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Fri, 30 Jul 2021 11:49:06 +0200 Subject: [PATCH 24/54] #6327 enable AsyncIOServletTest.testAsyncWriteClosed Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../org/eclipse/jetty/server/HttpOutput.java | 6 +++ .../jetty/http/client/AsyncIOServletTest.java | 53 +++++++++++++++++-- .../jetty/server/HttpOutputHelper.java | 22 ++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java index f8df5ea27a26..c061c04f7cc3 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java @@ -1516,6 +1516,12 @@ public void run() } } + // For testing + ApiState getApiState() + { + return _apiState; + } + private String stateString() { return String.format("s=%s,api=%s,sc=%b,e=%s", _state, _apiState, _softClose, _onError); diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java index d1c573059be6..60f3ffca1877 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java @@ -20,8 +20,11 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Deque; +import java.util.Objects; import java.util.Queue; +import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; @@ -65,6 +68,8 @@ import org.eclipse.jetty.server.HttpChannel; import org.eclipse.jetty.server.HttpInput; import org.eclipse.jetty.server.HttpInput.Content; +import org.eclipse.jetty.server.HttpOutput; +import org.eclipse.jetty.server.HttpOutputHelper; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandler.Context; @@ -74,8 +79,6 @@ import org.eclipse.jetty.util.compression.InflaterPool; import org.hamcrest.Matchers; import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -398,8 +401,6 @@ public void onError(Throwable t) @ParameterizedTest @ArgumentsSource(TransportProvider.class) - @Tag("Unstable") - @Disabled public void testAsyncWriteClosed(Transport transport) throws Exception { init(transport); @@ -431,7 +432,18 @@ public void onWritePossible() throws IOException // Wait for the failure to arrive to // the server while we are about to write. - sleep(2000); + try + { + Await.await().atMost(5, TimeUnit.SECONDS).until(() -> + { + out.write(new byte[0]); + return !HttpOutputHelper.getApiState(((HttpOutput)out)).equals("READY"); + }); + } + catch (Exception e) + { + throw new AssertionError(e); + } out.write(data); } @@ -1854,4 +1866,35 @@ public void stopServer() throws Exception super.stopServer(); } } + + static class Await + { + private Duration duration; + + public static Await await() + { + return new Await(); + } + + public Await atMost(long time, TimeUnit unit) + { + duration = Duration.ofMillis(unit.toMillis(time)); + return this; + } + + public void until(Callable<Boolean> condition) throws Exception + { + Objects.requireNonNull(duration); + long start = System.nanoTime(); + + while (true) + { + if (condition.call()) + return; + if (duration.minus(Duration.ofNanos(System.nanoTime() - start)).isNegative()) + throw new AssertionError("Duration expired"); + Thread.sleep(10); + } + } + } } diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java new file mode 100644 index 000000000000..d5bc9312bd65 --- /dev/null +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java @@ -0,0 +1,22 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.server; + +public class HttpOutputHelper +{ + public static String getApiState(HttpOutput httpOutput) + { + return httpOutput.getApiState().toString(); + } +} From 5dc856196e6e16488fafb82dc854bba09c3c9a10 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 2 Aug 2021 13:04:44 +0200 Subject: [PATCH 25/54] #6327 extract HttpOutput._apiState value from toString Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../org/eclipse/jetty/server/HttpOutput.java | 6 ----- .../jetty/http/client/AsyncIOServletTest.java | 5 ++--- .../jetty/server/HttpOutputHelper.java | 22 ------------------- 3 files changed, 2 insertions(+), 31 deletions(-) delete mode 100644 tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java index c061c04f7cc3..f8df5ea27a26 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java @@ -1516,12 +1516,6 @@ public void run() } } - // For testing - ApiState getApiState() - { - return _apiState; - } - private String stateString() { return String.format("s=%s,api=%s,sc=%b,e=%s", _state, _apiState, _softClose, _onError); diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java index 60f3ffca1877..358e6afa7c05 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java @@ -68,8 +68,6 @@ import org.eclipse.jetty.server.HttpChannel; import org.eclipse.jetty.server.HttpInput; import org.eclipse.jetty.server.HttpInput.Content; -import org.eclipse.jetty.server.HttpOutput; -import org.eclipse.jetty.server.HttpOutputHelper; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandler.Context; @@ -437,7 +435,8 @@ public void onWritePossible() throws IOException Await.await().atMost(5, TimeUnit.SECONDS).until(() -> { out.write(new byte[0]); - return !HttpOutputHelper.getApiState(((HttpOutput)out)).equals("READY"); + // Extract HttpOutput._apiState value from toString. + return !out.toString().split(",")[1].split("=")[1].equals("READY"); }); } catch (Exception e) diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java deleted file mode 100644 index d5bc9312bd65..000000000000 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/server/HttpOutputHelper.java +++ /dev/null @@ -1,22 +0,0 @@ -// -// ======================================================================== -// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. -// -// This program and the accompanying materials are made available under the -// terms of the Eclipse Public License v. 2.0 which is available at -// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 -// which is available at https://www.apache.org/licenses/LICENSE-2.0. -// -// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 -// ======================================================================== -// - -package org.eclipse.jetty.server; - -public class HttpOutputHelper -{ - public static String getApiState(HttpOutput httpOutput) - { - return httpOutput.getApiState().toString(); - } -} From 7867d51c919352a247875ec0fa93fb000412d27f Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 2 Aug 2021 13:05:44 +0200 Subject: [PATCH 26/54] #6327 replace Await helper with awaitility Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- jetty-util/pom.xml | 5 +++ .../jetty/util/BlockingArrayQueueTest.java | 36 +----------------- pom.xml | 5 +++ tests/test-http-client-transport/pom.xml | 5 +++ .../jetty/http/client/AsyncIOServletTest.java | 37 +------------------ 5 files changed, 18 insertions(+), 70 deletions(-) diff --git a/jetty-util/pom.xml b/jetty-util/pom.xml index bf58de61ba84..98b188a9e786 100644 --- a/jetty-util/pom.xml +++ b/jetty-util/pom.xml @@ -63,6 +63,11 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.eclipse.jetty.toolchain</groupId> <artifactId>jetty-perf-helper</artifactId> diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java index eac290032191..ba6da7943ad4 100644 --- a/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/BlockingArrayQueueTest.java @@ -13,13 +13,10 @@ package org.eclipse.jetty.util; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.ListIterator; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ThreadLocalRandom; @@ -30,7 +27,7 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; -import static org.eclipse.jetty.util.BlockingArrayQueueTest.Await.await; +import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -528,35 +525,4 @@ public void testDrainTo() throws Exception assertThat(queue.size(), Matchers.is(0)); assertThat(queue, Matchers.empty()); } - - static class Await - { - private Duration duration; - - public static Await await() - { - return new Await(); - } - - public Await atMost(long time, TimeUnit unit) - { - duration = Duration.ofMillis(unit.toMillis(time)); - return this; - } - - public void until(Callable<Boolean> condition) throws Exception - { - Objects.requireNonNull(duration); - long start = System.nanoTime(); - - while (true) - { - if (condition.call()) - return; - if (duration.minus(Duration.ofNanos(System.nanoTime() - start)).isNegative()) - throw new AssertionError("Duration expired"); - Thread.sleep(10); - } - } - } } diff --git a/pom.xml b/pom.xml index 3048dc8de693..2cfbc481c778 100644 --- a/pom.xml +++ b/pom.xml @@ -1131,6 +1131,11 @@ <artifactId>hamcrest</artifactId> <version>${hamcrest.version}</version> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <version>4.1.0</version> + </dependency> <dependency> <groupId>org.testcontainers</groupId> <artifactId>testcontainers-bom</artifactId> diff --git a/tests/test-http-client-transport/pom.xml b/tests/test-http-client-transport/pom.xml index 9c2aab3097d6..8c508d35092b 100644 --- a/tests/test-http-client-transport/pom.xml +++ b/tests/test-http-client-transport/pom.xml @@ -46,6 +46,11 @@ <artifactId>slf4j-api</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-alpn-java-client</artifactId> diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java index 358e6afa7c05..9e11dba26c0a 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java @@ -20,11 +20,8 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.time.Duration; import java.util.Deque; -import java.util.Objects; import java.util.Queue; -import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; @@ -81,6 +78,7 @@ import org.junit.jupiter.params.provider.ArgumentsSource; import static java.nio.ByteBuffer.wrap; +import static org.awaitility.Awaitility.await; import static org.eclipse.jetty.http.client.Transport.FCGI; import static org.eclipse.jetty.http.client.Transport.H2C; import static org.eclipse.jetty.http.client.Transport.HTTP; @@ -432,7 +430,7 @@ public void onWritePossible() throws IOException // the server while we are about to write. try { - Await.await().atMost(5, TimeUnit.SECONDS).until(() -> + await().atMost(5, TimeUnit.SECONDS).until(() -> { out.write(new byte[0]); // Extract HttpOutput._apiState value from toString. @@ -1865,35 +1863,4 @@ public void stopServer() throws Exception super.stopServer(); } } - - static class Await - { - private Duration duration; - - public static Await await() - { - return new Await(); - } - - public Await atMost(long time, TimeUnit unit) - { - duration = Duration.ofMillis(unit.toMillis(time)); - return this; - } - - public void until(Callable<Boolean> condition) throws Exception - { - Objects.requireNonNull(duration); - long start = System.nanoTime(); - - while (true) - { - if (condition.call()) - return; - if (duration.minus(Duration.ofNanos(System.nanoTime() - start)).isNegative()) - throw new AssertionError("Duration expired"); - Thread.sleep(10); - } - } - } } From 2868eea9ccdb1cf081ac3bd1812aa3bccb1d3a3b Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 2 Aug 2021 13:25:27 +0200 Subject: [PATCH 27/54] #6327 enable all disabled tests in HttpClientContinueTest Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../http/client/HttpClientContinueTest.java | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java index 1eb3d3e2be78..7bfdc6e2531a 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java @@ -25,6 +25,7 @@ import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import javax.servlet.ServletException; import javax.servlet.ServletInputStream; import javax.servlet.http.HttpServletRequest; @@ -45,11 +46,10 @@ import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.util.IO; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; +import static org.awaitility.Awaitility.await; import static org.eclipse.jetty.http.client.Transport.FCGI; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -319,12 +319,10 @@ public void onComplete(Result result) @ParameterizedTest @ArgumentsSource(TransportProvider.class) - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testExpect100ContinueWithContentWithResponseFailureBefore100Continue(Transport transport) throws Exception { init(transport); - long idleTimeout = 1000; + long idleTimeout = 100; scenario.startServer(new AbstractHandler() { @Override @@ -366,12 +364,10 @@ public void onComplete(Result result) @ParameterizedTest @ArgumentsSource(TransportProvider.class) - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testExpect100ContinueWithContentWithResponseFailureAfter100Continue(Transport transport) throws Exception { init(transport); - long idleTimeout = 1000; + long idleTimeout = 100; scenario.startServer(new AbstractHandler() { @Override @@ -474,10 +470,10 @@ public void onComplete(Result result) @ParameterizedTest @ArgumentsSource(TransportProvider.class) - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testExpect100ContinueWithDeferredContentRespond100Continue(Transport transport) throws Exception { + AtomicReference<Thread> handlerThread = new AtomicReference<>(); + CountDownLatch demandLatch = new CountDownLatch(3); init(transport); scenario.start(new AbstractHandler() { @@ -485,6 +481,7 @@ public void testExpect100ContinueWithDeferredContentRespond100Continue(Transport public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { baseRequest.setHandled(true); + handlerThread.set(Thread.currentThread()); // Send 100-Continue and echo the content IO.copy(request.getInputStream(), response.getOutputStream()); } @@ -496,8 +493,16 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques System.arraycopy(chunk1, 0, data, 0, chunk1.length); System.arraycopy(chunk2, 0, data, chunk1.length, chunk2.length); - CountDownLatch latch = new CountDownLatch(1); - AsyncRequestContent content = new AsyncRequestContent(); + CountDownLatch requestLatch = new CountDownLatch(1); + AsyncRequestContent content = new AsyncRequestContent() + { + @Override + public void demand() + { + super.demand(); + demandLatch.countDown(); + } + }; scenario.client.newRequest(scenario.newURI()) .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE)) .body(content) @@ -507,28 +512,32 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques public void onComplete(Result result) { assertArrayEquals(data, getContent()); - latch.countDown(); + requestLatch.countDown(); } }); - Thread.sleep(1000); + // Wait for the handler thread to be blocked in IO. + await().atMost(5, TimeUnit.SECONDS).until(() -> + { + Thread thread = handlerThread.get(); + return thread != null && thread.getState() == Thread.State.WAITING; + }); content.offer(ByteBuffer.wrap(chunk1)); - Thread.sleep(1000); + assertTrue(demandLatch.await(5, TimeUnit.SECONDS)); content.offer(ByteBuffer.wrap(chunk2)); content.close(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(requestLatch.await(5, TimeUnit.SECONDS)); } @ParameterizedTest @ArgumentsSource(TransportProvider.class) - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testExpect100ContinueWithInitialAndDeferredContentRespond100Continue(Transport transport) throws Exception { + AtomicReference<Thread> handlerThread = new AtomicReference<>(); init(transport); scenario.start(new AbstractHandler() { @@ -536,6 +545,7 @@ public void testExpect100ContinueWithInitialAndDeferredContentRespond100Continue public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { baseRequest.setHandled(true); + handlerThread.set(Thread.currentThread()); // Send 100-Continue and echo the content IO.copy(request.getInputStream(), response.getOutputStream()); } @@ -562,7 +572,12 @@ public void onComplete(Result result) } }); - Thread.sleep(1000); + // Wait for the handler thread to be blocked in IO. + await().atMost(5, TimeUnit.SECONDS).until(() -> + { + Thread thread = handlerThread.get(); + return thread != null && thread.getState() == Thread.State.WAITING; + }); content.offer(ByteBuffer.wrap(chunk2)); content.close(); From 37eb7909be7a7dfaa4c55b7122bb008d766b9181 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 2 Aug 2021 13:28:10 +0200 Subject: [PATCH 28/54] #6327 enable HttpClientStreamTest.testUploadWithDeferredContentProviderFromInputStream Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../http/client/HttpClientStreamTest.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java index 70639327cebc..a4f3b8008955 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java @@ -58,7 +58,6 @@ import org.eclipse.jetty.toolchain.test.MavenTestingUtils; import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.IO; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -666,7 +665,6 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, @ParameterizedTest @ArgumentsSource(TransportProvider.class) - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testUploadWithDeferredContentProviderFromInputStream(Transport transport) throws Exception { init(transport); @@ -680,8 +678,17 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, } }); - CountDownLatch latch = new CountDownLatch(1); - try (AsyncRequestContent content = new AsyncRequestContent()) + CountDownLatch demandLatch = new CountDownLatch(2); + CountDownLatch requestLatch = new CountDownLatch(1); + try (AsyncRequestContent content = new AsyncRequestContent() + { + @Override + public void demand() + { + demandLatch.countDown(); + super.demand(); + } + }) { scenario.client.newRequest(scenario.newURI()) .scheme(scenario.getScheme()) @@ -689,11 +696,11 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, .send(result -> { if (result.isSucceeded() && result.getResponse().getStatus() == 200) - latch.countDown(); + requestLatch.countDown(); }); // Make sure we provide the content *after* the request has been "sent". - Thread.sleep(1000); + assertTrue(demandLatch.await(5, TimeUnit.SECONDS)); try (ByteArrayInputStream input = new ByteArrayInputStream(new byte[1024])) { @@ -705,7 +712,7 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, } } } - assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(requestLatch.await(5, TimeUnit.SECONDS)); } @ParameterizedTest From 999b3ca11e0ad5d07ec569de70737e3bd2fdf2c7 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 2 Aug 2021 13:28:53 +0200 Subject: [PATCH 29/54] #6327 enable ResourceHandlerTest.testSlowBiggest Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../org/eclipse/jetty/server/handler/ResourceHandlerTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/handler/ResourceHandlerTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/handler/ResourceHandlerTest.java index ad2aef8768cc..9b79aba3cd10 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/handler/ResourceHandlerTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/handler/ResourceHandlerTest.java @@ -40,7 +40,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import static org.eclipse.jetty.http.HttpHeader.CONTENT_LENGTH; import static org.eclipse.jetty.http.HttpHeader.CONTENT_TYPE; @@ -277,7 +276,6 @@ public void testWelcomeRedirect() throws Exception } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testSlowBiggest() throws Exception { _connector.setIdleTimeout(9000); @@ -307,7 +305,7 @@ public void testSlowBiggest() throws Exception ByteBuffer buffer = null; while (true) { - Thread.sleep(25); + Thread.sleep(10); int len = in.read(array); if (len < 0) break; From 56eed8211e57fc2a5d7bddebecb15a2057b37fd9 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 2 Aug 2021 16:38:33 +0200 Subject: [PATCH 30/54] #6327 enable HttpClientTest.testRequestIdleTimeout Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../test/java/org/eclipse/jetty/client/HttpClientTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java index 2a33f7148839..7604ae1a28cb 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java @@ -91,7 +91,6 @@ import org.eclipse.jetty.util.SocketAddressResolver; import org.hamcrest.Matchers; import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -690,10 +689,9 @@ public void onComplete(Result result) @ParameterizedTest @ArgumentsSource(ScenarioProvider.class) - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testRequestIdleTimeout(Scenario scenario) throws Exception { - long idleTimeout = 1000; + long idleTimeout = 100; start(scenario, new AbstractHandler() { @Override From 8766bddb5072c086f505147c1ee62c744b90b00f Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Wed, 4 Aug 2021 10:16:57 +0200 Subject: [PATCH 31/54] #6327 improve test Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../jetty/http/client/AsyncIOServletTest.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java index 9e11dba26c0a..571839413d85 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java @@ -60,11 +60,13 @@ import org.eclipse.jetty.http2.api.Session; import org.eclipse.jetty.http2.client.http.HttpConnectionOverHTTP2; import org.eclipse.jetty.io.Connection; +import org.eclipse.jetty.io.EofException; import org.eclipse.jetty.logging.StacklessLogging; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpChannel; import org.eclipse.jetty.server.HttpInput; import org.eclipse.jetty.server.HttpInput.Content; +import org.eclipse.jetty.server.HttpOutput; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandler.Context; @@ -401,12 +403,7 @@ public void testAsyncWriteClosed(Transport transport) throws Exception { init(transport); - String text = "Now is the winter of our discontent. How Now Brown Cow. The quick brown fox jumped over the lazy dog.\n"; - for (int i = 0; i < 10; i++) - { - text = text + text; - } - byte[] data = text.getBytes(StandardCharsets.UTF_8); + byte[] data = new byte[1024]; CountDownLatch errorLatch = new CountDownLatch(1); scenario.start(new HttpServlet() @@ -432,17 +429,22 @@ public void onWritePossible() throws IOException { await().atMost(5, TimeUnit.SECONDS).until(() -> { - out.write(new byte[0]); - // Extract HttpOutput._apiState value from toString. - return !out.toString().split(",")[1].split("=")[1].equals("READY"); + try + { + if (out.isReady()) + ((HttpOutput)out).write(ByteBuffer.wrap(data)); + return false; + } + catch (EofException e) + { + return true; + } }); } catch (Exception e) { throw new AssertionError(e); } - - out.write(data); } @Override From 72505af84641d07515c9d40a474e6da922f0c887 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Wed, 4 Aug 2021 10:42:48 +0200 Subject: [PATCH 32/54] #6327 stop relying on timeouts to abort tested requests Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../http/client/HttpClientContinueTest.java | 95 ++++++++++++------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java index 7bfdc6e2531a..55cd8ae642a8 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientContinueTest.java @@ -14,6 +14,7 @@ package org.eclipse.jetty.http.client; import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -28,6 +29,7 @@ import java.util.concurrent.atomic.AtomicReference; import javax.servlet.ServletException; import javax.servlet.ServletInputStream; +import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -322,31 +324,37 @@ public void onComplete(Result result) public void testExpect100ContinueWithContentWithResponseFailureBefore100Continue(Transport transport) throws Exception { init(transport); - long idleTimeout = 100; + AtomicReference<org.eclipse.jetty.client.api.Request> clientRequestRef = new AtomicReference<>(); + CountDownLatch clientLatch = new CountDownLatch(1); + CountDownLatch serverLatch = new CountDownLatch(1); + scenario.startServer(new AbstractHandler() { @Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws ServletException { baseRequest.setHandled(true); + clientRequestRef.get().abort(new Exception("abort!")); try { - TimeUnit.MILLISECONDS.sleep(2 * idleTimeout); + if (!clientLatch.await(5, TimeUnit.SECONDS)) + throw new ServletException("Server timed out on client latch"); + serverLatch.countDown(); } - catch (InterruptedException x) + catch (InterruptedException e) { - throw new ServletException(x); + throw new ServletException(e); } } }); - scenario.startClient(httpClient -> httpClient.setIdleTimeout(2 * idleTimeout)); + scenario.startClient(); byte[] content = new byte[1024]; - CountDownLatch latch = new CountDownLatch(1); - scenario.client.newRequest(scenario.newURI()) + org.eclipse.jetty.client.api.Request clientRequest = scenario.client.newRequest(scenario.newURI()); + clientRequestRef.set(clientRequest); + clientRequest .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE)) .body(new BytesRequestContent(content)) - .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS) .send(new BufferingResponseListener() { @Override @@ -355,11 +363,12 @@ public void onComplete(Result result) assertTrue(result.isFailed()); assertNotNull(result.getRequestFailure()); assertNotNull(result.getResponseFailure()); - latch.countDown(); + clientLatch.countDown(); } }); - assertTrue(latch.await(3 * idleTimeout, TimeUnit.MILLISECONDS)); + assertTrue(clientLatch.await(5, TimeUnit.SECONDS)); + assertTrue(serverLatch.await(5, TimeUnit.SECONDS)); } @ParameterizedTest @@ -367,7 +376,9 @@ public void onComplete(Result result) public void testExpect100ContinueWithContentWithResponseFailureAfter100Continue(Transport transport) throws Exception { init(transport); - long idleTimeout = 100; + AtomicReference<org.eclipse.jetty.client.api.Request> clientRequestRef = new AtomicReference<>(); + CountDownLatch clientLatch = new CountDownLatch(1); + CountDownLatch serverLatch = new CountDownLatch(1); scenario.startServer(new AbstractHandler() { @Override @@ -376,9 +387,12 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques baseRequest.setHandled(true); // Send 100-Continue and consume the content IO.copy(request.getInputStream(), new ByteArrayOutputStream()); + clientRequestRef.get().abort(new Exception("abort!")); try { - TimeUnit.MILLISECONDS.sleep(2 * idleTimeout); + if (!clientLatch.await(5, TimeUnit.SECONDS)) + throw new ServletException("Server timed out on client latch"); + serverLatch.countDown(); } catch (InterruptedException x) { @@ -386,11 +400,12 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques } } }); - scenario.startClient(httpClient -> httpClient.setIdleTimeout(idleTimeout)); + scenario.startClient(); byte[] content = new byte[1024]; - CountDownLatch latch = new CountDownLatch(1); - scenario.client.newRequest(scenario.newURI()) + org.eclipse.jetty.client.api.Request clientRequest = scenario.client.newRequest(scenario.newURI()); + clientRequestRef.set(clientRequest); + clientRequest .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE)) .body(new BytesRequestContent(content)) .send(new BufferingResponseListener() @@ -401,11 +416,12 @@ public void onComplete(Result result) assertTrue(result.isFailed()); assertNull(result.getRequestFailure()); assertNotNull(result.getResponseFailure()); - latch.countDown(); + clientLatch.countDown(); } }); - assertTrue(latch.await(3 * idleTimeout, TimeUnit.MILLISECONDS)); + assertTrue(clientLatch.await(5, TimeUnit.SECONDS)); + assertTrue(serverLatch.await(5, TimeUnit.SECONDS)); } @ParameterizedTest @@ -472,8 +488,14 @@ public void onComplete(Result result) @ArgumentsSource(TransportProvider.class) public void testExpect100ContinueWithDeferredContentRespond100Continue(Transport transport) throws Exception { + byte[] chunk1 = new byte[]{0, 1, 2, 3}; + byte[] chunk2 = new byte[]{4, 5, 6, 7}; + byte[] data = new byte[chunk1.length + chunk2.length]; + System.arraycopy(chunk1, 0, data, 0, chunk1.length); + System.arraycopy(chunk2, 0, data, chunk1.length, chunk2.length); + + CountDownLatch serverLatch = new CountDownLatch(1); AtomicReference<Thread> handlerThread = new AtomicReference<>(); - CountDownLatch demandLatch = new CountDownLatch(3); init(transport); scenario.start(new AbstractHandler() { @@ -483,26 +505,21 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques baseRequest.setHandled(true); handlerThread.set(Thread.currentThread()); // Send 100-Continue and echo the content - IO.copy(request.getInputStream(), response.getOutputStream()); + + ServletOutputStream outputStream = response.getOutputStream(); + DataInputStream inputStream = new DataInputStream(request.getInputStream()); + // Block until the 1st chunk is fully received. + byte[] buf1 = new byte[chunk1.length]; + inputStream.readFully(buf1); + outputStream.write(buf1); + + serverLatch.countDown(); + IO.copy(inputStream, outputStream); } }); - byte[] chunk1 = new byte[]{0, 1, 2, 3}; - byte[] chunk2 = new byte[]{4, 5, 6, 7}; - byte[] data = new byte[chunk1.length + chunk2.length]; - System.arraycopy(chunk1, 0, data, 0, chunk1.length); - System.arraycopy(chunk2, 0, data, chunk1.length, chunk2.length); - CountDownLatch requestLatch = new CountDownLatch(1); - AsyncRequestContent content = new AsyncRequestContent() - { - @Override - public void demand() - { - super.demand(); - demandLatch.countDown(); - } - }; + AsyncRequestContent content = new AsyncRequestContent(); scenario.client.newRequest(scenario.newURI()) .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE)) .body(content) @@ -516,7 +533,7 @@ public void onComplete(Result result) } }); - // Wait for the handler thread to be blocked in IO. + // Wait for the handler thread to be blocked in the 1st IO. await().atMost(5, TimeUnit.SECONDS).until(() -> { Thread thread = handlerThread.get(); @@ -525,7 +542,13 @@ public void onComplete(Result result) content.offer(ByteBuffer.wrap(chunk1)); - assertTrue(demandLatch.await(5, TimeUnit.SECONDS)); + // Wait for the handler thread to be blocked in the 2nd IO. + assertTrue(serverLatch.await(5, TimeUnit.SECONDS)); + await().atMost(5, TimeUnit.SECONDS).until(() -> + { + Thread thread = handlerThread.get(); + return thread != null && thread.getState() == Thread.State.WAITING; + }); content.offer(ByteBuffer.wrap(chunk2)); content.close(); From 0a98a37cb6f7e7fb9059910058ea5ff46836a281 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Wed, 4 Aug 2021 10:57:09 +0200 Subject: [PATCH 33/54] #6327 wait for the request to be committed before sending the data Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../http/client/HttpClientStreamTest.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java index a4f3b8008955..6e49d148f4b5 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientStreamTest.java @@ -678,29 +678,22 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, } }); - CountDownLatch demandLatch = new CountDownLatch(2); - CountDownLatch requestLatch = new CountDownLatch(1); - try (AsyncRequestContent content = new AsyncRequestContent() - { - @Override - public void demand() - { - demandLatch.countDown(); - super.demand(); - } - }) + CountDownLatch requestSentLatch = new CountDownLatch(1); + CountDownLatch responseLatch = new CountDownLatch(1); + try (AsyncRequestContent content = new AsyncRequestContent()) { scenario.client.newRequest(scenario.newURI()) .scheme(scenario.getScheme()) .body(content) + .onRequestCommit((request) -> requestSentLatch.countDown()) .send(result -> { if (result.isSucceeded() && result.getResponse().getStatus() == 200) - requestLatch.countDown(); + responseLatch.countDown(); }); // Make sure we provide the content *after* the request has been "sent". - assertTrue(demandLatch.await(5, TimeUnit.SECONDS)); + assertTrue(requestSentLatch.await(5, TimeUnit.SECONDS)); try (ByteArrayInputStream input = new ByteArrayInputStream(new byte[1024])) { @@ -712,7 +705,7 @@ public void demand() } } } - assertTrue(requestLatch.await(5, TimeUnit.SECONDS)); + assertTrue(responseLatch.await(5, TimeUnit.SECONDS)); } @ParameterizedTest From 6881ee9a6acb4ff39816df54ab3f45ef48878bd6 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Wed, 4 Aug 2021 17:47:19 +0200 Subject: [PATCH 34/54] #6327 make SelectorManagerTest.testConnectTimeoutBeforeSuccessfulConnect always enabled, using latches and less wasteful timeouts Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../org/eclipse/jetty/io/SelectorManagerTest.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java index 68eb13bb47eb..6e461ac83ad0 100644 --- a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java +++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java @@ -55,15 +55,15 @@ public void dispose() throws Exception } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testConnectTimeoutBeforeSuccessfulConnect() throws Exception { ServerSocketChannel server = ServerSocketChannel.open(); server.bind(new InetSocketAddress("localhost", 0)); SocketAddress address = server.getLocalAddress(); - final AtomicLong timeoutConnection = new AtomicLong(); - final long connectTimeout = 1000; + CountDownLatch connectionFinishedLatch = new CountDownLatch(1); + AtomicLong timeoutConnection = new AtomicLong(); + long connectTimeout = 1000; SelectorManager selectorManager = new SelectorManager(executor, scheduler) { @Override @@ -88,6 +88,10 @@ protected boolean doFinishConnect(SelectableChannel channel) throws IOException { return false; } + finally + { + connectionFinishedLatch.countDown(); + } } @Override @@ -117,8 +121,7 @@ protected void connectionFailed(SelectableChannel channel, Throwable ex, Object SocketChannel client1 = SocketChannel.open(); client1.configureBlocking(false); client1.connect(address); - long timeout = connectTimeout * 2; - timeoutConnection.set(timeout); + timeoutConnection.set(connectTimeout * 110 / 100); final CountDownLatch latch1 = new CountDownLatch(1); selectorManager.connect(client1, new Callback() { @@ -132,7 +135,7 @@ public void failed(Throwable x) assertFalse(client1.isOpen()); // Wait for the first connect to finish, as the selector thread is waiting in finishConnect(). - Thread.sleep(timeout); + assertTrue(connectionFinishedLatch.await(5, TimeUnit.SECONDS)); // Verify that after the failure we can connect successfully. try (SocketChannel client2 = SocketChannel.open()) From 0108a197e8d8fba887678645a5568aad73916964 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Wed, 4 Aug 2021 17:57:02 +0200 Subject: [PATCH 35/54] #6327 enable disabled HttpConnectionLifecycleTest tests, using awaitility to remove wasteful sleeps Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- jetty-client/pom.xml | 5 +++++ .../jetty/client/HttpConnectionLifecycleTest.java | 12 ++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/jetty-client/pom.xml b/jetty-client/pom.xml index 7317069eb0e0..67702b3a5777 100644 --- a/jetty-client/pom.xml +++ b/jetty-client/pom.xml @@ -132,6 +132,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.kerby</groupId> <artifactId>kerb-simplekdc</artifactId> diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java index a28631cfe3fc..b1ec697f01ef 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java @@ -32,13 +32,12 @@ import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.logging.StacklessLogging; import org.eclipse.jetty.server.handler.AbstractHandler; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -213,8 +212,6 @@ public void onComplete(Result result) @ParameterizedTest @ArgumentsSource(ScenarioProvider.class) - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testBadRequestWithSlowRequestRemovesConnection(Scenario scenario) throws Exception { start(scenario, new EmptyServerHandler()); @@ -423,8 +420,6 @@ public void onComplete(Result result) @ParameterizedTest @ArgumentsSource(ScenarioProvider.class) - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testIdleConnectionIsClosedOnRemoteClose(Scenario scenario) throws Exception { start(scenario, new EmptyServerHandler()); @@ -448,10 +443,7 @@ public void testIdleConnectionIsClosedOnRemoteClose(Scenario scenario) throws Ex connector.stop(); // Give the connection some time to process the remote close - TimeUnit.SECONDS.sleep(1); - - assertEquals(0, idleConnections.size()); - assertEquals(0, activeConnections.size()); + await().atMost(5, TimeUnit.SECONDS).until(() -> idleConnections.size() == 0 && activeConnections.size() == 0); } @ParameterizedTest From 1a15f2c48d65c177e99064483bfe584ae7f6db3a Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Thu, 5 Aug 2021 09:28:28 +0200 Subject: [PATCH 36/54] #6327 enable more disabled tests, using awaitility and shortening wasteful sleeps Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../eclipse/jetty/client/http/HttpSenderOverHTTPTest.java | 5 ++--- .../java/org/eclipse/jetty/fcgi/server/HttpClientTest.java | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java index 1712e73a5065..8ec4f83dd194 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java @@ -35,8 +35,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; +import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -91,7 +91,6 @@ public void onSuccess(Request request) } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testSendNoRequestContentIncompleteFlush() throws Exception { ByteArrayEndPoint endPoint = new ByteArrayEndPoint("", 16); @@ -105,7 +104,7 @@ public void testSendNoRequestContentIncompleteFlush() throws Exception StringBuilder builder = new StringBuilder(endPoint.takeOutputString()); // Wait for the write to complete - TimeUnit.SECONDS.sleep(1); + await().atMost(5, TimeUnit.SECONDS).until(() -> endPoint.toEndPointString().contains(",flush=P,")); String chunk = endPoint.takeOutputString(); while (chunk.length() > 0) diff --git a/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java b/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java index b45ac747af6b..3216fca1b454 100644 --- a/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java +++ b/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java @@ -48,7 +48,6 @@ import org.eclipse.jetty.util.Callback; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; @@ -411,7 +410,6 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testRequestIdleTimeout() throws Exception { final long idleTimeout = 1000; @@ -423,7 +421,7 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, try { baseRequest.setHandled(true); - TimeUnit.MILLISECONDS.sleep(2 * idleTimeout); + TimeUnit.MILLISECONDS.sleep(idleTimeout); } catch (InterruptedException x) { @@ -437,7 +435,7 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, assertThrows(TimeoutException.class, () -> client.newRequest(host, port) .scheme(scheme) - .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS) + .idleTimeout(idleTimeout * 90 / 100, TimeUnit.MILLISECONDS) .timeout(3 * idleTimeout, TimeUnit.MILLISECONDS) .send()); From c818581185c6f3169a836f4bf882e2e5301cb7ae Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Thu, 5 Aug 2021 16:33:12 +0200 Subject: [PATCH 37/54] #6327 more latches, less sleeps Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../eclipse/jetty/io/SelectorManagerTest.java | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java index 6e461ac83ad0..d93d9dfb6531 100644 --- a/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java +++ b/jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java @@ -22,7 +22,6 @@ import java.nio.channels.SocketChannel; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.thread.QueuedThreadPool; @@ -30,7 +29,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -62,16 +60,14 @@ public void testConnectTimeoutBeforeSuccessfulConnect() throws Exception SocketAddress address = server.getLocalAddress(); CountDownLatch connectionFinishedLatch = new CountDownLatch(1); - AtomicLong timeoutConnection = new AtomicLong(); + CountDownLatch failedConnectionLatch = new CountDownLatch(1); long connectTimeout = 1000; SelectorManager selectorManager = new SelectorManager(executor, scheduler) { @Override protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key) { - SocketChannelEndPoint endPoint = new SocketChannelEndPoint((SocketChannel)channel, selector, key, getScheduler()); - endPoint.setIdleTimeout(connectTimeout / 2); - return endPoint; + return new SocketChannelEndPoint((SocketChannel)channel, selector, key, getScheduler()); } @Override @@ -79,9 +75,7 @@ protected boolean doFinishConnect(SelectableChannel channel) throws IOException { try { - long timeout = timeoutConnection.get(); - if (timeout > 0) - TimeUnit.MILLISECONDS.sleep(timeout); + assertTrue(failedConnectionLatch.await(connectTimeout * 2, TimeUnit.MILLISECONDS)); return super.doFinishConnect(channel); } catch (InterruptedException e) @@ -120,39 +114,36 @@ protected void connectionFailed(SelectableChannel channel, Throwable ex, Object { SocketChannel client1 = SocketChannel.open(); client1.configureBlocking(false); - client1.connect(address); - timeoutConnection.set(connectTimeout * 110 / 100); - final CountDownLatch latch1 = new CountDownLatch(1); + assertFalse(client1.connect(address)); selectorManager.connect(client1, new Callback() { @Override public void failed(Throwable x) { - latch1.countDown(); + failedConnectionLatch.countDown(); } }); - assertTrue(latch1.await(connectTimeout * 3, TimeUnit.MILLISECONDS)); + assertTrue(failedConnectionLatch.await(connectTimeout * 2, TimeUnit.MILLISECONDS)); assertFalse(client1.isOpen()); - // Wait for the first connect to finish, as the selector thread is waiting in finishConnect(). + // Wait for the first connect to finish, as the selector thread is waiting in doFinishConnect(). assertTrue(connectionFinishedLatch.await(5, TimeUnit.SECONDS)); // Verify that after the failure we can connect successfully. try (SocketChannel client2 = SocketChannel.open()) { client2.configureBlocking(false); - client2.connect(address); - timeoutConnection.set(0); - final CountDownLatch latch2 = new CountDownLatch(1); + assertFalse(client2.connect(address)); + CountDownLatch successfulConnectionLatch = new CountDownLatch(1); selectorManager.connect(client2, new Callback() { @Override public void succeeded() { - latch2.countDown(); + successfulConnectionLatch.countDown(); } }); - assertTrue(latch2.await(connectTimeout * 5, TimeUnit.MILLISECONDS)); + assertTrue(successfulConnectionLatch.await(connectTimeout * 2, TimeUnit.MILLISECONDS)); assertTrue(client2.isOpen()); } } From 15bef0c9ed9794719bee439bb858da519af7e2f7 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Thu, 5 Aug 2021 16:34:00 +0200 Subject: [PATCH 38/54] #6327 rework testRequestIdleTimeout and merge http(s) and fcgi test Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../eclipse/jetty/client/HttpClientTest.java | 43 --------------- .../jetty/fcgi/server/HttpClientTest.java | 41 -------------- .../jetty/http/client/HttpClientTest.java | 54 +++++++++++++++++++ 3 files changed, 54 insertions(+), 84 deletions(-) diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java index 7604ae1a28cb..bc6b1c39ed5c 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java @@ -40,7 +40,6 @@ import java.util.concurrent.Exchanger; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -49,7 +48,6 @@ import javax.servlet.AsyncContext; import javax.servlet.DispatcherType; import javax.servlet.ReadListener; -import javax.servlet.ServletException; import javax.servlet.ServletInputStream; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; @@ -687,47 +685,6 @@ public void onComplete(Result result) assertTrue(latch.await(5, TimeUnit.SECONDS)); } - @ParameterizedTest - @ArgumentsSource(ScenarioProvider.class) - public void testRequestIdleTimeout(Scenario scenario) throws Exception - { - long idleTimeout = 100; - start(scenario, new AbstractHandler() - { - @Override - public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws ServletException - { - try - { - baseRequest.setHandled(true); - TimeUnit.MILLISECONDS.sleep(2 * idleTimeout); - } - catch (InterruptedException x) - { - throw new ServletException(x); - } - } - }); - - String host = "localhost"; - int port = connector.getLocalPort(); - assertThrows(TimeoutException.class, () -> - client.newRequest(host, port) - .scheme(scenario.getScheme()) - .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS) - .timeout(3 * idleTimeout, TimeUnit.MILLISECONDS) - .send()); - - // Make another request without specifying the idle timeout, should not fail - ContentResponse response = client.newRequest(host, port) - .scheme(scenario.getScheme()) - .timeout(3 * idleTimeout, TimeUnit.MILLISECONDS) - .send(); - - assertNotNull(response); - assertEquals(200, response.getStatus()); - } - @ParameterizedTest @ArgumentsSource(ScenarioProvider.class) public void testSendToIPv6Address(Scenario scenario) throws Exception diff --git a/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java b/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java index 3216fca1b454..fee6a3d2617c 100644 --- a/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java +++ b/jetty-fcgi/fcgi-server/src/test/java/org/eclipse/jetty/fcgi/server/HttpClientTest.java @@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.zip.GZIPOutputStream; @@ -409,46 +408,6 @@ public void handle(String target, org.eclipse.jetty.server.Request baseRequest, assertArrayEquals(data, response.getContent()); } - @Test - public void testRequestIdleTimeout() throws Exception - { - final long idleTimeout = 1000; - start(new AbstractHandler() - { - @Override - public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws ServletException - { - try - { - baseRequest.setHandled(true); - TimeUnit.MILLISECONDS.sleep(idleTimeout); - } - catch (InterruptedException x) - { - throw new ServletException(x); - } - } - }); - - final String host = "localhost"; - final int port = connector.getLocalPort(); - assertThrows(TimeoutException.class, () -> - client.newRequest(host, port) - .scheme(scheme) - .idleTimeout(idleTimeout * 90 / 100, TimeUnit.MILLISECONDS) - .timeout(3 * idleTimeout, TimeUnit.MILLISECONDS) - .send()); - - // Make another request without specifying the idle timeout, should not fail - ContentResponse response = client.newRequest(host, port) - .scheme(scheme) - .timeout(3 * idleTimeout, TimeUnit.MILLISECONDS) - .send(); - - assertNotNull(response); - assertEquals(200, response.getStatus()); - } - @Test public void testConnectionIdleTimeout() throws Exception { diff --git a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientTest.java b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientTest.java index 4bb62d62dafe..92522a15da10 100644 --- a/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientTest.java +++ b/tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpClientTest.java @@ -22,9 +22,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.IntStream; +import javax.servlet.ServletException; import javax.servlet.ServletInputStream; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServlet; @@ -781,6 +783,58 @@ protected void service(String target, Request jettyRequest, HttpServletRequest r assertTrue(resultLatch.await(5, TimeUnit.SECONDS)); } + @ParameterizedTest + @ArgumentsSource(TransportProvider.class) + public void testRequestIdleTimeout(Transport transport) throws Exception + { + init(transport); + + CountDownLatch latch = new CountDownLatch(1); + long idleTimeout = 500; + scenario.start(new AbstractHandler() + { + @Override + public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws ServletException + { + try + { + baseRequest.setHandled(true); + if (target.equals("/1")) + assertTrue(latch.await(5, TimeUnit.SECONDS)); + else if (target.equals("/2")) + Thread.sleep(2 * idleTimeout); + else + fail("Unknown path: " + target); + } + catch (InterruptedException x) + { + throw new ServletException(x); + } + } + }); + + String host = "localhost"; + int port = scenario.getNetworkConnectorLocalPortInt().get(); + assertThrows(TimeoutException.class, () -> + scenario.client.newRequest(host, port) + .scheme(scenario.getScheme()) + .path("/1") + .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS) + .timeout(2 * idleTimeout, TimeUnit.MILLISECONDS) + .send()); + latch.countDown(); + + // Make another request without specifying the idle timeout, should not fail + ContentResponse response = scenario.client.newRequest(host, port) + .scheme(scenario.getScheme()) + .path("/2") + .timeout(3 * idleTimeout, TimeUnit.MILLISECONDS) + .send(); + + assertNotNull(response); + assertEquals(200, response.getStatus()); + } + private void sleep(long time) throws IOException { try From a99ff502d14cdd4bd8c7d7d9aad2d5534baf2737 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Fri, 6 Aug 2021 11:15:58 +0200 Subject: [PATCH 39/54] #6327 enable more disabled tests Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../jetty/server/ConnectionOpenCloseTest.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/ConnectionOpenCloseTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/ConnectionOpenCloseTest.java index 4f84a6d3c8a3..5af54a444505 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/ConnectionOpenCloseTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/ConnectionOpenCloseTest.java @@ -34,9 +34,7 @@ import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.hamcrest.Matchers; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -46,8 +44,6 @@ public class ConnectionOpenCloseTest extends AbstractHttpTest { @Test - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testOpenClose() throws Exception { server.setHandler(new AbstractHandler() @@ -97,8 +93,6 @@ public void onClosed(Connection connection) } @Test - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testOpenRequestClose() throws Exception { server.setHandler(new AbstractHandler() @@ -153,15 +147,13 @@ public void onClosed(Connection connection) assertTrue(closeLatch.await(5, TimeUnit.SECONDS)); // Wait some time to see if the callbacks are called too many times - TimeUnit.SECONDS.sleep(1); + TimeUnit.MILLISECONDS.sleep(200); assertEquals(2, callbacks.get()); } } @Test - @Tag("Slow") - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testSSLOpenRequestClose() throws Exception { SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); @@ -223,7 +215,7 @@ public void onClosed(Connection connection) assertTrue(closeLatch.await(5, TimeUnit.SECONDS)); // Wait some time to see if the callbacks are called too many times - TimeUnit.SECONDS.sleep(1); + TimeUnit.MILLISECONDS.sleep(200); assertEquals(4, callbacks.get()); } From 324fe5e3ee97c76d0db6125cfdedaf1008269196 Mon Sep 17 00:00:00 2001 From: Ludovic Orban <lorban@bitronix.be> Date: Mon, 9 Aug 2021 13:48:12 +0200 Subject: [PATCH 40/54] #6327 enable testDefaultServletSuccess Signed-off-by: Ludovic Orban <lorban@bitronix.be> --- .../eclipse/jetty/servlets/ThreadStarvationTest.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/jetty-servlets/src/test/java/org/eclipse/jetty/servlets/ThreadStarvationTest.java b/jetty-servlets/src/test/java/org/eclipse/jetty/servlets/ThreadStarvationTest.java index 729a00e13acc..18b0ba1a15ee 100644 --- a/jetty-servlets/src/test/java/org/eclipse/jetty/servlets/ThreadStarvationTest.java +++ b/jetty-servlets/src/test/java/org/eclipse/jetty/servlets/ThreadStarvationTest.java @@ -54,7 +54,6 @@ import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -71,10 +70,9 @@ public void dispose() throws Exception } @Test - @DisabledIfSystemProperty(named = "env", matches = "ci") // TODO: SLOW, needs review public void testDefaultServletSuccess() throws Exception { - int maxThreads = 10; + int maxThreads = 6; QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, maxThreads); threadPool.setDetailedDump(true); _server = new Server(threadPool); @@ -86,11 +84,11 @@ public void testDefaultServletSuccess() throws Exception Path resourcePath = Paths.get(directory.getPath(), resourceName); try (OutputStream output = Files.newOutputStream(resourcePath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { - byte[] chunk = new byte[1024]; + byte[] chunk = new byte[256 * 1024]; Arrays.fill(chunk, (byte)'X'); chunk[chunk.length - 2] = '\r'; chunk[chunk.length - 1] = '\n'; - for (int i = 0; i < 256 * 1024; ++i) + for (int i = 0; i < 1024; ++i) { output.write(chunk); } @@ -135,10 +133,9 @@ protected void onIncompleteFlush() "\r\n"; output.write(request.getBytes(StandardCharsets.UTF_8)); output.flush(); - Thread.sleep(100); } - // Wait for a the servlet to block. + // Wait for a thread on the servlet to block. assertTrue(writePending.await(5, TimeUnit.SECONDS)); long expected = Files.size(resourcePath); From 4af93b5e19120d0b1ccbca2a9e5269871f4635cd Mon Sep 17 00:00:00 2001 From: Simone Bordet <simone.bordet@gmail.com> Date: Fri, 13 Aug 2021 17:32:53 +0200 Subject: [PATCH 41/54] Issue #6476 - warn on exec (#6597) Fixes #6476 - Show message if JVM args are present but new JVM is spawned * Improved documentation by correctly redacting out `jetty-halt.xml`, an XML file that is only necessary for rendering the documentation. * Added WARN message when new JVM is spawned. * Updated documentation. * Updated --list-config to report whether a JVM is forked. * Added test case. Signed-off-by: Simone Bordet <simone.bordet@gmail.com> --- .../jetty/docs/JettyIncludeExtension.java | 11 +++++- .../operations-guide/modules/modules.adoc | 2 + .../start/start-configure.adoc | 38 +++++++++++++++++-- .../java/org/eclipse/jetty/start/Main.java | 10 +++++ .../org/eclipse/jetty/start/StartArgs.java | 16 +++----- .../tests/distribution/DistributionTests.java | 33 ++++++++++++++++ 6 files changed, 94 insertions(+), 16 deletions(-) diff --git a/documentation/jetty-asciidoctor-extensions/src/main/java/org/eclipse/jetty/docs/JettyIncludeExtension.java b/documentation/jetty-asciidoctor-extensions/src/main/java/org/eclipse/jetty/docs/JettyIncludeExtension.java index f9ace37ef6d7..903f6920a5eb 100644 --- a/documentation/jetty-asciidoctor-extensions/src/main/java/org/eclipse/jetty/docs/JettyIncludeExtension.java +++ b/documentation/jetty-asciidoctor-extensions/src/main/java/org/eclipse/jetty/docs/JettyIncludeExtension.java @@ -144,6 +144,7 @@ private String captureOutput(Document document, Map<String, Object> attributes, .map(line -> redact(line, run.getConfig().getMavenLocalRepository(), "/path/to/maven.repository")) .map(line -> redact(line, run.getConfig().getJettyHome().toString(), "/path/to/jetty.home")) .map(line -> redact(line, run.getConfig().getJettyBase().toString(), "/path/to/jetty.base")) + .map(line -> regexpRedact(line, "(^| )[^ ]+/etc/jetty-halt\\.xml", "")) .map(line -> redact(line, (String)document.getAttribute("project-version"), (String)document.getAttribute("version"))); lines = replace(lines, (String)attributes.get("replace")); lines = delete(lines, (String)attributes.get("delete")); @@ -160,6 +161,13 @@ private String redact(String line, String target, String replacement) return line; } + private String regexpRedact(String line, String regexp, String replacement) + { + if (regexp != null && replacement != null) + return line.replaceAll(regexp, replacement); + return line; + } + private Stream<String> replace(Stream<String> lines, String replace) { if (replace == null) @@ -178,8 +186,7 @@ private Stream<String> delete(Stream<String> lines, String delete) if (delete == null) return lines; Pattern regExp = Pattern.compile(delete); - return lines.filter(line -> !regExp.matcher(line).find()) - .filter(line -> !line.contains("jetty-halt.xml")); + return lines.filter(line -> !regExp.matcher(line).find()); } private Stream<String> denoteLineStart(Stream<String> lines) diff --git a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules.adoc b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules.adoc index 67c414717672..2b020bac4bb2 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/modules/modules.adoc @@ -396,6 +396,8 @@ When the `[exec]` section is present, the JVM running the Jetty start mechanism This is necessary because JVM options such as `-Xmx` (that specifies the max JVM heap size) cannot be changed in a running JVM. For an example, see xref:og-start-configure-custom-module-exec[this section]. +You can avoid that the Jetty start mechanism forks the second JVM, as explained in xref:og-start-configure-dry-run[this section]. + [[og-modules-directive-jpms]] ===== [jpms] diff --git a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/start/start-configure.adoc b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/start/start-configure.adoc index c6814ce45d63..e711d9018eae 100644 --- a/documentation/jetty-documentation/src/main/asciidoc/operations-guide/start/start-configure.adoc +++ b/documentation/jetty-documentation/src/main/asciidoc/operations-guide/start/start-configure.adoc @@ -163,10 +163,15 @@ $ java -jar $JETTY_HOME/start.jar --add-modules=jvm Since the module defines an `[exec]` section, it will fork _another_ JVM when Jetty is started. -This means that when you start Jetty, there will be _two_ JVMs running: one spawned by you when you run `java -jar $JETTY_HOME/start.jar`, and another spawned by the Jetty start mechanism with the JVM options you specified (that cannot be applied to an already running JVM). +This means that when you start Jetty, there will be _two_ JVMs running: one created by you when you run `java -jar $JETTY_HOME/start.jar`, and another forked by the Jetty start mechanism with the JVM options you specified (that cannot be applied to an already running JVM). Again, you can xref:og-start-configure-dry-run[display the JVM command line] to verify that it is correct. +[TIP] +==== +The second JVM forked by the Jetty start mechanism when one of the modules requires forking, for example a module that contains an `[exec]` section, may not be desirable, and may be avoided as explained in xref:og-start-configure-dry-run[this section]. +==== + [[og-start-configure-display]] ===== Displaying the Configuration @@ -205,7 +210,28 @@ Some option, such as `--jpms`, imply `--exec`, as it won't be possible to modify To start Jetty without forking a second JVM, the `--dry-run` option can be used to generate a command line that is then executed so that starting Jetty only spawns one JVM. -The `--dry-run` option is quite flexible and below you can find a few examples of how to use it to generate scripts or to create an arguments file that can be passed to the `java` executable. +IMPORTANT: You can use the `--dry-run` option as explained below to avoid forking a second JVM when using modules that have the `[exec]` section, or the `--exec` option, or when using the `--jpms` option. + +For example, using the `--dry-run` option with the `jvm.mod` introduced in xref:og-start-configure-custom-module-exec[this section] produces the following command line: + +---- +$ java -jar $JETTY_HOME/start.jar --dry-run +---- + +[source,options=nowrap] +---- +include::jetty[setupModules="src/main/asciidoc/operations-guide/start/jvm.mod",setupArgs="--add-modules=http,jvm",args="--dry-run",replace="( ),$1\\\n"] +---- + +You can then run the generated command line. + +For example, in the Linux `bash` shell you can run it by wrapping it into `$(\...)`: + +---- +$ $(java -jar $JETTY_HOME/start.jar --dry-run) +---- + +The `--dry-run` option is quite flexible and below you can find a few examples of how to use it to avoid forking a second JVM, or generating scripts or creating an arguments file that can be passed to (a possibly alternative) `java` executable. To display the `java` executable used to start Jetty: @@ -304,7 +330,13 @@ $ java -jar $JETTY_HOME/start.jar --dry-run=##opts,path,main,args## > /tmp/jvm_c $ /some/other/java @/tmp/jvm_cmd_line.txt ---- -Alternatively, they can be combined in a shell script: +Using `--dry-run=opts,path,main,args` can be used to avoid that the Jetty start mechanism forks a second JVM when using modules that require forking: + +---- +$ java $(java -jar $JETTY_HOME/start.jar --dry-run=opts,path,main,args) +---- + +The output of different `--dry-run` executions can be creatively combined in a shell script: [source,subs=quotes] ---- diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/Main.java b/jetty-start/src/main/java/org/eclipse/jetty/start/Main.java index ac56b37ea50e..82c7ad3df4a5 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/Main.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/Main.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; import org.eclipse.jetty.start.Props.Prop; import org.eclipse.jetty.start.config.CommandLineConfigSource; @@ -467,6 +468,15 @@ else if (args.isCreateFiles() || !args.getStartModules().isEmpty()) { CommandLineBuilder cmd = args.getMainArgs(StartArgs.ALL_PARTS); cmd.debug(); + + List<String> execModules = args.getEnabledModules().stream() + .map(name -> args.getAllModules().get(name)) + // Keep only the forking modules. + .filter(module -> !module.getJvmArgs().isEmpty()) + .map(Module::getName) + .collect(Collectors.toList()); + StartLog.warn("Forking second JVM due to forking module(s): %s. Use --dry-run to generate the command line to avoid forking.", execModules); + ProcessBuilder pbuilder = new ProcessBuilder(cmd.getArgs()); StartLog.endStartLog(); final Process process = pbuilder.start(); diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java b/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java index b9d3fb0298d2..812610fc1998 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java @@ -302,7 +302,7 @@ public void dumpEnvironment() // Jetty Environment System.out.println(); System.out.println("Jetty Environment:"); - System.out.println("-----------------"); + System.out.println("------------------"); dumpProperty(JETTY_VERSION_KEY); dumpProperty(JETTY_TAG_NAME_KEY); dumpProperty(JETTY_BUILDNUM_KEY); @@ -330,26 +330,20 @@ public void dumpEnvironment() public void dumpJvmArgs() { - System.out.println(); - System.out.println("JVM Arguments:"); - System.out.println("--------------"); if (jvmArgs.isEmpty()) - { - System.out.println(" (no jvm args specified)"); return; - } + + System.out.println(); + System.out.println("Forked JVM Arguments:"); + System.out.println("---------------------"); for (String jvmArgKey : jvmArgs) { String value = System.getProperty(jvmArgKey); if (value != null) - { System.out.printf(" %s = %s%n", jvmArgKey, value); - } else - { System.out.printf(" %s%n", jvmArgKey); - } } } diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java index 089d539fe79b..41129f76dcb7 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java @@ -931,4 +931,37 @@ public void testUnixDomain() throws Exception } } } + + @Test + public void testModuleWithExecEmitsWarning() throws Exception + { + String jettyVersion = System.getProperty("jettyVersion"); + JettyHomeTester distribution = JettyHomeTester.Builder.newInstance() + .jettyVersion(jettyVersion) + .mavenLocalRepository(System.getProperty("mavenRepoPath")) + .build(); + + Path jettyBase = distribution.getJettyBase(); + Path jettyBaseModules = jettyBase.resolve("modules"); + Files.createDirectories(jettyBaseModules); + Path execModule = jettyBaseModules.resolve("exec.mod"); + String module = "" + + "[exec]\n" + + "--show-version"; + Files.write(execModule, List.of(module), StandardOpenOption.CREATE); + + try (JettyHomeTester.Run run1 = distribution.start(List.of("--add-modules=http,exec"))) + { + assertTrue(run1.awaitFor(10, TimeUnit.SECONDS)); + assertEquals(0, run1.getExitValue()); + + int port = distribution.freePort(); + try (JettyHomeTester.Run run2 = distribution.start("jetty.http.port=" + port)) + { + assertTrue(run2.awaitConsoleLogsFor("Started Server@", 10, TimeUnit.SECONDS)); + assertTrue(run2.getLogs().stream() + .anyMatch(log -> log.contains("WARN") && log.contains("Forking"))); + } + } + } } From 2b0161e743ac4f536e2017148551487624c367d1 Mon Sep 17 00:00:00 2001 From: Greg Wilkins <gregw@webtide.com> Date: Sat, 14 Aug 2021 01:37:51 +1000 Subject: [PATCH 42/54] Fix #6476 Use dry run in jetty.sh (#6598) * Fix #6597 Use dry run in jetty.sh Use a --dry-run in jetty.sh to pre-expand the java arguments and thus avoid having two JVMs running in the case of exec. Also made a small change to allow script to check the current directory for JETTY_BASE, as that allows testing and runs in the same style as direct calls to start.jar Signed-off-by: Greg Wilkins <gregw@webtide.com> Co-authored-by: Simone Bordet <simone.bordet@gmail.com> --- jetty-home/src/main/resources/bin/jetty.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/jetty-home/src/main/resources/bin/jetty.sh b/jetty-home/src/main/resources/bin/jetty.sh index c128bbad2ce7..3a429107d13a 100755 --- a/jetty-home/src/main/resources/bin/jetty.sh +++ b/jetty-home/src/main/resources/bin/jetty.sh @@ -66,7 +66,8 @@ NAME=$(echo $(basename $0) | sed -e 's/^[SK][0-9]*//' -e 's/\.sh$//') # <Arg><Property name="jetty.home" default="."/>/webapps/jetty.war</Arg> # # JETTY_BASE -# Where your Jetty base directory is. If not set, the value from +# Where your Jetty base directory is. If not set, then the currently +# directory is checked, otherwise the value from # $JETTY_HOME will be used. # # JETTY_RUN @@ -238,7 +239,6 @@ then fi fi - ################################################## # No JETTY_HOME yet? We're out of luck! ################################################## @@ -247,20 +247,23 @@ if [ -z "$JETTY_HOME" ]; then exit 1 fi +RUN_DIR=$(pwd) cd "$JETTY_HOME" -JETTY_HOME=$PWD - +JETTY_HOME=$(pwd) ################################################## # Set JETTY_BASE ################################################## +export JETTY_BASE if [ -z "$JETTY_BASE" ]; then - JETTY_BASE=$JETTY_HOME + if [ -d "$RUN_DIR/start.d" -o -f "$RUN_DIR/start.ini" ]; then + JETTY_BASE=$RUN_DIR + else + JETTY_BASE=$JETTY_HOME + fi fi - cd "$JETTY_BASE" -JETTY_BASE=$PWD - +JETTY_BASE=$(pwd) ##################################################### # Check that jetty is where we think it is @@ -430,7 +433,7 @@ case "`uname`" in CYGWIN*) JETTY_START="`cygpath -w $JETTY_START`";; esac -RUN_ARGS=(${JAVA_OPTIONS[@]} -jar "$JETTY_START" ${JETTY_ARGS[*]}) +RUN_ARGS=$(echo $JAVA_OPTIONS ; "$JAVA" -jar "$JETTY_START" --dry-run=opts,path,main,args ${JETTY_ARGS[*]}) RUN_CMD=("$JAVA" ${RUN_ARGS[@]}) ##################################################### From dbc0ce7c134ce255ad8beae286f9707d46ab2e18 Mon Sep 17 00:00:00 2001 From: Simone Bordet <simone.bordet@gmail.com> Date: Fri, 13 Aug 2021 17:39:52 +0200 Subject: [PATCH 43/54] Fixes #6372 - Review socket options configuration (#6610) * Fixes #6372 - Review socket options configuration Introduced in ClientConnector: * tcpNoDelay * reusePort * receiveBufferSize * sendBufferSize Reworked configuration of socket options in ClientConnector. JMX-ified ClientConnector. Introduced reusePort in ServerConnector. Updated server modules with the new reusePort property. Signed-off-by: Simone Bordet <simone.bordet@gmail.com> --- .../org/eclipse/jetty/client/HttpClient.java | 4 + .../eclipse/jetty/client/HttpClientTest.java | 40 ++++ .../http/HttpClientTransportOverHTTP2.java | 2 + .../org/eclipse/jetty/io/ClientConnector.java | 188 ++++++++++++++++-- .../src/main/config/etc/jetty-http.xml | 1 + .../src/main/config/etc/jetty-ssl.xml | 1 + jetty-server/src/main/config/modules/http.mod | 3 + jetty-server/src/main/config/modules/ssl.mod | 3 + .../eclipse/jetty/server/ServerConnector.java | 28 ++- .../jetty/server/ServerConnectorTest.java | 58 ++++++ 10 files changed, 307 insertions(+), 21 deletions(-) diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java index 76caef9bb3d1..8eb987c34946 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java @@ -923,8 +923,10 @@ public void setMaxRedirects(int maxRedirects) /** * @return whether TCP_NODELAY is enabled + * @deprecated use {@link ClientConnector#isTCPNoDelay()} instead */ @ManagedAttribute(value = "Whether the TCP_NODELAY option is enabled", name = "tcpNoDelay") + @Deprecated public boolean isTCPNoDelay() { return tcpNoDelay; @@ -933,7 +935,9 @@ public boolean isTCPNoDelay() /** * @param tcpNoDelay whether TCP_NODELAY is enabled * @see java.net.Socket#setTcpNoDelay(boolean) + * @deprecated use {@link ClientConnector#setTCPNoDelay(boolean)} instead */ + @Deprecated public void setTCPNoDelay(boolean tcpNoDelay) { this.tcpNoDelay = tcpNoDelay; diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java index bc6b1c39ed5c..ab46fcdb2505 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java @@ -73,6 +73,7 @@ import org.eclipse.jetty.http.HttpHeaderValue; import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.HttpScheme; +import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.io.EndPoint; @@ -1909,6 +1910,45 @@ public long getLength() assertTrue(serverOnErrorLatch.await(5, TimeUnit.SECONDS), "serverOnErrorLatch didn't finish"); } + @ParameterizedTest + @ArgumentsSource(ScenarioProvider.class) + public void testBindAddress(Scenario scenario) throws Exception + { + String bindAddress = "127.0.0.2"; + start(scenario, new EmptyServerHandler() + { + @Override + protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) + { + assertEquals(bindAddress, request.getRemoteAddr()); + } + }); + + client.setBindAddress(new InetSocketAddress(bindAddress, 0)); + + CountDownLatch latch = new CountDownLatch(1); + ContentResponse response = client.newRequest("localhost", connector.getLocalPort()) + .scheme(scenario.getScheme()) + .path("/1") + .onRequestBegin(r -> + { + client.newRequest("localhost", connector.getLocalPort()) + .scheme(scenario.getScheme()) + .path("/2") + .send(result -> + { + assertTrue(result.isSucceeded()); + assertEquals(HttpStatus.OK_200, result.getResponse().getStatus()); + latch.countDown(); + }); + }) + .timeout(5, TimeUnit.SECONDS) + .send(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals(HttpStatus.OK_200, response.getStatus()); + } + private void assertCopyRequest(Request original) { Request copy = client.copyRequest((HttpRequest)original, original.getURI()); diff --git a/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpClientTransportOverHTTP2.java b/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpClientTransportOverHTTP2.java index f316cbedb0eb..001584b3a1f1 100644 --- a/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpClientTransportOverHTTP2.java +++ b/jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpClientTransportOverHTTP2.java @@ -93,6 +93,8 @@ protected void doStart() throws Exception client.setInputBufferSize(httpClient.getResponseBufferSize()); client.setUseInputDirectByteBuffers(httpClient.isUseInputDirectByteBuffers()); client.setUseOutputDirectByteBuffers(httpClient.isUseOutputDirectByteBuffers()); + client.setConnectBlocking(httpClient.isConnectBlocking()); + client.setBindAddress(httpClient.getBindAddress()); } addBean(client); super.doStart(); diff --git a/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java b/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java index f0a4c987e17f..414db0623366 100644 --- a/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java +++ b/jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java @@ -18,6 +18,7 @@ import java.net.ProtocolFamily; import java.net.SocketAddress; import java.net.SocketException; +import java.net.SocketOption; import java.net.StandardProtocolFamily; import java.net.StandardSocketOptions; import java.nio.channels.SelectableChannel; @@ -33,6 +34,8 @@ import org.eclipse.jetty.util.IO; import org.eclipse.jetty.util.JavaVersion; import org.eclipse.jetty.util.Promise; +import org.eclipse.jetty.util.annotation.ManagedAttribute; +import org.eclipse.jetty.util.annotation.ManagedObject; import org.eclipse.jetty.util.component.ContainerLifeCycle; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; @@ -41,6 +44,32 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * <p>The client-side component that connects to server sockets.</p> + * <p>ClientConnector delegates the handling of {@link SocketChannel}s + * to a {@link SelectorManager}, and centralizes the configuration of + * necessary components such as the executor, the scheduler, etc.</p> + * <p>ClientConnector offers a low-level API that can be used to + * connect {@link SocketChannel}s to listening servers via the + * {@link #connect(SocketAddress, Map)} method.</p> + * <p>However, a ClientConnector instance is typically just configured + * and then passed to an HttpClient transport, so that applications + * can use high-level APIs to make HTTP requests to servers:</p> + * <pre> + * // Create a ClientConnector instance. + * ClientConnector connector = new ClientConnector(); + * + * // Configure the ClientConnector. + * connector.setSelectors(1); + * connector.setSslContextFactory(new SslContextFactory.Client()); + * + * // Pass it to the HttpClient transport. + * HttpClientTransport transport = new HttpClientTransportDynamic(connector); + * HttpClient httpClient = new HttpClient(transport); + * httpClient.start(); + * </pre> + */ +@ManagedObject public class ClientConnector extends ContainerLifeCycle { public static final String CLIENT_CONNECTOR_CONTEXT_KEY = "org.eclipse.jetty.client.connector"; @@ -49,6 +78,12 @@ public class ClientConnector extends ContainerLifeCycle public static final String CONNECTION_PROMISE_CONTEXT_KEY = CLIENT_CONNECTOR_CONTEXT_KEY + ".connectionPromise"; private static final Logger LOG = LoggerFactory.getLogger(ClientConnector.class); + /** + * <p>Creates a ClientConnector configured to connect via Unix-Domain sockets to the given Unix-Domain path</p> + * + * @param path the Unix-Domain path to connect to + * @return a ClientConnector that connects to the given Unix-Domain path + */ public static ClientConnector forUnixDomain(Path path) { return new ClientConnector(SocketChannelWithAddress.Factory.forUnixDomain(path)); @@ -65,7 +100,11 @@ public static ClientConnector forUnixDomain(Path path) private Duration connectTimeout = Duration.ofSeconds(5); private Duration idleTimeout = Duration.ofSeconds(30); private SocketAddress bindAddress; + private boolean tcpNoDelay = true; private boolean reuseAddress = true; + private boolean reusePort; + private int receiveBufferSize = -1; + private int sendBufferSize = -1; public ClientConnector() { @@ -129,6 +168,10 @@ public void setSslContextFactory(SslContextFactory.Client sslContextFactory) this.sslContextFactory = sslContextFactory; } + /** + * @return the number of NIO selectors + */ + @ManagedAttribute("The number of NIO selectors") public int getSelectors() { return selectors; @@ -141,6 +184,10 @@ public void setSelectors(int selectors) this.selectors = selectors; } + /** + * @return whether {@link #connect(SocketAddress, Map)} operations are performed in blocking mode + */ + @ManagedAttribute("Whether connect operations are performed in blocking mode") public boolean isConnectBlocking() { return connectBlocking; @@ -151,6 +198,10 @@ public void setConnectBlocking(boolean connectBlocking) this.connectBlocking = connectBlocking; } + /** + * @return the timeout of {@link #connect(SocketAddress, Map)} operations + */ + @ManagedAttribute("The timeout of connect operations") public Duration getConnectTimeout() { return connectTimeout; @@ -163,6 +214,10 @@ public void setConnectTimeout(Duration connectTimeout) selectorManager.setConnectTimeout(connectTimeout.toMillis()); } + /** + * @return the max duration for which a connection can be idle (that is, without traffic of bytes in either direction) + */ + @ManagedAttribute("The duration for which a connection can be idle") public Duration getIdleTimeout() { return idleTimeout; @@ -173,26 +228,120 @@ public void setIdleTimeout(Duration idleTimeout) this.idleTimeout = idleTimeout; } + /** + * @return the address to bind a socket to before the connect operation + */ + @ManagedAttribute("The socket address to bind sockets to before the connect operation") public SocketAddress getBindAddress() { return bindAddress; } + /** + * <p>Sets the bind address of sockets before the connect operation.</p> + * <p>In multi-homed hosts, you may want to connect from a specific address:</p> + * <pre> + * clientConnector.setBindAddress(new InetSocketAddress("127.0.0.2", 0)); + * </pre> + * <p>Note the use of the port {@code 0} to indicate that a different ephemeral port + * should be used for each different connection.</p> + * <p>In the rare cases where you want to use the same port for all connections, + * you must also call {@link #setReusePort(boolean) setReusePort(true)}.</p> + * + * @param bindAddress the socket address to bind to before the connect operation + */ public void setBindAddress(SocketAddress bindAddress) { this.bindAddress = bindAddress; } + /** + * @return whether small TCP packets are sent without delay + */ + @ManagedAttribute("Whether small TCP packets are sent without delay") + public boolean isTCPNoDelay() + { + return tcpNoDelay; + } + + public void setTCPNoDelay(boolean tcpNoDelay) + { + this.tcpNoDelay = tcpNoDelay; + } + + /** + * @return whether rebinding is allowed with sockets in tear-down states + */ + @ManagedAttribute("Whether rebinding is allowed with sockets in tear-down states") public boolean getReuseAddress() { return reuseAddress; } + /** + * <p>Sets whether it is allowed to bind a socket to a socket address + * that may be in use by another socket in tear-down state, for example + * in TIME_WAIT state.</p> + * <p>This is useful when ClientConnector is restarted: an existing connection + * may still be using a network address (same host and same port) that is also + * chosen for a new connection.</p> + * + * @param reuseAddress whether rebinding is allowed with sockets in tear-down states + * @see #setReusePort(boolean) + */ public void setReuseAddress(boolean reuseAddress) { this.reuseAddress = reuseAddress; } + /** + * @return whether binding to same host and port is allowed + */ + @ManagedAttribute("Whether binding to same host and port is allowed") + public boolean isReusePort() + { + return reusePort; + } + + /** + * <p>Sets whether it is allowed to bind multiple sockets to the same + * socket address (same host and same port).</p> + * + * @param reusePort whether binding to same host and port is allowed + */ + public void setReusePort(boolean reusePort) + { + this.reusePort = reusePort; + } + + /** + * @return the receive buffer size in bytes, or -1 for the default value + */ + @ManagedAttribute("The receive buffer size in bytes") + public int getReceiveBufferSize() + { + return receiveBufferSize; + } + + public void setReceiveBufferSize(int receiveBufferSize) + { + this.receiveBufferSize = receiveBufferSize; + } + + /** + * @return the send buffer size in bytes, or -1 for the default value + */ + @ManagedAttribute("The send buffer size in bytes") + public int getSendBufferSize() + { + return sendBufferSize; + } + + public void setSendBufferSize(int sendBufferSize) + { + this.sendBufferSize = sendBufferSize; + } + @Override protected void doStart() throws Exception { @@ -246,10 +395,12 @@ public void connect(SocketAddress address, Map<String, Object> context) SocketChannelWithAddress channelWithAddress = factory.newSocketChannelWithAddress(address, context); channel = channelWithAddress.getSocketChannel(); address = channelWithAddress.getSocketAddress(); + + configure(channel); + SocketAddress bindAddress = getBindAddress(); if (bindAddress != null) bind(channel, bindAddress); - configure(channel); boolean connected = true; boolean blocking = isConnectBlocking() && address instanceof InetSocketAddress; @@ -306,33 +457,36 @@ public void accept(SocketChannel channel, Map<String, Object> context) } } - private void bind(SocketChannel channel, SocketAddress bindAddress) + private void bind(SocketChannel channel, SocketAddress bindAddress) throws IOException { - try - { - boolean reuseAddress = getReuseAddress(); - if (LOG.isDebugEnabled()) - LOG.debug("Binding to {} reusing address {}", bindAddress, reuseAddress); - channel.setOption(StandardSocketOptions.SO_REUSEADDR, reuseAddress); - channel.bind(bindAddress); - } - catch (Throwable x) - { - if (LOG.isDebugEnabled()) - LOG.debug("Could not bind {}", channel); - } + if (LOG.isDebugEnabled()) + LOG.debug("Binding {} to {}", channel, bindAddress); + channel.bind(bindAddress); } protected void configure(SocketChannel channel) throws IOException + { + setSocketOption(channel, StandardSocketOptions.TCP_NODELAY, isTCPNoDelay()); + setSocketOption(channel, StandardSocketOptions.SO_REUSEADDR, getReuseAddress()); + setSocketOption(channel, StandardSocketOptions.SO_REUSEPORT, isReusePort()); + int receiveBufferSize = getReceiveBufferSize(); + if (receiveBufferSize >= 0) + setSocketOption(channel, StandardSocketOptions.SO_RCVBUF, receiveBufferSize); + int sendBufferSize = getSendBufferSize(); + if (sendBufferSize >= 0) + setSocketOption(channel, StandardSocketOptions.SO_SNDBUF, sendBufferSize); + } + + private <T> void setSocketOption(SocketChannel channel, SocketOption<T> option, T value) { try { - channel.setOption(StandardSocketOptions.TCP_NODELAY, true); + channel.setOption(option, value); } catch (Throwable x) { if (LOG.isDebugEnabled()) - LOG.debug("Could not configure {}", channel); + LOG.debug("Could not configure {} to {} on {}", option, value, channel); } } diff --git a/jetty-server/src/main/config/etc/jetty-http.xml b/jetty-server/src/main/config/etc/jetty-http.xml index b827eac5e23e..2fed3ded2417 100644 --- a/jetty-server/src/main/config/etc/jetty-http.xml +++ b/jetty-server/src/main/config/etc/jetty-http.xml @@ -39,6 +39,7 @@ <Set name="acceptorPriorityDelta" property="jetty.http.acceptorPriorityDelta" /> <Set name="acceptQueueSize" property="jetty.http.acceptQueueSize" /> <Set name="reuseAddress"><Property name="jetty.http.reuseAddress" default="true"/></Set> + <Set name="reusePort"><Property name="jetty.http.reusePort" default="false"/></Set> <Set name="acceptedTcpNoDelay"><Property name="jetty.http.acceptedTcpNoDelay" default="true"/></Set> <Set name="acceptedReceiveBufferSize" property="jetty.http.acceptedReceiveBufferSize" /> <Set name="acceptedSendBufferSize" property="jetty.http.acceptedSendBufferSize" /> diff --git a/jetty-server/src/main/config/etc/jetty-ssl.xml b/jetty-server/src/main/config/etc/jetty-ssl.xml index 183445c4fb73..f58ee8c35730 100644 --- a/jetty-server/src/main/config/etc/jetty-ssl.xml +++ b/jetty-server/src/main/config/etc/jetty-ssl.xml @@ -32,6 +32,7 @@ <Set name="acceptorPriorityDelta" property="jetty.ssl.acceptorPriorityDelta"/> <Set name="acceptQueueSize" property="jetty.ssl.acceptQueueSize"/> <Set name="reuseAddress"><Property name="jetty.ssl.reuseAddress" default="true"/></Set> + <Set name="reusePort"><Property name="jetty.ssl.reusePort" default="false"/></Set> <Set name="acceptedTcpNoDelay"><Property name="jetty.ssl.acceptedTcpNoDelay" default="true"/></Set> <Set name="acceptedReceiveBufferSize" property="jetty.ssl.acceptedReceiveBufferSize" /> <Set name="acceptedSendBufferSize" property="jetty.ssl.acceptedSendBufferSize" /> diff --git a/jetty-server/src/main/config/modules/http.mod b/jetty-server/src/main/config/modules/http.mod index 1e1b02a85877..cc5c796df347 100644 --- a/jetty-server/src/main/config/modules/http.mod +++ b/jetty-server/src/main/config/modules/http.mod @@ -40,6 +40,9 @@ etc/jetty-http.xml ## Whether to enable the SO_REUSEADDR socket option. # jetty.http.reuseAddress=true +## Whether to enable the SO_REUSEPORT socket option. +# jetty.http.reusePort=false + ## Whether to enable the TCP_NODELAY socket option on accepted sockets. # jetty.http.acceptedTcpNoDelay=true diff --git a/jetty-server/src/main/config/modules/ssl.mod b/jetty-server/src/main/config/modules/ssl.mod index d9ed73a74d3c..83d68f931b95 100644 --- a/jetty-server/src/main/config/modules/ssl.mod +++ b/jetty-server/src/main/config/modules/ssl.mod @@ -42,6 +42,9 @@ etc/jetty-ssl-context.xml ## Whether to enable the SO_REUSEADDR socket option. # jetty.ssl.reuseAddress=true +## Whether to enable the SO_REUSEPORT socket option. +# jetty.ssl.reusePort=false + ## Whether to enable the TCP_NODELAY socket option on accepted sockets. # jetty.ssl.acceptedTcpNoDelay=true diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ServerConnector.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ServerConnector.java index a8aaf7e15f86..25592a2260d0 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/ServerConnector.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ServerConnector.java @@ -19,6 +19,7 @@ import java.net.ServerSocket; import java.net.Socket; import java.net.SocketException; +import java.net.StandardSocketOptions; import java.nio.channels.Channel; import java.nio.channels.SelectableChannel; import java.nio.channels.SelectionKey; @@ -77,6 +78,7 @@ public class ServerConnector extends AbstractNetworkConnector private volatile int _localPort = -1; private volatile int _acceptQueueSize = 0; private volatile boolean _reuseAddress = true; + private volatile boolean _reusePort = false; private volatile boolean _acceptedTcpNoDelay = true; private volatile int _acceptedReceiveBufferSize = -1; private volatile int _acceptedSendBufferSize = -1; @@ -332,8 +334,9 @@ protected ServerSocketChannel openAcceptChannel() throws IOException serverChannel = ServerSocketChannel.open(); try { - serverChannel.socket().setReuseAddress(getReuseAddress()); - serverChannel.socket().bind(bindAddress, getAcceptQueueSize()); + serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, getReuseAddress()); + serverChannel.setOption(StandardSocketOptions.SO_REUSEPORT, isReusePort()); + serverChannel.bind(bindAddress, getAcceptQueueSize()); } catch (Throwable e) { @@ -450,7 +453,7 @@ public void setAcceptQueueSize(int acceptQueueSize) } /** - * @return whether the server socket reuses addresses + * @return whether rebinding the server socket is allowed with sockets in tear-down states * @see ServerSocket#getReuseAddress() */ @ManagedAttribute("Server Socket SO_REUSEADDR") @@ -460,7 +463,7 @@ public boolean getReuseAddress() } /** - * @param reuseAddress whether the server socket reuses addresses + * @param reuseAddress whether rebinding the server socket is allowed with sockets in tear-down states * @see ServerSocket#setReuseAddress(boolean) */ public void setReuseAddress(boolean reuseAddress) @@ -468,6 +471,23 @@ public void setReuseAddress(boolean reuseAddress) _reuseAddress = reuseAddress; } + /** + * @return whether it is allowed to bind multiple server sockets to the same host and port + */ + @ManagedAttribute("Server Socket SO_REUSEPORT") + public boolean isReusePort() + { + return _reusePort; + } + + /** + * @param reusePort whether it is allowed to bind multiple server sockets to the same host and port + */ + public void setReusePort(boolean reusePort) + { + _reusePort = reusePort; + } + /** * @return whether the accepted socket gets {@link java.net.SocketOptions#TCP_NODELAY TCP_NODELAY} enabled. * @see Socket#getTcpNoDelay() diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/ServerConnectorTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/ServerConnectorTest.java index 062fb1645b75..cd5db4be13e6 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/ServerConnectorTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/ServerConnectorTest.java @@ -25,6 +25,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.concurrent.atomic.AtomicLong; @@ -32,6 +33,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.http.HttpTester; import org.eclipse.jetty.io.EndPoint; import org.eclipse.jetty.io.SocketChannelEndPoint; import org.eclipse.jetty.logging.StacklessLogging; @@ -50,6 +54,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -208,6 +213,59 @@ public void testReuseAddressFalse() throws Exception } } + @Test + public void testReusePort() throws Exception + { + int port; + try (ServerSocket server = new ServerSocket()) + { + server.setReuseAddress(true); + server.bind(new InetSocketAddress("localhost", 0)); + port = server.getLocalPort(); + } + + Server server = new Server(); + try + { + // Two connectors listening on the same port. + ServerConnector connector1 = new ServerConnector(server, 1, 1); + connector1.setReuseAddress(true); + connector1.setReusePort(true); + connector1.setPort(port); + server.addConnector(connector1); + ServerConnector connector2 = new ServerConnector(server, 1, 1); + connector2.setReuseAddress(true); + connector2.setReusePort(true); + connector2.setPort(port); + server.addConnector(connector2); + + server.setHandler(new AbstractHandler() + { + @Override + public void handle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) + { + jettyRequest.setHandled(true); + } + }); + + server.start(); + + try (SocketChannel client = SocketChannel.open(new InetSocketAddress("localhost", port))) + { + HttpTester.Request request = HttpTester.newRequest(); + request.put(HttpHeader.HOST, "localhost"); + client.write(request.generate()); + HttpTester.Response response = HttpTester.parseResponse(HttpTester.from(client)); + assertNotNull(response); + assertEquals(HttpStatus.OK_200, response.getStatus()); + } + } + finally + { + server.stop(); + } + } + @Test public void testAddFirstConnectionFactory() { From 9af67f8bff2b7b339df674ea29790d21ec366567 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts <lachlan@webtide.com> Date: Mon, 16 Aug 2021 10:45:26 +1000 Subject: [PATCH 44/54] Issue #6618 - azp claim should not be required for single value aud array Signed-off-by: Lachlan Roberts <lachlan@webtide.com> --- .../jetty/security/openid/JwtDecoder.java | 2 +- .../security/openid/OpenIdCredentials.java | 14 ++++--- .../openid/OpenIdCredentialsTest.java | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java diff --git a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/JwtDecoder.java b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/JwtDecoder.java index dfdbd511f8f2..69d03a478bc5 100644 --- a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/JwtDecoder.java +++ b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/JwtDecoder.java @@ -35,6 +35,7 @@ public class JwtDecoder * @param jwt the JWT to decode. * @return the map of claims encoded in the JWT. */ + @SuppressWarnings("unchecked") public static Map<String, Object> decode(String jwt) { if (LOG.isDebugEnabled()) @@ -54,7 +55,6 @@ public static Map<String, Object> decode(String jwt) Object parsedJwtHeader = json.fromJSON(jwtHeaderString); if (!(parsedJwtHeader instanceof Map)) throw new IllegalStateException("Invalid JWT header"); - @SuppressWarnings("unchecked") Map<String, Object> jwtHeader = (Map<String, Object>)parsedJwtHeader; if (LOG.isDebugEnabled()) LOG.debug("JWT Header: {}", jwtHeader); diff --git a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java index c87ef1604f26..2dee0d32faf6 100644 --- a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java +++ b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java @@ -15,6 +15,7 @@ import java.io.Serializable; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -95,7 +96,7 @@ public void redeemAuthCode(OpenIdConfiguration configuration) throws Exception claims = JwtDecoder.decode(idToken); if (LOG.isDebugEnabled()) LOG.debug("claims {}", claims); - validateClaims(configuration); + validateClaims(claims, configuration); } finally { @@ -105,14 +106,14 @@ public void redeemAuthCode(OpenIdConfiguration configuration) throws Exception } } - private void validateClaims(OpenIdConfiguration configuration) throws Exception + static void validateClaims(Map<String, Object> claims, OpenIdConfiguration configuration) throws Exception { // Issuer Identifier for the OpenID Provider MUST exactly match the value of the iss (issuer) Claim. if (!configuration.getIssuer().equals(claims.get("iss"))) throw new AuthenticationException("Issuer Identifier MUST exactly match the iss Claim"); // The aud (audience) Claim MUST contain the client_id value. - validateAudience(configuration); + validateAudience(claims, configuration); // If an azp (authorized party) Claim is present, verify that its client_id is the Claim Value. Object azp = claims.get("azp"); @@ -126,7 +127,7 @@ private void validateClaims(OpenIdConfiguration configuration) throws Exception throw new AuthenticationException("ID Token has expired"); } - private void validateAudience(OpenIdConfiguration configuration) throws AuthenticationException + private static void validateAudience(Map<String, Object> claims, OpenIdConfiguration configuration) throws AuthenticationException { Object aud = claims.get("aud"); String clientId = configuration.getClientId(); @@ -138,10 +139,11 @@ private void validateAudience(OpenIdConfiguration configuration) throws Authenti throw new AuthenticationException("Audience Claim MUST contain the client_id value"); else if (isList) { - if (!Arrays.asList((Object[])aud).contains(clientId)) + List<Object> list = Arrays.asList((Object[])aud); + if (!list.contains(clientId)) throw new AuthenticationException("Audience Claim MUST contain the client_id value"); - if (claims.get("azp") == null) + if (list.size() > 1 && claims.get("azp") == null) throw new AuthenticationException("A multi-audience ID token needs to contain an azp claim"); } else if (!isValidType) diff --git a/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java b/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java new file mode 100644 index 000000000000..b816d9752363 --- /dev/null +++ b/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java @@ -0,0 +1,40 @@ +// +// ======================================================================== +// Copyright (c) 1995-2021 Mort Bay Consulting Pty Ltd and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// ======================================================================== +// + +package org.eclipse.jetty.security.openid; + +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.jetty.client.HttpClient; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +public class OpenIdCredentialsTest +{ + @Test + public void testSingleAudienceValueInArray() throws Exception + { + String issuer = "myIssuer123"; + String clientId = "myClientId456"; + OpenIdConfiguration configuration = new OpenIdConfiguration(issuer, "", "", clientId, "", new HttpClient()); + + Map<String, Object> claims = new HashMap<>(); + claims.put("iss", issuer); + claims.put("aud", new String[]{clientId}); + claims.put("exp", System.currentTimeMillis() + 5000); + + assertDoesNotThrow(() -> OpenIdCredentials.validateClaims(claims, configuration)); + } +} From b91c3fdbb044e62d713ab0b581267dc91d9efcd3 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts <lachlan@webtide.com> Date: Mon, 16 Aug 2021 15:43:19 +1000 Subject: [PATCH 45/54] Issue #6617 - add support for the client_secret_basic authentication method Signed-off-by: Lachlan Roberts <lachlan@webtide.com> --- .../src/main/config/etc/jetty-openid.xml | 1 + .../src/main/config/modules/openid.mod | 3 +++ .../security/openid/OpenIdConfiguration.java | 23 ++++++++++++++++ .../security/openid/OpenIdCredentials.java | 26 +++++++++++++++---- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/jetty-openid/src/main/config/etc/jetty-openid.xml b/jetty-openid/src/main/config/etc/jetty-openid.xml index 5072c1604951..4e7925491210 100644 --- a/jetty-openid/src/main/config/etc/jetty-openid.xml +++ b/jetty-openid/src/main/config/etc/jetty-openid.xml @@ -26,6 +26,7 @@ <Arg><Property name="jetty.openid.provider.tokenEndpoint"/></Arg> <Arg><Property name="jetty.openid.clientId"/></Arg> <Arg><Property name="jetty.openid.clientSecret"/></Arg> + <Arg><Property name="jetty.openid.basicAuth" default="client_secret_post"/></Arg> <Arg><Ref refid="HttpClient"/></Arg> <Call name="addScopes"> <Arg> diff --git a/jetty-openid/src/main/config/modules/openid.mod b/jetty-openid/src/main/config/modules/openid.mod index 85f0de4c8899..2c2072bb9008 100644 --- a/jetty-openid/src/main/config/modules/openid.mod +++ b/jetty-openid/src/main/config/modules/openid.mod @@ -42,3 +42,6 @@ etc/jetty-openid.xml ## True if all certificates should be trusted by the default SslContextFactory # jetty.openid.sslContextFactory.trustAll=false + +## What authentication method to use with the Token Endpoint (client_secret_post, client_secret_basic). +# jetty.openid.basicAuth=client_secret_post \ No newline at end of file diff --git a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java index a1a82fe2b0e5..043a5f4ee802 100644 --- a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java +++ b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java @@ -45,6 +45,7 @@ public class OpenIdConfiguration extends ContainerLifeCycle private final String clientId; private final String clientSecret; private final List<String> scopes = new ArrayList<>(); + private final String authMethod; private String authEndpoint; private String tokenEndpoint; @@ -70,6 +71,22 @@ public OpenIdConfiguration(String provider, String clientId, String clientSecret */ public OpenIdConfiguration(String issuer, String authorizationEndpoint, String tokenEndpoint, String clientId, String clientSecret, HttpClient httpClient) + { + this(issuer, authorizationEndpoint, tokenEndpoint, clientId, clientSecret, "post", httpClient); + } + + /** + * Create an OpenID configuration for a specific OIDC provider. + * @param issuer The URL of the OpenID provider. + * @param authorizationEndpoint the URL of the OpenID provider's authorization endpoint if configured. + * @param tokenEndpoint the URL of the OpenID provider's token endpoint if configured. + * @param clientId OAuth 2.0 Client Identifier valid at the Authorization Server. + * @param clientSecret The client secret known only by the Client and the Authorization Server. + * @param authMethod Authentication method to use with the Token Endpoint. + * @param httpClient The {@link HttpClient} instance to use. + */ + public OpenIdConfiguration(String issuer, String authorizationEndpoint, String tokenEndpoint, + String clientId, String clientSecret, String authMethod, HttpClient httpClient) { this.issuer = issuer; this.clientId = clientId; @@ -77,6 +94,7 @@ public OpenIdConfiguration(String issuer, String authorizationEndpoint, String t this.authEndpoint = authorizationEndpoint; this.tokenEndpoint = tokenEndpoint; this.httpClient = httpClient != null ? httpClient : newHttpClient(); + this.authMethod = authMethod; if (this.issuer == null) throw new IllegalArgumentException("Issuer was not configured"); @@ -177,6 +195,11 @@ public String getTokenEndpoint() return tokenEndpoint; } + public String getAuthMethod() + { + return authMethod; + } + public void addScopes(String... scopes) { if (scopes != null) diff --git a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java index c87ef1604f26..de90391a6dca 100644 --- a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java +++ b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java @@ -14,12 +14,15 @@ package org.eclipse.jetty.security.openid; import java.io.Serializable; +import java.net.URI; import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.eclipse.jetty.client.api.Authentication; import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.util.BasicAuthentication; import org.eclipse.jetty.client.util.FormRequestContent; import org.eclipse.jetty.util.Fields; import org.eclipse.jetty.util.ajax.JSON; @@ -153,14 +156,27 @@ private Map<String, Object> claimAuthCode(OpenIdConfiguration configuration) thr { Fields fields = new Fields(); fields.add("code", authCode); - fields.add("client_id", configuration.getClientId()); - fields.add("client_secret", configuration.getClientSecret()); fields.add("redirect_uri", redirectUri); fields.add("grant_type", "authorization_code"); + + Request request = configuration.getHttpClient().POST(configuration.getTokenEndpoint()); + switch (configuration.getAuthMethod()) + { + case "client_secret_basic": + URI uri = URI.create(configuration.getTokenEndpoint()); + Authentication.Result authentication = new BasicAuthentication.BasicResult(uri, configuration.getClientId(), configuration.getClientSecret()); + authentication.apply(request); + break; + case "client_secret_post": + fields.add("client_id", configuration.getClientId()); + fields.add("client_secret", configuration.getClientSecret()); + break; + default: + throw new IllegalStateException(configuration.getAuthMethod()); + } + FormRequestContent formContent = new FormRequestContent(fields); - Request request = configuration.getHttpClient().POST(configuration.getTokenEndpoint()) - .body(formContent) - .timeout(10, TimeUnit.SECONDS); + request = request.body(formContent).timeout(10, TimeUnit.SECONDS); ContentResponse response = request.send(); String responseBody = response.getContentAsString(); if (LOG.isDebugEnabled()) From b0503091c223194b79425fc9a8a481f908120e6d Mon Sep 17 00:00:00 2001 From: Lachlan Roberts <lachlan@webtide.com> Date: Tue, 17 Aug 2021 09:24:02 +1000 Subject: [PATCH 46/54] Use correct auth method string in OpenIdConfiguration default constructor Signed-off-by: Lachlan Roberts <lachlan@webtide.com> --- .../org/eclipse/jetty/security/openid/OpenIdConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java index 043a5f4ee802..8a60ec3418c6 100644 --- a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java +++ b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdConfiguration.java @@ -72,7 +72,7 @@ public OpenIdConfiguration(String provider, String clientId, String clientSecret public OpenIdConfiguration(String issuer, String authorizationEndpoint, String tokenEndpoint, String clientId, String clientSecret, HttpClient httpClient) { - this(issuer, authorizationEndpoint, tokenEndpoint, clientId, clientSecret, "post", httpClient); + this(issuer, authorizationEndpoint, tokenEndpoint, clientId, clientSecret, "client_secret_post", httpClient); } /** From af316e58ffda20cd575fe48c43c317917293bd1f Mon Sep 17 00:00:00 2001 From: Lachlan Roberts <lachlan@webtide.com> Date: Tue, 17 Aug 2021 13:31:49 +1000 Subject: [PATCH 47/54] Issue #6618 - Use a new OpenIdCredentials constructor instead of static method. Signed-off-by: Lachlan Roberts <lachlan@webtide.com> --- .../security/openid/OpenIdCredentials.java | 21 +++++++++++++++---- .../openid/OpenIdCredentialsTest.java | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java index 2dee0d32faf6..ccc26d70989b 100644 --- a/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java +++ b/jetty-openid/src/main/java/org/eclipse/jetty/security/openid/OpenIdCredentials.java @@ -46,6 +46,14 @@ public class OpenIdCredentials implements Serializable private String authCode; private Map<String, Object> response; private Map<String, Object> claims; + private boolean verified = false; + + public OpenIdCredentials(Map<String, Object> claims) + { + this.redirectUri = null; + this.authCode = null; + this.claims = claims; + } public OpenIdCredentials(String authCode, String redirectUri) { @@ -96,7 +104,6 @@ public void redeemAuthCode(OpenIdConfiguration configuration) throws Exception claims = JwtDecoder.decode(idToken); if (LOG.isDebugEnabled()) LOG.debug("claims {}", claims); - validateClaims(claims, configuration); } finally { @@ -104,16 +111,22 @@ public void redeemAuthCode(OpenIdConfiguration configuration) throws Exception authCode = null; } } + + if (!verified) + { + validateClaims(configuration); + verified = true; + } } - static void validateClaims(Map<String, Object> claims, OpenIdConfiguration configuration) throws Exception + private void validateClaims(OpenIdConfiguration configuration) throws Exception { // Issuer Identifier for the OpenID Provider MUST exactly match the value of the iss (issuer) Claim. if (!configuration.getIssuer().equals(claims.get("iss"))) throw new AuthenticationException("Issuer Identifier MUST exactly match the iss Claim"); // The aud (audience) Claim MUST contain the client_id value. - validateAudience(claims, configuration); + validateAudience(configuration); // If an azp (authorized party) Claim is present, verify that its client_id is the Claim Value. Object azp = claims.get("azp"); @@ -127,7 +140,7 @@ static void validateClaims(Map<String, Object> claims, OpenIdConfiguration confi throw new AuthenticationException("ID Token has expired"); } - private static void validateAudience(Map<String, Object> claims, OpenIdConfiguration configuration) throws AuthenticationException + private void validateAudience(OpenIdConfiguration configuration) throws AuthenticationException { Object aud = claims.get("aud"); String clientId = configuration.getClientId(); diff --git a/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java b/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java index b816d9752363..18ac12841f46 100644 --- a/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java +++ b/jetty-openid/src/test/java/org/eclipse/jetty/security/openid/OpenIdCredentialsTest.java @@ -35,6 +35,6 @@ public void testSingleAudienceValueInArray() throws Exception claims.put("aud", new String[]{clientId}); claims.put("exp", System.currentTimeMillis() + 5000); - assertDoesNotThrow(() -> OpenIdCredentials.validateClaims(claims, configuration)); + assertDoesNotThrow(() -> new OpenIdCredentials(claims).redeemAuthCode(configuration)); } } From 2f805826153be1445cecc90f994c861c9e2501cf Mon Sep 17 00:00:00 2001 From: Greg Wilkins <gregw@webtide.com> Date: Tue, 17 Aug 2021 14:05:16 +1000 Subject: [PATCH 48/54] Fix flaky test from #6562 (#6627) Fix flaky test from #6562 Signed-off-by: Greg Wilkins <gregw@webtide.com> --- .../org/eclipse/jetty/server/HttpOutputTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java index af67693b8e5c..8e3c7e18a318 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/HttpOutputTest.java @@ -23,7 +23,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import javax.servlet.AsyncContext; import javax.servlet.ServletException; @@ -818,7 +817,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques @Test public void testEmptyBuffer() throws Exception { - AtomicBoolean committed = new AtomicBoolean(); + FuturePromise<Boolean> committed = new FuturePromise<>(); AbstractHandler handler = new AbstractHandler() { @Override @@ -827,7 +826,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques baseRequest.setHandled(true); response.setStatus(200); ((HttpOutput)response.getOutputStream()).write(ByteBuffer.wrap(new byte[0])); - committed.set(response.isCommitted()); + committed.succeeded(response.isCommitted()); } }; @@ -835,13 +834,13 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques handler.start(); String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); assertThat(response, containsString("HTTP/1.1 200 OK")); - assertThat(committed.get(), is(false)); + assertThat(committed.get(10, TimeUnit.SECONDS), is(false)); } @Test public void testEmptyBufferKnown() throws Exception { - AtomicBoolean committed = new AtomicBoolean(); + FuturePromise<Boolean> committed = new FuturePromise<>(); AbstractHandler handler = new AbstractHandler() { @Override @@ -851,7 +850,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques response.setStatus(200); response.setContentLength(0); ((HttpOutput)response.getOutputStream()).write(ByteBuffer.wrap(new byte[0])); - committed.set(response.isCommitted()); + committed.succeeded(response.isCommitted()); } }; @@ -860,7 +859,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques String response = _connector.getResponse("GET / HTTP/1.0\nHost: localhost:80\n\n"); assertThat(response, containsString("HTTP/1.1 200 OK")); assertThat(response, containsString("Content-Length: 0")); - assertThat(committed.get(), is(true)); + assertThat(committed.get(10, TimeUnit.SECONDS), is(true)); } @Test From a5b1845e60fa580531be151fb09ad0584df01f86 Mon Sep 17 00:00:00 2001 From: Greg Wilkins <gregw@webtide.com> Date: Tue, 17 Aug 2021 16:15:00 +1000 Subject: [PATCH 49/54] Disable ipv6 test for #6624 (#6625) (#6629) Temp disable of test that is breaking the build. --- .../test/java/org/eclipse/jetty/client/HttpClientTLSTest.java | 3 ++- .../java/org/eclipse/jetty/util/ssl/SslContextFactory.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java index ca01c19287fb..1efb74bb83e2 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java @@ -60,7 +60,6 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; -import org.eclipse.jetty.toolchain.test.Net; import org.eclipse.jetty.util.StringUtil; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; @@ -1029,6 +1028,7 @@ public void testForcedNonDomainSNI() throws Exception .send(); assertEquals(HttpStatus.OK_200, response2.getStatus()); + /* TODO Fix. See #6624 if (Net.isIpv6InterfaceAvailable()) { // Send a request with SNI "[::1]", we should get the certificate at alias=ip. @@ -1038,6 +1038,7 @@ public void testForcedNonDomainSNI() throws Exception assertEquals(HttpStatus.OK_200, response3.getStatus()); } + */ } @Test diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java b/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java index 41f36af02833..8132414e7780 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java @@ -2178,6 +2178,7 @@ private static List<SNIServerName> getSniServerNames(SSLEngine sslEngine, List<S String host = sslEngine.getPeerHost(); if (host != null) { + // TODO Must handle : somehow as java17 SNIHostName never handles: See #6624 // Must use the byte[] constructor, because the character ':' is forbidden when // using the String constructor (but typically present in IPv6 addresses). return List.of(new SNIHostName(host.getBytes(StandardCharsets.US_ASCII))); From a7a2ae6f84f53481e32a3f7ee982d6b04676ae26 Mon Sep 17 00:00:00 2001 From: Jan Bartel <janb@webtide.com> Date: Tue, 17 Aug 2021 16:58:35 +1000 Subject: [PATCH 50/54] Issue #6586 - remove unnecessary dependencies (#6599) * Issue #6586 Remove unnecessary dependencies Signed-off-by: Jan Bartel <janb@webtide.com> --- apache-jsp/pom.xml | 6 ------ jetty-jndi/pom.xml | 9 +++++++-- jetty-webapp/pom.xml | 7 ------- .../java/org/eclipse/jetty/webapp/JaasConfiguration.java | 6 +----- .../eclipse/jetty/webapp/JettyWebXmlConfiguration.java | 2 +- .../java/org/eclipse/jetty/webapp/JmxConfiguration.java | 6 +----- .../java/org/eclipse/jetty/webapp/JndiConfiguration.java | 6 +----- .../java/org/eclipse/jetty/webapp/JspConfiguration.java | 6 +----- 8 files changed, 12 insertions(+), 36 deletions(-) diff --git a/apache-jsp/pom.xml b/apache-jsp/pom.xml index 460dd96a1ccc..7bc846eb24f5 100644 --- a/apache-jsp/pom.xml +++ b/apache-jsp/pom.xml @@ -80,12 +80,6 @@ <groupId>org.mortbay.jasper</groupId> <artifactId>apache-jsp</artifactId> </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-annotations</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-servlet</artifactId> diff --git a/jetty-jndi/pom.xml b/jetty-jndi/pom.xml index 7caf00bf284e..7bbd856a06b3 100644 --- a/jetty-jndi/pom.xml +++ b/jetty-jndi/pom.xml @@ -60,14 +60,19 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-webapp</artifactId> + <artifactId>jetty-server</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty.orbit</groupId> <artifactId>javax.mail.glassfish</artifactId> - <version>1.4.1.v201005082020</version> <scope>provided</scope> <exclusions> <exclusion> diff --git a/jetty-webapp/pom.xml b/jetty-webapp/pom.xml index 94a4a4b28437..17d4dc26586e 100644 --- a/jetty-webapp/pom.xml +++ b/jetty-webapp/pom.xml @@ -80,13 +80,6 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> - - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-jmx</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-slf4j-impl</artifactId> diff --git a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JaasConfiguration.java b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JaasConfiguration.java index c4ea95f41a89..a635445bae4a 100644 --- a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JaasConfiguration.java +++ b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JaasConfiguration.java @@ -13,8 +13,6 @@ package org.eclipse.jetty.webapp; -import java.util.ServiceLoader; - import org.eclipse.jetty.util.Loader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,9 +22,7 @@ * <p>This configuration configures the WebAppContext server/system classes to * be able to see the org.eclipse.jetty.jaas package. * This class is defined in the webapp package, as it implements the {@link Configuration} interface, - * which is unknown to the jaas package. However, the corresponding {@link ServiceLoader} - * resource is defined in the jaas package, so that this configuration only be - * loaded if the jetty-jaas jars are on the classpath. + * which is unknown to the jaas package. * </p> */ public class JaasConfiguration extends AbstractConfiguration diff --git a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JettyWebXmlConfiguration.java b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JettyWebXmlConfiguration.java index aba74957f58b..7dccaaea820a 100644 --- a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JettyWebXmlConfiguration.java +++ b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JettyWebXmlConfiguration.java @@ -24,7 +24,7 @@ /** * JettyWebConfiguration. * - * Looks for XmlConfiguration files in WEB-INF. Searches in order for the first of jetty6-web.xml, jetty-web.xml or web-jetty.xml + * Looks for XmlConfiguration files in WEB-INF. Searches in order for the first of jetty8-web.xml, jetty-web.xml or web-jetty.xml */ public class JettyWebXmlConfiguration extends AbstractConfiguration { diff --git a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JmxConfiguration.java b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JmxConfiguration.java index 2eebbbebad68..ed9cef8fd595 100644 --- a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JmxConfiguration.java +++ b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JmxConfiguration.java @@ -13,8 +13,6 @@ package org.eclipse.jetty.webapp; -import java.util.ServiceLoader; - import org.eclipse.jetty.util.Loader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,9 +22,7 @@ * <p>This configuration configures the WebAppContext server/system classes to * be able to see the org.eclipse.jetty.jmx package. This class is defined * in the webapp package, as it implements the {@link Configuration} interface, - * which is unknown to the jmx package. However, the corresponding {@link ServiceLoader} - * resource is defined in the jmx package, so that this configuration only be - * loaded if the jetty-jmx jars are on the classpath. + * which is unknown to the jmx package. * </p> */ public class JmxConfiguration extends AbstractConfiguration diff --git a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JndiConfiguration.java b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JndiConfiguration.java index 5c446b1b2097..2de34e0c6fff 100644 --- a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JndiConfiguration.java +++ b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JndiConfiguration.java @@ -13,8 +13,6 @@ package org.eclipse.jetty.webapp; -import java.util.ServiceLoader; - import org.eclipse.jetty.util.Loader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,9 +22,7 @@ * <p>This configuration configures the WebAppContext system/server classes to * be able to see the org.eclipse.jetty.jaas package. * This class is defined in the webapp package, as it implements the {@link Configuration} interface, - * which is unknown to the jndi package. However, the corresponding {@link ServiceLoader} - * resource is defined in the jndi package, so that this configuration only be - * loaded if the jetty-jndi jars are on the classpath. + * which is unknown to the jndi package. * </p> */ public class JndiConfiguration extends AbstractConfiguration diff --git a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JspConfiguration.java b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JspConfiguration.java index 12333f5bbd1e..dd6d1e3cb6cc 100644 --- a/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JspConfiguration.java +++ b/jetty-webapp/src/main/java/org/eclipse/jetty/webapp/JspConfiguration.java @@ -13,8 +13,6 @@ package org.eclipse.jetty.webapp; -import java.util.ServiceLoader; - import org.eclipse.jetty.util.Loader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,9 +22,7 @@ * <p>This configuration configures the WebAppContext server/system classes to * be able to see the org.eclipse.jetty.jsp and org.eclipse.jetty.apache packages. * This class is defined in the webapp package, as it implements the {@link Configuration} interface, - * which is unknown to the jsp package. However, the corresponding {@link ServiceLoader} - * resource is defined in the jsp package, so that this configuration only be - * loaded if the jetty-jsp jars are on the classpath. + * which is unknown to the jsp package. * </p> */ public class JspConfiguration extends AbstractConfiguration From 50798ff504d64f9290be70f5649518ce1f581cb7 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts <lachlan@webtide.com> Date: Wed, 18 Aug 2021 10:43:50 +1000 Subject: [PATCH 51/54] Issue #6617 - change jetty property name to jetty.openid.authMethod Signed-off-by: Lachlan Roberts <lachlan@webtide.com> --- jetty-openid/src/main/config/etc/jetty-openid.xml | 2 +- jetty-openid/src/main/config/modules/openid.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jetty-openid/src/main/config/etc/jetty-openid.xml b/jetty-openid/src/main/config/etc/jetty-openid.xml index 4e7925491210..8e252c22f809 100644 --- a/jetty-openid/src/main/config/etc/jetty-openid.xml +++ b/jetty-openid/src/main/config/etc/jetty-openid.xml @@ -26,7 +26,7 @@ <Arg><Property name="jetty.openid.provider.tokenEndpoint"/></Arg> <Arg><Property name="jetty.openid.clientId"/></Arg> <Arg><Property name="jetty.openid.clientSecret"/></Arg> - <Arg><Property name="jetty.openid.basicAuth" default="client_secret_post"/></Arg> + <Arg><Property name="jetty.openid.authMethod" default="client_secret_post"/></Arg> <Arg><Ref refid="HttpClient"/></Arg> <Call name="addScopes"> <Arg> diff --git a/jetty-openid/src/main/config/modules/openid.mod b/jetty-openid/src/main/config/modules/openid.mod index 2c2072bb9008..7f04767a2d92 100644 --- a/jetty-openid/src/main/config/modules/openid.mod +++ b/jetty-openid/src/main/config/modules/openid.mod @@ -44,4 +44,4 @@ etc/jetty-openid.xml # jetty.openid.sslContextFactory.trustAll=false ## What authentication method to use with the Token Endpoint (client_secret_post, client_secret_basic). -# jetty.openid.basicAuth=client_secret_post \ No newline at end of file +# jetty.openid.authMethod=client_secret_post From 1acf54ba59d61d5fb90a40124df237a87717d311 Mon Sep 17 00:00:00 2001 From: Olivier Lamy <oliver.lamy@gmail.com> Date: Thu, 19 Aug 2021 09:06:52 +1000 Subject: [PATCH 52/54] Upgrade h2spec-maven-plugin to 1.0.6 (#6616) Signed-off-by: Olivier Lamy <oliver.lamy@gmail.com> --- pom.xml | 2 +- .../org/eclipse/jetty/tests/distribution/DistributionTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2cfbc481c778..eee4038483db 100644 --- a/pom.xml +++ b/pom.xml @@ -981,7 +981,7 @@ <plugin> <groupId>org.mortbay.jetty</groupId> <artifactId>h2spec-maven-plugin</artifactId> - <version>1.0.5</version> + <version>1.0.6</version> </plugin> </plugins> </pluginManagement> diff --git a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java index 41129f76dcb7..1201ac5c170d 100644 --- a/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java +++ b/tests/test-distribution/src/test/java/org/eclipse/jetty/tests/distribution/DistributionTests.java @@ -843,7 +843,7 @@ public void testBeforeDirectiveInModule() throws Exception // Protocol "h2" must not be enabled because the // http2 Jetty module was not explicitly enabled. assertFalse(run3.getLogs().stream() - .anyMatch(log -> log.contains("h2"))); + .anyMatch(log -> log.contains("h2")), "Full logs: " + String.join("", run3.getLogs())); } } } From 9f896c6390a64e2d4f1f0e7cd2bb8bc243679bd0 Mon Sep 17 00:00:00 2001 From: Simone Bordet <simone.bordet@gmail.com> Date: Tue, 17 Aug 2021 18:35:04 +0200 Subject: [PATCH 53/54] Fixes #6624 - Non-domain SNI on java17 Java 17 only allows letter|digit|hyphen characters for SNI names. While we could bypass this restriction on the client, when the SNI bytes arrive to the server they will be verified and if not allowed the TLS handshake will fail. Signed-off-by: Simone Bordet <simone.bordet@gmail.com> (cherry picked from commit 693663a4ce3ed0f35cc7da66760e02c9e3bc2d97) --- .../jetty/client/HttpClientTLSTest.java | 49 ++++++++++++++----- .../jetty/util/ssl/SslContextFactory.java | 2 +- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java index 1efb74bb83e2..d7de9d72e18f 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java @@ -60,14 +60,16 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.toolchain.test.Net; import org.eclipse.jetty.util.StringUtil; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledOnJre; +import org.junit.jupiter.api.condition.EnabledForJreRange; import org.junit.jupiter.api.condition.JRE; import static org.hamcrest.MatcherAssert.assertThat; @@ -365,7 +367,7 @@ public void handshakeSucceeded(Event event) // Excluded in JDK 11+ because resumed sessions cannot be compared // using their session IDs even though they are resumed correctly. - @EnabledOnJre({JRE.JAVA_8, JRE.JAVA_9, JRE.JAVA_10}) + @EnabledForJreRange(max = JRE.JAVA_10) @Test public void testHandshakeSucceededWithSessionResumption() throws Exception { @@ -445,7 +447,7 @@ public void handshakeSucceeded(Event event) // Excluded in JDK 11+ because resumed sessions cannot be compared // using their session IDs even though they are resumed correctly. - @EnabledOnJre({JRE.JAVA_8, JRE.JAVA_9, JRE.JAVA_10}) + @EnabledForJreRange(max = JRE.JAVA_10) @Test public void testClientRawCloseDoesNotInvalidateSession() throws Exception { @@ -1013,7 +1015,6 @@ public void testForcedNonDomainSNI() throws Exception // Force TLS-level hostName verification, as we want to receive the correspondent certificate. clientTLS.setEndpointIdentificationAlgorithm("HTTPS"); startClient(clientTLS); - clientTLS.setSNIProvider(SslContextFactory.Client.SniProvider.NON_DOMAIN_SNI_PROVIDER); // Send a request with SNI "localhost", we should get the certificate at alias=localhost. @@ -1027,18 +1028,40 @@ public void testForcedNonDomainSNI() throws Exception .scheme(HttpScheme.HTTPS.asString()) .send(); assertEquals(HttpStatus.OK_200, response2.getStatus()); + } - /* TODO Fix. See #6624 - if (Net.isIpv6InterfaceAvailable()) + @Test + @EnabledForJreRange(max = JRE.JAVA_16, disabledReason = "Since Java 17, SNI host names can only have letter|digit|hyphen characters.") + public void testForcedNonDomainSNIWithIPv6() throws Exception + { + Assumptions.assumeTrue(Net.isIpv6InterfaceAvailable()); + + SslContextFactory.Server serverTLS = new SslContextFactory.Server(); + serverTLS.setKeyStorePath("src/test/resources/keystore_sni_non_domain.p12"); + serverTLS.setKeyStorePassword("storepwd"); + serverTLS.setSNISelector((keyType, issuers, session, sniHost, certificates) -> { - // Send a request with SNI "[::1]", we should get the certificate at alias=ip. - ContentResponse response3 = client.newRequest("[::1]", connector.getLocalPort()) - .scheme(HttpScheme.HTTPS.asString()) - .send(); + // We have forced the client to send the non-domain SNI. + assertNotNull(sniHost); + return serverTLS.sniSelect(keyType, issuers, session, sniHost, certificates); + }); + startServer(serverTLS, new EmptyServerHandler()); - assertEquals(HttpStatus.OK_200, response3.getStatus()); - } - */ + SslContextFactory.Client clientTLS = new SslContextFactory.Client(); + // Trust any certificate received by the server. + clientTLS.setTrustStorePath("src/test/resources/keystore_sni_non_domain.p12"); + clientTLS.setTrustStorePassword("storepwd"); + // Force TLS-level hostName verification, as we want to receive the correspondent certificate. + clientTLS.setEndpointIdentificationAlgorithm("HTTPS"); + startClient(clientTLS); + clientTLS.setSNIProvider(SslContextFactory.Client.SniProvider.NON_DOMAIN_SNI_PROVIDER); + + // Send a request with SNI "[::1]", we should get the certificate at alias=ip. + ContentResponse response3 = client.newRequest("[::1]", connector.getLocalPort()) + .scheme(HttpScheme.HTTPS.asString()) + .send(); + + assertEquals(HttpStatus.OK_200, response3.getStatus()); } @Test diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java b/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java index 8132414e7780..c3470f4124ac 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java @@ -2178,9 +2178,9 @@ private static List<SNIServerName> getSniServerNames(SSLEngine sslEngine, List<S String host = sslEngine.getPeerHost(); if (host != null) { - // TODO Must handle : somehow as java17 SNIHostName never handles: See #6624 // Must use the byte[] constructor, because the character ':' is forbidden when // using the String constructor (but typically present in IPv6 addresses). + // Since Java 17, only letter|digit|hyphen characters are allowed, even by the byte[] constructor. return List.of(new SNIHostName(host.getBytes(StandardCharsets.US_ASCII))); } } From b2c420a124392b3e79a6abf263ec00f51e387409 Mon Sep 17 00:00:00 2001 From: Jan Bartel <janb@webtide.com> Date: Fri, 20 Aug 2021 11:40:18 +1000 Subject: [PATCH 54/54] Issue #6327 Ensure TempDirTest runs in envs other than CI (#6638) * Issue #6327 Ensure TempDirTest runs in envs other than CI Signed-off-by: Jan Bartel <janb@webtide.com> * pass sysproperty env=ci to surefire run Signed-off-by: Olivier Lamy <oliver.lamy@gmail.com> Co-authored-by: Olivier Lamy <oliver.lamy@gmail.com> --- .../test/java/org/eclipse/jetty/webapp/TempDirTest.java | 7 +++++-- pom.xml | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/TempDirTest.java b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/TempDirTest.java index 97803758d4a3..4f95f09fd1d9 100644 --- a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/TempDirTest.java +++ b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/TempDirTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -289,9 +290,11 @@ public void jettyBaseWorkExists() throws Exception /** * ServletContext.TEMPDIR has invalid <code>String</code> directory value (wrong permission to write into it) - * IllegalStateException + * + * Note that if run in the CI environment, the test will fail, because it runs as root, + * so we _will_ have permission to write to this directory. */ - @Disabled("Jenkins will run as root so we do have permission to write to this directory.") + @DisabledIfSystemProperty(named = "env", matches = "ci") @Test public void attributeWithInvalidPermissions() { diff --git a/pom.xml b/pom.xml index eee4038483db..13cbedc949b4 100644 --- a/pom.xml +++ b/pom.xml @@ -1458,6 +1458,9 @@ <version>${maven.surefire.version}</version> <configuration> <excludedGroups>external, large-disk-resource</excludedGroups> + <systemPropertyVariables> + <env>ci</env> + </systemPropertyVariables> </configuration> </plugin> </plugins>