From 48597ba06f525884d780f6ff04fb919ca52c8d57 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 22 Feb 2022 03:00:37 -0800 Subject: [PATCH] flate: Fix rare huffman only (-2) corruption (#503) * flate: Fix rare huffman only (-2) corruption Fix case where all of the last 3 tokens outputs were 15 bits and remainder was also 15 bits. Very rare. Found by fuzz test. --- flate/huffman_bit_writer.go | 41 +++++++++++++++++++++++++--------- flate/testdata/regression.zip | Bin 475404 -> 483763 bytes 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/flate/huffman_bit_writer.go b/flate/huffman_bit_writer.go index 8c1bf42ff8..25f6d1108f 100644 --- a/flate/huffman_bit_writer.go +++ b/flate/huffman_bit_writer.go @@ -1034,6 +1034,9 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { } } if abs < max { + if debugDeflate { + fmt.Println("stored", abs, "<", max) + } // No chance we can compress this... w.writeStoredHeader(len(input), eof) w.writeBytes(input) @@ -1058,6 +1061,9 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // Store bytes, if we don't get a reasonable improvement. if storable && ssize <= estBits { + if debugDeflate { + fmt.Println("stored,", ssize, "<=", estBits) + } w.writeStoredHeader(len(input), eof) w.writeBytes(input) return @@ -1068,7 +1074,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { if estBits < reuseSize { if debugDeflate { - //fmt.Println("not reusing, reuse:", reuseSize/8, "> new:", estBits/8, "- header est:", w.lastHeader/8) + fmt.Println("NOT reusing, reuse:", reuseSize/8, "> new:", estBits/8, "header est:", w.lastHeader/8, "bytes") } // We owe an EOB w.writeCode(w.literalEncoding.codes[endBlockMarker]) @@ -1102,6 +1108,9 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // Go 1.16 LOVES having these on stack. At least 1.5x the speed. bits, nbits, nbytes := w.bits, w.nbits, w.nbytes + if debugDeflate { + count -= int(nbytes)*8 + int(nbits) + } // Unroll, write 3 codes/loop. // Fastest number of unrolls. for len(input) > 3 { @@ -1111,13 +1120,16 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) bits >>= (n * 8) & 63 nbits -= n * 8 - nbytes += uint8(n) + nbytes += n } if nbytes >= bufferFlushSize { if w.err != nil { nbytes = 0 return } + if debugDeflate { + count += int(nbytes) * 8 + } _, w.err = w.writer.Write(w.bytes[:nbytes]) nbytes = 0 } @@ -1133,13 +1145,6 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // Remaining... for _, t := range input { - // Bitwriting inlined, ~30% speedup - c := encoding[t] - bits |= uint64(c.code) << (nbits & 63) - nbits += c.len - if debugDeflate { - count += int(c.len) - } if nbits >= 48 { binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits @@ -1151,17 +1156,33 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { nbytes = 0 return } + if debugDeflate { + count += int(nbytes) * 8 + } _, w.err = w.writer.Write(w.bytes[:nbytes]) nbytes = 0 } } + // Bitwriting inlined, ~30% speedup + c := encoding[t] + bits |= uint64(c.code) << (nbits & 63) + nbits += c.len + if debugDeflate { + count += int(c.len) + } } // Restore... w.bits, w.nbits, w.nbytes = bits, nbits, nbytes if debugDeflate { - fmt.Println("wrote", count/8, "bytes") + nb := count + int(nbytes)*8 + int(nbits) + fmt.Println("wrote", nb, "bits,", nb/8, "bytes.") + } + // Flush if needed to have space. + if w.nbits >= 48 { + w.writeOutBits() } + if eof || sync { w.writeCode(w.literalEncoding.codes[endBlockMarker]) w.lastHeader = 0 diff --git a/flate/testdata/regression.zip b/flate/testdata/regression.zip index a7be57703cdc61f1366600f3e1abe9fd5c25e564..73cf84036d975bc6c9345f6851c8807f52ed63c2 100644 GIT binary patch delta 8473 zcmb`tRa2b}6D*3mThKso5ALuC?hxGF-5s7lVBr#A;qDsTg1fszaCZxC`#pE(W>PfeV2TL1;i^76D-4@StFzkf7}AG}O~MiylxZ zprGDAprGiXprH8qIM_^hEqHm@+041w%=tKZElfE$c=$~$%(+asx!5gvc-Z;OxVg^_ z3`$u>G{6!na%`RbA)SQZ6+q9Ik6ZFo;VokM(U0_h4!sY3!*R}1vaur45Wh`-CwEuU z5|&P7%l$3`E?Rf7d@kTc6}gr$R3VApSM5b#>eV$}1>cKJ(7876r4 zm6i2@-2dNW4dA|fdH>*aj^CEJ^wLO>uJlnj=XGrRyswG>;A;Yn^Fq3(A!mKT6@J8X z`iJWD&t1*Bnyi2=`;0Ju@mHxM#*@z9yQ}~#u?=IFkD=>LLxFM{-QI4SnL2Uix2#3e7K4N7@i2Y#d$xzjy+c|{gDw@LGUdWm{z>)Y;mx9swB_uiiwcz1nwn40d2R{tFx zc42<$Q*yDP@z7oM^8Ep>=TM=ghC`|u34ant-D?lWi-1S>+UVwOcYQz2uM7Bh$qALB zb2Bht(%kig$urdV;^gT+XkZ&F_%H{&p}1m9-d?}GUhQA(1WgG)#N~J<`+GYtlN+{? zu(!Lmg6j59#aj}aiuHzyiP(1~j%HN~+B#@6V>Q-^G1^2D@XYyM@9GcYo!1yzw*S)P zoE|yiqg7vacX`q|8ov>Qtq&4+ZdYz^&)Dc`PK;%D9bg3t(YQy+wg^h*>8Aku(yS!S zdeK&_%5VK1cht*zk-!)i?JL^Pf7?sO%{JqmC2_AY(8^bcK_y$5A!F!SM1jGwpDJuba}2vr ziG1%kjyvD{dtW^WQD-%o9$OuSfhvB{IBokHi7 zCpvE;a1n{wtQdN!{5-v3qybjP_O&8kx=79?2s6e@@XG%8&L2ggu`86ibKaA6MZO|6 z-guHAPSLDM8WyFb6mO=yg81k`qoKA}4znU%H()TQ+C@%%8m;H|m6)?&=$?5+^z=ZXbZXcs5EB zNmgnr$+xj>m}Px)8&Y|d6#wdt4TRdE#N3O2T4jscpFW!LH9o%#L8TTCMTqI9pRnow z!azUW^~}Co!iKq*ZoLX@4G-tf!sCJ)5I`jj{=*u)<>lCd-C7bVQCf1V4XA?PC zO0qxLjON2Hg!&;VVVwV>Hd<-juqiy}F?*Ew*PPRL&nTjgcLLhn2`Z|0uPH`aOsWN= z2p|}_bA40d10317sDa#{7??{tGHL18lxfSHkXIMHX*x{fmL!3UkRa0utVKkz$iNcs zP;~kiUAA%Rt68(ivgh4OBvI1H7WNGI>7gFq7+FRJ7>BYTm_@~L7AF=6|Pq%CcTxcp<++A?uazv?K`F(;$bw^TXj;pNeZSQ(YS`{<(iqM z7}{tbgfa!MAUr_S1>u%&PzHDRuB!!PE~T_Ff+y%Wn_WWB@>&P#Tw~}%GvvL9xUOjZ z3)(n$$!uud=5;>PNXJVQ#Cm``_dxOJSK005@M1Xml>r;^Q&As^jhRy{+Z#go8R>Vf zw!X^I)gUrkB1~yKF?FrgOe&AuaukBg-qq^}HGagb_XXgR=mNmfvo+8laA2V_hMQj? zZHj~_nEV8tzZj{?Sx2`Fi!KDfXCcr?Vl+)-Vl{8VbECNua};peM#jSne_TT(^6H`U z7)SUlk?Q*wax)6B5tXuw{-X0Gr(&}X$EUD8S{~l0~3B8PO1Q1)7N^lD(tf+-?# zNh%gMZ2-<_fdiQPgi?tRK6)XoxQz~F%OAaAC!OE6SH5U7rNt6)G$V~&R|rt4g8tfJ zXn)Q|n_W~;XxkC`BPGYbp-Xw5`&bNhD3l$Xg_(FjC{v-Ppr<1K-;jx1PtN}{s5ju`okqvTX!TXzCKd*@#~BZ>^zxd zwKv&KvzJA*K(m{Api?&TlnWVo{Lqbak0pGIe5_PvXk}+uk_xAzEg#olK>is;N)3fK zl1G)BpdCk;dE0}SD8*K(9#R;9w|zy{fKejWJD?!c>L7zuExM|^D5dk+dd@C>&>1^H z%NPjkoqD1*{FGO4wTgt6JNHg9EtN!q2zI09qY7dUA>6rUl!Z%IsOlI2%Llo1CFUa8 zDpg5y6XYBq2e|DU2-A~sA{~fg3;Lr|V~JhUBG_QTpqN6gDB__IhW*4D#-GsCcL~A6 z;s%HeiK(Kwq6Iya5+bE5@?Dcy-z4Npu>b@Zku*bRb$s7%=jf3b>Z4;_G@J007N>nu zJQNglDJ}C%&@}!vNQYzA{Vu|iu}7K=L=<61MhTQFh4MMJWI0sma3)jqzrH(t ztyN-aLSciWaE|9vma@S~5Jh&w4hh!wa4b0%Tb2Y!t@XucaFK=CT^f`V3^$S169D>r z%zUnKSd+Tsd021JA=Kje$!&fF6z}+Xy)Pix{l!rW5w7nfN$p|QeCXE*D#+m@+`sg8 z8W^%AeZ=B~g5Y`mTu_gc)aFNLwv(#mMXCbs74HmNzH_! zFNkrPSTR4CL51@i+2NpZ97QU9M`ciS|4m59Ti@e@lcI~ znpDkyhGhDvIp%%?*C^c^Gov4Q9)G?0xn8Us&Bvr}4E}G9)FZnXW%2OiEO(N4C>$x< zFMLrZynr9qTzKj^$T{??>;Pq<>I&BRKzTA12Z?GDT^4*&KnO<%Tur80$eCH`K5YC# zKbkdb0uEfb8f2Js04_MkY6d!-9&YanmuVx_mY^c>21yUvcVrg&=#ggHK$R#~-8*Qv zYCBEyC@>yR#Mu*87Pin1ah`-Iz+b879u>b(Y)cWRRDox}eZAznI3W8j>Wq%I95L`| zlI_+P7J0DW1I+v>p$-(V!a$%&{Z){SNTf4|m&I&{!YV>IRs2+_0q{6rtf^m#>RU#_Cxemz z@s;%E%(V|{6~c9J?q;D!A;OQshJ1=;L$-%AoUr~Fp-Yv48ydi5=W$ykwPRDlt--?W z4wCB__s3rl%*Q3V7|bik3pxp+t+E+8| zHoo){2-7(;1ITG)&aGrYYveO6&dkaw+oK0PQu24p&XI~jVp8B_G%$(zxL2Smr3dml zY4qofJOiK_-7B3CwB-;Ny+k;0DkLp*w?a*4685wV@wpQ zI0+w%K#l;d6X{@aOAf^i*D;-w+tDk)xm{?dP8h!U0!g>x^WFJoUl=V0mh8X+nGY-0 zrzS$Xg|@qowAY6Wy|l3$H}A?zOLOKD`z#7Ce@97vE8bXo>C)@da=%2QwWBkl)k~Qp z;=@)g8@e}mh+Ok{nEjwqXUU9_TT+9^b{s7y$<*`^WH*;ZD3@E@H&aIl6|~L^ZV!CI zOPBWe2i%}r3uI;)6#U#jn5iUnwH-lEj-+B33|8*f#+_vCmhW5d|>NF=MnB&;{y@4cUKh*8z~p z)I;vQk@r5CDCr2(m+zVu(oqw9~{vf5%d7(d!z&LyAk0PZDyib#%+3p846qUKRaCPlTV z^d~wl3MvG(D7%sFY)}%b_gj~{8X$Ux9M*Ut!}B@mzK-E={nv(pr9T3-_7L5YNtz6M zedJdOa6(9mmVXdVx33*#=G?-bjjLM^#JY>tKb$s{oi9%sYo6)Z^+(A%3nV4Rc!9hq zEKB7@Q+2*6JHKhDu4u7qKCs!p6s1q^&ezP(o-ojx`yw*y2eG9^RF`a8Ne5aPE`;=hj!AqGfyno19$0Bt16zzPmE^I--^Zh&jImx(r;#(@Euw)6Bgou>(82>;t&_w2zJZ z&wsf>H|gw3Y)H@f6mP%2f9MD~Jd$qCGM%}SyvGd9mi)8q%;B?tbGoJ8x4BjC>oW7H z4ZIf7QpyJcf~B1*A!g&#bXf<7g35M*oKXV=&G}w@WOfy5X~ZD(F{Zlr!?E{tXY=eq z`bkNCU4mi_aveRQ0H6@RUFatI6`@v?WNhxA9@l7ET;ss_#}qVmn@6A#NSK^(@8z|~ z(`A%~Gb8@PYLVULU54puw!tQh#APwzBwAJ%AA4kB0n{z%r>ECC)jy|4lzqpbI54WS z(vWfvnAXDEkz;t^P&6t3n6SbEx=((+X4v>R-=SENW;>9$o8cryMI$Z~+jeWpjKND_<1 zYZzf=NI>`FQDovV=cJ0{Xf9|(^>zHTU%JnY0_?BbyN_wf=6m`;H9nk{uqaG*^;ku5 z5Y=*P??0W~15S3*4zJaYg>FA8jO@?zXG$3S6lNbr;*)THzZi?;={nITlu7?AYYzi^#m4VZ}$s;G1IJ+bBRl7F<)3=021mNAEgF4;#(Huj|1xCuc@6WeqP zZ%}&`{2g6NPMK$|^5Xn@Rh{6H3ZR#cCs3f2D6)j|0xHnyI4gDIk_6U-bFKD`0zsI^ zs!=~C#W{g#S>`kq1(Bgj(Bb-sMtGY<49+iFQ7m8n3yFsx%@-|&7CK|6-8yV`T+)>-6CiKGixog@|MM=mSg~L-%$3yeGN!iv;Mp*T4oa}ei zVmGg&fZ~L+P(B!kbMNz71^c8>E<<%EEk9HgYAh{rU39pHcrvy^l77NcO!2#nZTyi| z>B)wxX7XiUX=#%#jAJ?o#+D!N*+0veoj5yO4V{VHZGq*4Lrzhs05_{#Md<|w-19sq zbcV4?s~p7k3x_4RJX98Tb_wVCVTE=K4#AijAg4k^KF!i#p+Q!_23w`lE~oh>LBkV! zuXlUb@Y;n}OIi`R)ukqkOm~2>`=LP(wsA<$B9_*f$`Q4_v4%WKlyEUz5g)y^zn~O3 zlKCm0dwhJqXh9<5gj0GpWk-SnDT(E`5_^S4pw&{-lh=oqEtK!7DMaf_|wVAveka@_lLR!wIct-cTb5WEM*^R$d*iavQo|) zTXv5Deh)0ze_}Pcpo%5PVSX|v&R2ewIUa}9_~bj<{IKOsm)2bAw^mr&e8_H;hiv+jcBdT_D$+Gx``D<@dF-b%^gX6Qhgja#YPK+h`X`CVmUg#Ea-vHg ze&4|1F2#>9RXGW_5Mq?a}>Su{<$Lh zt1mmI`w)n0^YqU9vEZ0p8XTLd(GXMK3F*Dtd)764)ooH9N%m)>zo|^+jcW|?u~$uk zdYd-*(B$Ukx{gR7=<%I!B+3GQO?(Ln0sS&7h$PQk| zIDU_NS&R3G5ZeDU#OJX9<5cp{X__GKbLGF~VAfC`L99KRd$Zqu_)M zP#Y1vCy;9y4-JQh1eFe~ezv3)_?mvLtt*!s#Sv`;cY4emrf~;oDPmB>(_wUV z#s{aRi1;igC+P3j^wM?BsqCxXp)rQxlS=p9menNOo`R@YUk#Ipdm$lguK)PDIw4|U z{4!XHmZ$F-oX)W#qfft@O$92aw||*uQ%E^WyU$eMQWZ&^_#qjEFG?c1o#-I-E~Wn- zn?XS*UCuWr{0pn1YU2-e-Pm00ZFzL|y{CBA{3%2|9T#YmC{TxJ zq*6b2N%GxShNNodRG=+oHLVW|#IaxK?%Z%)EhpWlFb~BJI|Be8)RrHYkdfQCee+uN z+rVc_>gD#7Vr>UmNZ%VXIE3Q7aZo^Bdbau1web9VvtQ=w--I4|!R0EwAgoh_k=O^cxDb$FMCnxh2i_O&irZmL2Zg zt9n;yp$C$34-`PKOjQeSIZ6CxeA-OL1Te#uh}+{vp$r2U#<{<{B6u^-VRKp&Eam zsKMzN7zr^bhtT6D+>)l`5E3S8dD-ijr%SB66H8~|Cjb~+M-vNb{AsfOjbe#RwZY}{ zGU%~fKG-dRKs1soi%E40`v8b`&2bMyCIo@o9HBYCj_@Y9i=Z3Ov0jU^nTQx6I+}wk&KHf}l zO)Z96Ir5Jpe*6bPQx36tM{#_7Sb98TZ=AOv-w#MzW!Yc~R4sT|+kLYA*x@=0 zd&$LqvfKouAZf5t7)V}tc=A7MeruJ@%p{oA0FdZDv}1=(5ata}yhLuSD@(w*%3OgI zG6w3G4~^wrh8wm5R<9_zWjr6d$Vhg&#@Wm7&)dpQ*RcgMIV=zY79!iTPd`9PWeg@& zM?VYMWs7+SXkx%+rgEJA(myilLzsDj&6SSSu+y2jLibLtQv_0Jca#u+5x$nN-|2E- z11*1Zaoyk381K}vOqd$+bZ*{XZ1nAAWNcaNhw}fjSD|j(nX!;s8yONwflQknR0UjHRb zmsj^STv&W;h6{S5;ec#Sdx}TJO2i_$Y|CI$@V)ZVaN@D+d+>3(W+n>Pr%1l-hp!Nc zpS^K8qpwD2ic_)$IWzjx5IGJF=B!dQ^sN@)UsjzEd1i$SqA%giiK}r6W0s{Xpmn&+ zfmanuKIUA@*t6!#*3jE1RH0Y9@$<+?0On&1w6F1l;pCuY{%JStMQnyA)uiwEZ5~cG zI$|Q@k5Efmj8v;%7J_baGEHN84KxLco`q(4&Y|1#bE$Z&q>_iJMikLgiC^QFN-8** zUlw<|#J#v<&D9>r9Thf&Fn&aRy|_mo$uf}t%C;b?5f_WpOsS%i9nypBM$Gz5?@%aSL*~6h zcAI<@*XcCto{AQ1ToQiQ6*cu3KS2UbMURtXO@|l5nt4nuzTPx7+6m1LgnqYNJonEh z$0VT?h)pP)|J7sY5?Yd$I^9i@x<`M=*~_w;UvZy( zw3FHS`ebb>Or5L)$Db>6?+c}=U2&5k7y;G0zC`$$-p6N9tE6{hS#%;W%e-Xu{!XqS zr&Rw4eCAt#&EGh3;L?uv-~VIJR)bH|!E&})4Zio)Ri>z08&v1&AY3YwYJI4tnt!%y zY`2T!rheR}&J|B@_*mN#t5!C76`L)yZB~`+1v?jGIx4ntOk(Y~2j>E>Lo(+4`f~uB}ZwDDa9z8=O@KDyxbI!IKa`WgEemSkmj@zR(gSq zHf*|MUiFye=axUlz#E}bZ-o4u1(}g!PBt{D%mp;>Z1G|`Gk)Af0Vsioq75dX>;12^ zWo3Ru-d855GKNlBOxEZx&5Qjt%4L=0HbNw9C^>*K$V!fs}OgW^;p!45luS# zT7-`HC+5%$iT+W;^J28;3U1J;olPs{vTA$5#It;qNK=)T=P6{}*P_{Q<^KT%{-8FspVfI%=qeS?o`DMUDZZ!Z0C$Za?-39FXMTa(P9G1jz)f41_tP&RU{kr{d_!>S+^F;Qku&G@ z=jD4C7VsKm^!~c({~Gmi6SnydG&71IPQuti!{9;v-;D|Xc@zE%ivRGR`hT`5sK~=3 U{C{)E|0N|X6qKJK;(yiu0S%-_jsO4v delta 50 zcmdmdShlB0wxNZwg{g(Pg{6hHg>4JFj@|a!IQBD4BCKo-K)?ZnXP6imwCvb{JOF<3 B3uFKQ