compaction: prevent huge multi-level compactions

[sumeerbhola]

CockroachDB running with multi-level compactions sees huge ones like the following. When each compaction can take 100+ seconds, we are unable to schedule new compactions (including those out of L0), which causes L0 sub-levels to rise and the lack of compactions out of L0 causes admission control to severely throttle traffic (see https://cockroachlabs.slack.com/archives/C067NG669KJ/p1701464855212159?thread_ts=1701462083.011679&cid=C067NG669KJ)

I231201 20:19:15.856331 2901155 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 96006 [JOB 13324] compacted(default) multilevelL4 [18416634] (3.9KB) Score=1.59 + L5 [18414708 18414709 18414710 18414711 18414712 18415613 18415615 18415616 18415617 18415618 18415619 18415620 18415621 18415626 18415629 18415630 18415631 18415632 18415633 18415634 18415635 18415636 18415638 18415640 18415648 18415649 18415650 18415651 18415652 18415655 18415656 18415657 18415658 18415661 18415662 18415663 18415666 18415667 18415668 18415669] (304MB) Score=1.04 + L6 [18220672 18220680 18220710 16893393 16922170 16922167 16922178 16922207 16922237 16922247 16922256 16922325 16922354 16922361 16922399 16922424 16922445 16928991 16929010 16929074 16929088 16929107 16929158 16929200 16929240 16929284 16929298 16929316 16929364 16929811 16929866 16929882 16929893 16929923 16925675 16925534 16925590 16925508 16925510 16925512 16925515 16925547 16925467 16925481 16925507 16925527 16925560 16925586 16925598 16925617 16925649 16925672 16925682 16925691 18238329 18238569 16875135 16875148 16875278 16875396 16879808 18413940 18413979 18332121 16879865 16879873 16879888 18414201 18414229 16888272 18332742 18332749 18332756 16886831 16886842 16886841 16886847 16886851 16886859 16886869 16886899 16886932 16886963 16886991 16887021 16887051 16887063 16887066 16887073 16887078 16887104 16887130 16887161 16887183 16887200 16887232 16887237 16887245 16887251 16887257 16887285 16887323 16887345 16934226 18242059 18242101 18409574 18408733 18412762 18245912 16876172 16876209 18245969 18246075 16876247 16876298 16876348 16876394 18396050 18396051 18397571 18397574 16942324 18219902 16878967 16878973 16879004 16879023 16879035 18245989 18246076 18246137 18246220 18246263 18246325 18246373 16879064 16879090 16879112 18399339 18399340 18399341 16886332 18400802 16879236 16879244 18401241 18220130 18220166 16883610 16972077 16906094 16902600 16902606 16902611 16902614 16902618 16902620 16902725 16902832 16913182 16913188 16913213 16913253 16913276 16913299 16913319 16913338 16913348 16913356 16913375 16913403 16913450 16913472 16913477 16913493 16913517 16913571 16941430 16941439 16941496 16941511 16941519 16941553 16941590 16941604 16941620 16941628 16941673 16941686 16941700 16941717 16942585 16942600 16942640 16942688 16942713 16942738 16942752 16942764 16942777 16942847 16942875 16942887 16942896 16942924 16942948 16942975 16943040 16943043 16943057 16943062 16943072 16943098 16943160 16943173 16943207 16943230 16943263 16943287 16943318 16943348 16943386 16943431 16889109 16893817 16893819 16893850 16893882 16893901 18246230 18246267 18246338 16893991 16894001 18333073 18333089 18401486 16894142 18246624 18246927 18246978 18247072 18247134 18247194 16897654 16897703 16897749 16897755 16897800 16897848 16897870 16897873 16897899 16898002 16898020 16898021 16898022 16898059 16898125 16898141 16898182 16898245 16898266 16898286 16898308 16898340 16898389 16898422 16898457 16898472 16898550 16898590 18220160 18220176 18220193 18220093 16899833 16899871 16899875 16899877 16899892 16899898 16899910 16899931 16899932 16899951 16899967 16900006 16900011 16900040 16900075 16900091 16900108 16900123 16900157 16900169 16900223 16900225 16900256 16900297 16900315 16900317 16900342 16900367 16900385 16900388 16902793 16902737 16902628 16902686 16902693 16902704 18225547 18225605 18225943 18225997 18220189 18220204 16959104 16959150 16959178 16959405 16877594 16877622 18239066 18239127 16862618 16862503 16862716 16862526 16862555 16862563 16862580 16862594 16862608 16862622 16862671 16862690 16862708 16862737 16862770 16862800 16862900 16862914 16862928 16862941 16862969 16863002 16863021 16863031 16863046 16863056 16863063 16863069 16863075 16863080 16863086 16863091 16863095 16867437 16865543 18240269 16865647 16865680 16865701 16865711 16865744 16865778 16865801 16865807 16865820 16865917 16866001 16866172 16866178 16866316 16866360 16866428 16866438 16866514 16866530 16866539 16866581 16866597 16866601 16866604 16866624 18220569 18220570 18389735 16980922 16980935 16980968 16980993 16981001 16981008 16981014 16981036 16981795 16981117 16981129 16865495 16867467 16868179 16869269 16871428 16873333 16873349 18247712 18247716 18247731 18247738 18247739 18247740 18247741 18247746 16873487 16873500 16873531 16873540 17048638 17048644 18258245 18258246 18258247 18258248 18258249 18258250 18258251 18258252 18258253 18236939 17115052 17115035 17115066 17115067 17119151 17119156 17119634 17119711 17119781 17119793 17119868 17119945 17119973 18219804 18411625 16889996 16889918 16889921 16889925 16889930 16889935 16889953 16889956 16889977 16890126 16893338 16893351 16893373 16893388 16893397 16893569 17427645 17427676 17431180 18229464 18229473 18229479 18229516 18229546 18229598 18229699 18230021 18230035 18230044 18230076 18230107 18230165 18230233 18230260 18230279 18230324 18230337 18230349 18230391 18230412 18230472 18230523 18230553 18230599 17458353 17456755 18219628 18219643 18219651 18219659 16984690 16984597 16984601 16984605 16984609 16984612 16984638 18220645 18220655 16899179 16899111 18244561 16899168 16902829 16902836 16902845 16902862 16902874 16902885 16902891 16902907 16902916 16902941 16902962 16902974 16902985 16902993 16903023 18401863 16915284 16915311 16915324 16915343 16915370 16915409 16915457 16915717 18402000 18402001 16915883 16915918 16915949 16915978 16916003 18333492 18333493 18333514 18333536 16916097 16916175 16916216 16916250 16916278] (24GB) Score=0.98 -> L6 [18416786 18416817 18416851 18416885 18416912 18416926 18416967 18417001 18417029 18417047 18417078 18417094 18417113 18417141 18417173 18417185 18417238 18417268 18417294 18417326 18417353 18417388 18417413 18417433 18417461 18417483 18417523 18417542 18417565 18417595 18417631 18417647 18417662 18417666 18417674 18417682 18417695 18417702 18417710 18417716 18417728 18417735 18417744 18417762 18417785 18417835 18417847 18417891 18417914 18417938 18417983 18418005 18418022 18418122 18418148 18418187 18418204 18418217 18418250 18418291 18418322 18418362 18418379 18418387 18418399 18418406 18418412 18418434 18418442 18418453 18418499 18418526 18418542 18418579 18418608 18418632 18418670 18418700 18418710 18418749 18418765 18418783 18418817 18418882 18418939 18418967 18418989 18419022 18419052 18419067 18419113 18419156 18419191 18419221 18419232 18419239 18419269 18419276 18419288 18419308 18419314 18419321 18419341 18419348 18419354 18419377 18419384 18419394 18419410 18419417 18419424 18419443 18419450 18419474 18419499 18419527 18419565 18419599 18419629 18419665 18419726 18419759 18419785 18419830 18419878 18419932 18420045 18420162 18420352 18420358 18420368 18420394 18420412 18420419 18420449 18420460 18420493 18420539 18420565 18420589 18420628 18420645 18420678 18420702 18420716 18420773 18420811 18420848 18420896 18420934 18420958 18420988 18420993 18421016 18421027 18421035 18421048 18421076 18421089 18421155 18421188 18421229 18421282 18421309 18421331 18421377 18421414 18421450 18421504 18421542 18421566 18421610 18421627 18421637 18421673 18421703 18421711 18421776 18421824 18421868 18421889 18421930 18421949 18421962 18422009 18422050 18422067 18422081 18422130 18422168 18422196 18422242 18422302 18422339 18422381 18422393 18422408 18422428 18422432 18422456 18422516 18422569 18422597 18422616 18422655 18422669 18422702 18422747 18422800 18422842 18422905 18422956 18423048 18423081 18423112 18423172 18423224 18423270 18423300 18423352 18423384 18423415 18423482 18423530 18423560 18423608 18423665 18423693 18423720 18423778 18423835 18423864 18423901 18423949 18423975 18424007 18424048 18424106 18424169 18424214 18424235 18424284 18424324 18424378 18424401 18424448 18424459 18424501 18424540 18424602 18424656 18424680 18424717 18424738 18424774 18424826 18424874 18424901 18424916 18424945 18424980 18425026 18425050 18425089 18425097 18425109 18425140 18425160 18425205 18425254 18425296 18425310 18425326 18425368 18425408 18425458 18425479 18425527 18425546 18425555 18425588 18425651 18425701 18425745 18425767 18425779 18425802 18425854 18425918 18425944 18425974 18425997 18426013 18426054 18426079 18426142 18426219 18426234 18426257 18426298 18426362 18426431 18426469 18426493 18426518 18426561 18426628 18426692 18426729] (19GB), in 1147.6s (1147.7s total), output rate 17MB/s

Even sub 1GB compactions can be disruptive wrt AC behavior, because they prevent more incremental compaction completion e.g.

I231201 20:29:27.462310 5032934 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 114538  [JOB 18574] compacted(default) multilevelL2 [18430848] (6.2MB) Score=3.62 + L3 [18426363 18426374 18426380 18426385 18426388 18426392 18430198 18430204 18423291] (74MB) Score=0.77 + L4 [18396429 18396430 18256818 18343713 18343721 18399081 18399082 18330840 18414291 18425943 18425953 18425972 18426416 18399555 18399083 18402124 18402125 18279392 18404459 18404460 18404461 18403512 18407869 18343676 18343685 18343692 18276331 18276332 18294188] (570MB) Score=1.07 -> L4 [18433207 18433217 18433225 18433231 18433239 18433255 18433267 18433283 18433295 18433298 18433307 18433315 18433323 18433328 18433336 18433344 18433354 18433368 18433382 18433396 18433405 18433410 18433416 18433422 18433429 18433434 18433441 18433446 18433458 18433466 18433478 18433491 18433504 18433512 18433517 18433524 18433530 18433541 18433548 18433555 18433562 18433571 18433587 18433608 18433615 18433623 18433631 18433641 18433647 18433654 18433661] (650MB), in 37.5s (37.5s total), output rate 17MB/s
I231201 20:29:29.805967 5021501 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 114562  [JOB 18567] compacted(default) multilevelL2 [18432616] (6.2MB) Score=3.56 + L3 [18425600 18425609 18425615 18425623 18429891 18429894 18429900 18430098] (73MB) Score=0.78 + L4 [18361729 18361730 18268855 18383408 18383409 18383410 18383411 18383412 18392945 18425718 18392947 18392948 18425770 18425775 18425698 18383301 18383302 18379181 18425693 18425702 18425716 18428997 18429003 18301808 18246710 18391922 18391923 18427440 18427460 18384603 18384604 18384605 18278197] (653MB) Score=1.07 -> L4 [18433154 18433170 18433184 18433194 18433203 18433213 18433220 18433226 18433235 18433247 18433258 18433294 18433304 18433314 18433324 18433330 18433340 18433345 18433355 18433362 18433374 18433399 18433403 18433409 18433417 18433420 18433427 18433431 18433438 18433443 18433450 18433462 18433474 18433500 18433503 18433507 18433514 18433526 18433531 18433540 18433545 18433551 18433556 18433566 18433580 18433588 18433604 18433614 18433618 18433625 18433628 18433635 18433639 18433645 18433650 18433656 18433665 18433670 18433678 18433683] (732MB), in 43.2s (43.3s total), output rate 17MB/s
I231201 20:29:38.166347 5014298 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 114667  [JOB 18561] compacted(default) multilevelL2 [18430177] (6.6MB) Score=3.52 + L3 [18428750 18428759 18428417 18428419 18428422 18428451 18428460 18428471 18428479 18428485 18428487 18428490] (75MB) Score=0.80 + L4 [18394249 18389374 18389375 18338417 18414124 18414136 18290130 18428941 18428524 18290135 18290136 18290137 18389505 18428518 18428530 18423970 18423977 18428298 18428309 18423572 18423582 18398185 18398186 18339599 18429988 18429994 18428503 18425924 18425934 18391648 18424021 18424032 18424057 18321586 18284924 18393493 18393494 18393490 18393492 18383741 18383742 18383743 18410137 18410161 18383526] (841MB) Score=1.07 -> L4 [18433122 18433131 18433145 18433156 18433171 18433195 18433206 18433219 18433223 18433236 18433238 18433253 18433269 18433292 18433300 18433313 18433321 18433326 18433333 18433347 18433365 18433370 18433384 18433397 18433400 18433406 18433411 18433421 18433425 18433433 18433440 18433447 18433456 18433469 18433487 18433501 18433511 18433515 18433519 18433525 18433533 18433543 18433549 18433554 18433558 18433563 18433575 18433591 18433606 18433620 18433626 18433634 18433636 18433640 18433646 18433651 18433657 18433662 18433672 18433680 18433687 18433692 18433699 18433708 18433719 18433740 18433748 18433753 18433763 18433772 18433780] (922MB), in 53.9s (53.9s total), output rate 17MB/s
I231201 20:29:39.384018 5084598 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 114688  [JOB 18598] compacted(default) multilevelL2 [18431874] (3.8MB) Score=4.04 + L3 [18429137 18429168 18429177 18429182 18429186] (54MB) Score=0.73 + L4 [18427230 18427252 18427258 18396683 18396684 18429178 18379759 18379760 18388796 18426247 18425092 18425095 18392965 18392966 18253673 18236698 18426920 18426944 18425085 18426658 18426674 18426687 18424222 18424231 18219264 18402485 18424200 18405827 18404536 18404538 18381829] (541MB) Score=1.08 -> L4 [18433430 18433437 18433451 18433467 18433480 18433498 18433502 18433510 18433518 18433523 18433528 18433534 18433539 18433546 18433552 18433559 18433561 18433567 18433574 18433586 18433600 18433611 18433619 18433627 18433633 18433642 18433648 18433655 18433660 18433666 18433673 18433675 18433682 18433686 18433691 18433701 18433710 18433725 18433738 18433742 18433745 18433747 18433752 18433760 18433764 18433768 18433774 18433779 18433782 18433787 18433793] (599MB), in 33.1s (33.2s total), output rate 18MB/s
I231201 20:29:43.159515 5066128 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 114743  [JOB 18592] compacted(default) multilevelL2 [18431885] (6.1MB) Score=3.97 + L3 [18428912 18428918 18428925 18428506 18428511 18428515 18430939] (80MB) Score=0.74 + L4 [18396308 18396309 18397503 18397505 18397506 18397507 18397508 18397509 18397510 18397513 18397514 18397515 18397516 18397517 18380074 18380075 18380076 18390243 18423972 18423979 18406234 18406235 18409999 18410008 18410018 18405402 18405403 18301351 18389860 18389861 18382990 18382991 18379284 18417623 18417628 18405266 18405714 18405715 18300796] (678MB) Score=1.08 -> L4 [18433348 18433353 18433358 18433376 18433392 18433398 18433404 18433408 18433414 18433424 18433432 18433439 18433445 18433448 18433457 18433471 18433483 18433499 18433505 18433513 18433521 18433527 18433532 18433537 18433544 18433550 18433557 18433565 18433576 18433584 18433596 18433613 18433624 18433630 18433632 18433638 18433643 18433653 18433659 18433664 18433669 18433674 18433684 18433693 18433703 18433716 18433736 18433744 18433749 18433755 18433762 18433769 18433776 18433781 18433786 18433788 18433794 18433798 18433804 18433810 18433813 18433833 18433847] (764MB), in 43.4s (43.4s total), output rate 18MB/s
I231201 20:29:55.181551 5127531 3@pebble/event.go:703 ⋮ [n6,s6,pebble] 114955  [JOB 18612] compacted(default) 
multilevelL2 [18430457] (3.5MB) Score=4.08 + L3 [18427933 18427919 18427934 18429981 18429986] (54MB) Score=0.71 + L4 [18376778 18376782 18420247 18401189 18401190 18401191 18273364 18376390 18427986 18427992 18427996 18392568 18391163 18391164 18321767 18420220 18249328 18405180 18405719 18405720 18405721 18395511 18395512 18395513 18395514 18384944] (487MB) Score=1.08 -> L4 [18433658 18433663 18433667 18433671 18433676 18433685 18433690 18433706 18433715 18433727 18433741 18433750 18433754 18433759 18433765 18433773 18433777 18433783 18433792 18433795 18433800 18433807 18433812 18433823 18433840 18433846 18433859 18433862 18433869 18433870 18433882 18433886 18433891 18433898 18433902 18433907 18433913 18433922 18433937 18433947 18433957 18433966 18433970 18433979 18433985 18433998] (544MB), in 28.9s (29.0s total), output rate 19MB/s