Skip to content

Commit

Permalink
Improve performance by usually starting from earliest-added items
Browse files Browse the repository at this point in the history
  • Loading branch information
@pxpeterxu
pxpeterxu committed Nov 6, 2021
1 parent cc8447f commit ec9d327
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 146 deletions.
10 changes: 7 additions & 3 deletions lib/commands/cleanJobsInSet-3.lua
View file
Expand Up @@ -36,8 +36,12 @@ local rangeEnd = -1

-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeEnd = limit - 1
rangeStart = -1 - limit + 1
rangeEnd = -1
end

local jobIds = rcall(command, setKey, rangeStart, rangeEnd)
Expand Down Expand Up @@ -91,8 +95,8 @@ while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do

if deletedCount < limit then
-- We didn't delete enough. Look for more to delete
rangeStart = rangeStart + limit
rangeEnd = rangeEnd + limit
rangeStart = rangeStart - limit
rangeEnd = rangeEnd - limit
jobIds = rcall(command, setKey, rangeStart, rangeEnd)
end
end
Expand Down
12 changes: 12 additions & 0 deletions test/test_queue.js
View file
Expand Up @@ -2853,6 +2853,18 @@ describe('Queue', () => {
expect(len).to.be.eql(2);
});

it('shouldn\'t clean anything if all jobs are in grace period', async () => {
await queue.add({ some: 'data' });
await queue.add({ some: 'data' });

const cleaned = await queue.clean(5000, 'wait', 1);
expect(cleaned.length).to.be.eql(0);

const cleaned2 = await queue.clean(5000, 'wait');
expect(cleaned2.length).to.be.eql(0);
expect(await queue.count()).to.be.eql(2);
});

it('should properly clean jobs from the priority set', done => {
const client = new redis(6379, '127.0.0.1', {});
queue.add({ some: 'data' }, { priority: 5 });
Expand Down
147 changes: 4 additions & 143 deletions yarn.lock
View file
Expand Up @@ -1290,7 +1290,7 @@ caching-transform@^4.0.0:
package-hash "^4.0.0"
write-file-atomic "^3.0.0"

call-bind@^1.0.0, call-bind@^1.0.2:
call-bind@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c"
integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==
Expand Down Expand Up @@ -2151,37 +2151,6 @@ error-ex@^1.3.1:
dependencies:
is-arrayish "^0.2.1"

es-abstract@^1.17.0-next.0, es-abstract@^1.18.0-next.2:
version "1.18.0"
resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0.tgz#ab80b359eecb7ede4c298000390bc5ac3ec7b5a4"
integrity sha512-LJzK7MrQa8TS0ja2w3YNLzUgJCGPdPOV1yVvezjNnS89D+VR08+Szt2mz3YB2Dck/+w5tfIq/RoUAFqJJGM2yw==
dependencies:
call-bind "^1.0.2"
es-to-primitive "^1.2.1"
function-bind "^1.1.1"
get-intrinsic "^1.1.1"
has "^1.0.3"
has-symbols "^1.0.2"
is-callable "^1.2.3"
is-negative-zero "^2.0.1"
is-regex "^1.1.2"
is-string "^1.0.5"
object-inspect "^1.9.0"
object-keys "^1.1.1"
object.assign "^4.1.2"
string.prototype.trimend "^1.0.4"
string.prototype.trimstart "^1.0.4"
unbox-primitive "^1.0.0"

es-to-primitive@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a"
integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==
dependencies:
is-callable "^1.1.4"
is-date-object "^1.0.1"
is-symbol "^1.0.2"

es6-error@^4.0.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d"
Expand Down Expand Up @@ -2617,13 +2586,6 @@ fn-name@~2.0.1:
resolved "https://registry.yarnpkg.com/fn-name/-/fn-name-2.0.1.tgz#5214d7537a4d06a4a301c0cc262feb84188002e7"
integrity sha1-UhTXU3pNBqSjAcDMJi/rhBiAAuc=

for-each@^0.3.3:
version "0.3.3"
resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e"
integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==
dependencies:
is-callable "^1.1.3"

for-in@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
Expand Down Expand Up @@ -2746,7 +2708,7 @@ get-func-name@^2.0.0:
resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41"
integrity sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=

get-intrinsic@^1.0.2, get-intrinsic@^1.1.1:
get-intrinsic@^1.0.2:
version "1.1.1"
resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6"
integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==
Expand Down Expand Up @@ -2955,11 +2917,6 @@ has-ansi@^2.0.0:
dependencies:
ansi-regex "^2.0.0"

has-bigints@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.1.tgz#64fe6acb020673e3b78db035a5af69aa9d07b113"
integrity sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==

has-flag@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-1.0.0.tgz#9d9e793165ce017a00f00418c43f942a7b1d11fa"
Expand All @@ -2975,7 +2932,7 @@ has-flag@^4.0.0:
resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==

has-symbols@^1.0.1, has-symbols@^1.0.2:
has-symbols@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423"
integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==
Expand Down Expand Up @@ -3279,11 +3236,6 @@ is-arrayish@^0.2.1:
resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=

is-bigint@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.1.tgz#6923051dfcbc764278540b9ce0e6b3213aa5ebc2"
integrity sha512-J0ELF4yHFxHy0cmSxZuheDOz2luOdVvqjwmEcj8H/L1JHeuEDSDbeRP+Dk9kFVk5RTFzbucJ2Kb9F7ixY2QaCg==

is-binary-path@~2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09"
Expand All @@ -3303,11 +3255,6 @@ is-buffer@^1.1.5:
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==

is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.3.tgz#8b1e0500b73a1d76c70487636f368e519de8db8e"
integrity sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==

is-cidr@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/is-cidr/-/is-cidr-4.0.2.tgz#94c7585e4c6c77ceabf920f8cde51b8c0fda8814"
Expand Down Expand Up @@ -3336,11 +3283,6 @@ is-data-descriptor@^1.0.0:
dependencies:
kind-of "^6.0.0"

is-date-object@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e"
integrity sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==

is-descriptor@^0.1.0:
version "0.1.6"
resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-0.1.6.tgz#366d8240dde487ca51823b1ab9f07a10a78251ca"
Expand Down Expand Up @@ -3418,11 +3360,6 @@ is-nan@^1.3.0:
call-bind "^1.0.0"
define-properties "^1.1.3"

is-negative-zero@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24"
integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==

is-number-object@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.4.tgz#36ac95e741cf18b283fc1ddf5e83da798e3ec197"
Expand Down Expand Up @@ -3513,14 +3450,6 @@ is-promise@^2.1.0:
resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1"
integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==

is-regex@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.2.tgz#81c8ebde4db142f2cf1c53fc86d6a45788266251"
integrity sha512-axvdhb5pdhEVThqJzYXwMlVuZwC+FF2DpcOhTS+y/8jVq4trxyPgfcwIxIKiyeuLlSQYKkmUaPQJ8ZE4yNKXDg==
dependencies:
call-bind "^1.0.2"
has-symbols "^1.0.1"

is-regexp@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069"
Expand All @@ -3541,13 +3470,6 @@ is-string@^1.0.5:
resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6"
integrity sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==

is-symbol@^1.0.2, is-symbol@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.3.tgz#38e1014b9e6329be0de9d24a414fd7441ec61937"
integrity sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==
dependencies:
has-symbols "^1.0.1"

is-text-path@^1.0.0, is-text-path@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/is-text-path/-/is-text-path-1.0.1.tgz#4e1aa0fb51bfbcb3e92688001397202c1775b66e"
Expand Down Expand Up @@ -5007,12 +4929,7 @@ object-copy@^0.1.0:
define-property "^0.2.5"
kind-of "^3.0.3"

object-inspect@^1.9.0:
version "1.10.2"
resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.10.2.tgz#b6385a3e2b7cae0b5eafcf90cddf85d128767f30"
integrity sha512-gz58rdPpadwztRrPjZE9DZLOABUpTGdcANUgOwBFO1C+HZZhePoP83M65WGDmbpwFYJSWqavbl4SgDn4k8RYTA==

object-keys@^1.0.12, object-keys@^1.1.1:
object-keys@^1.0.12:
version "1.1.1"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
Expand All @@ -5024,25 +4941,6 @@ object-visit@^1.0.0:
dependencies:
isobject "^3.0.0"

object.assign@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940"
integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==
dependencies:
call-bind "^1.0.0"
define-properties "^1.1.3"
has-symbols "^1.0.1"
object-keys "^1.1.1"

object.getownpropertydescriptors@^2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz#1bd63aeacf0d5d2d2f31b5e393b03a7c601a23f7"
integrity sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==
dependencies:
call-bind "^1.0.2"
define-properties "^1.1.3"
es-abstract "^1.18.0-next.2"

object.pick@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/object.pick/-/object.pick-1.3.0.tgz#87a10ac4c1694bd2e1cbf53591a66141fb5dd747"
Expand Down Expand Up @@ -6344,22 +6242,6 @@ string-width@^4.1.0, string-width@^4.2.0:
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.0"

string.prototype.trimend@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80"
integrity sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==
dependencies:
call-bind "^1.0.2"
define-properties "^1.1.3"

string.prototype.trimstart@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz#b36399af4ab2999b4c9c648bd7a3fb2bb26feeed"
integrity sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==
dependencies:
call-bind "^1.0.2"
define-properties "^1.1.3"

string_decoder@^1.1.1:
version "1.3.0"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
Expand Down Expand Up @@ -6742,16 +6624,6 @@ uglify-js@^3.1.4:
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.13.4.tgz#592588bb9f47ae03b24916e2471218d914955574"
integrity sha512-kv7fCkIXyQIilD5/yQy8O+uagsYIOt5cZvs890W40/e/rvjMSzJw81o9Bg0tkURxzZBROtDQhW2LFjOGoK3RZw==

unbox-primitive@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"
integrity sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==
dependencies:
function-bind "^1.1.1"
has-bigints "^1.0.1"
has-symbols "^1.0.2"
which-boxed-primitive "^1.0.2"

union-value@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847"
Expand Down Expand Up @@ -6879,17 +6751,6 @@ wcwidth@^1.0.0:
dependencies:
defaults "^1.0.3"

which-boxed-primitive@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"
integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==
dependencies:
is-bigint "^1.0.1"
is-boolean-object "^1.1.0"
is-number-object "^1.0.4"
is-string "^1.0.5"
is-symbol "^1.0.3"

which-module@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
Expand Down

0 comments on commit ec9d327

Please sign in to comment.