chore: merge master

docs/gitbook/SUMMARY.md

@@ -49,7 +49,9 @@
 
 * [Introduction](bullmq-pro/introduction.md)
 * [Install](bullmq-pro/install.md)
-* [Groups](bullmq-pro/groups.md)
+* [Observables](bullmq-pro/observables.md)
+* [Groups](bullmq-pro/groups/README.md)
+  * [Rate limiting](bullmq-pro/groups/rate-limiting.md)
 
 ## Bull
 

docs/gitbook/api/bullmq.job.md

@@ -27,6 +27,7 @@ export declare class Job<DataType = any, ReturnType = any, NameType extends stri
 |  [id?](./bullmq.job.id.md) |  | string | <i>(Optional)</i> |
 |  [name](./bullmq.job.name.md) |  | NameType | The name of the Job |
 |  [opts](./bullmq.job.opts.md) |  | [JobsOptions](./bullmq.jobsoptions.md) | The options object for this job. |
+|  [parent?](./bullmq.job.parent.md) |  | { id: string; queueKey: string; } | <i>(Optional)</i> |
 |  [parentKey?](./bullmq.job.parentkey.md) |  | string | <i>(Optional)</i> Fully qualified key (including the queue prefix) pointing to the parent of this job. |
 |  [processedOn?](./bullmq.job.processedon.md) |  | number | <i>(Optional)</i> Timestamp for when the job was processed. |
 |  [progress](./bullmq.job.progress.md) |  | number \| object | The progress a job has performed so far. |

docs/gitbook/api/bullmq.job.movetocompleted.md

@@ -9,7 +9,7 @@ Moves a job to the completed queue. Returned job to be used with Queue.prototype
 <b>Signature:</b>
 
 ```typescript
-moveToCompleted(returnValue: ReturnType, token: string, fetchNext?: boolean): Promise<[JobJsonRaw, string] | []>;
+moveToCompleted(returnValue: ReturnType, token: string, fetchNext?: boolean): Promise<JobData | []>;
 ```
 
 ## Parameters
@@ -22,7 +22,7 @@ moveToCompleted(returnValue: ReturnType, token: string, fetchNext?: boolean): Pr
 
 <b>Returns:</b>
 
-Promise&lt;\[[JobJsonRaw](./bullmq.jobjsonraw.md)<!-- -->, string\] \| \[\]&gt;
+Promise&lt;[JobData](./bullmq.jobdata.md) \| \[\]&gt;
 
 Returns the jobData of the next job in the waiting queue.
 

docs/gitbook/api/bullmq.job.parent.md

@@ -0,0 +1,14 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [bullmq](./bullmq.md) &gt; [Job](./bullmq.job.md) &gt; [parent](./bullmq.job.parent.md)
+
+## Job.parent property
+
+<b>Signature:</b>
+
+```typescript
+parent?: {
+        id: string;
+        queueKey: string;
+    };
+```

docs/gitbook/api/bullmq.jobdata.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [bullmq](./bullmq.md) &gt; [JobData](./bullmq.jobdata.md)
+
+## JobData type
+
+<b>Signature:</b>
+
+```typescript
+export declare type JobData = [JobJsonRaw | number, string?];
+```
+<b>References:</b> [JobJsonRaw](./bullmq.jobjsonraw.md)
+

docs/gitbook/api/bullmq.jobjsonraw.md

@@ -21,6 +21,7 @@ export interface JobJsonRaw
 |  [id](./bullmq.jobjsonraw.id.md) | string |  |
 |  [name](./bullmq.jobjsonraw.name.md) | string |  |
 |  [opts](./bullmq.jobjsonraw.opts.md) | string |  |
+|  [parent?](./bullmq.jobjsonraw.parent.md) | string | <i>(Optional)</i> |
 |  [parentKey?](./bullmq.jobjsonraw.parentkey.md) | string | <i>(Optional)</i> |
 |  [processedOn?](./bullmq.jobjsonraw.processedon.md) | string | <i>(Optional)</i> |
 |  [progress](./bullmq.jobjsonraw.progress.md) | string |  |

docs/gitbook/api/bullmq.jobjsonraw.parent.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [bullmq](./bullmq.md) &gt; [JobJsonRaw](./bullmq.jobjsonraw.md) &gt; [parent](./bullmq.jobjsonraw.parent.md)
+
+## JobJsonRaw.parent property
+
+<b>Signature:</b>
+
+```typescript
+parent?: string;
+```

docs/gitbook/api/bullmq.md

@@ -45,6 +45,7 @@
 |  [isRedisInstance(obj)](./bullmq.isredisinstance.md) |  |
 |  [jobIdForGroup(jobOpts, data, queueOpts)](./bullmq.jobidforgroup.md) |  |
 |  [lengthInUtf8Bytes(str)](./bullmq.lengthinutf8bytes.md) | Checks the size of string for ascii/non-ascii characters |
+|  [raw2jobData(raw)](./bullmq.raw2jobdata.md) |  |
 |  [removeAllQueueData(client, queueName, prefix)](./bullmq.removeallqueuedata.md) |  |
 |  [tryCatch(fn, ctx, args)](./bullmq.trycatch.md) |  |
 
@@ -109,6 +110,7 @@
 |  [EntryId](./bullmq.entryid.md) |  |
 |  [EntryRaw](./bullmq.entryraw.md) |  |
 |  [FlowQueuesOpts](./bullmq.flowqueuesopts.md) |  |
+|  [JobData](./bullmq.jobdata.md) |  |
 |  [MinimalQueue](./bullmq.minimalqueue.md) |  |
 |  [ParentOpts](./bullmq.parentopts.md) |  |
 |  [Processor](./bullmq.processor.md) | An async function that receives <code>Job</code>s and handles them. |

docs/gitbook/api/bullmq.raw2jobdata.md

@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [bullmq](./bullmq.md) &gt; [raw2jobData](./bullmq.raw2jobdata.md)
+
+## raw2jobData() function
+
+<b>Signature:</b>
+
+```typescript
+export declare function raw2jobData(raw: any[]): [JobJsonRaw | number, string?] | [];
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  raw | any\[\] |  |
+
+<b>Returns:</b>
+
+\[[JobJsonRaw](./bullmq.jobjsonraw.md) \| number, string?\] \| \[\]
+

docs/gitbook/api/bullmq.redisoptions.md

@@ -7,7 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-export declare type RedisOptions = BaseRedisOptions & {
+export declare type RedisOptions = (BaseRedisOptions | ClusterOptions) & {
     skipVersionCheck?: boolean;
 };
 ```

docs/gitbook/api/bullmq.scripts.md

@@ -31,7 +31,6 @@ export declare class Scripts
 |  [moveToDelayed(queue, jobId, timestamp)](./bullmq.scripts.movetodelayed.md) | <code>static</code> |  |
 |  [moveToDelayedArgs(queue, jobId, timestamp)](./bullmq.scripts.movetodelayedargs.md) | <code>static</code> |  |
 |  [moveToFailedArgs(queue, job, failedReason, removeOnFailed, token, fetchNext)](./bullmq.scripts.movetofailedargs.md) | <code>static</code> |  |
-|  [moveToFinished(queue, job, val, propVal, shouldRemove, target, token, fetchNext)](./bullmq.scripts.movetofinished.md) | <code>static</code> |  |
 |  [moveToFinishedArgs(queue, job, val, propVal, shouldRemove, target, token, fetchNext)](./bullmq.scripts.movetofinishedargs.md) | <code>static</code> |  |
 |  [moveToWaitingChildren(queue, jobId, token, opts)](./bullmq.scripts.movetowaitingchildren.md) | <code>static</code> | Move parent job to waiting-children state. |
 |  [moveToWaitingChildrenArgs(queue, jobId, token, opts)](./bullmq.scripts.movetowaitingchildrenargs.md) | <code>static</code> |  |

docs/gitbook/api/bullmq.scripts.movetoactive.md

@@ -7,7 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-static moveToActive<T, R, N extends string>(worker: Worker<T, R, N>, token: string, jobId?: string): Promise<[] | [number, undefined] | [JobJsonRaw, string]>;
+static moveToActive<T, R, N extends string>(worker: Worker<T, R, N>, token: string, jobId?: string): Promise<[] | [number | JobJsonRaw, string?]>;
 ```
 
 ## Parameters
@@ -20,5 +20,5 @@ static moveToActive<T, R, N extends string>(worker: Worker<T, R, N>, token: stri
 
 <b>Returns:</b>
 
-Promise&lt;\[\] \| \[number, undefined\] \| \[[JobJsonRaw](./bullmq.jobjsonraw.md)<!-- -->, string\]&gt;
+Promise&lt;\[\] \| \[number \| [JobJsonRaw](./bullmq.jobjsonraw.md)<!-- -->, string?\]&gt;
 

docs/gitbook/api/bullmq.scripts.movetocompleted.md

@@ -7,7 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-static moveToCompleted(queue: MinimalQueue, job: Job, returnvalue: any, removeOnComplete: boolean | number, token: string, fetchNext: boolean): Promise<[JobJsonRaw, string] | []>;
+static moveToCompleted(queue: MinimalQueue, job: Job, returnvalue: any, removeOnComplete: boolean | number, token: string, fetchNext: boolean): Promise<JobData | []>;
 ```
 
 ## Parameters
@@ -23,5 +23,5 @@ static moveToCompleted(queue: MinimalQueue, job: Job, returnvalue: any, removeOn
 
 <b>Returns:</b>
 
-Promise&lt;\[[JobJsonRaw](./bullmq.jobjsonraw.md)<!-- -->, string\] \| \[\]&gt;
+Promise&lt;[JobData](./bullmq.jobdata.md) \| \[\]&gt;
 

docs/gitbook/api/bullmq.worker_2.createjob.md

@@ -0,0 +1,23 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [bullmq](./bullmq.md) &gt; [Worker\_2](./bullmq.worker_2.md) &gt; [createJob](./bullmq.worker_2.createjob.md)
+
+## Worker\_2.createJob() method
+
+<b>Signature:</b>
+
+```typescript
+protected createJob(data: JobJsonRaw, jobId: string): Job<any, any, string>;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  data | [JobJsonRaw](./bullmq.jobjsonraw.md) |  |
+|  jobId | string |  |
+
+<b>Returns:</b>
+
+[Job](./bullmq.job.md)<!-- -->&lt;any, any, string&gt;
+

docs/gitbook/api/bullmq.worker_2.md

@@ -37,6 +37,7 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
 |  --- | --- | --- |
 |  [callProcessJob(job, token)](./bullmq.worker_2.callprocessjob.md) |  |  |
 |  [close(force)](./bullmq.worker_2.close.md) |  | Closes the worker and related redis connections.<!-- -->This method waits for current jobs to finalize before returning. |
+|  [createJob(data, jobId)](./bullmq.worker_2.createjob.md) |  |  |
 |  [delay()](./bullmq.worker_2.delay.md) |  | This function is exposed only for testing purposes. |
 |  [getNextJob(token, { block })](./bullmq.worker_2.getnextjob.md) |  | Returns a promise that resolves to the next job in queue. |
 |  [isPaused()](./bullmq.worker_2.ispaused.md) |  | Checks if worker is paused. |

docs/gitbook/bullmq-pro/groups.md → docs/gitbook/bullmq-pro/groups/README.md

@@ -4,9 +4,9 @@ Groups allows you to use only one queue yet distribute the jobs among groups so
 
 For example, imagine that you have 1 queue for processing video transcoding for all your users, you may have thousands of users in your application. You need to offload the transcoding operation since it is lengthy and CPU consuming. If you have many users that want to transcode many files, then in a non-grouped queue one user could fill the queue with jobs and the rest of the users will need to wait for that user to complete all its jobs before their jobs get processed.
 
-Groups resolves this problem since jobs will be processed in a "[round-robin](https://en.wikipedia.org/wiki/Round-robin\_item\_allocation)" fashion among all the users. 
+Groups resolves this problem since jobs will be processed in a "[round-robin](https://en.wikipedia.org/wiki/Round-robin\_item\_allocation)" fashion among all the users.
 
-![](<../.gitbook/assets/image (1).png>)
+![](<../../.gitbook/assets/image (1).png>)
 
 If you have several workers or a concurrency factor larger than one, jobs will be processed in parallel, but they will be picked up from the groups as mentioned before following a round-robin ordering.
 
@@ -23,32 +23,39 @@ Another way to see groups is like "virtual" queues. So instead of having one que
 In order to use the group functionality just use the group property in the job options when adding a job:
 
 ```typescript
-  import { QueuePro } from '@taskforcesh/bullmq-pro'
-
-  const queue = new QueuePro();
+import { QueuePro } from '@taskforcesh/bullmq-pro';
 
-  const job1 = await queue.add('test', { foo: 'bar1' }, {
+const queue = new QueuePro();
+
+const job1 = await queue.add(
+  'test',
+  { foo: 'bar1' },
+  {
     group: {
       id: 1,
     },
-  });
-
-  const job2 = await queue.add('test', { foo: 'bar2' }, {
+  },
+);
+
+const job2 = await queue.add(
+  'test',
+  { foo: 'bar2' },
+  {
     group: {
       id: 2,
     },
-  });
-
+  },
+);
 ```
 
 In order to process the jobs, just use a pro worker as you normally do with standard workers:
 
 ```typescript
-import { WorkerPro } from '@taskforcesh/bullmq-pro'
+import { WorkerPro } from '@taskforcesh/bullmq-pro';
 
 const worker = new WorkerPro('test', async job => {
   // Do something usefull.
-  
+
   // You can also do something different depending on the group
   await doSomethingSpecialForMyGroup(job.opts.group);
 });

docs/gitbook/bullmq-pro/groups/rate-limiting.md

@@ -0,0 +1,28 @@
+# Rate limiting
+
+A useful feature when using groups is to be able to rate limit the groups independently of each other, so you can evenly process the jobs belonging to many groups and still limit how many jobs per group are allowed to be processed by unit of time.
+
+The way the rate limiting works is that when the jobs for a given group exceed the maximum amount of jobs per unit of time that particular group gets rate limited. The jobs that belongs to this particular group will not be processed until the rate limit expires.
+
+For example "group 2" is rate limited in the following chart:
+
+![Rate limited group](<../../.gitbook/assets/image (3).png>)
+
+While one or more groups are rate limited, the rest of the jobs belonging to non rate limited groups will continue to be consumed normally or until they also get rate limited.
+
+The rate limit is configured on the worker instances:
+
+```typescript
+import { WorkerPro } from '@taskforcesh/bullmq-pro';
+
+const worker = new WorkerPro('myQueue', processFn, {
+    groups: {
+      limit: {
+        max: 100,  // Limit to 100 jobs per second per group
+        duration 1000,
+      }
+    },
+    connection
+});
+```
+

docs/gitbook/bullmq-pro/introduction.md

@@ -12,4 +12,4 @@ There is a growing number of features that will be implemented in BullMQ Pro, yo
 
 BullMQ Pro is licensed per organization with unlimited use on any projects. The current price is 95$/mo or 950$/year. You can try it for free by generating a new token on the "BullMQ Pro" tab in your account:
 
-![](<../.gitbook/assets/image (2).png>)
+![](<../.gitbook/assets/image (2) (1).png>)

docs/gitbook/bullmq-pro/observables.md

@@ -0,0 +1,8 @@
+# Observables
+
+Instead of returning regular promises, you can also return an Observable, this allows for some more advanced uses cases:
+
+* It makes possible to cleanly cancel a running job.
+* You can define a "Time to live" (TTL) so that jobs that take too long time will be automatically cancelled.
+* Since the last value returned by the observable is persisted, you could retry a job and continue where you left of, for example if the job implements a state machine or similar.
+

docs/gitbook/changelog.md

@@ -1,3 +1,37 @@
+## [1.57.4](https://github.com/taskforcesh/bullmq/compare/v1.57.3...v1.57.4) (2021-12-14)
+
+
+### Bug Fixes
+
+* **move-to-active:** add try catch in moveToActive call ([#933](https://github.com/taskforcesh/bullmq/issues/933)) ([bab45b0](https://github.com/taskforcesh/bullmq/commit/bab45b05d08c625557e2df65921e12f48081d39c))
+* **redis-connection:** consider cluster redisOptions config ([#934](https://github.com/taskforcesh/bullmq/issues/934)) ([5130f63](https://github.com/taskforcesh/bullmq/commit/5130f63ad969efa9649ab8f9abf36a72e8f553f4))
+
+## [1.57.3](https://github.com/taskforcesh/bullmq/compare/v1.57.2...v1.57.3) (2021-12-14)
+
+
+### Bug Fixes
+
+* remove debug console.error ([#932](https://github.com/taskforcesh/bullmq/issues/932)) ([271aac3](https://github.com/taskforcesh/bullmq/commit/271aac3417bc7f76ac02435b456552677b2847db))
+
+## [1.57.2](https://github.com/taskforcesh/bullmq/compare/v1.57.1...v1.57.2) (2021-12-11)
+
+
+### Bug Fixes
+
+* **connection:** check instance options to console log deprecation message ([#927](https://github.com/taskforcesh/bullmq/issues/927)) ([fc1e2b9](https://github.com/taskforcesh/bullmq/commit/fc1e2b9f3f20db53f9dc7ecdfa4644f02acc9f83))
+
+
+### Performance Improvements
+
+* **add-job:** save parent data as json ([#859](https://github.com/taskforcesh/bullmq/issues/859)) ([556d4ee](https://github.com/taskforcesh/bullmq/commit/556d4ee427090f60270945a7fd438e2595bb43e9))
+
+## [1.57.1](https://github.com/taskforcesh/bullmq/compare/v1.57.0...v1.57.1) (2021-12-11)
+
+
+### Bug Fixes
+
+* **worker:** better handling of block timeout ([be4c933](https://github.com/taskforcesh/bullmq/commit/be4c933ae0a7a790d24a081b2ed4e7e1c0216e47))
+
 # [1.57.0](https://github.com/taskforcesh/bullmq/compare/v1.56.0...v1.57.0) (2021-12-08)
 
 
@@ -37,7 +71,7 @@
 
 ### Bug Fixes
 
-* **stalled:** save finidhedOn when job stalled more than allowable limit ([#900](https://github.com/taskforcesh/bullmq/issues/900)) ([eb89edf](https://github.com/taskforcesh/bullmq/commit/eb89edf2f4eb85dedb1485de32e79331940a654f))
+* **stalled:** save finishedOn when job stalled more than allowable limit ([#900](https://github.com/taskforcesh/bullmq/issues/900)) ([eb89edf](https://github.com/taskforcesh/bullmq/commit/eb89edf2f4eb85dedb1485de32e79331940a654f))
 
 ## [1.54.5](https://github.com/taskforcesh/bullmq/compare/v1.54.4...v1.54.5) (2021-11-26)