diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..95c2601a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +Ruby version: +Rails version: +Sidekiq / Pro / Enterprise version(s): + +Please include your initializer, sidekiq.yml, and any error message with the full backtrace. + +If you are using an old version, have you checked the changelogs to see if your issue has been fixed in a later version? + +https://github.com/mperham/sidekiq/blob/main/Changes.md +https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md +https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md diff --git a/.github/contributing.md b/.github/contributing.md new file mode 100644 index 00000000..dffd847a --- /dev/null +++ b/.github/contributing.md @@ -0,0 +1,47 @@ +# Contributing + +## Issues + +When opening an issue: + +* include the full **backtrace** with your error +* include your Sidekiq initializer +* list versions you are using: Ruby, Rails, Sidekiq, OS, etc. + +It's always better to include more info rather than less. + +## Code + +It's always best to open an issue before investing a lot of time into a +fix or new functionality. Functionality must meet my design goals and +vision for the project to be accepted; I would be happy to discuss how +your idea can best fit into Sidekiq. + +### Local development setup + +You need Redis installed and a Ruby version that fulfills the requirements in +`sidekiq.gemspec`. Then: + +``` +bundle install +``` + +And in order to run the tests and linter checks: + +``` +bundle exec rake +``` + +## Legal + +By submitting a Pull Request, you disavow any rights or claims to any changes +submitted to the Sidekiq project and assign the copyright of +those changes to Contributed Systems LLC. + +If you cannot or do not want to reassign those rights (your employment +contract for your employer may not allow this), you should not submit a PR. +Open an issue and someone else can do the work. + +This is a legal way of saying "If you submit a PR to us, that code becomes ours". +99.9% of the time that's what you intend anyways; we hope it doesn't scare you +away from contributing. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..0a462adc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,39 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ruby: ["2.6", "2.7", "3.0", "3.1"] + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, + # change this to (see https://github.com/ruby/setup-ruby#versioning): + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true # 'bundle install' and cache gems + ruby-version: ${{ matrix.ruby }} + - name: Run tests + run: bundle exec rake diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d2af119d --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.rvmrc +.ruby-version +tags +*.swp +dump.rdb +.rbx +coverage/ +vendor/ +.bundle/ +.sass-cache/ +tmp/ +pkg/*.gem +.byebug_history +Gemfile.lock diff --git a/.standard.yml b/.standard.yml new file mode 100644 index 00000000..22990b4f --- /dev/null +++ b/.standard.yml @@ -0,0 +1,15 @@ +ruby_version: 2.5.0 +fix: true +parallel: true +ignore: + - test/**/* + - examples/**/* + - myapp/**/* + - 'lib/sidekiq.rb': + - Lint/InheritException + - 'lib/sidekiq/extensions/**/*': + - Style/MissingRespondToMissing + - 'lib/**/*': + - Lint/RescueException + - Security/YAMLLoad + - Style/GlobalVars diff --git a/.standard_todo.yml b/.standard_todo.yml new file mode 100644 index 00000000..57e47d31 --- /dev/null +++ b/.standard_todo.yml @@ -0,0 +1,6 @@ +# Auto generated files with errors to ignore. +# Remove from this list as you refactor files. +--- +ignore: +- lib/sidekiq/delay_extensions/generic_proxy.rb: + - Style/MissingRespondToMissing diff --git a/COMM-LICENSE.txt b/COMM-LICENSE.txt new file mode 100644 index 00000000..880a1d37 --- /dev/null +++ b/COMM-LICENSE.txt @@ -0,0 +1,97 @@ +END-USER LICENSE AGREEMENT + +------------------------------------------------------------------------------ + +IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND CONTRIBUTED SYSTEMS. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING SIDEKIQ PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT. + +------------------------------------------------------------------------------ + +In order to use the Software under this Agreement, you must receive a “Source URL” at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement. + +1. License Grant + +1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold. + +1.2 Unlimited Organization License. If you purchased an Organization License (included with the Sidekiq Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. + +1.3 Limited Enterprise License. If you purchased a Limited License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. + +1.4 Enterprise Site License. If you purchased a Site License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. + +1.5 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer's server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA. + +1.6 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. + +1.7 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). + +2. Modifications. Contributed Systems shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software. + +3. Restricted Uses. + +3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Contributed Systems; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Contributed Systems in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Contributed Systems; (j) use the Software to develop a product which is competitive with any Contributed Systems product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Contributed Systems in writing. If your unique Source URL is ever published, Contributed Systems reserves the right to terminate your access without notice. + +3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF. + +The Open Source version of the Software (“LGPL Version”) is licensed +under the terms of the GNU Lesser General Public License version 3.0 +(“LGPL”) and not under this EULA. + +4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Contributed Systems and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. + +5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Contributed Systems). + +6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the Sidekiq Pro support page (https://github.com/mperham/sidekiq/wiki/Commercial-Support), support and maintenance services may be included with the purchase of your license subscription. + +7. Term of Agreement. + +7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Contributed Systems may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise. + +7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. + +7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Contributed Systems. + +8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Contributed Systems does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Contributed Systems shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Contributed Systems to do so. +To the maximum extent permitted by applicable law, Contributed Systems disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose. + +9. Limitation of Liability. + +In no event will Contributed Systems be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if Contributed Systems has been advised of the possibility of such damages. + +In no event will Contributed Systems' liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit. + +10. Remedies. Your exclusive remedy and Contributed Systems' entire liability for breach of this Agreement shall be limited, at Contributed Systems' sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Contributed Systems, payable in accordance with Contributed Systems' refund policy. + +11. Acknowledgements. + +11.1 Consent to the Use of Data. You agree that Contributed Systems and its affiliates may collect and use technical information gathered as part of the product support services. Contributed Systems may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you. + +11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software. + +11.3 Government End Users. If the Software and related documentation are supplied to or purchased by or on behalf of the United States Government, then the Software is deemed to be "commercial software" as that term is used in the Federal Acquisition Regulation system. Rights of the United States shall not exceed the minimum rights set forth in FAR 52.227-19 for "restricted computer software". All other terms and conditions of this Agreement apply. + +12. Third Party Software. Examples included in Software may provide links to third party libraries or code (collectively “Third Party Software”) to implement various functions. Third Party Software does not comprise part of the Software. In some cases, access to Third Party Software may be included along with the Software delivery as a convenience for demonstration purposes. Such source code and libraries may be included in the “…/examples” source tree delivered with the Software and do not comprise the Software. Licensee acknowledges (1) that some part of Third Party Software may require additional licensing of copyright and patents from the owners of such, and (2) that distribution of any of the Software referencing or including any portion of a Third Party Software may require appropriate licensing from such third parties. + + +13. Miscellaneous + +13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral. + +13.2 Amendment. Contributed Systems reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/mperham/sidekiq/blob/main/COMM-LICENSE. + +13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Contributed Systems and any attempted assignment without such consent shall be void. + +13.4 Export Compliance. You agree to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software. + +13.5 Indemnification. You agree to defend, indemnify, and hold harmless Contributed Systems from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement. + +13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. + +13.7 Attorneys' Fees and Costs. The prevailing party in any action to enforce this Agreement will be entitled to recover its attorneys' fees and costs in connection with such action. + +13.8 Severability. If any provision of this Agreement is held by a court of competent jurisdiction to be invalid, illegal, or unenforceable, the remainder of this Agreement will remain in full force and effect. + +13.9 Waiver. Failure or neglect by either party to enforce at any time any of the provisions of this licence Agreement shall not be construed or deemed to be a waiver of that party's rights under this Agreement. + +13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs. + +14. Contact Information. If you have any questions about this EULA, or if you want to contact Contributed Systems for any reason, please direct correspondence to info@contribsys.com. diff --git a/Changes.md b/Changes.md new file mode 100644 index 00000000..1891785b --- /dev/null +++ b/Changes.md @@ -0,0 +1,1864 @@ +# Sidekiq Changes + +[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md) + +HEAD +--------- + +- Fix sidekiq.yml YAML load errors on Ruby 3.1 [#5141] +- Sharding support for `perform_bulk` [#5129] +- Refactor job logger for SPEEEEEEED + +6.4.0 +--------- + +- **SECURITY**: Validate input to avoid possible DoS in Web UI. +- Add **strict argument checking** [#5071] + Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`. + Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning. + This warning will switch to an exception in Sidekiq 7.0. +- Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076] +- Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally] +```ruby +SomeJob.perform_async(args...) +SomeJob.perform_sync(args...) +SomeJob.perform_inline(args...) +``` + You can also dynamically redirect a job to run synchronously: +```ruby +SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline +``` +- Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055] +``` +bin/rails generate sidekiq:job ProcessOrderJob +``` +- Fix job retries losing CurrentAttributes [#5090] +- Tweak shutdown to give long-running threads time to cleanup [#5095] + +6.3.1 +--------- + +- Fix keyword arguments error with CurrentAttributes on Ruby 3.0 [#5048] + +6.3.0 +--------- + +- **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions + which use jQuery will break. +- **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger` + output in jobs now shows up in the Sidekiq console. Remove any logger + hacks in your initializer and see if it Just Works™ now. [#5021] +- **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better + reflect industry standard terminology. You can now do this: +```ruby +class MyJob + include Sidekiq::Job + sidekiq_options ... + def perform(args) + end +end +``` +- **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982] +```ruby +# config/initializers/sidekiq.rb +require "sidekiq/middleware/current_attributes" +Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton +``` +- **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk, + similar to `Sidekiq::Client.push_bulk` [#5042] +```ruby +MyJob.perform_bulk([[1], [2], [3]]) +``` +- Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003] +- Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044] +- Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985] +- Run existing signal traps, if any, before running Sidekiq's trap [#4991] +- Fix fetch bug when using weighted queues which caused Sidekiq to stop + processing queues randomly [#5031] + +6.2.2 +--------- + +- Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957] +- Minimize scheduler load on Redis at scale [#4882] +- Improve logging of delay jobs [#4904, BuonOno] +- Minor CSS improvements for buttons and tables, design PRs always welcome! +- Tweak Web UI `Cache-Control` header [#4966] +- Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955] + +6.2.1 +--------- + +- Update RTT warning logic to handle transient RTT spikes [#4851] +- Fix very low priority CVE on unescaped queue name [#4852] +- Add note about sessions and Rails apps in API mode + +6.2.0 +--------- + +- Store Redis RTT and log if poor [#4824] +- Add process/thread stats to Busy page [#4806] +- Improve Web UI on mobile devices [#4840] +- **Refactor Web UI session usage** [#4804] + Numerous people have hit "Forbidden" errors and struggled with Sidekiq's + Web UI session requirement. If you have code in your initializer for + Web sessions, it's quite possible it will need to be removed. Here's + an overview: +``` +Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app, +make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so +Sidekiq can reuse the Rails session: + + Rails.application.routes.draw do + mount Sidekiq::Web => "/sidekiq" + .... + end + +If this is a bare Rack app, use a session middleware before Sidekiq::Web: + + # first, use IRB to create a shared secret key for sessions and commit it + require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) } + + # now, update your Rack app to include the secret with a session cookie middleware + use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400 + run Sidekiq::Web + +If this is a Rails app in API mode, you need to enable sessions. + + https://guides.rubyonrails.org/api_app.html#using-session-middlewares +``` + +6.1.3 +--------- + +- Warn if Redis is configured to evict data under memory pressure [#4752] +- Add process RSS on the Busy page [#4717] + +6.1.2 +--------- + +- Improve readability in dark mode Web UI [#4674] +- Fix Web UI crash with corrupt session [#4672] +- Allow middleware to yield arguments [#4673, @eugeneius] +- Migrate CI from CircleCI to GitHub Actions [#4677] + +6.1.1 +--------- + +- Jobs are now sorted by age in the Busy Workers table. [#4641] +- Fix "check all" JS logic in Web UI [#4619] + +6.1.0 +--------- + +- Web UI - Dark Mode fixes [#4543, natematykiewicz] +- Ensure `Rack::ContentLength` is loaded as middleware for correct Web UI responses [#4541] +- Avoid exception dumping SSL store in Redis connection logging [#4532] +- Better error messages in Sidekiq::Client [#4549] +- Remove rack-protection, reimplement CSRF protection [#4588] +- Require redis-rb 4.2 [#4591] +- Update to jquery 1.12.4 [#4593] +- Refactor internal fetch logic and API [#4602] + +6.0.7 +--------- + +- Refactor systemd integration to work better with custom binaries [#4511] +- Don't connect to Redis at process exit if not needed [#4502] +- Remove Redis connection naming [#4479] +- Fix Redis Sentinel password redaction [#4499] +- Add Vietnamese locale (vi) [#4528] + +6.0.6 +--------- + +- **Integrate with systemd's watchdog and notification features** [#4488] + Set `Type=notify` in [sidekiq.service](https://github.com/mperham/sidekiq/blob/4b8a8bd3ae42f6e48ae1fdaf95ed7d7af18ed8bb/examples/systemd/sidekiq.service#L30-L39). The integration works automatically. +- Use `setTimeout` rather than `setInterval` to avoid thundering herd [#4480] +- Fix edge case where a job can be pushed without a queue. +- Flush job stats at exit [#4498] +- Check RAILS_ENV before RACK_ENV [#4493] +- Add Lithuanian locale [#4476] + +6.0.5 +--------- + +- Fix broken Web UI response when using NewRelic and Rack 2.1.2+. [#4440] +- Update APIs to use `UNLINK`, not `DEL`. [#4449] +- Fix Ruby 2.7 warnings [#4412] +- Add support for `APP_ENV` [[95fa5d9]](https://github.com/mperham/sidekiq/commit/95fa5d90192148026e52ca2902f1b83c70858ce8) + +6.0.4 +--------- + +- Fix ActiveJob's `sidekiq_options` integration [#4404] +- Sidekiq Pro users will now see a Pause button next to each queue in + the Web UI, allowing them to pause queues manually [#4374, shayonj] +- Fix Sidekiq::Workers API unintentional change in 6.0.2 [#4387] + + +6.0.3 +--------- + +- Fix `Sidekiq::Client.push_bulk` API which was erroneously putting + invalid `at` values in the job payloads [#4321] + +6.0.2 +--------- + +- Fix Sidekiq Enterprise's rolling restart functionality, broken by refactoring in 6.0.0. [#4334] +- More internal refactoring and performance tuning [fatkodima] + +6.0.1 +--------- + +- **Performance tuning**, Sidekiq should be 10-15% faster now [#4303, 4299, + 4269, fatkodima] +- **Dark Mode support in Web UI** (further design polish welcome!) [#4227, mperham, + fatkodima, silent-e] +- **Job-specific log levels**, allowing you to turn on debugging for + problematic workers. [fatkodima, #4287] +```ruby +MyWorker.set(log_level: :debug).perform_async(...) +``` +- **Ad-hoc job tags**. You can tag your jobs with, e.g, subdomain, tenant, country, + locale, application, version, user/client, "alpha/beta/pro/ent", types of jobs, + teams/people responsible for jobs, additional metadata, etc. + Tags are shown on different pages with job listings. Sidekiq Pro users + can filter based on them [fatkodima, #4280] +```ruby +class MyWorker + include Sidekiq::Worker + sidekiq_options tags: ['bank-ops', 'alpha'] + ... +end +``` +- Fetch scheduled jobs in batches before pushing into specific queues. + This will decrease enqueueing time of scheduled jobs by a third. [fatkodima, #4273] +``` +ScheduledSet with 10,000 jobs +Before: 56.6 seconds +After: 39.2 seconds +``` +- Compress error backtraces before pushing into Redis, if you are + storing error backtraces, this will halve the size of your RetrySet + in Redis [fatkodima, #4272] +``` +RetrySet with 100,000 jobs +Before: 261 MB +After: 129 MB +``` +- Support display of ActiveJob 6.0 payloads in the Web UI [#4263] +- Add `SortedSet#scan` for pattern based scanning. For large sets this API will be **MUCH** faster + than standard iteration using each. [fatkodima, #4262] +```ruby + Sidekiq::DeadSet.new.scan("UnreliableApi") do |job| + job.retry + end +``` +- Dramatically speed up SortedSet#find\_job(jid) by using Redis's ZSCAN + support, approx 10x faster. [fatkodima, #4259] +``` +zscan 0.179366 0.047727 0.227093 ( 1.161376) +enum 8.522311 0.419826 8.942137 ( 9.785079) +``` +- Respect rails' generators `test_framework` option and gracefully handle extra `worker` suffix on generator [fatkodima, #4256] +- Add ability to sort 'Enqueued' page on Web UI by position in the queue [fatkodima, #4248] +- Support `Client.push_bulk` with different delays [fatkodima, #4243] +```ruby +Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2]], "at" => [1.minute.from_now.to_f, 5.minutes.from_now.to_f]) +``` +- Easier way to test enqueuing specific ActionMailer and ActiveRecord delayed jobs. Instead of manually + parsing embedded class, you can now test by fetching jobs for specific classes. [fatkodima, #4292] +```ruby +assert_equal 1, Sidekiq::DelayExtensions::DelayedMailer.jobs_for(FooMailer).size +``` +- Add `sidekiqmon` to gemspec executables [#4242] +- Gracefully handle `Sidekiq.logger = nil` [#4240] +- Inject Sidekiq::LogContext module if user-supplied logger does not include it [#4239] + +6.0 +--------- + +This release has major breaking changes. Read and test carefully in production. + +- With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq + features/internals like the retry subsystem. [#4213, pirj] +```ruby +class MyJob < ActiveJob::Base + queue_as :myqueue + sidekiq_options retry: 10, backtrace: 20 + def perform(...) + end +end +``` +- Logging has been redesigned to allow for pluggable log formatters: +```ruby +Sidekiq.configure_server do |config| + config.log_formatter = Sidekiq::Logger::Formatters::JSON.new +end +``` +See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for more details. +- **BREAKING CHANGE** Validate proper usage of the `REDIS_PROVIDER` + variable. This variable is meant to hold the name of the environment + variable which contains your Redis URL, so that you can switch Redis + providers quickly and easily with a single variable change. It is not + meant to hold the actual Redis URL itself. If you want to manually set + the Redis URL (not recommended as it implies you have no failover), + then you may set `REDIS_URL` directly. [#3969] +- **BREAKING CHANGE** Increase default shutdown timeout from 8 seconds + to 25 seconds. Both Heroku and ECS now use 30 second shutdown timeout + by default and we want Sidekiq to take advantage of this time. If you + have deployment scripts which depend on the old default timeout, use `-t 8` to + get the old behavior. [#3968] +- **BREAKING CHANGE** Remove the daemonization, logfile and pidfile + arguments to Sidekiq. Use a proper process supervisor (e.g. systemd or + foreman) to manage Sidekiq. See the Deployment wiki page for links to + more resources. +- Integrate the StandardRB code formatter to ensure consistent code + styling. [#4114, gearnode] + +5.2.9 +--------- + +- Release Rack lock due to a cascade of CVEs. [#4566] + Pro-tip: don't lock Rack. + +5.2.8 +--------- + +- Lock to Rack 2.0.x to prevent future incompatibilities +- Fix invalid reference in `sidekiqctl` + +5.2.7 +--------- + +- Fix stale `enqueued_at` when retrying [#4149] +- Move build to [Circle CI](https://circleci.com/gh/mperham/sidekiq) [#4120] + +5.2.6 +--------- + +- Fix edge case where a job failure during Redis outage could result in a lost job [#4141] +- Better handling of malformed job arguments in payload [#4095] +- Restore bootstap's dropdown css component [#4099, urkle] +- Display human-friendly time diff for longer queue latencies [#4111, interlinked] +- Allow `Sidekiq::Worker#set` to be chained + +5.2.5 +--------- + +- Fix default usage of `config/sidekiq.yml` [#4077, Tensho] + +5.2.4 +--------- + +- Add warnings for various deprecations and changes coming in Sidekiq 6.0. + See the 6-0 branch. [#4056] +- Various improvements to the Sidekiq test suite and coverage [#4026, #4039, Tensho] + +5.2.3 +--------- + +- Warning message on invalid REDIS\_PROVIDER [#3970] +- Add `sidekiqctl status` command [#4003, dzunk] +- Update elapsed time calculatons to use monotonic clock [#3999] +- Fix a few issues with mobile Web UI styling [#3973, navied] +- Jobs with `retry: false` now go through the global `death_handlers`, + meaning you can take action on failed ephemeral jobs. [#3980, Benjamin-Dobell] +- Fix race condition in defining Workers. [#3997, mattbooks] + +5.2.2 +--------- + +- Raise error for duplicate queue names in config to avoid unexpected fetch algorithm change [#3911] +- Fix concurrency bug on JRuby [#3958, mattbooks] +- Add "Kill All" button to the retries page [#3938] + +5.2.1 +----------- + +- Fix concurrent modification error during heartbeat [#3921] + +5.2.0 +----------- + +- **Decrease default concurrency from 25 to 10** [#3892] +- Verify connection pool sizing upon startup [#3917] +- Smoother scheduling for large Sidekiq clusters [#3889] +- Switch Sidekiq::Testing impl from alias\_method to Module#prepend, for resiliency [#3852] +- Update Sidekiq APIs to use SCAN for scalability [#3848, ffiller] +- Remove concurrent-ruby gem dependency [#3830] +- Optimize Web UI's bootstrap.css [#3914] + +5.1.3 +----------- + +- Fix version comparison so Ruby 2.2.10 works. [#3808, nateberkopec] + +5.1.2 +----------- + +- Add link to docs in Web UI footer +- Fix crash on Ctrl-C in Windows [#3775, Bernica] +- Remove `freeze` calls on String constants. This is superfluous with Ruby + 2.3+ and `frozen_string_literal: true`. [#3759] +- Fix use of AR middleware outside of Rails [#3787] +- Sidekiq::Worker `sidekiq_retry_in` block can now return nil or 0 to use + the default backoff delay [#3796, dsalahutdinov] + +5.1.1 +----------- + +- Fix Web UI incompatibility with Redis 3.x gem [#3749] + +5.1.0 +----------- + +- **NEW** Global death handlers - called when your job exhausts all + retries and dies. Now you can take action when a job fails permanently. [#3721] +- **NEW** Enable ActiveRecord query cache within jobs by default [#3718, sobrinho] + This will prevent duplicate SELECTS; cache is cleared upon any UPDATE/INSERT/DELETE. + See the issue for how to bypass the cache or disable it completely. +- Scheduler timing is now more accurate, 15 -> 5 seconds [#3734] +- Exceptions during the :startup event will now kill the process [#3717] +- Make `Sidekiq::Client.via` reentrant [#3715] +- Fix use of Sidekiq logger outside of the server process [#3714] +- Tweak `constantize` to better match Rails class lookup. [#3701, caffeinated-tech] + +5.0.5 +----------- + +- Update gemspec to allow newer versions of the Redis gem [#3617] +- Refactor Worker.set so it can be memoized [#3602] +- Fix display of Redis URL in web footer, broken in 5.0.3 [#3560] +- Update `Sidekiq::Job#display_args` to avoid mutation [#3621] + +5.0.4 +----------- + +- Fix "slow startup" performance regression from 5.0.2. [#3525] +- Allow users to disable ID generation since some redis providers disable the CLIENT command. [#3521] + +5.0.3 +----------- + +- Fix overriding `class_attribute` core extension from ActiveSupport with Sidekiq one [PikachuEXE, #3499] +- Allow job logger to be overridden [AlfonsoUceda, #3502] +- Set a default Redis client identifier for debugging [#3516] +- Fix "Uninitialized constant" errors on startup with the delayed extensions [#3509] + +5.0.2 +----------- + +- fix broken release, thanks @nateberkopec + +5.0.1 +----------- + +- Fix incorrect server identity when daemonizing [jwilm, #3496] +- Work around error running Web UI against Redis Cluster [#3492] +- Remove core extensions, Sidekiq is now monkeypatch-free! [#3474] +- Reimplement Web UI's HTTP\_ACCEPT\_LANGUAGE parsing because the spec is utterly + incomprehensible for various edge cases. [johanlunds, natematykiewicz, #3449] +- Update `class_attribute` core extension to avoid warnings +- Expose `job_hash_context` from `Sidekiq::Logging` to support log customization + +5.0.0 +----------- + +- **BREAKING CHANGE** Job dispatch was refactored for safer integration with + Rails 5. The **Logging** and **RetryJobs** server middleware were removed and + functionality integrated directly into Sidekiq::Processor. These aren't + commonly used public APIs so this shouldn't impact most users. +``` +Sidekiq::Middleware::Server::RetryJobs -> Sidekiq::JobRetry +Sidekiq::Middleware::Server::Logging -> Sidekiq::JobLogger +``` +- Quieting Sidekiq is now done via the TSTP signal, the USR1 signal is deprecated. +- The `delay` extension APIs are no longer available by default, you + must opt into them. +- The Web UI is now BiDi and can render RTL languages like Arabic, Farsi and Hebrew. +- Rails 3.2 and Ruby 2.0 and 2.1 are no longer supported. +- The `SomeWorker.set(options)` API was re-written to avoid thread-local state. [#2152] +- Sidekiq Enterprise's encrypted jobs now display "[encrypted data]" in the Web UI instead + of random hex bytes. +- Please see the [5.0 Upgrade notes](5.0-Upgrade.md) for more detail. + +4.2.10 +----------- + +- Scheduled jobs can now be moved directly to the Dead queue via API [#3390] +- Fix edge case leading to job duplication when using Sidekiq Pro's + reliability feature [#3388] +- Fix error class name display on retry page [#3348] +- More robust latency calculation [#3340] + +4.2.9 +----------- + +- Rollback [#3303] which broke Heroku Redis users [#3311] +- Add support for TSTP signal, for Sidekiq 5.0 forward compatibility. [#3302] + +4.2.8 +----------- + +- Fix rare edge case with Redis driver that can create duplicate jobs [#3303] +- Fix Rails 5 loading issue [#3275] +- Restore missing tooltips to timestamps in Web UI [#3310] +- Work on **Sidekiq 5.0** is now active! [#3301] + +4.2.7 +----------- + +- Add new integration testing to verify code loading and job execution + in development and production modes with Rails 4 and 5 [#3241] +- Fix delayed extensions in development mode [#3227, DarthSim] +- Use Worker's `retry` default if job payload does not have a retry + attribute [#3234, mlarraz] + +4.2.6 +----------- + +- Run Rails Executor when in production [#3221, eugeneius] + +4.2.5 +----------- + +- Re-enable eager loading of all code when running non-development Rails 5. [#3203] +- Better root URL handling for zany web servers [#3207] + +4.2.4 +----------- + +- Log errors coming from the Rails 5 reloader. [#3212, eugeneius] +- Clone job data so middleware changes don't appear in Busy tab + +4.2.3 +----------- + +- Disable use of Rails 5's Reloader API in non-development modes, it has proven + to be unstable under load [#3154] +- Allow disabling of Sidekiq::Web's cookie session to handle the + case where the app provides a session already [#3180, inkstak] +```ruby +Sidekiq::Web.set :sessions, false +``` +- Fix Web UI sharding support broken in 4.2.2. [#3169] +- Fix timestamps not updating during UI polling [#3193, shaneog] +- Relax rack-protection version to >= 1.5.0 +- Provide consistent interface to exception handlers, changing the structure of the context hash. [#3161] + +4.2.2 +----------- + +- Fix ever-increasing cookie size with nginx [#3146, cconstantine] +- Fix so Web UI works without trailing slash [#3158, timdorr] + +4.2.1 +----------- + +- Ensure browser does not cache JSON/AJAX responses. [#3136] +- Support old Sinatra syntax for setting config [#3139] + +4.2.0 +----------- + +- Enable development-mode code reloading. **With Rails 5.0+, you don't need + to restart Sidekiq to pick up your Sidekiq::Worker changes anymore!** [#2457] +- **Remove Sinatra dependency**. Sidekiq's Web UI now uses Rack directly. + Thank you to Sidekiq's newest committer, **badosu**, for writing the code + and doing a lot of testing to ensure compatibility with many different + 3rd party plugins. If your Web UI works with 4.1.4 but fails with + 4.2.0, please open an issue. [#3075] +- Allow tuning of concurrency with the `RAILS_MAX_THREADS` env var. [#2985] + This is the same var used by Puma so you can tune all of your systems + the same way: +```sh +web: RAILS_MAX_THREADS=5 bundle exec puma ... +worker: RAILS_MAX_THREADS=10 bundle exec sidekiq ... +``` +Using `-c` or `config/sidekiq.yml` overrides this setting. I recommend +adjusting your `config/database.yml` to use it too so connections are +auto-scaled: +```yaml + pool: <%= ENV['RAILS_MAX_THREADS'] || 5 %> +``` + +4.1.4 +----------- + +- Unlock Sinatra so a Rails 5.0 compatible version may be used [#3048] +- Fix race condition on startup with JRuby [#3043] + + +4.1.3 +----------- + +- Please note the Redis 3.3.0 gem has a [memory leak](https://github.com/redis/redis-rb/issues/612), + Redis 3.2.2 is recommended until that issue is fixed. +- Sinatra 1.4.x is now a required dependency, avoiding cryptic errors + and old bugs due to people not upgrading Sinatra for years. [#3042] +- Fixed race condition in heartbeat which could rarely lead to lingering + processes on the Busy tab. [#2982] +```ruby +# To clean up lingering processes, modify this as necessary to connect to your Redis. +# After 60 seconds, lingering processes should disappear from the Busy page. + +require 'redis' +r = Redis.new(url: "redis://localhost:6379/0") +# uncomment if you need a namespace +#require 'redis-namespace' +#r = Redis::Namespace.new("foo", r) +r.smembers("processes").each do |pro| + r.expire(pro, 60) + r.expire("#{pro}:workers", 60) +end +``` + + +4.1.2 +----------- + +- Fix Redis data leak with worker data when a busy Sidekiq process + crashes. You can find and expire leaked data in Redis with this +script: +```bash +$ redis-cli keys "*:workers" | while read LINE ; do TTL=`redis-cli expire "$LINE" 60`; echo "$LINE"; done; +``` + Please note that `keys` can be dangerous to run on a large, busy Redis. Caveat runner. +- Freeze all string literals with Ruby 2.3. [#2741] +- Client middleware can now stop bulk job push. [#2887] + +4.1.1 +----------- + +- Much better behavior when Redis disappears and comes back. [#2866] +- Update FR locale [dbachet] +- Don't fill logfile in case of Redis downtime [#2860] +- Allow definition of a global retries_exhausted handler. [#2807] +```ruby +Sidekiq.configure_server do |config| + config.default_retries_exhausted = -> (job, ex) do + Sidekiq.logger.info "#{job['class']} job is now dead" + end +end +``` + +4.1.0 +----------- + +- Tag quiet processes in the Web UI [#2757, jcarlson] +- Pass last exception to sidekiq\_retries\_exhausted block [#2787, Nowaker] +```ruby +class MyWorker + include Sidekiq::Worker + sidekiq_retries_exhausted do |job, exception| + end +end +``` +- Add native support for ActiveJob's `set(options)` method allowing +you to override worker options dynamically. This should make it +even easier to switch between ActiveJob and Sidekiq's native APIs [#2780] +```ruby +class MyWorker + include Sidekiq::Worker + sidekiq_options queue: 'default', retry: true + + def perform(*args) + # do something + end +end + +MyWorker.set(queue: 'high', retry: false).perform_async(1) +``` + +4.0.2 +----------- + +- Better Japanese translations +- Remove `json` gem dependency from gemspec. [#2743] +- There's a new testing API based off the `Sidekiq::Queues` namespace. All + assertions made against the Worker class still work as expected. + [#2676, brandonhilkert] +```ruby +assert_equal 0, Sidekiq::Queues["default"].size +HardWorker.perform_async("log") +assert_equal 1, Sidekiq::Queues["default"].size +assert_equal "log", Sidekiq::Queues["default"].first['args'][0] +Sidekiq::Queues.clear_all +``` + +4.0.1 +----------- + +- Yank new queue-based testing API [#2663] +- Fix invalid constant reference in heartbeat + +4.0.0 +----------- + +- Sidekiq's internals have been completely overhauled for performance + and to remove dependencies. This has resulted in major speedups, as + [detailed on my blog](http://www.mikeperham.com/2015/10/14/optimizing-sidekiq/). +- See the [4.0 upgrade notes](4.0-Upgrade.md) for more detail. + +3.5.4 +----------- + +- Ensure exception message is a string [#2707] +- Revert racy Process.kill usage in sidekiqctl + +3.5.3 +----------- + +- Adjust shutdown event to run in parallel with the rest of system shutdown. [#2635] + +3.5.2 +----------- + +- **Sidekiq 3 is now in maintenance mode**, only major bugs will be fixed. +- The exception triggering a retry is now passed into `sidekiq_retry_in`, + allowing you to retry more frequently for certain types of errors. + [#2619, kreynolds] +```ruby + sidekiq_retry_in do |count, ex| + case ex + when RuntimeError + 5 * count + else + 10 * count + end + end +``` + +3.5.1 +----------- + +- **FIX MEMORY LEAK** Under rare conditions, threads may leak [#2598, gazay] +- Add Ukrainian locale [#2561, elrakita] +- Disconnect and retry Redis operations if we see a READONLY error [#2550] +- Add server middleware testing harness; see [wiki](https://github.com/mperham/sidekiq/wiki/Testing#testing-server-middleware) [#2534, ryansch] + +3.5.0 +----------- + +- Polished new banner! [#2522, firedev] +- Upgrade to Celluloid 0.17. [#2420, digitalextremist] +- Activate sessions in Sinatra for CSRF protection, requires Rails + monkeypatch due to rails/rails#15843. [#2460, jc00ke] + +3.4.2 +----------- + +- Don't allow `Sidekiq::Worker` in ActiveJob::Base classes. [#2424] +- Safer display of job data in Web UI [#2405] +- Fix CSRF vulnerability in Web UI, thanks to Egor Homakov for + reporting. [#2422] If you are running the Web UI as a standalone Rack app, + ensure you have a [session middleware +configured](https://github.com/mperham/sidekiq/wiki/Monitoring#standalone): +```ruby +use Rack::Session::Cookie, :secret => "some unique secret string here" +``` + +3.4.1 +----------- + +- Lock to Celluloid 0.16 + + +3.4.0 +----------- + +- Set a `created_at` attribute when jobs are created, set `enqueued_at` only + when they go into a queue. Fixes invalid latency calculations with scheduled jobs. + [#2373, mrsimo] +- Don't log timestamp on Heroku [#2343] +- Run `shutdown` event handlers in reverse order of definition [#2374] +- Rename and rework `poll_interval` to be simpler, more predictable [#2317, cainlevy] + The new setting is `average_scheduled_poll_interval`. To configure + Sidekiq to look for scheduled jobs every 5 seconds, just set it to 5. +```ruby +Sidekiq.configure_server do |config| + config.average_scheduled_poll_interval = 5 +end +``` + +3.3.4 +----------- + +- **Improved ActiveJob integration** - Web UI now shows ActiveJobs in a + nicer format and job logging shows the actual class name, requires + Rails 4.2.2+ [#2248, #2259] +- Add Sidekiq::Process#dump\_threads API to trigger TTIN output [#2247] +- Web UI polling now uses Ajax to avoid page reload [#2266, davydovanton] +- Several Web UI styling improvements [davydovanton] +- Add Tamil, Hindi translations for Web UI [ferdinandrosario, tejasbubane] +- Fix Web UI to work with country-specific locales [#2243] +- Handle circular error causes [#2285, eugenk] + +3.3.3 +----------- + +- Fix crash on exit when Redis is down [#2235] +- Fix duplicate logging on startup +- Undeprecate delay extension for ActionMailer 4.2+ . [#2186] + +3.3.2 +----------- + +- Add Sidekiq::Stats#queues back +- Allows configuration of dead job set size and timeout [#2173, jonhyman] +- Refactor scheduler enqueuing so Sidekiq Pro can override it. [#2159] + +3.3.1 +----------- + +- Dumb down ActionMailer integration so it tries to deliver if possible [#2149] +- Stringify Sidekiq.default\_worker\_options's keys [#2126] +- Add random integer to process identity [#2113, michaeldiscala] +- Log Sidekiq Pro's Batch ID if available [#2076] +- Refactor Processor Redis usage to avoid redis/redis-rb#490 [#2094] +- Move /dashboard/stats to /stats. Add /stats/queues. [moserke, #2099] +- Add processes count to /stats [ismaelga, #2141] +- Greatly improve speed of Sidekiq::Stats [ismaelga, #2142] +- Add better usage text for `sidekiqctl`. +- `Sidekiq::Logging.with_context` is now a stack so you can set your + own job context for logging purposes [grosser, #2110] +- Remove usage of Google Fonts in Web UI so it loads in China [#2144] + +3.3.0 +----------- + +- Upgrade to Celluloid 0.16 [#2056] +- Fix typo for generator test file name [dlackty, #2016] +- Add Sidekiq::Middleware::Chain#prepend [seuros, #2029] + +3.2.6 +----------- + +- Deprecate delay extension for ActionMailer 4.2+ . [seuros, #1933] +- Poll interval tuning now accounts for dead processes [epchris, #1984] +- Add non-production environment to Web UI page titles [JacobEvelyn, #2004] + +3.2.5 +----------- + +- Lock Celluloid to 0.15.2 due to bugs in 0.16.0. This prevents the + "hang on shutdown" problem with Celluloid 0.16.0. + +3.2.4 +----------- + +- Fix issue preventing ActionMailer sends working in some cases with + Rails 4. [pbhogan, #1923] + +3.2.3 +----------- + +- Clean invalid bytes from error message before converting to JSON (requires Ruby 2.1+) [#1705] +- Add queues list for each process to the Busy page. [davetoxa, #1897] +- Fix for crash caused by empty config file. [jordan0day, #1901] +- Add Rails Worker generator, `rails g sidekiq:worker User` will create `app/workers/user_worker.rb`. [seuros, #1909] +- Fix Web UI rendering with huge job arguments [jhass, #1918] +- Minor refactoring of Sidekiq::Client internals, for Sidekiq Pro. [#1919] + +3.2.2 +----------- + +- **This version of Sidekiq will no longer start on Ruby 1.9.** Sidekiq + 3 does not support MRI 1.9 but we've allowed it to run before now. +- Fix issue which could cause Sidekiq workers to disappear from the Busy + tab while still being active [#1884] +- Add "Back to App" button in Web UI. You can set the button link via + `Sidekiq::Web.app_url = 'http://www.mysite.com'` [#1875, seuros] +- Add process tag (`-g tag`) to the Busy page so you can differentiate processes at a glance. [seuros, #1878] +- Add "Kill" button to move retries directly to the DJQ so they don't retry. [seuros, #1867] + +3.2.1 +----------- + +- Revert eager loading change for Rails 3.x apps, as it broke a few edge + cases. + +3.2.0 +----------- + +- **Fix issue which caused duplicate job execution in Rails 3.x** + This issue is caused by [improper exception handling in ActiveRecord](https://github.com/rails/rails/blob/3-2-stable/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb#L281) which changes Sidekiq's Shutdown exception into a database + error, making Sidekiq think the job needs to be retried. **The fix requires Ruby 2.1**. [#1805] +- Update how Sidekiq eager loads Rails application code [#1791, jonleighton] +- Change logging timestamp to show milliseconds. +- Reverse sorting of Dead tab so newer jobs are listed first [#1802] + +3.1.4 +----------- + +- Happy π release! +- Self-tuning Scheduler polling, we use heartbeat info to better tune poll\_interval [#1630] +- Remove all table column width rules, hopefully get better column formatting [#1747] +- Handle edge case where YAML can't be decoded in dev mode [#1761] +- Fix lingering jobs in Busy page on Heroku [#1764] + +3.1.3 +----------- + +- Use ENV['DYNO'] on Heroku for hostname display, rather than an ugly UUID. [#1742] +- Show per-process labels on the Busy page, for feature tagging [#1673] + + +3.1.2 +----------- + +- Suitably chastised, @mperham reverts the Bundler change. + + +3.1.1 +----------- + +- Sidekiq::CLI now runs `Bundler.require(:default, environment)` to boot all gems + before loading any app code. +- Sort queues by name in Web UI [#1734] + + +3.1.0 +----------- + +- New **remote control** feature: you can remotely trigger Sidekiq to quiet + or terminate via API, without signals. This is most useful on JRuby + or Heroku which does not support the USR1 'quiet' signal. Now you can + run a rake task like this at the start of your deploy to quiet your + set of Sidekiq processes. [#1703] +```ruby +namespace :sidekiq do + task :quiet => :environment do + Sidekiq::ProcessSet.new.each(&:quiet!) + end +end +``` +- The Web UI can use the API to quiet or stop all processes via the Busy page. +- The Web UI understands and hides the `Sidekiq::DelayExtensions::Delay*` + classes, instead showing `Class.method` as the Job. [#1718] +- Polish the Dashboard graphs a bit, update Rickshaw [brandonhilkert, #1725] +- The poll interval is now configurable in the Web UI [madebydna, #1713] +- Delay extensions can be removed so they don't conflict with + DelayedJob: put `Sidekiq.remove_delay!` in your initializer. [devaroop, #1674] + + +3.0.2 +----------- + +- Revert gemfile requirement of Ruby 2.0. JRuby 1.7 calls itself Ruby + 1.9.3 and broke with this requirement. + +3.0.1 +----------- + +- Revert pidfile behavior from 2.17.5: Sidekiq will no longer remove its own pidfile + as this is a race condition when restarting. [#1470, #1677] +- Show warning on the Queues page if a queue is paused [#1672] +- Only activate the ActiveRecord middleware if ActiveRecord::Base is defined on boot. [#1666] +- Add ability to disable jobs going to the DJQ with the `dead` option. +```ruby +sidekiq_options :dead => false, :retry => 5 +``` +- Minor fixes + + +3.0.0 +----------- + +Please see [3.0-Upgrade.md](3.0-Upgrade.md) for more comprehensive upgrade notes. + +- **Dead Job Queue** - jobs which run out of retries are now moved to a dead + job queue. These jobs must be retried manually or they will expire + after 6 months or 10,000 jobs. The Web UI contains a "Dead" tab + exposing these jobs. Use `sidekiq_options :retry => false` if you +don't wish jobs to be retried or put in the DJQ. Use +`sidekiq_options :retry => 0` if you don't want jobs to retry but go +straight to the DJQ. +- **Process Lifecycle Events** - you can now register blocks to run at + certain points during the Sidekiq process lifecycle: startup, quiet and + shutdown. +```ruby +Sidekiq.configure_server do |config| + config.on(:startup) do + # do something + end +end +``` +- **Global Error Handlers** - blocks of code which handle errors that + occur anywhere within Sidekiq, not just within middleware. +```ruby +Sidekiq.configure_server do |config| + config.error_handlers << proc {|ex,ctx| ... } +end +``` +- **Process Heartbeat** - each Sidekiq process will ping Redis every 5 + seconds to give a summary of the Sidekiq population at work. +- The Workers tab is now renamed to Busy and contains a list of live + Sidekiq processes and jobs in progress based on the heartbeat. +- **Shardable Client** - Sidekiq::Client instances can use a custom + Redis connection pool, allowing very large Sidekiq installations to scale by + sharding: sending different jobs to different Redis instances. +```ruby +client = Sidekiq::Client.new(ConnectionPool.new { Redis.new }) +client.push(...) +``` +```ruby +Sidekiq::Client.via(ConnectionPool.new { Redis.new }) do + FooWorker.perform_async + BarWorker.perform_async +end +``` + **Sharding support does require a breaking change to client-side +middleware, see 3.0-Upgrade.md.** +- New Chinese, Greek, Swedish and Czech translations for the Web UI. +- Updated most languages translations for the new UI features. +- **Remove official Capistrano integration** - this integration has been + moved into the [capistrano-sidekiq](https://github.com/seuros/capistrano-sidekiq) gem. +- **Remove official support for MRI 1.9** - Things still might work but + I no longer actively test on it. +- **Remove built-in support for Redis-to-Go**. + Heroku users: `heroku config:set REDIS_PROVIDER=REDISTOGO_URL` +- **Remove built-in error integration for Airbrake, Honeybadger, ExceptionNotifier and Exceptional**. + Each error gem should provide its own Sidekiq integration. Update your error gem to the latest + version to pick up Sidekiq support. +- Upgrade to connection\_pool 2.0 which now creates connections lazily. +- Remove deprecated Sidekiq::Client.registered\_\* APIs +- Remove deprecated support for the old Sidekiq::Worker#retries\_exhausted method. +- Removed 'sidekiq/yaml\_patch', this was never documented or recommended. +- Removed --profile option, #1592 +- Remove usage of the term 'Worker' in the UI for clarity. Users would call both threads and + processes 'workers'. Instead, use "Thread", "Process" or "Job". + +2.17.7 +----------- + +- Auto-prune jobs older than one hour from the Workers page [#1508] +- Add Sidekiq::Workers#prune which can perform the auto-pruning. +- Fix issue where a job could be lost when an exception occurs updating + Redis stats before the job executes [#1511] + +2.17.6 +----------- + +- Fix capistrano integration due to missing pidfile. [#1490] + +2.17.5 +----------- + +- Automatically use the config file found at `config/sidekiq.yml`, if not passed `-C`. [#1481] +- Store 'retried\_at' and 'failed\_at' timestamps as Floats, not Strings. [#1473] +- A `USR2` signal will now reopen _all_ logs, using IO#reopen. Thus, instead of creating a new Logger object, + Sidekiq will now just update the existing Logger's file descriptor [#1163]. +- Remove pidfile when shutting down if started with `-P` [#1470] + +2.17.4 +----------- + +- Fix JID support in inline testing, #1454 +- Polish worker arguments display in UI, #1453 +- Marshal arguments fully to avoid worker mutation, #1452 +- Support reverse paging sorted sets, #1098 + + +2.17.3 +----------- + +- Synchronously terminates the poller and fetcher to fix a race condition in bulk requeue during shutdown [#1406] + +2.17.2 +----------- + +- Fix bug where strictly prioritized queues might be processed out of + order [#1408]. A side effect of this change is that it breaks a queue + declaration syntax that worked, although only because of a bug—it was + never intended to work and never supported. If you were declaring your + queues as a comma-separated list, e.g. `sidekiq -q critical,default,low`, + you must now use the `-q` flag before each queue, e.g. + `sidekiq -q critical -q default -q low`. + +2.17.1 +----------- + +- Expose `delay` extension as `sidekiq_delay` also. This allows you to + run Delayed::Job and Sidekiq in the same process, selectively porting + `delay` calls to `sidekiq_delay`. You just need to ensure that + Sidekiq is required **before** Delayed::Job in your Gemfile. [#1393] +- Bump redis client required version to 3.0.6 +- Minor CSS fixes for Web UI + +2.17.0 +----------- + +- Change `Sidekiq::Client#push_bulk` to return an array of pushed `jid`s. [#1315, barelyknown] +- Web UI refactoring to use more API internally (yummy dogfood!) +- Much faster Sidekiq::Job#delete performance for larger queue sizes +- Further capistrano 3 fixes +- Many misc minor fixes + +2.16.1 +----------- + +- Revert usage of `resolv-replace`. MRI's native DNS lookup releases the GIL. +- Fix several Capistrano 3 issues +- Escaping dynamic data like job args and error messages in Sidekiq Web UI. [#1299, lian] + +2.16.0 +----------- + +- Deprecate `Sidekiq::Client.registered_workers` and `Sidekiq::Client.registered_queues` +- Refactor Sidekiq::Client to be instance-based [#1279] +- Pass all Redis options to the Redis driver so Unix sockets + can be fully configured. [#1270, salimane] +- Allow sidekiq-web extensions to add locale paths so extensions + can be localized. [#1261, ondrejbartas] +- Capistrano 3 support [#1254, phallstrom] +- Use Ruby's `resolv-replace` to enable pure Ruby DNS lookups. + This ensures that any DNS resolution that takes place in worker + threads won't lock up the entire VM on MRI. [#1258] + +2.15.2 +----------- + +- Iterating over Sidekiq::Queue and Sidekiq::SortedSet will now work as + intended when jobs are deleted [#866, aackerman] +- A few more minor Web UI fixes [#1247] + +2.15.1 +----------- + +- Fix several Web UI issues with the Bootstrap 3 upgrade. + +2.15.0 +----------- + +- The Core Sidekiq actors are now monitored. If any crash, the + Sidekiq process logs the error and exits immediately. This is to + help prevent "stuck" Sidekiq processes which are running but don't + appear to be doing any work. [#1194] +- Sidekiq's testing behavior is now dynamic. You can choose between + `inline` and `fake` behavior in your tests. See +[Testing](https://github.com/mperham/sidekiq/wiki/Testing) for detail. [#1193] +- The Retries table has a new column for the error message. +- The Web UI topbar now contains the status and live poll button. +- Orphaned worker records are now auto-vacuumed when you visit the + Workers page in the Web UI. +- Sidekiq.default\_worker\_options allows you to configure default + options for all Sidekiq worker types. + +```ruby +Sidekiq.default_worker_options = { 'queue' => 'default', 'backtrace' => true } +``` +- Added two Sidekiq::Client class methods for compatibility with resque-scheduler: + `enqueue_to_in` and `enqueue_in` [#1212] +- Upgrade Web UI to Bootstrap 3.0. [#1211, jeffboek] + +2.14.1 +----------- + +- Fix misc Web UI issues due to ERB conversion. +- Bump redis-namespace version due to security issue. + +2.14.0 +----------- + +- Removed slim gem dependency, Web UI now uses ERB [Locke23rus, #1120] +- Fix more race conditions in Web UI actions +- Don't reset Job enqueued\_at when retrying +- Timestamp tooltips in the Web UI should use UTC +- Fix invalid usage of handle\_exception causing issues in Airbrake + [#1134] + + +2.13.1 +----------- + +- Make Sidekiq::Middleware::Chain Enumerable +- Make summary bar and graphs responsive [manishval, #1025] +- Adds a job status page for scheduled jobs [jonhyman] +- Handle race condition in retrying and deleting jobs in the Web UI +- The Web UI relative times are now i18n. [MadRabbit, #1088] +- Allow for default number of retry attempts to be set for + `Sidekiq::Middleware::Server::RetryJobs` middleware. [czarneckid] [#1091] + +```ruby +Sidekiq.configure_server do |config| + config.server_middleware do |chain| + chain.add Sidekiq::Middleware::Server::RetryJobs, :max_retries => 10 + end +end +``` + + +2.13.0 +----------- + +- Adding button to move scheduled job to main queue [guiceolin, #1020] +- fix i18n support resetting saved locale when job is retried [#1011] +- log rotation via USR2 now closes the old logger [#1008] +- Add ability to customize retry schedule, like so [jmazzi, #1027] + +```ruby +class MyWorker + include Sidekiq::Worker + sidekiq_retry_in { |count| count * 2 } +end +``` +- Redesign Worker#retries\_exhausted callback to use same form as above [jmazzi, #1030] + +```ruby +class MyWorker + include Sidekiq::Worker + sidekiq_retries_exhausted do |msg| + Rails.logger.error "Failed to process #{msg['class']} with args: #{msg['args']}" + end +end +``` + +2.12.4 +----------- + +- Fix error in previous release which crashed the Manager when a + Processor died. + +2.12.3 +----------- + +- Revert back to Celluloid's TaskFiber for job processing which has proven to be more + stable than TaskThread. [#985] +- Avoid possible lockup during hard shutdown [#997] + +At this point, if you are experiencing stability issues with Sidekiq in +Ruby 1.9, please try Ruby 2.0. It seems to be more stable. + +2.12.2 +----------- + +- Relax slim version requirement to >= 1.1.0 +- Refactor historical stats to use TTL, not explicit cleanup. [grosser, #971] + +2.12.1 +----------- + +- Force Celluloid 0.14.1 as 0.14.0 has a serious bug. [#954] +- Scheduled and Retry jobs now use Sidekiq::Client to push + jobs onto the queue, so they use client middleware. [dimko, #948] +- Record the timestamp when jobs are enqueued. Add + Sidekiq::Job#enqueued\_at to query the time. [mariovisic, #944] +- Add Sidekiq::Queue#latency - calculates diff between now and + enqueued\_at for the oldest job in the queue. +- Add testing method `perform_one` that dequeues and performs a single job. + This is mainly to aid testing jobs that spawn other jobs. [fumin, #963] + +2.12.0 +----------- + +- Upgrade to Celluloid 0.14, remove the use of Celluloid's thread + pool. This should halve the number of threads in each Sidekiq + process, thus requiring less resources. [#919] +- Abstract Celluloid usage to Sidekiq::Actor for testing purposes. +- Better handling for Redis downtime when fetching jobs and shutting + down, don't print exceptions every second and print success message + when Redis is back. +- Fix unclean shutdown leading to duplicate jobs [#897] +- Add Korean locale [#890] +- Upgrade test suite to Minitest 5 +- Remove usage of `multi_json` as `json` is now robust on all platforms. + +2.11.2 +----------- + +- Fix Web UI when used without Rails [#886] +- Add Sidekiq::Stats#reset [#349] +- Add Norwegian locale. +- Updates for the JA locale. + +2.11.1 +----------- + +- Fix timeout warning. +- Add Dutch web UI locale. + +2.11.0 +----------- + +- Upgrade to Celluloid 0.13. [#834] +- Remove **timeout** support from `sidekiq_options`. Ruby's timeout + is inherently unsafe in a multi-threaded application and was causing + stability problems for many. See http://bit.ly/OtYpK +- Add Japanese locale for Web UI [#868] +- Fix a few issues with Web UI i18n. + +2.10.1 +----------- + +- Remove need for the i18n gem. (brandonhilkert) +- Improve redis connection info logging on startup for debugging +purposes [#858] +- Revert sinatra/slim as runtime dependencies +- Add `find_job` method to sidekiq/api + + +2.10.0 +----------- + +- Refactor algorithm for putting scheduled jobs onto the queue [#843] +- Fix scheduler thread dying due to incorrect error handling [#839] +- Fix issue which left stale workers if Sidekiq wasn't shutdown while +quiet. [#840] +- I18n for web UI. Please submit translations of `web/locales/en.yml` for +your own language. [#811] +- 'sinatra', 'slim' and 'i18n' are now gem dependencies for Sidekiq. + + +2.9.0 +----------- + +- Update 'sidekiq/testing' to work with any Sidekiq::Client call. It + also serializes the arguments as using Redis would. [#713] +- Raise a Sidekiq::Shutdown error within workers which don't finish within the hard + timeout. This is to prevent unwanted database transaction commits. [#377] +- Lazy load Redis connection pool, you no longer need to specify + anything in Passenger or Unicorn's after_fork callback [#794] +- Add optional Worker#retries_exhausted hook after max retries failed. [jkassemi, #780] +- Fix bug in pagination link to last page [pitr, #774] +- Upstart scripts for multiple Sidekiq instances [dariocravero, #763] +- Use select via pipes instead of poll to catch signals [mrnugget, #761] + +2.8.0 +----------- + +- I18n support! Sidekiq can optionally save and restore the Rails locale + so it will be properly set when your jobs execute. Just include + `require 'sidekiq/middleware/i18n'` in your sidekiq initializer. [#750] +- Fix bug which could lose messages when using namespaces and the message +needs to be requeued in Redis. [#744] +- Refactor Redis namespace support [#747]. The redis namespace can no longer be + passed via the config file, the only supported way is via Ruby in your + initializer: + +```ruby +sidekiq_redis = { :url => 'redis://localhost:3679', :namespace => 'foo' } +Sidekiq.configure_server { |config| config.redis = sidekiq_redis } +Sidekiq.configure_client { |config| config.redis = sidekiq_redis } +``` + +A warning is printed out to the log if a namespace is found in your sidekiq.yml. + + +2.7.5 +----------- + +- Capistrano no longer uses daemonization in order to work with JRuby [#719] +- Refactor signal handling to work on Ruby 2.0 [#728, #730] +- Fix dashboard refresh URL [#732] + +2.7.4 +----------- + +- Fixed daemonization, was broken by some internal refactoring in 2.7.3 [#727] + +2.7.3 +----------- + +- Real-time dashboard is now the default web page +- Make config file optional for capistrano +- Fix Retry All button in the Web UI + +2.7.2 +----------- + +- Remove gem signing infrastructure. It was causing Sidekiq to break +when used via git in Bundler. This is why we can't have nice things. [#688] + + +2.7.1 +----------- + +- Fix issue with hard shutdown [#680] + + +2.7.0 +----------- + +- Add -d daemonize flag, capistrano recipe has been updated to use it [#662] +- Support profiling via `ruby-prof` with -p. When Sidekiq is stopped + via Ctrl-C, it will output `profile.html`. You must add `gem 'ruby-prof'` to your Gemfile for it to work. +- Dynamically update Redis stats on dashboard [brandonhilkert] +- Add Sidekiq::Workers API giving programmatic access to the current + set of active workers. + +``` +workers = Sidekiq::Workers.new +workers.size => 2 +workers.each do |name, work| + # name is a unique identifier per Processor instance + # work is a Hash which looks like: + # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg } +end +``` + +- Allow environment-specific sections within the config file which +override the global values [dtaniwaki, #630] + +``` +--- +:concurrency: 50 +:verbose: false +staging: + :verbose: true + :concurrency: 5 +``` + + +2.6.5 +----------- + +- Several reliability fixes for job requeueing upon termination [apinstein, #622, #624] +- Fix typo in capistrano recipe +- Add `retry_queue` option so retries can be given lower priority [ryanlower, #620] + +```ruby +sidekiq_options queue: 'high', retry_queue: 'low' +``` + +2.6.4 +----------- + +- Fix crash upon empty queue [#612] + +2.6.3 +----------- + +- sidekiqctl exits with non-zero exit code upon error [jmazzi] +- better argument validation in Sidekiq::Client [karlfreeman] + +2.6.2 +----------- + +- Add Dashboard beacon indicating when stats are updated. [brandonhilkert, #606] +- Revert issue with capistrano restart. [#598] + +2.6.1 +----------- + +- Dashboard now live updates summary stats also. [brandonhilkert, #605] +- Add middleware chain APIs `insert_before` and `insert_after` for fine + tuning the order of middleware. [jackrg, #595] + +2.6.0 +----------- + +- Web UI much more mobile friendly now [brandonhilkert, #573] +- Enable live polling for every section in Web UI [brandonhilkert, #567] +- Add Stats API [brandonhilkert, #565] +- Add Stats::History API [brandonhilkert, #570] +- Add Dashboard to Web UI with live and historical stat graphs [brandonhilkert, #580] +- Add option to log output to a file, reopen log file on USR2 signal [mrnugget, #581] + +2.5.4 +----------- + +- `Sidekiq::Client.push` now accepts the worker class as a string so the + Sidekiq client does not have to load your worker classes at all. [#524] +- `Sidekiq::Client.push_bulk` now works with inline testing. +- **Really** fix status icon in Web UI this time. +- Add "Delete All" and "Retry All" buttons to Retries in Web UI + + +2.5.3 +----------- + +- Small Web UI fixes +- Add `delay_until` so you can delay jobs until a specific timestamp: + +```ruby +Auction.delay_until(@auction.ends_at).close(@auction.id) +``` + +This is identical to the existing Sidekiq::Worker method, `perform_at`. + +2.5.2 +----------- + +- Remove asset pipeline from Web UI for much faster, simpler runtime. [#499, #490, #481] +- Add -g option so the procline better identifies a Sidekiq process, defaults to File.basename(Rails.root). [#486] + + sidekiq 2.5.1 myapp [0 of 25 busy] + +- Add splay to retry time so groups of failed jobs don't fire all at once. [#483] + +2.5.1 +----------- + +- Fix issues with core\_ext + +2.5.0 +----------- + +- REDESIGNED WEB UI! [unity, cavneb] +- Support Honeybadger for error delivery +- Inline testing runs the client middleware before executing jobs [#465] +- Web UI can now remove jobs from queue. [#466, dleung] +- Web UI can now show the full message, not just 100 chars [#464, dleung] +- Add APIs for manipulating the retry and job queues. See sidekiq/api. [#457] + + +2.4.0 +----------- + +- ActionMailer.delay.method now only tries to deliver if method returns a valid message. +- Logging now uses "MSG-#{Job ID}", not a random msg ID +- Allow generic Redis provider as environment variable. [#443] +- Add ability to customize sidekiq\_options with delay calls [#450] + +```ruby +Foo.delay(:retry => false).bar +Foo.delay(:retry => 10).bar +Foo.delay(:timeout => 10.seconds).bar +Foo.delay_for(5.minutes, :timeout => 10.seconds).bar +``` + +2.3.3 +----------- + +- Remove option to disable Rails hooks. [#401] +- Allow delay of any module class method + +2.3.2 +----------- + +- Fix retry. 2.3.1 accidentally disabled it. + +2.3.1 +----------- + +- Add Sidekiq::Client.push\_bulk for bulk adding of jobs to Redis. + My own simple test case shows pushing 10,000 jobs goes from 5 sec to 1.5 sec. +- Add support for multiple processes per host to Capistrano recipe +- Re-enable Celluloid::Actor#defer to fix stack overflow issues [#398] + +2.3.0 +----------- + +- Upgrade Celluloid to 0.12 +- Upgrade Twitter Bootstrap to 2.1.0 +- Rescue more Exceptions +- Change Job ID to be Hex, rather than Base64, for HTTP safety +- Use `Airbrake#notify_or_ignore` + +2.2.1 +----------- + +- Add support for custom tabs to Sidekiq::Web [#346] +- Change capistrano recipe to run 'quiet' before deploy:update\_code so + it is run upon both 'deploy' and 'deploy:migrations'. [#352] +- Rescue Exception rather than StandardError to catch and log any sort + of Processor death. + +2.2.0 +----------- + +- Roll back Celluloid optimizations in 2.1.0 which caused instability. +- Add extension to delay any arbitrary class method to Sidekiq. + Previously this was limited to ActiveRecord classes. + +```ruby +SomeClass.delay.class_method(1, 'mike', Date.today) +``` + +- Sidekiq::Client now generates and returns a random, 128-bit Job ID 'jid' which + can be used to track the processing of a Job, e.g. for calling back to a webhook + when a job is finished. + +2.1.1 +----------- + +- Handle networking errors causing the scheduler thread to die [#309] +- Rework exception handling to log all Processor and actor death (#325, subelsky) +- Clone arguments when calling worker so modifications are discarded. (#265, hakanensari) + +2.1.0 +----------- + +- Tune Celluloid to no longer run message processing within a Fiber. + This gives us a full Thread stack and also lowers Sidekiq's memory + usage. +- Add pagination within the Web UI [#253] +- Specify which Redis driver to use: *hiredis* or *ruby* (default) +- Remove FailureJobs and UniqueJobs, which were optional middleware + that I don't want to support in core. [#302] + +2.0.3 +----------- +- Fix sidekiq-web's navbar on mobile devices and windows under 980px (ezkl) +- Fix Capistrano task for first deploys [#259] +- Worker subclasses now properly inherit sidekiq\_options set in + their superclass [#221] +- Add random jitter to scheduler to spread polls across POLL\_INTERVAL + window. [#247] +- Sidekiq has a new mailing list: sidekiq@librelist.org See README. + +2.0.2 +----------- + +- Fix "Retry Now" button on individual retry page. (ezkl) + +2.0.1 +----------- + +- Add "Clear Workers" button to UI. If you kill -9 Sidekiq, the workers + set can fill up with stale entries. +- Update sidekiq/testing to support new scheduled jobs API: + + ```ruby + require 'sidekiq/testing' + DirectWorker.perform_in(10.seconds, 1, 2) + assert_equal 1, DirectWorker.jobs.size + assert_in_delta 10.seconds.from_now.to_f, DirectWorker.jobs.last['at'], 0.01 + ``` + +2.0.0 +----------- + +- **SCHEDULED JOBS**! + +You can now use `perform_at` and `perform_in` to schedule jobs +to run at arbitrary points in the future, like so: + +```ruby + SomeWorker.perform_in(5.days, 'bob', 13) + SomeWorker.perform_at(5.days.from_now, 'bob', 13) +``` + +It also works with the delay extensions: + +```ruby + UserMailer.delay_for(5.days).send_welcome_email(user.id) +``` + +The time is approximately when the job will be placed on the queue; +it is not guaranteed to run at precisely at that moment in time. + +This functionality is meant for one-off, arbitrary jobs. I still +recommend `whenever` or `clockwork` if you want cron-like, +recurring jobs. See `examples/scheduling.rb` + +I want to specially thank @yabawock for his work on sidekiq-scheduler. +His extension for Sidekiq 1.x filled an obvious functional gap that I now think is +useful enough to implement in Sidekiq proper. + +- Fixed issues due to Redis 3.x API changes. Sidekiq now requires + the Redis 3.x client. +- Inline testing now round trips arguments through JSON to catch + serialization issues (betelgeuse) + +1.2.1 +----------- + +- Sidekiq::Worker now has access to Sidekiq's standard logger +- Fix issue with non-StandardErrors leading to Processor exhaustion +- Fix issue with Fetcher slowing Sidekiq shutdown +- Print backtraces for all threads upon TTIN signal [#183] +- Overhaul retries Web UI with new index page and bulk operations [#184] + +1.2.0 +----------- + +- Full or partial error backtraces can optionally be stored as part of the retry + for display in the web UI if you aren't using an error service. [#155] + +```ruby +class Worker + include Sidekiq::Worker + sidekiq_options :backtrace => [true || 10] +end +``` +- Add timeout option to kill a worker after N seconds (blackgold9) + +```ruby +class HangingWorker + include Sidekiq::Worker + sidekiq_options :timeout => 600 + def perform + # will be killed if it takes longer than 10 minutes + end +end +``` + +- Fix delayed extensions not available in workers [#152] +- In test environments add the `#drain` class method to workers. This method + executes all previously queued jobs. (panthomakos) +- Sidekiq workers can be run inline during tests, just `require 'sidekiq/testing/inline'` (panthomakos) +- Queues can now be deleted from the Sidekiq web UI [#154] +- Fix unnecessary shutdown delay due to Retry Poller [#174] + +1.1.4 +----------- + +- Add 24 hr expiry for basic keys set in Redis, to avoid any possible leaking. +- Only register workers in Redis while working, to avoid lingering + workers [#156] +- Speed up shutdown significantly. + +1.1.3 +----------- + +- Better network error handling when fetching jobs from Redis. + Sidekiq will retry once per second until it can re-establish + a connection. (ryanlecompte) +- capistrano recipe now uses `bundle_cmd` if set [#147] +- handle multi\_json API changes (sferik) + +1.1.2 +----------- + +- Fix double restart with cap deploy [#137] + +1.1.1 +----------- + +- Set procline for easy monitoring of Sidekiq status via "ps aux" +- Fix race condition on shutdown [#134] +- Fix hang with cap sidekiq:start [#131] + +1.1.0 +----------- + +- The Sidekiq license has switched from GPLv3 to LGPLv3! +- Sidekiq::Client.push now returns whether the actual Redis + operation succeeded or not. [#123] +- Remove UniqueJobs from the default middleware chain. Its + functionality, while useful, is unexpected for new Sidekiq + users. You can re-enable it with the following config. + Read #119 for more discussion. + +```ruby +Sidekiq.configure_client do |config| + require 'sidekiq/middleware/client/unique_jobs' + config.client_middleware do |chain| + chain.add Sidekiq::Middleware::Client::UniqueJobs + end +end +Sidekiq.configure_server do |config| + require 'sidekiq/middleware/server/unique_jobs' + config.server_middleware do |chain| + chain.add Sidekiq::Middleware::Server::UniqueJobs + end +end +``` + +1.0.0 +----------- + +Thanks to all Sidekiq users and contributors for helping me +get to this big milestone! + +- Default concurrency on client-side to 5, not 25 so we don't + create as many unused Redis connections, same as ActiveRecord's + default pool size. +- Ensure redis= is given a Hash or ConnectionPool. + +0.11.2 +----------- + +- Implement "safe shutdown". The messages for any workers that + are still busy when we hit the TERM timeout will be requeued in + Redis so the messages are not lost when the Sidekiq process exits. + [#110] +- Work around Celluloid's small 4kb stack limit [#115] +- Add support for a custom Capistrano role to limit Sidekiq to + a set of machines. [#113] + +0.11.1 +----------- + +- Fix fetch breaking retry when used with Redis namespaces. [#109] +- Redis connection now just a plain ConnectionPool, not CP::Wrapper. +- Capistrano initial deploy fix [#106] +- Re-implemented weighted queues support (ryanlecompte) + +0.11.0 +----------- + +- Client-side API changes, added sidekiq\_options for Sidekiq::Worker. + As a side effect of this change, the client API works on Ruby 1.8. + It's not officially supported but should work [#103] +- NO POLL! Sidekiq no longer polls Redis, leading to lower network + utilization and lower latency for message processing. +- Add --version CLI option + +0.10.1 +----------- + +- Add details page for jobs in retry queue (jcoene) +- Display relative timestamps in web interface (jcoene) +- Capistrano fixes (hinrik, bensie) + +0.10.0 +----------- + +- Reworked capistrano recipe to make it more fault-tolerant [#94]. +- Automatic failure retry! Sidekiq will now save failed messages + and retry them, with an exponential backoff, over about 20 days. + Did a message fail to process? Just deploy a bug fix in the next + few days and Sidekiq will retry the message eventually. + +0.9.1 +----------- + +- Fix missed deprecations, poor method name in web UI + +0.9.0 +----------- + +- Add -t option to configure the TERM shutdown timeout +- TERM shutdown timeout is now configurable, defaults to 5 seconds. +- USR1 signal now stops Sidekiq from accepting new work, + capistrano sends USR1 at start of deploy and TERM at end of deploy + giving workers the maximum amount of time to finish. +- New Sidekiq::Web rack application available +- Updated Sidekiq.redis API + +0.8.0 +----------- + +- Remove :namespace and :server CLI options (mperham) +- Add ExceptionNotifier support (masterkain) +- Add capistrano support (mperham) +- Workers now log upon start and finish (mperham) +- Messages for terminated workers are now automatically requeued (mperham) +- Add support for Exceptional error reporting (bensie) + +0.7.0 +----------- + +- Example chef recipe and monitrc script (jc00ke) +- Refactor global configuration into Sidekiq.configure\_server and + Sidekiq.configure\_client blocks. (mperham) +- Add optional middleware FailureJobs which saves failed jobs to a + 'failed' queue (fbjork) +- Upon shutdown, workers are now terminated after 5 seconds. This is to + meet Heroku's hard limit of 10 seconds for a process to shutdown. (mperham) +- Refactor middleware API for simplicity, see sidekiq/middleware/chain. (mperham) +- Add `delay` extensions for ActionMailer and ActiveRecord. (mperham) +- Added config file support. See test/config.yml for an example file. (jc00ke) +- Added pidfile for tools like monit (jc00ke) + +0.6.0 +----------- + +- Resque-compatible processing stats in redis (mperham) +- Simple client testing support in sidekiq/testing (mperham) +- Plain old Ruby support via the -r cli flag (mperham) +- Refactored middleware support, introducing ability to add client-side middleware (ryanlecompte) +- Added middleware for ignoring duplicate jobs (ryanlecompte) +- Added middleware for displaying jobs in resque-web dashboard (maxjustus) +- Added redis namespacing support (maxjustus) + +0.5.1 +----------- + +- Initial release! diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..e1fdc703 --- /dev/null +++ b/Gemfile @@ -0,0 +1,28 @@ +source "https://rubygems.org" + +gemspec + +gem "sidekiq", ">= 6.4.1" +gem "rake" +gem "redis-namespace" +gem "rails", ">= 6.0.2" +gem "sqlite3", platforms: :ruby +gem "activerecord-jdbcsqlite3-adapter", platforms: :jruby + +# mail dependencies +gem "net-smtp", platforms: :mri, require: false + +group :test do + gem "minitest" + gem "simplecov" + gem "codecov", require: false +end + +group :development, :test do + gem "standard", require: false +end + +group :load_test do + gem "hiredis" + gem "toxiproxy" +end diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d92e06b7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) Contributed Systems LLC + +Sidekiq is an Open Source project licensed under the terms of +the LGPLv3 license. Please see +for license text. + +Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license. +You can find the commercial license in COMM-LICENSE.txt. +Please see https://sidekiq.org for purchasing options. diff --git a/README.md b/README.md new file mode 100644 index 00000000..0acf8f59 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +Sidekiq +============== + +[![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq) +![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg) + +Simple, efficient background processing for Ruby. + +Sidekiq uses threads to handle many jobs at the same time in the +same process. It does not require Rails but will integrate tightly with +Rails to make background processing dead simple. + +Performance +--------------- + +Version | Latency | Garbage created for 10k jobs | Time to process 100k jobs | Throughput | Ruby +-----------------|------|---------|---------|------------------------|----- +Sidekiq 6.0.2 | 3 ms | 156 MB | 14.0 sec| **7100 jobs/sec** | MRI 2.6.3 +Sidekiq 6.0.0 | 3 ms | 156 MB | 19 sec | 5200 jobs/sec | MRI 2.6.3 +Sidekiq 4.0.0 | 10 ms | 151 MB | 22 sec | 4500 jobs/sec | +Sidekiq 3.5.1 | 22 ms | 1257 MB | 125 sec | 800 jobs/sec | +Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec | +DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec | + +This benchmark can be found in `bin/sidekiqload` and assumes a Redis network latency of 1ms. + +Requirements +----------------- + +- Redis: 4.0+ +- Ruby: MRI 2.5+ or JRuby 9.2+. + +Sidekiq 6.0 supports Rails 5.0+ but does not require it. + + +Installation +----------------- + + gem install sidekiq + + +Getting Started +----------------- + +See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process. +You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about +Sidekiq and see its features in action. Here's the Web UI: + +![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png) + + +Want to Upgrade? +------------------- + +I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more +features, a commercial-friendly license and allow you to support high +quality open source development all at the same time. Please see the +[Sidekiq](https://sidekiq.org/) homepage for more detail. + +Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest +features and changes to Sidekiq and its bigger siblings. + + +Problems? +----------------- + +**Please do not directly email any Sidekiq committers with questions or problems.** A community is best served when discussions are held in public. + +If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages. +Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea. + +Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at https://sidekiq.org; email support@contribsys.com for help. + +Useful resources: + +* Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki). +* Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account. +* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A. + +Every Friday morning is Sidekiq happy hour: I video chat and answer questions. +See the [Sidekiq support page](https://sidekiq.org/support.html) for details. + +Contributing +----------------- + +Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md). + + +License +----------------- + +Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details. + + +Author +----------------- + +Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com) diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..a5dd6f6d --- /dev/null +++ b/Rakefile @@ -0,0 +1,10 @@ +require "bundler/gem_tasks" +require "rake/testtask" +require "standard/rake" + +Rake::TestTask.new(:test) do |test| + test.warning = true + test.pattern = "test/**/test_*.rb" +end + +task default: [:standard, :test] diff --git a/code_of_conduct.md b/code_of_conduct.md new file mode 100644 index 00000000..e3c8d3dd --- /dev/null +++ b/code_of_conduct.md @@ -0,0 +1,50 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project maintainer at mperham AT gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/lib/sidekiq/delay_extensions.rb b/lib/sidekiq/delay_extensions.rb new file mode 100644 index 00000000..07010b24 --- /dev/null +++ b/lib/sidekiq/delay_extensions.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "sidekiq" + +module Sidekiq + module DelayExtensions + def self.enable_delay! + if defined?(::ActiveSupport) + require "sidekiq/delay_extensions/active_record" + require "sidekiq/delay_extensions/action_mailer" + + # Need to patch Psych so it can autoload classes whose names are serialized + # in the delayed YAML. + Psych::Visitors::ToRuby.prepend(Sidekiq::DelayExtensions::PsychAutoload) + + ActiveSupport.on_load(:active_record) do + include Sidekiq::DelayExtensions::ActiveRecord + end + ActiveSupport.on_load(:action_mailer) do + extend Sidekiq::DelayExtensions::ActionMailer + end + end + + require "sidekiq/delay_extensions/class_methods" + Module.__send__(:include, Sidekiq::DelayExtensions::Klass) + + require "sidekiq/delay_extensions/api" + Sidekiq::JobRecord.prepend(Sidekiq::DelayExtensions::JobRecord) + end + + module PsychAutoload + def resolve_class(klass_name) + return nil if !klass_name || klass_name.empty? + # constantize + names = klass_name.split("::") + names.shift if names.empty? || names.first.empty? + + names.inject(Object) do |constant, name| + constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name) + end + rescue NameError + super + end + end + end +end diff --git a/lib/sidekiq/delay_extensions/action_mailer.rb b/lib/sidekiq/delay_extensions/action_mailer.rb new file mode 100644 index 00000000..30761d0e --- /dev/null +++ b/lib/sidekiq/delay_extensions/action_mailer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "sidekiq/delay_extensions/generic_proxy" + +module Sidekiq + module DelayExtensions + ## + # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email + # delivery to Sidekiq. + # + # @example + # UserMailer.delay.send_welcome_email(new_user) + # UserMailer.delay_for(5.days).send_welcome_email(new_user) + # UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user) + class DelayedMailer + include Sidekiq::Worker + + def perform(yml) + (target, method_name, args) = YAML.load(yml) + msg = target.public_send(method_name, *args) + # The email method can return nil, which causes ActionMailer to return + # an undeliverable empty message. + if msg + msg.deliver_now + else + raise "#{target.name}##{method_name} returned an undeliverable mail object" + end + end + end + + module ActionMailer + def sidekiq_delay(options = {}) + Proxy.new(DelayedMailer, self, options) + end + + def sidekiq_delay_for(interval, options = {}) + Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f)) + end + + def sidekiq_delay_until(timestamp, options = {}) + Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f)) + end + alias_method :delay, :sidekiq_delay + alias_method :delay_for, :sidekiq_delay_for + alias_method :delay_until, :sidekiq_delay_until + end + end +end diff --git a/lib/sidekiq/delay_extensions/active_record.rb b/lib/sidekiq/delay_extensions/active_record.rb new file mode 100644 index 00000000..94bf03de --- /dev/null +++ b/lib/sidekiq/delay_extensions/active_record.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require "sidekiq/delay_extensions/generic_proxy" + +module Sidekiq + module DelayExtensions + ## + # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method + # execution to Sidekiq. + # + # @example + # User.recent_signups.each { |user| user.delay.mark_as_awesome } + # + # Please note, this is not recommended as this will serialize the entire + # object to Redis. Your Sidekiq jobs should pass IDs, not entire instances. + # This is here for backwards compatibility with Delayed::Job only. + class DelayedModel + include Sidekiq::Worker + + def perform(yml) + (target, method_name, args) = YAML.load(yml) + target.__send__(method_name, *args) + end + end + + module ActiveRecord + def sidekiq_delay(options = {}) + Proxy.new(DelayedModel, self, options) + end + + def sidekiq_delay_for(interval, options = {}) + Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f)) + end + + def sidekiq_delay_until(timestamp, options = {}) + Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f)) + end + alias_method :delay, :sidekiq_delay + alias_method :delay_for, :sidekiq_delay_for + alias_method :delay_until, :sidekiq_delay_until + end + end +end diff --git a/lib/sidekiq/delay_extensions/api.rb b/lib/sidekiq/delay_extensions/api.rb new file mode 100644 index 00000000..0237df8d --- /dev/null +++ b/lib/sidekiq/delay_extensions/api.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "sidekiq/api" + +module Sidekiq + module DelayExtensions + module JobRecord + def display_class + # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI + @klass ||= self["display_class"] || begin + case klass + when /\ASidekiq::DelayExtensions::Delayed/ + safe_load(args[0], klass) do |target, method, _| + "#{target}.#{method}" + end + else + super + end + end + end + + def display_args + # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI + @display_args ||= case klass + when /\ASidekiq::DelayExtensions::Delayed/ + safe_load(args[0], args) do |_, _, arg, kwarg| + if !kwarg || kwarg.empty? + arg + else + [arg, kwarg] + end + end + else + super + end + end + end + end +end diff --git a/lib/sidekiq/delay_extensions/class_methods.rb b/lib/sidekiq/delay_extensions/class_methods.rb new file mode 100644 index 00000000..f40b7f57 --- /dev/null +++ b/lib/sidekiq/delay_extensions/class_methods.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require "sidekiq/delay_extensions/generic_proxy" + +module Sidekiq + module DelayExtensions + ## + # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method + # execution to Sidekiq. + # + # @example + # User.delay.delete_inactive + # Wikipedia.delay.download_changes_for(Date.today) + # + class DelayedClass + include Sidekiq::Worker + + def perform(yml) + (target, method_name, args) = YAML.load(yml) + target.__send__(method_name, *args) + end + end + + module Klass + def sidekiq_delay(options = {}) + Proxy.new(DelayedClass, self, options) + end + + def sidekiq_delay_for(interval, options = {}) + Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f)) + end + + def sidekiq_delay_until(timestamp, options = {}) + Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f)) + end + alias_method :delay, :sidekiq_delay + alias_method :delay_for, :sidekiq_delay_for + alias_method :delay_until, :sidekiq_delay_until + end + end +end + +Module.__send__(:include, Sidekiq::DelayExtensions::Klass) unless defined?(::Rails) diff --git a/lib/sidekiq/delay_extensions/generic_proxy.rb b/lib/sidekiq/delay_extensions/generic_proxy.rb new file mode 100644 index 00000000..47d6c4ae --- /dev/null +++ b/lib/sidekiq/delay_extensions/generic_proxy.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require "yaml" + +module Sidekiq + module DelayExtensions + SIZE_LIMIT = 8_192 + + class Proxy < BasicObject + def initialize(performable, target, options = {}) + @performable = performable + @target = target + @opts = options + end + + def method_missing(name, *args) + # Sidekiq has a limitation in that its message must be JSON. + # JSON can't round trip real Ruby objects so we use YAML to + # serialize the objects to a String. The YAML will be converted + # to JSON and then deserialized on the other side back into a + # Ruby object. + obj = [@target, name, args] + marshalled = ::YAML.dump(obj) + if marshalled.size > SIZE_LIMIT + ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" } + end + @performable.client_push({"class" => @performable, + "args" => [marshalled], + "display_class" => "#{@target}.#{name}"}.merge(@opts)) + end + end + end +end diff --git a/lib/sidekiq/delay_extensions/testing.rb b/lib/sidekiq/delay_extensions/testing.rb new file mode 100644 index 00000000..088a2c6f --- /dev/null +++ b/lib/sidekiq/delay_extensions/testing.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "sidekiq/testing" + +module Sidekiq + Sidekiq::DelayExtensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::DelayExtensions::DelayedMailer) + Sidekiq::DelayExtensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::DelayExtensions::DelayedModel) +end diff --git a/lib/sidekiq/delay_extensions/version.rb b/lib/sidekiq/delay_extensions/version.rb new file mode 100644 index 00000000..2a136464 --- /dev/null +++ b/lib/sidekiq/delay_extensions/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Sidekiq + module DelayExtensions + VERSION = "6.4.1" + end +end diff --git a/myapp/.gitignore b/myapp/.gitignore new file mode 100644 index 00000000..ec0b1960 --- /dev/null +++ b/myapp/.gitignore @@ -0,0 +1,30 @@ +# See https://help.github.com/articles/ignoring-files for more about ignoring files. +# +# If you find yourself ignoring temporary files generated by your text editor +# or operating system, you probably want to add a global ignore instead: +# git config --global core.excludesfile '~/.gitignore_global' + +# Ignore bundler config. +/.bundle + +# Ignore the default SQLite database. +/db/*.sqlite3 +/db/*.sqlite3-journal + +# Ignore all logfiles and tempfiles. +/log/* +/tmp/* +!/log/.keep +!/tmp/.keep + +# Ignore uploaded files in development +/storage/* + +/node_modules +/yarn-error.log + +/public/assets +.byebug_history + +# Ignore master key for decrypting credentials and more. +/config/master.key diff --git a/myapp/Gemfile b/myapp/Gemfile new file mode 100644 index 00000000..1b6cc51e --- /dev/null +++ b/myapp/Gemfile @@ -0,0 +1,11 @@ +source 'https://rubygems.org' + +gem 'sidekiq-delay_extensions', :path => '..' +gem 'rails' +gem 'puma' + +# Ubuntu required packages to install rails +# apt-get install build-essential patch ruby-dev zlib1g-dev liblzma-dev libsqlite3-dev +platforms :ruby do + gem 'sqlite3' +end diff --git a/myapp/Rakefile b/myapp/Rakefile new file mode 100644 index 00000000..e85f9139 --- /dev/null +++ b/myapp/Rakefile @@ -0,0 +1,6 @@ +# Add your own tasks in files placed in lib/tasks ending in .rake, +# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. + +require_relative 'config/application' + +Rails.application.load_tasks diff --git a/myapp/app/assets/config/manifest.js b/myapp/app/assets/config/manifest.js new file mode 100644 index 00000000..e69de29b diff --git a/myapp/app/controllers/application_controller.rb b/myapp/app/controllers/application_controller.rb new file mode 100644 index 00000000..09705d12 --- /dev/null +++ b/myapp/app/controllers/application_controller.rb @@ -0,0 +1,2 @@ +class ApplicationController < ActionController::Base +end diff --git a/myapp/app/controllers/work_controller.rb b/myapp/app/controllers/work_controller.rb new file mode 100644 index 00000000..fcf1b4b3 --- /dev/null +++ b/myapp/app/controllers/work_controller.rb @@ -0,0 +1,44 @@ +class WorkController < ApplicationController + def index + @count = rand(100) + puts "Adding #{@count} jobs" + @count.times do |x| + HardWorker.perform_async('bubba', 0.01, x) + end + end + + def email + UserMailer.delay_for(30.seconds).greetings(Time.now) + render :plain => 'enqueued' + end + + def bulk + Sidekiq::Client.push_bulk('class' => HardWorker, + 'args' => [['bob', 1, 1], ['mike', 1, 2]]) + render :plain => 'enbulked' + end + + def long + 50.times do |x| + HardWorker.perform_async('bob', 15, x) + end + render :plain => 'enqueued' + end + + def crash + HardWorker.perform_async('crash', 1, Time.now.to_f) + render :plain => 'enqueued' + end + + def delayed_post + p = Post.first + unless p + p = Post.create!(:title => "Title!", :body => 'Body!') + p2 = Post.create!(:title => "Other!", :body => 'Second Body!') + else + p2 = Post.second + end + p.delay.long_method(p2) + render :plain => 'enqueued' + end +end diff --git a/myapp/app/helpers/application_helper.rb b/myapp/app/helpers/application_helper.rb new file mode 100644 index 00000000..de6be794 --- /dev/null +++ b/myapp/app/helpers/application_helper.rb @@ -0,0 +1,2 @@ +module ApplicationHelper +end diff --git a/myapp/app/jobs/application_job.rb b/myapp/app/jobs/application_job.rb new file mode 100644 index 00000000..a009ace5 --- /dev/null +++ b/myapp/app/jobs/application_job.rb @@ -0,0 +1,2 @@ +class ApplicationJob < ActiveJob::Base +end diff --git a/myapp/app/jobs/exit_job.rb b/myapp/app/jobs/exit_job.rb new file mode 100644 index 00000000..a9d78de1 --- /dev/null +++ b/myapp/app/jobs/exit_job.rb @@ -0,0 +1,11 @@ +class ExitJob < ApplicationJob + queue_as :default + + def perform(*args) + Sidekiq.logger.warn "Success" + Thread.new do + sleep 0.1 + exit(0) + end + end +end diff --git a/myapp/app/jobs/some_job.rb b/myapp/app/jobs/some_job.rb new file mode 100644 index 00000000..ec2c210c --- /dev/null +++ b/myapp/app/jobs/some_job.rb @@ -0,0 +1,8 @@ +class SomeJob < ApplicationJob + queue_as :default + + def perform(*args) + puts "What's up?!?!" + # Do something later + end +end diff --git a/myapp/app/mailers/.gitkeep b/myapp/app/mailers/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/myapp/app/mailers/user_mailer.rb b/myapp/app/mailers/user_mailer.rb new file mode 100644 index 00000000..becd58e9 --- /dev/null +++ b/myapp/app/mailers/user_mailer.rb @@ -0,0 +1,9 @@ +class UserMailer < ActionMailer::Base + default from: "sidekiq@example.com" + + def greetings(now) + @now = now + @hostname = `hostname`.strip + mail(:to => 'mperham@gmail.com', :subject => 'Ahoy Matey!') + end +end diff --git a/myapp/app/models/.gitkeep b/myapp/app/models/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/myapp/app/models/exiter.rb b/myapp/app/models/exiter.rb new file mode 100644 index 00000000..61ef5147 --- /dev/null +++ b/myapp/app/models/exiter.rb @@ -0,0 +1,9 @@ +class Exiter + def self.run + Sidekiq.logger.warn "Success" + Thread.new do + sleep 0.1 + exit(0) + end + end +end diff --git a/myapp/app/models/post.rb b/myapp/app/models/post.rb new file mode 100644 index 00000000..490d0dee --- /dev/null +++ b/myapp/app/models/post.rb @@ -0,0 +1,9 @@ +class Post < ActiveRecord::Base + def long_method(other_post) + puts "Running long method with #{self.id} and #{other_post.id}" + end + + def self.testing + Sidekiq.logger.info "Test" + end +end diff --git a/myapp/app/views/layouts/application.html.erb b/myapp/app/views/layouts/application.html.erb new file mode 100644 index 00000000..0f2f3601 --- /dev/null +++ b/myapp/app/views/layouts/application.html.erb @@ -0,0 +1,15 @@ + + + + Myapp + <%= csrf_meta_tags %> + <%= csp_meta_tag %> + + <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %> + <%= javascript_include_tag 'application', 'data-turbolinks-track': 'reload' %> + + + + <%= yield %> + + diff --git a/myapp/app/views/user_mailer/greetings.html.erb b/myapp/app/views/user_mailer/greetings.html.erb new file mode 100644 index 00000000..bdc17008 --- /dev/null +++ b/myapp/app/views/user_mailer/greetings.html.erb @@ -0,0 +1,3 @@ +

+ Hi Mike, it's <%= @now %> and I'm at <%= @hostname %>. +

diff --git a/myapp/app/views/work/index.html.erb b/myapp/app/views/work/index.html.erb new file mode 100644 index 00000000..bcbe068d --- /dev/null +++ b/myapp/app/views/work/index.html.erb @@ -0,0 +1 @@ +Added <%= @count %> jobs! diff --git a/myapp/app/workers/exit_worker.rb b/myapp/app/workers/exit_worker.rb new file mode 100644 index 00000000..41f95cd1 --- /dev/null +++ b/myapp/app/workers/exit_worker.rb @@ -0,0 +1,11 @@ +class ExitWorker + include Sidekiq::Worker + + def perform + logger.warn "Success" + Thread.new do + sleep 0.1 + exit(0) + end + end +end diff --git a/myapp/app/workers/hard_worker.rb b/myapp/app/workers/hard_worker.rb new file mode 100644 index 00000000..83409cf9 --- /dev/null +++ b/myapp/app/workers/hard_worker.rb @@ -0,0 +1,10 @@ +class HardWorker + include Sidekiq::Worker + sidekiq_options :backtrace => 5 + + def perform(name, count, salt) + raise name if name == 'crash' + logger.info Time.now + sleep count + end +end diff --git a/myapp/app/workers/lazy_worker.rb b/myapp/app/workers/lazy_worker.rb new file mode 100644 index 00000000..157cf72c --- /dev/null +++ b/myapp/app/workers/lazy_worker.rb @@ -0,0 +1,6 @@ +class LazyWorker + include Sidekiq::Worker + + def perform + end +end diff --git a/myapp/bin/bundle b/myapp/bin/bundle new file mode 100755 index 00000000..f19acf5b --- /dev/null +++ b/myapp/bin/bundle @@ -0,0 +1,3 @@ +#!/usr/bin/env ruby +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) +load Gem.bin_path('bundler', 'bundle') diff --git a/myapp/bin/rails b/myapp/bin/rails new file mode 100755 index 00000000..5badb2fd --- /dev/null +++ b/myapp/bin/rails @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby +begin + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') +end +APP_PATH = File.expand_path('../config/application', __dir__) +require_relative '../config/boot' +require 'rails/commands' diff --git a/myapp/bin/rake b/myapp/bin/rake new file mode 100755 index 00000000..d87d5f57 --- /dev/null +++ b/myapp/bin/rake @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby +begin + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') +end +require_relative '../config/boot' +require 'rake' +Rake.application.run diff --git a/myapp/bin/setup b/myapp/bin/setup new file mode 100755 index 00000000..94fd4d79 --- /dev/null +++ b/myapp/bin/setup @@ -0,0 +1,36 @@ +#!/usr/bin/env ruby +require 'fileutils' +include FileUtils + +# path to your application root. +APP_ROOT = File.expand_path('..', __dir__) + +def system!(*args) + system(*args) || abort("\n== Command #{args} failed ==") +end + +chdir APP_ROOT do + # This script is a starting point to setup your application. + # Add necessary setup steps to this file. + + puts '== Installing dependencies ==' + system! 'gem install bundler --conservative' + system('bundle check') || system!('bundle install') + + # Install JavaScript dependencies if using Yarn + # system('bin/yarn') + + # puts "\n== Copying sample files ==" + # unless File.exist?('config/database.yml') + # cp 'config/database.yml.sample', 'config/database.yml' + # end + + puts "\n== Preparing database ==" + system! 'bin/rails db:setup' + + puts "\n== Removing old logs and tempfiles ==" + system! 'bin/rails log:clear tmp:clear' + + puts "\n== Restarting application server ==" + system! 'bin/rails restart' +end diff --git a/myapp/config.ru b/myapp/config.ru new file mode 100644 index 00000000..f7ba0b52 --- /dev/null +++ b/myapp/config.ru @@ -0,0 +1,5 @@ +# This file is used by Rack-based servers to start the application. + +require_relative 'config/environment' + +run Rails.application diff --git a/myapp/config/application.rb b/myapp/config/application.rb new file mode 100644 index 00000000..d2e6429b --- /dev/null +++ b/myapp/config/application.rb @@ -0,0 +1,34 @@ +require_relative 'boot' + +require 'rails' +%w( + active_record/railtie + action_controller/railtie + action_view/railtie + action_mailer/railtie + active_job/railtie + sprockets/railtie +).each do |railtie| + begin + require railtie + rescue LoadError + end +end + +# Require the gems listed in Gemfile, including any gems +# you've limited to :test, :development, or :production. +Bundler.require(*Rails.groups) + +module Myapp + class Application < Rails::Application + # Load the defaults for whichever Rails version we are using, we need to + # ensure we work with all defaults. + config.load_defaults "#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}" + + # Settings in config/environments/* take precedence over those specified here. + # Application configuration can go into files in config/initializers + # -- all .rb files in that directory are automatically loaded after loading + # the framework and any gems in your application. + config.active_job.queue_adapter = :sidekiq + end +end diff --git a/myapp/config/boot.rb b/myapp/config/boot.rb new file mode 100644 index 00000000..30f5120d --- /dev/null +++ b/myapp/config/boot.rb @@ -0,0 +1,3 @@ +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) + +require 'bundler/setup' # Set up gems listed in the Gemfile. diff --git a/myapp/config/database.yml b/myapp/config/database.yml new file mode 100644 index 00000000..0d02f249 --- /dev/null +++ b/myapp/config/database.yml @@ -0,0 +1,25 @@ +# SQLite version 3.x +# gem install sqlite3 +# +# Ensure the SQLite 3 gem is defined in your Gemfile +# gem 'sqlite3' +# +default: &default + adapter: sqlite3 + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + timeout: 5000 + +development: + <<: *default + database: db/development.sqlite3 + +# Warning: The database defined as "test" will be erased and +# re-generated from your development database when you run "rake". +# Do not set this db to the same as development or production. +test: + <<: *default + database: db/test.sqlite3 + +production: + <<: *default + database: db/production.sqlite3 diff --git a/myapp/config/environment.rb b/myapp/config/environment.rb new file mode 100644 index 00000000..426333bb --- /dev/null +++ b/myapp/config/environment.rb @@ -0,0 +1,5 @@ +# Load the Rails application. +require_relative 'application' + +# Initialize the Rails application. +Rails.application.initialize! diff --git a/myapp/config/environments/development.rb b/myapp/config/environments/development.rb new file mode 100644 index 00000000..9a842b56 --- /dev/null +++ b/myapp/config/environments/development.rb @@ -0,0 +1,61 @@ +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # In the development environment your application's code is reloaded on + # every request. This slows down response time but is perfect for development + # since you don't have to restart the web server when you make code changes. + config.cache_classes = false + + # Do not eager load code on boot. + config.eager_load = false + + # Show full error reports. + config.consider_all_requests_local = true + + # Enable/disable caching. By default caching is disabled. + # Run rails dev:cache to toggle caching. + if Rails.root.join('tmp', 'caching-dev.txt').exist? + config.action_controller.perform_caching = true + + config.cache_store = :memory_store + config.public_file_server.headers = { + 'Cache-Control' => "public, max-age=#{2.days.to_i}" + } + else + config.action_controller.perform_caching = false + + config.cache_store = :null_store + end + + # Store uploaded files on the local file system (see config/storage.yml for options) + #config.active_storage.service = :local + + # Don't care if the mailer can't send. + config.action_mailer.raise_delivery_errors = false + + config.action_mailer.perform_caching = false + + # Print deprecation notices to the Rails logger. + config.active_support.deprecation = :log + + # Raise an error on page load if there are pending migrations. + config.active_record.migration_error = :page_load + + # Highlight code that triggered database queries in logs. + config.active_record.verbose_query_logs = true + + # Debug mode disables concatenation and preprocessing of assets. + # This option may cause significant delays in view rendering with a large + # number of complex assets. + # config.assets.debug = true + + # Suppress logger output for asset requests. + # config.assets.quiet = true + + # Raises error for missing translations + # config.action_view.raise_on_missing_translations = true + + # Use an evented file watcher to asynchronously detect changes in source code, + # routes, locales, etc. This feature depends on the listen gem. + #config.file_watcher = ActiveSupport::EventedFileUpdateChecker +end diff --git a/myapp/config/environments/production.rb b/myapp/config/environments/production.rb new file mode 100644 index 00000000..2a20538b --- /dev/null +++ b/myapp/config/environments/production.rb @@ -0,0 +1,94 @@ +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? + + # Compress JavaScripts and CSS. + config.assets.js_compressor = :uglifier + # config.assets.css_compressor = :sass + + # Do not fallback to assets pipeline if a precompiled asset is missed. + config.assets.compile = false + + # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb + + # Enable serving of images, stylesheets, and JavaScripts from an asset server. + # config.action_controller.asset_host = 'http://assets.example.com' + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Store uploaded files on the local file system (see config/storage.yml for options) + #config.active_storage.service = :local + + # Mount Action Cable outside main process or domain + # config.action_cable.mount_path = nil + # config.action_cable.url = 'wss://example.com/cable' + # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Use the lowest log level to ensure availability of diagnostic information + # when problems arise. + config.log_level = :debug + + # Prepend all log lines with the following tags. + config.log_tags = [ :request_id ] + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + + # Use a real queuing backend for Active Job (and separate queues per environment) + # config.active_job.queue_adapter = :resque + # config.active_job.queue_name_prefix = "myapp_#{Rails.env}" + + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + # config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = true + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + + # Use a different logger for distributed setups. + # require 'syslog/logger' + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false +end diff --git a/myapp/config/environments/test.rb b/myapp/config/environments/test.rb new file mode 100644 index 00000000..ccf4335d --- /dev/null +++ b/myapp/config/environments/test.rb @@ -0,0 +1,46 @@ +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # The test environment is used exclusively to run your application's + # test suite. You never need to work with it otherwise. Remember that + # your test database is "scratch space" for the test suite and is wiped + # and recreated between test runs. Don't rely on the data there! + config.cache_classes = true + + # Do not eager load code on boot. This avoids loading your whole application + # just for the purpose of running a single test. If you are using a tool that + # preloads Rails for running tests, you may have to set it to true. + config.eager_load = false + + # Configure public file server for tests with Cache-Control for performance. + config.public_file_server.enabled = true + config.public_file_server.headers = { + 'Cache-Control' => "public, max-age=#{1.hour.to_i}" + } + + # Show full error reports and disable caching. + config.consider_all_requests_local = true + config.action_controller.perform_caching = false + + # Raise exceptions instead of rendering exception templates. + config.action_dispatch.show_exceptions = false + + # Disable request forgery protection in test environment. + config.action_controller.allow_forgery_protection = false + + # Store uploaded files on the local file system in a temporary directory + #config.active_storage.service = :test + + config.action_mailer.perform_caching = false + + # Tell Action Mailer not to deliver emails to the real world. + # The :test delivery method accumulates sent emails in the + # ActionMailer::Base.deliveries array. + config.action_mailer.delivery_method = :test + + # Print deprecation notices to the stderr. + config.active_support.deprecation = :stderr + + # Raises error for missing translations + # config.action_view.raise_on_missing_translations = true +end diff --git a/myapp/config/initializers/assets.rb b/myapp/config/initializers/assets.rb new file mode 100644 index 00000000..ff2d2299 --- /dev/null +++ b/myapp/config/initializers/assets.rb @@ -0,0 +1,11 @@ +# Be sure to restart your server when you modify this file. + +# Version of your assets, change this if you want to expire all your assets. +# Rails.application.config.assets.version = '1.0' + +# Add additional assets to the asset load path +# Rails.application.config.assets.paths << Emoji.images_path + +# Precompile additional assets. +# application.js, application.css, and all non-JS/CSS in app/assets folder are already added. +# Rails.application.config.assets.precompile += %w( search.js ) diff --git a/myapp/config/initializers/backtrace_silencers.rb b/myapp/config/initializers/backtrace_silencers.rb new file mode 100644 index 00000000..59385cdf --- /dev/null +++ b/myapp/config/initializers/backtrace_silencers.rb @@ -0,0 +1,7 @@ +# Be sure to restart your server when you modify this file. + +# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. +# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } + +# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. +# Rails.backtrace_cleaner.remove_silencers! diff --git a/myapp/config/initializers/cookies_serializer.rb b/myapp/config/initializers/cookies_serializer.rb new file mode 100644 index 00000000..5a6a32d3 --- /dev/null +++ b/myapp/config/initializers/cookies_serializer.rb @@ -0,0 +1,5 @@ +# Be sure to restart your server when you modify this file. + +# Specify a serializer for the signed and encrypted cookie jars. +# Valid options are :json, :marshal, and :hybrid. +Rails.application.config.action_dispatch.cookies_serializer = :json diff --git a/myapp/config/initializers/filter_parameter_logging.rb b/myapp/config/initializers/filter_parameter_logging.rb new file mode 100644 index 00000000..4a994e1e --- /dev/null +++ b/myapp/config/initializers/filter_parameter_logging.rb @@ -0,0 +1,4 @@ +# Be sure to restart your server when you modify this file. + +# Configure sensitive parameters which will be filtered from the log file. +Rails.application.config.filter_parameters += [:password] diff --git a/myapp/config/initializers/inflections.rb b/myapp/config/initializers/inflections.rb new file mode 100644 index 00000000..ac033bf9 --- /dev/null +++ b/myapp/config/initializers/inflections.rb @@ -0,0 +1,16 @@ +# Be sure to restart your server when you modify this file. + +# Add new inflection rules using the following format. Inflections +# are locale specific, and you may define rules for as many different +# locales as you wish. All of these examples are active by default: +# ActiveSupport::Inflector.inflections(:en) do |inflect| +# inflect.plural /^(ox)$/i, '\1en' +# inflect.singular /^(ox)en/i, '\1' +# inflect.irregular 'person', 'people' +# inflect.uncountable %w( fish sheep ) +# end + +# These inflection rules are supported but not enabled by default: +# ActiveSupport::Inflector.inflections(:en) do |inflect| +# inflect.acronym 'RESTful' +# end diff --git a/myapp/config/initializers/mime_types.rb b/myapp/config/initializers/mime_types.rb new file mode 100644 index 00000000..dc189968 --- /dev/null +++ b/myapp/config/initializers/mime_types.rb @@ -0,0 +1,4 @@ +# Be sure to restart your server when you modify this file. + +# Add new mime types for use in respond_to blocks: +# Mime::Type.register "text/richtext", :rtf diff --git a/myapp/config/initializers/secret_token.rb b/myapp/config/initializers/secret_token.rb new file mode 100644 index 00000000..007d2f86 --- /dev/null +++ b/myapp/config/initializers/secret_token.rb @@ -0,0 +1,7 @@ +# Be sure to restart your server when you modify this file. + +# Your secret key for verifying the integrity of signed cookies. +# If you change this key, all old signed cookies will become invalid! +# Make sure the secret is at least 30 characters and all random, +# no regular words or you'll be exposed to dictionary attacks. +Myapp::Application.config.secret_token = 'bdd335500c81ba1f279f9dd8198d1f334445d0193168ed73c1282502180dca27e3e102ec345e99b2acac6a63f7fe29da69c60cc9e76e8da34fb5d4989db24cd8' diff --git a/myapp/config/initializers/session_store.rb b/myapp/config/initializers/session_store.rb new file mode 100644 index 00000000..bf64a296 --- /dev/null +++ b/myapp/config/initializers/session_store.rb @@ -0,0 +1,3 @@ +# Be sure to restart your server when you modify this file. + +Rails.application.config.session_store :cookie_store, key: '_myapp_session' diff --git a/myapp/config/initializers/sidekiq.rb b/myapp/config/initializers/sidekiq.rb new file mode 100644 index 00000000..8053418e --- /dev/null +++ b/myapp/config/initializers/sidekiq.rb @@ -0,0 +1,44 @@ +Sidekiq.configure_client do |config| + config.redis = { :size => 2 } +end +Sidekiq.configure_server do |config| + config.on(:startup) { } + config.on(:quiet) { } + config.on(:shutdown) do + #result = RubyProf.stop + + ## Write the results to a file + ## Requires railsexpress patched MRI build + # brew install qcachegrind + #File.open("callgrind.profile", "w") do |f| + #RubyProf::CallTreePrinter.new(result).print(f, :min_percent => 1) + #end + end +end + +class EmptyWorker + include Sidekiq::Worker + + def perform + end +end + +class TimedWorker + include Sidekiq::Worker + + def perform(start) + now = Time.now.to_f + puts "Latency: #{now - start} sec" + end +end + +Sidekiq::DelayExtensions.enable_delay! + +module Myapp + class Current < ActiveSupport::CurrentAttributes + attribute :tenant_id + end +end + +require "sidekiq/middleware/current_attributes" +Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton \ No newline at end of file diff --git a/myapp/config/initializers/wrap_parameters.rb b/myapp/config/initializers/wrap_parameters.rb new file mode 100644 index 00000000..bbfc3961 --- /dev/null +++ b/myapp/config/initializers/wrap_parameters.rb @@ -0,0 +1,14 @@ +# Be sure to restart your server when you modify this file. + +# This file contains settings for ActionController::ParamsWrapper which +# is enabled by default. + +# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. +ActiveSupport.on_load(:action_controller) do + wrap_parameters format: [:json] +end + +# To enable root element in JSON for ActiveRecord objects. +# ActiveSupport.on_load(:active_record) do +# self.include_root_in_json = true +# end diff --git a/myapp/config/locales/en.yml b/myapp/config/locales/en.yml new file mode 100644 index 00000000..06539571 --- /dev/null +++ b/myapp/config/locales/en.yml @@ -0,0 +1,23 @@ +# Files in the config/locales directory are used for internationalization +# and are automatically loaded by Rails. If you want to use locales other +# than English, add the necessary files in this directory. +# +# To use the locales, use `I18n.t`: +# +# I18n.t 'hello' +# +# In views, this is aliased to just `t`: +# +# <%= t('hello') %> +# +# To use a different locale, set it with `I18n.locale`: +# +# I18n.locale = :es +# +# This would use the information in config/locales/es.yml. +# +# To learn more, please read the Rails Internationalization guide +# available at http://guides.rubyonrails.org/i18n.html. + +en: + hello: "Hello world" diff --git a/myapp/config/routes.rb b/myapp/config/routes.rb new file mode 100644 index 00000000..c32e3f26 --- /dev/null +++ b/myapp/config/routes.rb @@ -0,0 +1,15 @@ +# turns off browser asset caching so we can test CSS changes quickly +ENV['SIDEKIQ_WEB_TESTING'] = '1' + +require 'sidekiq/web' +Sidekiq::Web.app_url = '/' + +Rails.application.routes.draw do + mount Sidekiq::Web => '/sidekiq' + get "work" => "work#index" + get "work/email" => "work#email" + get "work/post" => "work#delayed_post" + get "work/long" => "work#long" + get "work/crash" => "work#crash" + get "work/bulk" => "work#bulk" +end diff --git a/myapp/config/secrets.yml b/myapp/config/secrets.yml new file mode 100644 index 00000000..0093078d --- /dev/null +++ b/myapp/config/secrets.yml @@ -0,0 +1,22 @@ +# Be sure to restart your server when you modify this file. + +# Your secret key is used for verifying the integrity of signed cookies. +# If you change this key, all old signed cookies will become invalid! + +# Make sure the secret is at least 30 characters and all random, +# no regular words or you'll be exposed to dictionary attacks. +# You can use `rake secret` to generate a secure secret key. + +# Make sure the secrets in this file are kept private +# if you're sharing your code publicly. + +development: + secret_key_base: c96477a6cbbc1304f41d7b26d5acb28187a8e8a16308a6d0d0310433e86054fe0bbc6d23a5c9269bd21b1a45650dd0ace354db80b9647bffb1c8bb600a00a1ae + +test: + secret_key_base: ff0824c8b17ce583c8402dbe9968fec3a169dd6a294d4768fdef22cb083a048a2b4bababc5f37ca3a5d3c1b034e13fd703490671cb32db2983c4abb65e67fb09 + +# Do not keep production secrets in the repository, +# instead read values from the environment. +production: + secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> diff --git a/myapp/config/sidekiq.yml b/myapp/config/sidekiq.yml new file mode 100644 index 00000000..b56158ff --- /dev/null +++ b/myapp/config/sidekiq.yml @@ -0,0 +1,3 @@ +--- +:labels: + - some_label diff --git a/myapp/db/migrate/20120123214055_create_posts.rb b/myapp/db/migrate/20120123214055_create_posts.rb new file mode 100644 index 00000000..ca423815 --- /dev/null +++ b/myapp/db/migrate/20120123214055_create_posts.rb @@ -0,0 +1,10 @@ +class CreatePosts < ActiveRecord::Migration[6.0] + def change + create_table :posts do |t| + t.string :title + t.string :body + + t.timestamps + end + end +end diff --git a/myapp/db/schema.rb b/myapp/db/schema.rb new file mode 100644 index 00000000..5c5c5ced --- /dev/null +++ b/myapp/db/schema.rb @@ -0,0 +1,22 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema.define(version: 2012_01_23_214055) do + + create_table "posts", force: :cascade do |t| + t.string "title" + t.string "body" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + end + +end diff --git a/myapp/db/seeds.rb b/myapp/db/seeds.rb new file mode 100644 index 00000000..4edb1e85 --- /dev/null +++ b/myapp/db/seeds.rb @@ -0,0 +1,7 @@ +# This file should contain all the record creation needed to seed the database with its default values. +# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup). +# +# Examples: +# +# cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }]) +# Mayor.create(name: 'Emanuel', city: cities.first) diff --git a/myapp/lib/assets/.gitkeep b/myapp/lib/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/myapp/lib/tasks/.gitkeep b/myapp/lib/tasks/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/myapp/log/.gitkeep b/myapp/log/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/myapp/script/rails b/myapp/script/rails new file mode 100755 index 00000000..f8da2cff --- /dev/null +++ b/myapp/script/rails @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application. + +APP_PATH = File.expand_path('../../config/application', __FILE__) +require File.expand_path('../../config/boot', __FILE__) +require 'rails/commands' diff --git a/myapp/simple.ru b/myapp/simple.ru new file mode 100644 index 00000000..8e007471 --- /dev/null +++ b/myapp/simple.ru @@ -0,0 +1,19 @@ +# Easiest way to run Sidekiq::Web. +# Run with "bundle exec rackup simple.ru" + +require 'sidekiq/web' + +# A Web process always runs as client, no need to configure server +Sidekiq.configure_client do |config| + config.redis = { url: 'redis://localhost:6379/0', size: 1 } +end + +Sidekiq::Client.push('class' => "HardWorker", 'args' => []) + +# In a multi-process deployment, all Web UI instances should share +# this secret key so they can all decode the encrypted browser cookies +# and provide a working session. +# Rails does this in /config/initializers/secret_token.rb +secret_key = SecureRandom.hex(32) +use Rack::Session::Cookie, secret: secret_key, same_site: true, max_age: 86400 +run Sidekiq::Web diff --git a/sidekiq-delay_extensions.gemspec b/sidekiq-delay_extensions.gemspec new file mode 100644 index 00000000..1e6013d2 --- /dev/null +++ b/sidekiq-delay_extensions.gemspec @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative "lib/sidekiq/delay_extensions/version" + +Gem::Specification.new do |spec| + spec.name = "sidekiq-delay_extensions" + spec.version = Sidekiq::DelayExtensions::VERSION + spec.authors = ["Mike Perham", "Benjamin Fleischer"] + spec.email = ["mperham@gmail.com", "github@benjaminfleischer.com"] + + spec.summary = "Sidekiq Delay Extensions" + spec.description = "Extracted from Sidekiq 6.0" + spec.homepage = "https://github.com/gemhome/sidekiq-delay_extensions/wiki/Delayed-extensions" + spec.license = "LGPL-3.0" + + spec.files = Dir.glob("{bin,lib,config}/**/*") + %w[Gemfile sidekiq-delay_extensions.gemspec README.md Changes.md LICENSE] + + spec.bindir = "exe" + spec.executables = [] + spec.require_paths = ["lib"] + spec.required_ruby_version = ">= 2.5.0" + + spec.metadata = { + "homepage_uri" => "https://github.com/gemhome/sidekiq-delay_extensions/wiki/Delayed-extensions", + "bug_tracker_uri" => "https://github.com/gemhome/sidekiq-delay_extensions/issues", + "documentation_uri" => "https://github.com/gemhome/sidekiq-delay_extensions/wiki", + "changelog_uri" => "https://github.com/gemhome/sidekiq-delay_extensions/blob/main/Changes.md", + "source_code_uri" => "https://github.com/gemhome/sidekiq-delay_extensions" + } + + spec.add_dependency "sidekiq", ">= 6.4.1" +end diff --git a/test/config.yml b/test/config.yml new file mode 100644 index 00000000..d1270add --- /dev/null +++ b/test/config.yml @@ -0,0 +1,7 @@ +--- +:verbose: false +:require: ./test/fake_env.rb +:concurrency: 50 +:queues: + - [<%="very_"%>often, 2] + - [seldom, 1] diff --git a/test/config__FILE__and__dir__.yml b/test/config__FILE__and__dir__.yml new file mode 100644 index 00000000..27bbe354 --- /dev/null +++ b/test/config__FILE__and__dir__.yml @@ -0,0 +1,5 @@ +--- +:require: ./test/fake_env.rb + +:__FILE__: <%= __FILE__ %> +:__dir__: <%= __dir__ %> diff --git a/test/config_empty.yml b/test/config_empty.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/test/config_empty.yml @@ -0,0 +1 @@ +--- diff --git a/test/config_environment.yml b/test/config_environment.yml new file mode 100644 index 00000000..b53ab451 --- /dev/null +++ b/test/config_environment.yml @@ -0,0 +1,10 @@ +--- +:concurrency: 50 + +staging: + :verbose: false + :require: ./test/fake_env.rb + :concurrency: 50 + :queues: + - [<%="very_"%>often, 2] + - [seldom, 1] diff --git a/test/config_queues_without_weights.yml b/test/config_queues_without_weights.yml new file mode 100644 index 00000000..4d5370dc --- /dev/null +++ b/test/config_queues_without_weights.yml @@ -0,0 +1,4 @@ +--- +:queues: + - [queue_1] + - [queue_2] diff --git a/test/config_string.yml b/test/config_string.yml new file mode 100644 index 00000000..9eb50306 --- /dev/null +++ b/test/config_string.yml @@ -0,0 +1,8 @@ +--- +verbose: false +require: ./test/fake_env.rb +concurrency: 50 +queues: + - [<%="very_"%>often, 2] + - [seldom, 1] + diff --git a/test/config_with_alias.yml b/test/config_with_alias.yml new file mode 100644 index 00000000..7f9f8bb0 --- /dev/null +++ b/test/config_with_alias.yml @@ -0,0 +1,12 @@ +--- +:production: &production + :verbose: true + :require: ./test/fake_env.rb + :concurrency: 50 + :queues: + - [<%="very_"%>often, 2] + - [seldom, 1] + +:staging: + <<: *production + :verbose: false \ No newline at end of file diff --git a/test/config_with_internal_options.yml b/test/config_with_internal_options.yml new file mode 100644 index 00000000..fe5da7b6 --- /dev/null +++ b/test/config_with_internal_options.yml @@ -0,0 +1,11 @@ +--- +:verbose: false +:timeout: 10 +:require: ./test/fake_env.rb +:concurrency: 50 +:tag: tag +:queues: + - [often] + - [seldom] + +:strict: false diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb new file mode 100644 index 00000000..f7fb6414 --- /dev/null +++ b/test/dummy/config/application.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require "rails" +require "active_model/railtie" +require "active_job/railtie" +require "active_record/railtie" +require "action_controller/railtie" +require "action_mailer/railtie" +require "action_view/railtie" +require "rails/test_unit/railtie" + +module Dummy + class Application < Rails::Application + config.root = File.expand_path("../..", __FILE__) + config.eager_load = false + config.logger = Logger.new('/dev/null') + config.load_defaults "6.0" + end +end diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml new file mode 100644 index 00000000..29cc2b2f --- /dev/null +++ b/test/dummy/config/database.yml @@ -0,0 +1,11 @@ +development: + adapter: sqlite3 + database: db/development.sqlite3 + pool: 5 + timeout: 5000 + +test: + adapter: sqlite3 + database: db/test.sqlite3 + pool: 5 + timeout: 5000 diff --git a/test/dummy/config/environment.rb b/test/dummy/config/environment.rb new file mode 100644 index 00000000..875ab0f2 --- /dev/null +++ b/test/dummy/config/environment.rb @@ -0,0 +1,3 @@ +require_relative 'application' + +Rails.application.initialize! diff --git a/test/dummy/config/sidekiq.yml b/test/dummy/config/sidekiq.yml new file mode 100644 index 00000000..b9e5211b --- /dev/null +++ b/test/dummy/config/sidekiq.yml @@ -0,0 +1,3 @@ +--- +:require: ./test/fake_env.rb +:concurrency: 25 diff --git a/test/dummy/tmp/.keep b/test/dummy/tmp/.keep new file mode 100644 index 00000000..e69de29b diff --git a/test/fake_env.rb b/test/fake_env.rb new file mode 100644 index 00000000..8e9b8f90 --- /dev/null +++ b/test/fake_env.rb @@ -0,0 +1 @@ +# frozen_string_literal: true diff --git a/test/fixtures/en.yml b/test/fixtures/en.yml new file mode 100644 index 00000000..26500216 --- /dev/null +++ b/test/fixtures/en.yml @@ -0,0 +1,2 @@ +en: + translated_text: 'Changed text from add locals' diff --git a/test/helper.rb b/test/helper.rb new file mode 100644 index 00000000..f283c6a9 --- /dev/null +++ b/test/helper.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'bundler/setup' +Bundler.require(:default, :test) + +require 'minitest/pride' +require 'minitest/autorun' + +$TESTING = true +# disable minitest/parallel threads +ENV["MT_CPU"] = "0" +ENV["N"] = "0" +# Disable any stupid backtrace cleansers +ENV["BACKTRACE"] = "1" + +if ENV["COVERAGE"] + require 'simplecov' + SimpleCov.start do + enable_coverage :branch + add_filter "/test/" + add_filter "/myapp/" + end + if ENV['CI'] + require 'codecov' + SimpleCov.formatter = SimpleCov::Formatter::Codecov + end +end + +ENV['REDIS_URL'] ||= 'redis://localhost/15' + +Sidekiq.logger = ::Logger.new(STDOUT) +Sidekiq.logger.level = Logger::ERROR + +def capture_logging(lvl=Logger::INFO) + old = Sidekiq.logger + begin + out = StringIO.new + logger = ::Logger.new(out) + logger.level = lvl + Sidekiq.logger = logger + yield + out.string + ensure + Sidekiq.logger = old + end +end diff --git a/test/test_api.rb b/test/test_api.rb new file mode 100644 index 00000000..101986df --- /dev/null +++ b/test/test_api.rb @@ -0,0 +1,680 @@ +# frozen_string_literal: true +require_relative 'helper' +require 'sidekiq/api' +require 'active_job' +require 'action_mailer' + +describe 'API' do + before do + Sidekiq.redis {|c| c.flushdb } + end + + describe "stats" do + it "is initially zero" do + s = Sidekiq::Stats.new + assert_equal 0, s.processed + assert_equal 0, s.failed + assert_equal 0, s.enqueued + assert_equal 0, s.default_queue_latency + assert_equal 0, s.workers_size + end + + describe "processed" do + it "returns number of processed jobs" do + Sidekiq.redis { |conn| conn.set("stat:processed", 5) } + s = Sidekiq::Stats.new + assert_equal 5, s.processed + end + end + + describe "failed" do + it "returns number of failed jobs" do + Sidekiq.redis { |conn| conn.set("stat:failed", 5) } + s = Sidekiq::Stats.new + assert_equal 5, s.failed + end + end + + describe "reset" do + before do + Sidekiq.redis do |conn| + conn.set('stat:processed', 5) + conn.set('stat:failed', 10) + end + end + + it 'will reset all stats by default' do + Sidekiq::Stats.new.reset + s = Sidekiq::Stats.new + assert_equal 0, s.failed + assert_equal 0, s.processed + end + + it 'can reset individual stats' do + Sidekiq::Stats.new.reset('failed') + s = Sidekiq::Stats.new + assert_equal 0, s.failed + assert_equal 5, s.processed + end + + it 'can accept anything that responds to #to_s' do + Sidekiq::Stats.new.reset(:failed) + s = Sidekiq::Stats.new + assert_equal 0, s.failed + assert_equal 5, s.processed + end + + it 'ignores anything other than "failed" or "processed"' do + Sidekiq::Stats.new.reset((1..10).to_a, ['failed']) + s = Sidekiq::Stats.new + assert_equal 0, s.failed + assert_equal 5, s.processed + end + end + + describe "workers_size" do + it 'retrieves the number of busy workers' do + Sidekiq.redis do |c| + c.sadd("processes", "process_1") + c.sadd("processes", "process_2") + c.hset("process_1", "busy", 1) + c.hset("process_2", "busy", 2) + end + s = Sidekiq::Stats.new + assert_equal 3, s.workers_size + end + end + + describe "queues" do + it "is initially empty" do + s = Sidekiq::Stats::Queues.new + assert_equal 0, s.lengths.size + end + + it "returns a hash of queue and size in order" do + Sidekiq.redis do |conn| + conn.rpush 'queue:foo', '{}' + conn.sadd 'queues', 'foo' + + 3.times { conn.rpush 'queue:bar', '{}' } + conn.sadd 'queues', 'bar' + end + + s = Sidekiq::Stats::Queues.new + assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths + assert_equal "bar", s.lengths.first.first + + assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths + end + end + + describe "enqueued" do + it 'handles latency for good jobs' do + Sidekiq.redis do |conn| + conn.rpush 'queue:default', "{\"enqueued_at\": #{Time.now.to_f}}" + conn.sadd 'queues', 'default' + end + s = Sidekiq::Stats.new + assert s.default_queue_latency > 0 + q = Sidekiq::Queue.new + assert q.latency > 0 + end + + it 'handles latency for incomplete jobs' do + Sidekiq.redis do |conn| + conn.rpush 'queue:default', '{}' + conn.sadd 'queues', 'default' + end + s = Sidekiq::Stats.new + assert_equal 0, s.default_queue_latency + q = Sidekiq::Queue.new + assert_equal 0, q.latency + end + + it "returns total enqueued jobs" do + Sidekiq.redis do |conn| + conn.rpush 'queue:foo', '{}' + conn.sadd 'queues', 'foo' + + 3.times { conn.rpush 'queue:bar', '{}' } + conn.sadd 'queues', 'bar' + end + + s = Sidekiq::Stats.new + assert_equal 4, s.enqueued + end + end + + describe "over time" do + before do + require 'active_support/core_ext/time/conversions' + @before = Time::DATE_FORMATS[:default] + Time::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S" + end + + after do + Time::DATE_FORMATS[:default] = @before + end + + describe "history" do + it "does not allow invalid input" do + assert_raises(ArgumentError) { Sidekiq::Stats::History.new(-1) } + assert_raises(ArgumentError) { Sidekiq::Stats::History.new(0) } + assert_raises(ArgumentError) { Sidekiq::Stats::History.new(2000) } + assert Sidekiq::Stats::History.new(200) + end + end + + describe "processed" do + it 'retrieves hash of dates' do + Sidekiq.redis do |c| + c.incrby("stat:processed:2012-12-24", 4) + c.incrby("stat:processed:2012-12-25", 1) + c.incrby("stat:processed:2012-12-26", 6) + c.incrby("stat:processed:2012-12-27", 2) + end + Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do + s = Sidekiq::Stats::History.new(2) + assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1 }, s.processed) + + s = Sidekiq::Stats::History.new(3) + assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed) + + s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25")) + assert_equal({ "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed) + end + end + end + + describe "failed" do + it 'retrieves hash of dates' do + Sidekiq.redis do |c| + c.incrby("stat:failed:2012-12-24", 4) + c.incrby("stat:failed:2012-12-25", 1) + c.incrby("stat:failed:2012-12-26", 6) + c.incrby("stat:failed:2012-12-27", 2) + end + Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do + s = Sidekiq::Stats::History.new(2) + assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed + + s = Sidekiq::Stats::History.new(3) + assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed + + s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25")) + assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed + end + end + end + end + end + + describe 'with an empty database' do + it 'shows queue as empty' do + q = Sidekiq::Queue.new + assert_equal 0, q.size + assert_equal 0, q.latency + end + + before do + ActiveJob::Base.queue_adapter = :sidekiq + ActiveJob::Base.logger = nil + end + + class ApiMailer < ActionMailer::Base + def test_email(*) + end + end + + class ApiJob < ActiveJob::Base + def perform(*) + end + end + + class ApiWorker + include Sidekiq::Worker + end + + class WorkerWithTags + include Sidekiq::Worker + sidekiq_options tags: ['foo'] + end + + it 'can enumerate jobs' do + q = Sidekiq::Queue.new + Time.stub(:now, Time.new(2012, 12, 26)) do + ApiWorker.perform_async(1, 'mike') + assert_equal [ApiWorker.name], q.map(&:klass) + + job = q.first + assert_equal 24, job.jid.size + assert_equal [1, 'mike'], job.args + assert_equal Time.new(2012, 12, 26), job.enqueued_at + end + assert q.latency > 10_000_000 + + q = Sidekiq::Queue.new('other') + assert_equal 0, q.size + end + + it 'enumerates jobs in descending score order' do + # We need to enqueue more than 50 items, which is the page size when retrieving + # from Redis to ensure everything is sorted: the pages and the items withing them. + 51.times { ApiWorker.perform_in(100, 1, 'foo') } + + set = Sidekiq::ScheduledSet.new.to_a + + assert_equal set.sort_by { |job| -job.score }, set + end + + it 'has no enqueued_at time for jobs enqueued in the future' do + job_id = ApiWorker.perform_in(100, 1, 'foo') + job = Sidekiq::ScheduledSet.new.find_job(job_id) + assert_nil job.enqueued_at + end + + it 'unwraps delayed jobs' do + Sidekiq::DelayExtensions.enable_delay! + Sidekiq::Queue.delay.foo(1,2,3) + q = Sidekiq::Queue.new + x = q.first + assert_equal "Sidekiq::Queue.foo", x.display_class + assert_equal [1,2,3], x.display_args + end + + it 'handles previous (raw Array) error_backtrace format' do + add_retry + job = Sidekiq::RetrySet.new.first + assert_equal ['line1', 'line2'], job.error_backtrace + end + + it 'handles previous (marshalled Array) error_backtrace format' do + backtrace = ['line1', 'line2'] + serialized = Marshal.dump(backtrace) + compressed = Zlib::Deflate.deflate(serialized) + encoded = Base64.encode64(compressed) + + payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1], 'queue' => 'default', 'jid' => 'jid', 'error_backtrace' => encoded) + Sidekiq.redis do |conn| + conn.zadd('retry', Time.now.to_f.to_s, payload) + end + + job = Sidekiq::RetrySet.new.first + assert_equal backtrace, job.error_backtrace + end + + describe "Rails unwrapping" do + SERIALIZED_JOBS = { + "5.x" => [ + '{"class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ApiJob","queue":"default","args":[{"job_class":"ApiJob","job_id":"f1bde53f-3852-4ae4-a879-c12eacebbbb0","provider_job_id":null,"queue_name":"default","priority":null,"arguments":[1,2,3],"executions":0,"locale":"en"}],"retry":true,"jid":"099eee72911085a511d0e312","created_at":1568305542.339916,"enqueued_at":1568305542.339947}', + '{"class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::DeliveryJob","queue":"mailers","args":[{"job_class":"ActionMailer::DeliveryJob","job_id":"19cc0115-3d1c-4bbe-a51e-bfa1385895d1","provider_job_id":null,"queue_name":"mailers","priority":null,"arguments":["ApiMailer","test_email","deliver_now",1,2,3],"executions":0,"locale":"en"}],"retry":true,"jid":"37436e5504936400e8cf98db","created_at":1568305542.370133,"enqueued_at":1568305542.370241}', + ], + "6.x" => [ + '{"class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ApiJob","queue":"default","args":[{"job_class":"ApiJob","job_id":"ff2b48d4-bdce-4825-af6b-ef8c11ab651e","provider_job_id":null,"queue_name":"default","priority":null,"arguments":[1,2,3],"executions":0,"exception_executions":{},"locale":"en","timezone":"UTC","enqueued_at":"2019-09-12T16:28:37Z"}],"retry":true,"jid":"ce121bf77b37ae81fe61b6dc","created_at":1568305717.9469702,"enqueued_at":1568305717.947005}', + '{"class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::MailDeliveryJob","queue":"mailers","args":[{"job_class":"ActionMailer::MailDeliveryJob","job_id":"2f967da1-a389-479c-9a4e-5cc059e6d65c","provider_job_id":null,"queue_name":"mailers","priority":null,"arguments":["ApiMailer","test_email","deliver_now",{"args":[1,2,3],"_aj_symbol_keys":["args"]}],"executions":0,"exception_executions":{},"locale":"en","timezone":"UTC","enqueued_at":"2019-09-12T16:28:37Z"}],"retry":true,"jid":"469979df52bb9ef9f48b49e1","created_at":1568305717.9457421,"enqueued_at":1568305717.9457731}', + ], + }.each_pair do |ver,jobs| + it "unwraps ActiveJob #{ver} jobs" do + #ApiJob.perform_later(1,2,3) + #puts Sidekiq::Queue.new.first.value + x = Sidekiq::JobRecord.new(jobs[0], "default") + assert_equal ApiJob.name, x.display_class + assert_equal [1,2,3], x.display_args + end + + it "unwraps ActionMailer #{ver} jobs" do + #ApiMailer.test_email(1,2,3).deliver_later + #puts Sidekiq::Queue.new("mailers").first.value + x = Sidekiq::JobRecord.new(jobs[1], "mailers") + assert_equal "#{ApiMailer.name}#test_email", x.display_class + assert_equal [1,2,3], x.display_args + end + end + end + + it 'has no enqueued_at time for jobs enqueued in the future' do + job_id = ApiWorker.perform_in(100, 1, 'foo') + job = Sidekiq::ScheduledSet.new.find_job(job_id) + assert_nil job.enqueued_at + end + + it 'returns tags field for jobs' do + job_id = ApiWorker.perform_async + assert_equal [], Sidekiq::Queue.new.find_job(job_id).tags + + job_id = WorkerWithTags.perform_async + assert_equal ['foo'], Sidekiq::Queue.new.find_job(job_id).tags + end + + it 'can delete jobs' do + q = Sidekiq::Queue.new + ApiWorker.perform_async(1, 'mike') + assert_equal 1, q.size + + x = q.first + assert_equal ApiWorker.name, x.display_class + assert_equal [1,'mike'], x.display_args + + assert_equal [true], q.map(&:delete) + assert_equal 0, q.size + end + + it "can move scheduled job to queue" do + remain_id = ApiWorker.perform_in(100, 1, 'jason') + job_id = ApiWorker.perform_in(100, 1, 'jason') + job = Sidekiq::ScheduledSet.new.find_job(job_id) + q = Sidekiq::Queue.new + job.add_to_queue + queued_job = q.find_job(job_id) + refute_nil queued_job + assert_equal queued_job.jid, job_id + assert_nil Sidekiq::ScheduledSet.new.find_job(job_id) + refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id) + end + + it "handles multiple scheduled jobs when moving to queue" do + jids = Sidekiq::Client.push_bulk('class' => ApiWorker, + 'args' => [[1, 'jason'], [2, 'jason']], + 'at' => Time.now.to_f) + assert_equal 2, jids.size + (remain_id, job_id) = jids + job = Sidekiq::ScheduledSet.new.find_job(job_id) + q = Sidekiq::Queue.new + job.add_to_queue + queued_job = q.find_job(job_id) + refute_nil queued_job + assert_equal queued_job.jid, job_id + assert_nil Sidekiq::ScheduledSet.new.find_job(job_id) + refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id) + end + + it 'can kill a scheduled job' do + job_id = ApiWorker.perform_in(100, 1, '{"foo":123}') + job = Sidekiq::ScheduledSet.new.find_job(job_id) + ds = Sidekiq::DeadSet.new + assert_equal 0, ds.size + job.kill + assert_equal 1, ds.size + end + + it 'can find a scheduled job by jid' do + 10.times do |idx| + ApiWorker.perform_in(idx, 1) + end + + job_id = ApiWorker.perform_in(5, 1) + job = Sidekiq::ScheduledSet.new.find_job(job_id) + assert_equal job_id, job.jid + + ApiWorker.perform_in(100, 1, 'jid' => 'jid_in_args') + assert_nil Sidekiq::ScheduledSet.new.find_job('jid_in_args') + end + + it 'can remove jobs when iterating over a sorted set' do + # scheduled jobs must be greater than SortedSet#each underlying page size + 51.times do + ApiWorker.perform_in(100, 'aaron') + end + set = Sidekiq::ScheduledSet.new + set.map(&:delete) + assert_equal set.size, 0 + end + + it 'can remove jobs when iterating over a queue' do + # initial queue size must be greater than Queue#each underlying page size + 51.times do + ApiWorker.perform_async(1, 'aaron') + end + q = Sidekiq::Queue.new + q.map(&:delete) + assert_equal q.size, 0 + end + + it 'can find job by id in queues' do + q = Sidekiq::Queue.new + job_id = ApiWorker.perform_async(1, 'jason') + job = q.find_job(job_id) + refute_nil job + assert_equal job_id, job.jid + end + + it 'can clear a queue' do + q = Sidekiq::Queue.new + 2.times { ApiWorker.perform_async(1, 'mike') } + q.clear + + Sidekiq.redis do |conn| + refute conn.smembers('queues').include?('foo') + refute conn.exists?('queue:foo') + end + end + + it 'can fetch by score' do + same_time = Time.now.to_f + add_retry('bob1', same_time) + add_retry('bob2', same_time) + r = Sidekiq::RetrySet.new + assert_equal 2, r.fetch(same_time).size + end + + it 'can fetch by score and jid' do + same_time = Time.now.to_f + add_retry('bob1', same_time) + add_retry('bob2', same_time) + r = Sidekiq::RetrySet.new + assert_equal 1, r.fetch(same_time, 'bob1').size + end + + it 'can fetch by score range' do + same_time = Time.now.to_f + add_retry('bob1', same_time) + add_retry('bob2', same_time + 1) + add_retry('bob3', same_time + 2) + r = Sidekiq::RetrySet.new + range = (same_time..(same_time + 1)) + assert_equal 2, r.fetch(range).size + end + + it 'can fetch by score range and jid' do + same_time = Time.now.to_f + add_retry('bob1', same_time) + add_retry('bob2', same_time + 1) + add_retry('bob3', same_time + 2) + r = Sidekiq::RetrySet.new + range = (same_time..(same_time + 1)) + jobs = r.fetch(range, 'bob2') + assert_equal 1, jobs.size + assert_equal jobs[0].jid, 'bob2' + end + + it 'shows empty retries' do + r = Sidekiq::RetrySet.new + assert_equal 0, r.size + end + + it 'can enumerate retries' do + add_retry + + r = Sidekiq::RetrySet.new + assert_equal 1, r.size + array = r.to_a + assert_equal 1, array.size + + retri = array.first + assert_equal 'ApiWorker', retri.klass + assert_equal 'default', retri.queue + assert_equal 'bob', retri.jid + assert_in_delta Time.now.to_f, retri.at.to_f, 0.02 + end + + it 'requires a jid to delete an entry' do + start_time = Time.now.to_f + add_retry('bob2', Time.now.to_f) + assert_raises(ArgumentError) do + Sidekiq::RetrySet.new.delete(start_time) + end + end + + it 'can delete a single retry from score and jid' do + same_time = Time.now.to_f + add_retry('bob1', same_time) + add_retry('bob2', same_time) + r = Sidekiq::RetrySet.new + assert_equal 2, r.size + Sidekiq::RetrySet.new.delete(same_time, 'bob1') + assert_equal 1, r.size + end + + it 'can retry a retry' do + add_retry + r = Sidekiq::RetrySet.new + assert_equal 1, r.size + r.first.retry + assert_equal 0, r.size + assert_equal 1, Sidekiq::Queue.new('default').size + job = Sidekiq::Queue.new('default').first + assert_equal 'bob', job.jid + assert_equal 1, job['retry_count'] + end + + it 'can clear retries' do + add_retry + add_retry('test') + r = Sidekiq::RetrySet.new + assert_equal 2, r.size + r.clear + assert_equal 0, r.size + end + + it 'can scan retries' do + add_retry + add_retry('test') + r = Sidekiq::RetrySet.new + assert_instance_of Enumerator, r.scan('Worker') + assert_equal 2, r.scan('ApiWorker').to_a.size + assert_equal 1, r.scan('*test*').to_a.size + end + + it 'can enumerate processes' do + identity_string = "identity_string" + odata = { + 'pid' => 123, + 'hostname' => Socket.gethostname, + 'key' => identity_string, + 'identity' => identity_string, + 'started_at' => Time.now.to_f - 15, + 'queues' => ['foo', 'bar'] + } + + time = Time.now.to_f + Sidekiq.redis do |conn| + conn.multi do |transaction| + transaction.sadd('processes', odata['key']) + transaction.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time) + transaction.sadd('processes', 'fake:pid') + end + end + + ps = Sidekiq::ProcessSet.new.to_a + assert_equal 1, ps.size + data = ps.first + assert_equal 10, data['busy'] + assert_equal time, data['beat'] + assert_equal 123, data['pid'] + assert_equal ['foo', 'bar'], data.queues + data.quiet! + data.stop! + signals_string = "#{odata['key']}-signals" + assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) } + assert_equal "TSTP", Sidekiq.redis{|c| c.lpop(signals_string) } + end + + it 'can enumerate workers' do + w = Sidekiq::Workers.new + assert_equal 0, w.size + w.each do + assert false + end + + hn = Socket.gethostname + key = "#{hn}:#{$$}" + pdata = { 'pid' => $$, 'hostname' => hn, 'started_at' => Time.now.to_i } + Sidekiq.redis do |conn| + conn.sadd('processes', key) + conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f) + end + + s = "#{key}:workers" + data = Sidekiq.dump_json({ 'payload' => "{}", 'queue' => 'default', 'run_at' => Time.now.to_i }) + Sidekiq.redis do |c| + c.hmset(s, '1234', data) + end + + w.each do |p, x, y| + assert_equal key, p + assert_equal "1234", x + assert_equal 'default', y['queue'] + assert_equal({}, y['payload']) + assert_equal Time.now.year, Time.at(y['run_at']).year + end + + s = "#{key}:workers" + data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) }) + Sidekiq.redis do |c| + c.multi do |transaction| + transaction.hmset(s, '5678', data) + transaction.hmset("b#{s}", '5678', data) + end + end + + assert_equal ['5678', '1234'], w.map { |_, tid, _| tid } + end + + it 'can reschedule jobs' do + add_retry('foo1') + add_retry('foo2') + + retries = Sidekiq::RetrySet.new + assert_equal 2, retries.size + refute(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?) + + retries.each do |retri| + retri.reschedule(Time.now + 15) if retri.jid == 'foo1' + retri.reschedule(Time.now.to_f + 10) if retri.jid == 'foo2' + end + + assert_equal 2, retries.size + assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?) + assert(retries.map { |r| r.score > (Time.now.to_f + 14) }.any?) + end + + it 'prunes processes which have died' do + data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f } + key = "#{data['hostname']}:#{data['pid']}" + Sidekiq.redis do |conn| + conn.sadd('processes', key) + conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f) + end + + ps = Sidekiq::ProcessSet.new + assert_equal 1, ps.size + assert_equal 1, ps.to_a.size + + Sidekiq.redis do |conn| + conn.sadd('processes', "bar:987") + conn.sadd('processes', "bar:986") + end + + ps = Sidekiq::ProcessSet.new + assert_equal 1, ps.size + assert_equal 1, ps.to_a.size + end + + def add_retry(jid = 'bob', at = Time.now.to_f) + payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f, 'error_backtrace' => ['line1', 'line2']) + Sidekiq.redis do |conn| + conn.zadd('retry', at.to_s, payload) + end + end + end +end diff --git a/test/test_extensions.rb b/test/test_extensions.rb new file mode 100644 index 00000000..4cec62df --- /dev/null +++ b/test/test_extensions.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true +require_relative 'helper' +require 'sidekiq/api' +require 'active_record' +require 'action_mailer' +Sidekiq::DelayExtensions.enable_delay! + +describe Sidekiq::DelayExtensions do + before do + Sidekiq.redis {|c| c.flushdb } + end + + class MyModel < ActiveRecord::Base + def self.long_class_method + raise "Should not be called!" + end + end + + it 'allows delayed execution of ActiveRecord class methods' do + assert_equal [], Sidekiq::Queue.all.map(&:name) + q = Sidekiq::Queue.new + assert_equal 0, q.size + MyModel.delay.long_class_method + assert_equal ['default'], Sidekiq::Queue.all.map(&:name) + assert_equal 1, q.size + end + + it 'uses and stringifies specified options' do + assert_equal [], Sidekiq::Queue.all.map(&:name) + q = Sidekiq::Queue.new('notdefault') + assert_equal 0, q.size + MyModel.delay(queue: :notdefault).long_class_method + assert_equal ['notdefault'], Sidekiq::Queue.all.map(&:name) + assert_equal ['MyModel.long_class_method'], q.map(&:display_class) + assert_equal 1, q.size + end + + it 'allows delayed scheduling of AR class methods' do + ss = Sidekiq::ScheduledSet.new + assert_equal 0, ss.size + MyModel.delay_for(5.days).long_class_method + assert_equal 1, ss.size + end + + it 'allows until delayed scheduling of AR class methods' do + ss = Sidekiq::ScheduledSet.new + assert_equal 0, ss.size + MyModel.delay_until(1.day.from_now).long_class_method + assert_equal 1, ss.size + end + + class UserMailer < ActionMailer::Base + def greetings(a, b) + raise "Should not be called!" + end + end + + it 'allows delayed delivery of ActionMailer mails' do + assert_equal [], Sidekiq::Queue.all.map(&:name) + q = Sidekiq::Queue.new + assert_equal 0, q.size + UserMailer.delay.greetings(1, 2) + assert_equal ['default'], Sidekiq::Queue.all.map(&:name) + assert_equal 1, q.size + end + + it 'allows delayed scheduling of AM mails' do + ss = Sidekiq::ScheduledSet.new + assert_equal 0, ss.size + UserMailer.delay_for(5.days).greetings(1, 2) + assert_equal 1, ss.size + end + + it 'allows until delay scheduling of AM mails' do + ss = Sidekiq::ScheduledSet.new + assert_equal 0, ss.size + UserMailer.delay_until(5.days.from_now).greetings(1, 2) + assert_equal 1, ss.size + end + + class SomeClass + def self.doit(arg) + end + end + + it 'allows delay of any ole class method' do + q = Sidekiq::Queue.new + assert_equal 0, q.size + SomeClass.delay.doit(Date.today) + assert_equal 1, q.size + end + + module SomeModule + def self.doit(arg) + end + end + + it 'logs large payloads' do + output = capture_logging(Logger::WARN) do + SomeClass.delay.doit('a' * 8192) + end + assert_match(/#{SomeClass}.doit job argument is/, output) + end + + it 'allows delay of any module class method' do + q = Sidekiq::Queue.new + assert_equal 0, q.size + SomeModule.delay.doit(Date.today) + assert_equal 1, q.size + end + +end diff --git a/test/test_testing_fake.rb b/test/test_testing_fake.rb new file mode 100644 index 00000000..3d3baaa7 --- /dev/null +++ b/test/test_testing_fake.rb @@ -0,0 +1,365 @@ +# frozen_string_literal: true +require_relative 'helper' + +describe 'Sidekiq::Testing.fake' do + class PerformError < RuntimeError; end + + class DirectWorker + include Sidekiq::Worker + def perform(a, b) + a + b + end + end + + class EnqueuedWorker + include Sidekiq::Worker + def perform(a, b) + a + b + end + end + + class StoredWorker + include Sidekiq::Worker + def perform(error) + raise PerformError if error + end + end + + before do + require 'sidekiq/delay_extensions/testing' + Sidekiq::Testing.fake! + EnqueuedWorker.jobs.clear + DirectWorker.jobs.clear + end + + after do + Sidekiq::Testing.disable! + Sidekiq::Queues.clear_all + end + + it 'stubs the async call' do + assert_equal 0, DirectWorker.jobs.size + assert DirectWorker.perform_async(1, 2) + assert_in_delta Time.now.to_f, DirectWorker.jobs.last['enqueued_at'], 0.1 + assert_equal 1, DirectWorker.jobs.size + assert DirectWorker.perform_in(10, 1, 2) + refute DirectWorker.jobs.last['enqueued_at'] + assert_equal 2, DirectWorker.jobs.size + assert DirectWorker.perform_at(10, 1, 2) + assert_equal 3, DirectWorker.jobs.size + assert_in_delta 10.seconds.from_now.to_f, DirectWorker.jobs.last['at'], 0.1 + end + + describe 'delayed' do + require 'action_mailer' + class FooMailer < ActionMailer::Base + def bar(str) + str + end + end + + before do + Sidekiq::DelayExtensions.enable_delay! + end + + it 'stubs the delay call on mailers' do + assert_equal 0, Sidekiq::DelayExtensions::DelayedMailer.jobs.size + FooMailer.delay.bar('hello!') + assert_equal 1, Sidekiq::DelayExtensions::DelayedMailer.jobs.size + end + + class Something + def self.foo(x) + end + end + + it 'stubs the delay call on classes' do + assert_equal 0, Sidekiq::DelayExtensions::DelayedClass.jobs.size + Something.delay.foo(Date.today) + assert_equal 1, Sidekiq::DelayExtensions::DelayedClass.jobs.size + end + + class BarMailer < ActionMailer::Base + def foo(str) + str + end + end + + it 'returns enqueued jobs for specific classes' do + assert_equal 0, Sidekiq::DelayExtensions::DelayedClass.jobs.size + FooMailer.delay.bar('hello!') + BarMailer.delay.foo('hello!') + assert_equal 2, Sidekiq::DelayExtensions::DelayedMailer.jobs.size + assert_equal 1, Sidekiq::DelayExtensions::DelayedMailer.jobs_for(FooMailer).size + assert_equal 1, Sidekiq::DelayExtensions::DelayedMailer.jobs_for(BarMailer).size + end + end + + it 'stubs the enqueue call' do + assert_equal 0, EnqueuedWorker.jobs.size + assert Sidekiq::Client.enqueue(EnqueuedWorker, 1, 2) + assert_equal 1, EnqueuedWorker.jobs.size + end + + it 'stubs the enqueue_to call' do + assert_equal 0, EnqueuedWorker.jobs.size + assert Sidekiq::Client.enqueue_to('someq', EnqueuedWorker, 1, 2) + assert_equal 1, Sidekiq::Queues['someq'].size + end + + it 'executes all stored jobs' do + assert StoredWorker.perform_async(false) + assert StoredWorker.perform_async(true) + + assert_equal 2, StoredWorker.jobs.size + assert_raises PerformError do + StoredWorker.drain + end + assert_equal 0, StoredWorker.jobs.size + end + + class SpecificJidWorker + include Sidekiq::Worker + sidekiq_class_attribute :count + self.count = 0 + def perform(worker_jid) + return unless worker_jid == self.jid + self.class.count += 1 + end + end + + it 'execute only jobs with assigned JID' do + 4.times do |i| + jid = SpecificJidWorker.perform_async(nil) + if i % 2 == 0 + SpecificJidWorker.jobs[-1]["args"] = ["wrong_jid"] + else + SpecificJidWorker.jobs[-1]["args"] = [jid] + end + end + + SpecificJidWorker.perform_one + assert_equal 0, SpecificJidWorker.count + + SpecificJidWorker.perform_one + assert_equal 1, SpecificJidWorker.count + + SpecificJidWorker.drain + assert_equal 2, SpecificJidWorker.count + end + + it 'round trip serializes the job arguments' do + assert StoredWorker.perform_async(:mike) + job = StoredWorker.jobs.first + assert_equal "mike", job['args'].first + StoredWorker.clear + end + + it 'perform_one runs only one job' do + DirectWorker.perform_async(1, 2) + DirectWorker.perform_async(3, 4) + assert_equal 2, DirectWorker.jobs.size + + DirectWorker.perform_one + assert_equal 1, DirectWorker.jobs.size + + DirectWorker.clear + end + + it 'perform_one raise error upon empty queue' do + DirectWorker.clear + assert_raises Sidekiq::EmptyQueueError do + DirectWorker.perform_one + end + end + + class FirstWorker + include Sidekiq::Worker + sidekiq_class_attribute :count + self.count = 0 + def perform + self.class.count += 1 + end + end + + class SecondWorker + include Sidekiq::Worker + sidekiq_class_attribute :count + self.count = 0 + def perform + self.class.count += 1 + end + end + + class ThirdWorker + include Sidekiq::Worker + sidekiq_class_attribute :count + def perform + FirstWorker.perform_async + SecondWorker.perform_async + end + end + + it 'clears jobs across all workers' do + Sidekiq::Worker.jobs.clear + FirstWorker.count = 0 + SecondWorker.count = 0 + + assert_equal 0, FirstWorker.jobs.size + assert_equal 0, SecondWorker.jobs.size + + FirstWorker.perform_async + SecondWorker.perform_async + + assert_equal 1, FirstWorker.jobs.size + assert_equal 1, SecondWorker.jobs.size + + Sidekiq::Worker.clear_all + + assert_equal 0, FirstWorker.jobs.size + assert_equal 0, SecondWorker.jobs.size + + assert_equal 0, FirstWorker.count + assert_equal 0, SecondWorker.count + end + + it 'drains jobs across all workers' do + Sidekiq::Worker.jobs.clear + FirstWorker.count = 0 + SecondWorker.count = 0 + + assert_equal 0, FirstWorker.jobs.size + assert_equal 0, SecondWorker.jobs.size + + assert_equal 0, FirstWorker.count + assert_equal 0, SecondWorker.count + + FirstWorker.perform_async + SecondWorker.perform_async + + assert_equal 1, FirstWorker.jobs.size + assert_equal 1, SecondWorker.jobs.size + + Sidekiq::Worker.drain_all + + assert_equal 0, FirstWorker.jobs.size + assert_equal 0, SecondWorker.jobs.size + + assert_equal 1, FirstWorker.count + assert_equal 1, SecondWorker.count + end + + it 'drains jobs across all workers even when workers create new jobs' do + Sidekiq::Worker.jobs.clear + FirstWorker.count = 0 + SecondWorker.count = 0 + + assert_equal 0, ThirdWorker.jobs.size + + assert_equal 0, FirstWorker.count + assert_equal 0, SecondWorker.count + + ThirdWorker.perform_async + + assert_equal 1, ThirdWorker.jobs.size + + Sidekiq::Worker.drain_all + + assert_equal 0, ThirdWorker.jobs.size + + assert_equal 1, FirstWorker.count + assert_equal 1, SecondWorker.count + end + + it 'drains jobs of workers with symbolized queue names' do + Sidekiq::Worker.jobs.clear + + AltQueueWorker.perform_async(5,6) + assert_equal 1, AltQueueWorker.jobs.size + + Sidekiq::Worker.drain_all + assert_equal 0, AltQueueWorker.jobs.size + end + + it 'can execute a job' do + DirectWorker.execute_job(DirectWorker.new, [2, 3]) + end + + describe 'queue testing' do + before do + require 'sidekiq/delay_extensions/testing' + Sidekiq::Testing.fake! + end + + after do + Sidekiq::Testing.disable! + Sidekiq::Queues.clear_all + end + + class QueueWorker + include Sidekiq::Worker + def perform(a, b) + a + b + end + end + + class AltQueueWorker + include Sidekiq::Worker + sidekiq_options queue: :alt + def perform(a, b) + a + b + end + end + + it 'finds enqueued jobs' do + assert_equal 0, Sidekiq::Queues["default"].size + + QueueWorker.perform_async(1, 2) + QueueWorker.perform_async(1, 2) + AltQueueWorker.perform_async(1, 2) + + assert_equal 2, Sidekiq::Queues["default"].size + assert_equal [1, 2], Sidekiq::Queues["default"].first["args"] + + assert_equal 1, Sidekiq::Queues["alt"].size + end + + it 'clears out all queues' do + assert_equal 0, Sidekiq::Queues["default"].size + + QueueWorker.perform_async(1, 2) + QueueWorker.perform_async(1, 2) + AltQueueWorker.perform_async(1, 2) + + Sidekiq::Queues.clear_all + + assert_equal 0, Sidekiq::Queues["default"].size + assert_equal 0, QueueWorker.jobs.size + assert_equal 0, Sidekiq::Queues["alt"].size + assert_equal 0, AltQueueWorker.jobs.size + end + + it 'finds jobs enqueued by client' do + Sidekiq::Client.push( + 'class' => 'NonExistentWorker', + 'queue' => 'missing', + 'args' => [1] + ) + + assert_equal 1, Sidekiq::Queues["missing"].size + end + + it 'respects underlying array changes' do + # Rspec expect change() syntax saves a reference to + # an underlying array. When the array containing jobs is + # derived, Rspec test using `change(QueueWorker.jobs, :size).by(1)` + # won't pass. This attempts to recreate that scenario + # by saving a reference to the jobs array and ensuring + # it changes properly on enqueueing + jobs = QueueWorker.jobs + assert_equal 0, jobs.size + QueueWorker.perform_async(1, 2) + assert_equal 1, jobs.size + end + end +end diff --git a/test/test_testing_inline.rb b/test/test_testing_inline.rb new file mode 100644 index 00000000..e356fb30 --- /dev/null +++ b/test/test_testing_inline.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true +require_relative 'helper' + +describe 'Sidekiq::Testing.inline' do + class InlineError < RuntimeError; end + class ParameterIsNotString < RuntimeError; end + + class InlineWorker + include Sidekiq::Worker + def perform(pass) + raise ArgumentError, "no jid" unless jid + raise InlineError unless pass + end + end + + class InlineWorkerWithTimeParam + include Sidekiq::Worker + def perform(time) + raise ParameterIsNotString unless time.is_a?(String) || time.is_a?(Numeric) + end + end + + before do + require 'sidekiq/delay_extensions/testing' + require 'sidekiq/testing/inline' + Sidekiq::Testing.inline! + end + + after do + Sidekiq::Testing.disable! + end + + it 'stubs the async call when in testing mode' do + assert InlineWorker.perform_async(true) + + assert_raises InlineError do + InlineWorker.perform_async(false) + end + end + + describe 'delay' do + require 'action_mailer' + class InlineFooMailer < ActionMailer::Base + def bar(str) + raise InlineError + end + end + + class InlineFooModel + def self.bar(str) + raise InlineError + end + end + + before do + Sidekiq::DelayExtensions.enable_delay! + end + + it 'stubs the delay call on mailers' do + assert_raises InlineError do + InlineFooMailer.delay.bar('three') + end + end + + it 'stubs the delay call on models' do + assert_raises InlineError do + InlineFooModel.delay.bar('three') + end + end + end + + it 'stubs the enqueue call when in testing mode' do + assert Sidekiq::Client.enqueue(InlineWorker, true) + + assert_raises InlineError do + Sidekiq::Client.enqueue(InlineWorker, false) + end + end + + it 'stubs the push_bulk call when in testing mode' do + assert Sidekiq::Client.push_bulk({'class' => InlineWorker, 'args' => [[true], [true]]}) + + assert_raises InlineError do + Sidekiq::Client.push_bulk({'class' => InlineWorker, 'args' => [[true], [false]]}) + end + end + + it 'should relay parameters through json' do + assert Sidekiq::Client.enqueue(InlineWorkerWithTimeParam, Time.now.to_f) + end + +end