From 85a1efd77b8b571cd0fb695364918d4e0a7eb551 Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Thu, 22 Apr 2021 00:20:00 +0200 Subject: [PATCH] FEATURE: Setting for populating posts with images This adds the new `include_images_in_posts` setting to `dev.yml` (disabled by default). If it's enabled some posts will contain images from https://picsum.photos/. The images get cached locally to speed up subsequent calls of `dev:populate`. It also moves `redis-cli flushall` to the beginning of the `dev:populate` rake task. Otherwise it would delete jobs from the Sidekiq queue that are required for processing new posts. But it deletes the `UserEmail` jobs in order to prevent lots of errors when mailcatcher isn't running. --- config/dev.yml | 1 + lib/discourse_dev/post.rb | 19 +++++--- lib/discourse_dev/tasks/dev.rake | 2 +- lib/discourse_dev/topic.rb | 16 ++++++- lib/faker/discourse_markdown.rb | 82 ++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 lib/faker/discourse_markdown.rb diff --git a/config/dev.yml b/config/dev.yml index c4f2e84..1175a9d 100644 --- a/config/dev.yml +++ b/config/dev.yml @@ -6,6 +6,7 @@ start_date: "01/01/2020" max_likes_count: 10 auth_plugin_enabled: true allow_anonymous_to_impersonate: false +include_images_in_posts: false new_user: username: new_user email: new_user@example.com diff --git a/lib/discourse_dev/post.rb b/lib/discourse_dev/post.rb index f6c76e1..27e70aa 100644 --- a/lib/discourse_dev/post.rb +++ b/lib/discourse_dev/post.rb @@ -25,7 +25,7 @@ def initialize(topic, count = DEFAULT_COUNT) def data { topic_id: topic.id, - raw: Faker::Markdown.sandwich(sentences: 5), + raw: Faker::DiscourseMarkdown.sandwich(sentences: 5), created_at: Faker::Time.between(from: topic.last_posted_at, to: DateTime.now), skip_validations: true, skip_guardian: true @@ -33,6 +33,8 @@ def data end def create! + user = self.user + data = Faker::DiscourseMarkdown.with_user(user.id) { self.data } post = PostCreator.new(user, data).create! topic.reload generate_likes(post) @@ -86,12 +88,15 @@ def self.add_replies!(args) count.times do |i| @index = i begin - reply = { - topic_id: topic.id, - raw: Faker::Markdown.sandwich(sentences: 5), - skip_validations: true - } - PostCreator.new(User.random, reply).create! + user = User.random + reply = Faker::DiscourseMarkdown.with_user(user.id) do + { + topic_id: topic.id, + raw: Faker::DiscourseMarkdown.sandwich(sentences: 5), + skip_validations: true + } + end + PostCreator.new(user, reply).create! rescue ActiveRecord::RecordNotSaved => e puts e end diff --git a/lib/discourse_dev/tasks/dev.rake b/lib/discourse_dev/tasks/dev.rake index 3f77a49..faddf02 100644 --- a/lib/discourse_dev/tasks/dev.rake +++ b/lib/discourse_dev/tasks/dev.rake @@ -25,12 +25,12 @@ end desc 'Populate sample content for development environment' task 'dev:populate' => ['db:load_config'] do |_, args| + system("redis-cli flushall") Rake::Task['groups:populate'].invoke Rake::Task['users:populate'].invoke Rake::Task['categories:populate'].invoke Rake::Task['tags:populate'].invoke Rake::Task['topics:populate'].invoke - system("redis-cli flushall") end desc 'Repopulate sample datas in development environment' diff --git a/lib/discourse_dev/topic.rb b/lib/discourse_dev/topic.rb index 2f54164..43b31ff 100644 --- a/lib/discourse_dev/topic.rb +++ b/lib/discourse_dev/topic.rb @@ -30,7 +30,7 @@ def data { title: title[0, SiteSetting.max_topic_title_length], - raw: Faker::Markdown.sandwich(sentences: 5), + raw: Faker::DiscourseMarkdown.sandwich(sentences: 5), category: @category.id, created_at: Faker::Time.between(from: DiscourseDev.config.start_date, to: DateTime.now), tags: tags, @@ -63,7 +63,8 @@ def tags def create! @category = Category.random - topic = data + user = self.user + topic = Faker::DiscourseMarkdown.with_user(user.id) { data } post = PostCreator.new(user, topic).create! if topic[:title] == "Coolest thing you have seen today" @@ -75,6 +76,11 @@ def create! Post.new(post.topic, reply_count).populate! end + def populate! + super + delete_unwanted_sidekiq_jobs + end + def user return User.random if @category.groups.blank? @@ -89,5 +95,11 @@ def current_count category_definition_topic_ids = ::Category.pluck(:topic_id) ::Topic.where(archetype: :regular).where.not(id: category_definition_topic_ids).count end + + def delete_unwanted_sidekiq_jobs + Sidekiq::ScheduledSet.new.each do |job| + job.delete if job.item["class"] == "Jobs::UserEmail" + end + end end end diff --git a/lib/faker/discourse_markdown.rb b/lib/faker/discourse_markdown.rb new file mode 100644 index 0000000..a965d8f --- /dev/null +++ b/lib/faker/discourse_markdown.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'faker' +require 'net/http' +require 'json' + +module Faker + class DiscourseMarkdown < Markdown + class << self + attr_writer(:user_id) + + def user_id + @user_id || ::Discourse::SYSTEM_USER_ID + end + + def with_user(user_id) + current_user_id = self.user_id + self.user_id = user_id + begin + yield + ensure + self.user_id = current_user_id + end + end + + def image + image = next_image + image_file = load_image(image) + + upload = ::UploadCreator.new( + image_file, + image[:filename], + origin: image[:url] + ).create_for(user_id) + + ::UploadMarkdown.new(upload).to_markdown + end + + private + + def next_image + if @images.blank? + @next_page = (@next_page || 0) + 1 + url = URI("https://picsum.photos/v2/list?page=#{@next_page}&limit=100") + response = Net::HTTP.get(url) + json = JSON.parse(response) + @images = json.sort_by { |image| image["id"] } + end + + image = @images.pop + { filename: "#{image['id']}.jpg", url: "#{image['download_url']}.jpg" } + end + + def image_cache_dir + @image_cache_dir ||= ::File.join(Rails.root, "tmp", "discourse_dev", "images") + end + + def load_image(image) + cache_path = ::File.join(image_cache_dir, image[:filename]) + + if !::File.exists?(cache_path) + FileUtils.mkdir_p(image_cache_dir) + temp_file = ::FileHelper.download( + image[:url], + max_file_size: [SiteSetting.max_image_size_kb.kilobytes, 10.megabytes].max, + tmp_file_name: "image", + follow_redirect: true + ) + FileUtils.cp(temp_file, cache_path) + end + + ::File.open(cache_path) + end + + def available_methods + methods = super + methods << :image if ::DiscourseDev.config.include_images_in_posts + methods + end + end + end +end