FEATURE: Setting for populating posts with images (#4)

This adds the new `include_images_in_posts` setting to `dev.yml` (disabled by default). If it's enabled some posts will contain images from https://picsum.photos/. The images get cached locally to speed up subsequent calls of `dev:populate`.

It also moves `redis-cli flushall` to the beginning of the `dev:populate` rake task. Otherwise it would delete jobs from the Sidekiq queue that are required for processing new posts. But it deletes the `UserEmail` jobs in order to prevent lots of errors when mailcatcher isn't running.

config/dev.yml

@@ -6,6 +6,7 @@ start_date: "01/01/2020"
 max_likes_count: 10
 auth_plugin_enabled: true
 allow_anonymous_to_impersonate: false
+include_images_in_posts: false
 new_user:
    username: new_user
    email: new_user@example.com

lib/discourse_dev/post.rb

@@ -25,14 +25,16 @@ def initialize(topic, count = DEFAULT_COUNT)
     def data
       {
         topic_id: topic.id,
-        raw: Faker::Markdown.sandwich(sentences: 5),
+        raw: Faker::DiscourseMarkdown.sandwich(sentences: 5),
         created_at: Faker::Time.between(from: topic.last_posted_at, to: DateTime.now),
         skip_validations: true,
         skip_guardian: true
       }
     end
 
     def create!
+      user = self.user
+      data = Faker::DiscourseMarkdown.with_user(user.id) { self.data }
       post = PostCreator.new(user, data).create!
       topic.reload
       generate_likes(post)
@@ -86,12 +88,15 @@ def self.add_replies!(args)
       count.times do |i|
         @index = i
         begin
-          reply = {
-            topic_id: topic.id,
-            raw: Faker::Markdown.sandwich(sentences: 5),
-            skip_validations: true
-          }
-          PostCreator.new(User.random, reply).create!
+          user = User.random
+          reply = Faker::DiscourseMarkdown.with_user(user.id) do
+            {
+              topic_id: topic.id,
+              raw: Faker::DiscourseMarkdown.sandwich(sentences: 5),
+              skip_validations: true
+            }
+          end
+          PostCreator.new(user, reply).create!
         rescue ActiveRecord::RecordNotSaved => e
           puts e
         end

lib/discourse_dev/tasks/dev.rake

@@ -25,12 +25,12 @@ end
 
 desc 'Populate sample content for development environment'
 task 'dev:populate' => ['db:load_config'] do |_, args|
+  system("redis-cli flushall")
   Rake::Task['groups:populate'].invoke
   Rake::Task['users:populate'].invoke
   Rake::Task['categories:populate'].invoke
   Rake::Task['tags:populate'].invoke
   Rake::Task['topics:populate'].invoke
-  system("redis-cli flushall")
 end
 
 desc 'Repopulate sample datas in development environment'

lib/discourse_dev/topic.rb

@@ -30,7 +30,7 @@ def data
 
       {
         title: title[0, SiteSetting.max_topic_title_length],
-        raw: Faker::Markdown.sandwich(sentences: 5),
+        raw: Faker::DiscourseMarkdown.sandwich(sentences: 5),
         category: @category.id,
         created_at: Faker::Time.between(from: DiscourseDev.config.start_date, to: DateTime.now),
         tags: tags,
@@ -63,7 +63,8 @@ def tags
 
     def create!
       @category = Category.random
-      topic = data
+      user = self.user
+      topic = Faker::DiscourseMarkdown.with_user(user.id) { data }
       post = PostCreator.new(user, topic).create!
 
       if topic[:title] == "Coolest thing you have seen today"
@@ -75,6 +76,11 @@ def create!
       Post.new(post.topic, reply_count).populate!
     end
 
+    def populate!
+      super
+      delete_unwanted_sidekiq_jobs
+    end
+
     def user
       return User.random if @category.groups.blank?
 
@@ -89,5 +95,11 @@ def current_count
       category_definition_topic_ids = ::Category.pluck(:topic_id)
       ::Topic.where(archetype: :regular).where.not(id: category_definition_topic_ids).count
     end
+
+    def delete_unwanted_sidekiq_jobs
+      Sidekiq::ScheduledSet.new.each do |job|
+        job.delete if job.item["class"] == "Jobs::UserEmail"
+      end
+    end
   end
 end

lib/faker/discourse_markdown.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'faker'
+require 'net/http'
+require 'json'
+
+module Faker
+  class DiscourseMarkdown < Markdown
+    class << self
+      attr_writer(:user_id)
+
+      def user_id
+        @user_id || ::Discourse::SYSTEM_USER_ID
+      end
+
+      def with_user(user_id)
+        current_user_id = self.user_id
+        self.user_id = user_id
+        begin
+          yield
+        ensure
+          self.user_id = current_user_id
+        end
+      end
+
+      def image
+        image = next_image
+        image_file = load_image(image)
+
+        upload = ::UploadCreator.new(
+          image_file,
+          image[:filename],
+          origin: image[:url]
+        ).create_for(user_id)
+
+        ::UploadMarkdown.new(upload).to_markdown
+      end
+
+      private
+
+      def next_image
+        if @images.blank?
+          @next_page = (@next_page || 0) + 1
+          url = URI("https://picsum.photos/v2/list?page=#{@next_page}&limit=100")
+          response = Net::HTTP.get(url)
+          json = JSON.parse(response)
+          @images = json.sort_by { |image| image["id"] }
+        end
+
+        image = @images.pop
+        { filename: "#{image['id']}.jpg", url: "#{image['download_url']}.jpg" }
+      end
+
+      def image_cache_dir
+        @image_cache_dir ||= ::File.join(Rails.root, "tmp", "discourse_dev", "images")
+      end
+
+      def load_image(image)
+        cache_path = ::File.join(image_cache_dir, image[:filename])
+
+        if !::File.exists?(cache_path)
+          FileUtils.mkdir_p(image_cache_dir)
+          temp_file = ::FileHelper.download(
+            image[:url],
+            max_file_size: [SiteSetting.max_image_size_kb.kilobytes, 10.megabytes].max,
+            tmp_file_name: "image",
+            follow_redirect: true
+          )
+          FileUtils.cp(temp_file, cache_path)
+        end
+
+        ::File.open(cache_path)
+      end
+
+      def available_methods
+        methods = super
+        methods << :image if ::DiscourseDev.config.include_images_in_posts
+        methods
+      end
+    end
+  end
+end