Fix duplicate assets and multiple pipeline runs

dev-mode/sbt-plugin/src/main/scala/play/sbt/PlaySettings.scala

@@ -209,6 +209,11 @@ object PlaySettings {
       (dirs * "routes").get ++ (dirs * "*.routes").get
     },
     inConfig(Compile)(externalizedSettings),
+    // Override sbt-web's exportedMappings to avoid multiple assets pipeline runs and duplicated files in dist packages
+    // - https://github.com/playframework/playframework/issues/5765
+    // - https://github.com/playframework/playframework/issues/5242
+    Assets / WebKeys.exportedMappings     := Nil,
+    TestAssets / WebKeys.exportedMappings := Nil,
   )
 
   /**

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/app/assets/coffeescripts/cscript.coffee

@@ -0,0 +1,28 @@
+# Assignment:
+number   = 46
+opposite = true
+
+# Conditions:
+number = -42 if opposite
+
+# Functions:
+square = (x) -> x * x
+
+# Arrays:
+list = [1, 2, 3, 4, 5]
+
+# Objects:
+math =
+  root:   Math.sqrt
+  square: square
+  cube:   (x) -> x * square x
+
+# Splats:
+race = (winner, runners...) ->
+  print winner, runners
+
+# Existence:
+alert "I knew it!" if elvis?
+
+# Array comprehensions:
+cubes = (math.cube num for num in list)

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/app/controllers/Application.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) from 2022 The Play Framework Contributors <https://github.com/playframework>, 2011-2021 Lightbend Inc. <https://www.lightbend.com>
+ */
+
+package controllers;
+
+import play.*;
+import play.mvc.*;
+
+public class Application extends Controller {
+    public Result index() {
+        return ok("hello world");
+    }
+}

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/build.sbt

@@ -0,0 +1,119 @@
+// Copyright (C) from 2022 The Play Framework Contributors <https://github.com/playframework>, 2011-2021 Lightbend Inc. <https://www.lightbend.com>
+
+import com.typesafe.sbt.web.pipeline.Pipeline
+import com.typesafe.sbt.web.PathMapping
+import java.nio.file._
+import scala.sys.process.Process
+
+val transform = taskKey[Pipeline.Stage]("transformer")
+
+lazy val root = (project in file("."))
+  .enablePlugins(PlayJava)
+  .settings(
+    name := "assets-pipeline",
+    scalaVersion  := ScriptedTools.scalaVersionFromJavaProperties(),
+    updateOptions := updateOptions.value.withLatestSnapshots(false),
+    update / evictionWarningOptions ~= (_.withWarnTransitiveEvictions(false).withWarnDirectEvictions(false)),
+    PlayKeys.playInteractionMode := play.sbt.StaticPlayNonBlockingInteractionMode,
+    // In sbt 1.4 extraLoggers got deprecated in favor of extraAppenders (new in sbt 1.4) and as a consequence logManager switched to use extraAppenders.
+    // To be able to run the tests in sbt 1.2+ however we can't use extraAppenders yet and to run the tests in sbt 1.4+ we need to make logManager use extraLoggers again.
+    // https://github.com/sbt/sbt/commit/2e9805b9d01c6345214c14264c61692d9c21651c#diff-6d9589bfb3f1247d2eace99bab7e928590337680d1aebd087d9da286586fba77R455
+    logManager := sbt.internal.LogManager.defaults(extraLoggers.value, ConsoleOut.systemOut),
+    extraLoggers ~= (fn => BufferLogger +: fn(_)),
+    libraryDependencies += guice,
+    // can't use test directory since scripted calls its script "test"
+    Test / sourceDirectory := baseDirectory.value / "tests",
+    Test / scalaSource := baseDirectory.value / "tests",
+    transform := { (mappings: Seq[PathMapping]) =>
+      streams.value.log.info("Running transform")
+      mappings
+    },
+    Assets / pipelineStages := Seq(transform),
+    InputKey[Unit]("verifyResourceContains") := {
+      val args       = Def.spaceDelimited("<path> <status> <words> ...").parsed
+      val path       = args.head
+      val status     = args.tail.head.toInt
+      val assertions = args.tail.tail
+      ScriptedTools.verifyResourceContains(path, status, assertions)
+    },
+    InputKey[Unit]("checkLogPipelineStages") := {
+      val transformCount = BufferLogger.messages.count(_ == "Running transform")
+      if (transformCount != 1) {
+        sys.error(
+          s"""sbt web pipeline stage "transform" found $transformCount time(s) in logs, should run exactly once however.
+             |Output:
+             |    ${BufferLogger.messages.reverse.mkString("\n    ")}""".stripMargin
+        )
+      }
+      val csCount = BufferLogger.messages.count(_ == "CoffeeScript compiling on 1 source(s)")
+      if (csCount != 1) {
+        sys.error(
+          s"""sbt web pipeline stage "coffeescript" found $csCount time(s) in logs, should run exactly once however.
+             |Output:
+             |    ${BufferLogger.messages.reverse.mkString("\n    ")}""".stripMargin
+        )
+      }
+    },
+    InputKey[Unit]("resetBufferLoggerHelper") := {
+      BufferLogger.resetMessages()
+    },
+    InputKey[Unit]("countFiles") := {
+      val args = Def.spaceDelimited("<filename> <expectedCount> [subDirPath]").parsed
+      val originalBaseDir = (ThisBuild / baseDirectory).value
+
+      if (args.length < 2 || args.length > 3) {
+        sys.error("Usage: countFiles <filename> <expectedCount> [subDirPath]")
+      } else {
+        val filename = args(0)
+        val expectedCount = args(1).toInt
+        val baseDir = if (args.length == 3) originalBaseDir.toPath.resolve(args(2)).normalize() else originalBaseDir.toPath
+
+        if (!Files.exists(baseDir) || !Files.isDirectory(baseDir)) {
+          sys.error(s"The path '$baseDir' is not a valid directory.")
+        }
+
+        val matcher = FileSystems.getDefault.getPathMatcher("glob:**/" + filename)
+
+        val fileCount = Files.walk(baseDir)
+          .filter(Files.isRegularFile(_))
+          .filter(matcher.matches(_))
+          .count()
+
+        if (fileCount != expectedCount) {
+          sys.error(s"Expected $expectedCount files named $filename, but found $fileCount.")
+        } else {
+          println(s"Found $fileCount files named $filename, as expected.")
+        }
+      }
+    },
+    InputKey[Unit]("checkUnzipListing") := {
+      val args = Def.spaceDelimited("<zipfile> <difffile>").parsed
+      val baseDir = (ThisBuild / baseDirectory).value
+
+      if (args.length != 2) {
+        sys.error("Usage: checkUnzipListing <zipfile> <difffile>")
+      } else {
+        val zipfile = args(0)
+        val difffile = args(1)
+
+        val unzipcmd = s"unzip -l $zipfile" // We assume the system has unzip installed...
+        val unzipOutput = Process(unzipcmd, baseDir).!!
+
+        val difffile_content = IO.readLines(new File(difffile)).mkString("\n") + "\n"
+
+        println(s"\nComparing unzip listing of file $zipfile with contents of $difffile")
+        println(s"### $zipfile")
+        print(unzipOutput)
+        println(s"### $difffile")
+        print(difffile_content)
+        println(s"###")
+
+        if (unzipOutput != difffile_content) {
+          sys.error(s"Unzip listing ('$unzipcmd') does not match expected content!")
+        } else {
+          println(s"Listing of $zipfile as expected.")
+        }
+        println()
+      }
+    },
+  )

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/conf/application.conf

@@ -0,0 +1,2 @@
+# This is the main configuration file for the application.
+# https://www.playframework.com/documentation/latest/ConfigFile

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/conf/routes

@@ -0,0 +1,4 @@
+# Copyright (C) from 2022 The Play Framework Contributors <https://github.com/playframework>, 2011-2021 Lightbend Inc. <https://www.lightbend.com>
+
+# Home page
+GET     /                           controllers.Application.index()

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/expected-assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-assets.jar.txt

@@ -0,0 +1,17 @@
+Archive:  target/universal/stage/lib/assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-assets.jar
+  Length      Date    Time    Name
+---------  ---------- -----   ----
+       25  2010-01-01 00:00   META-INF/MANIFEST.MF
+        0  2010-01-01 00:00   public/
+        0  2010-01-01 00:00   public/coffeescripts/
+        0  2010-01-01 00:00   public/images/
+        0  2010-01-01 00:00   public/javascripts/
+        0  2010-01-01 00:00   public/stylesheets/
+      414  2010-01-01 00:00   public/coffeescripts/cscript.coffee
+      912  2010-01-01 00:00   public/coffeescripts/cscript.js
+     1289  2010-01-01 00:00   public/coffeescripts/cscript.js.map
+      687  2010-01-01 00:00   public/images/favicon.png
+    95957  2010-01-01 00:00   public/javascripts/jquery-1.11.3.min.js
+        0  2010-01-01 00:00   public/stylesheets/main.css
+---------                     -------
+    99284                     12 files

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/expected-assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-sans-externalized.jar.txt

@@ -0,0 +1,18 @@
+Archive:  target/universal/stage/lib/assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-sans-externalized.jar
+  Length      Date    Time    Name
+---------  ---------- -----   ----
+       25  2010-01-01 00:00   META-INF/MANIFEST.MF
+        0  2010-01-01 00:00   controllers/
+        0  2010-01-01 00:00   controllers/javascript/
+        0  2010-01-01 00:00   router/
+      449  2010-01-01 00:00   controllers/Application.class
+     1616  2010-01-01 00:00   controllers/ReverseApplication.class
+     1973  2010-01-01 00:00   controllers/javascript/ReverseApplication.class
+      669  2010-01-01 00:00   controllers/routes$javascript.class
+      641  2010-01-01 00:00   controllers/routes.class
+     3853  2010-01-01 00:00   router/Routes$$anonfun$routes$1.class
+     8869  2010-01-01 00:00   router/Routes.class
+     2075  2010-01-01 00:00   router/RoutesPrefix$.class
+     1126  2010-01-01 00:00   router/RoutesPrefix.class
+---------                     -------
+    21296                     13 files

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/project/BufferLogger.scala

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) from 2022 The Play Framework Contributors <https://github.com/playframework>, 2011-2021 Lightbend Inc. <https://www.lightbend.com>
+ */
+
+import org.apache.logging.log4j.core.{ LogEvent => Log4JLogEvent, _ }
+import org.apache.logging.log4j.core.appender.AbstractAppender
+import org.apache.logging.log4j.core.filter.LevelRangeFilter
+import org.apache.logging.log4j.core.layout.PatternLayout
+import org.apache.logging.log4j.core.Filter.Result
+import org.apache.logging.log4j.Level
+
+object BufferLogger
+    extends AbstractAppender(
+      "FakeAppender",
+      LevelRangeFilter.createFilter(Level.ERROR, Level.ERROR, Result.NEUTRAL, Result.DENY),
+      PatternLayout.createDefaultLayout()
+    ) {
+  @volatile var messages = List.empty[String]
+
+  def append(event: Log4JLogEvent): Unit = {
+    if (event.getLevel == Level.INFO) {
+      synchronized {
+        messages = event.getMessage.getFormattedMessage :: messages
+      }
+    }
+  }
+
+  def resetMessages(): Unit = {
+    messages = List.empty[String]
+  }
+}

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/project/plugins.sbt

@@ -0,0 +1,7 @@
+// Copyright (C) from 2022 The Play Framework Contributors <https://github.com/playframework>, 2011-2021 Lightbend Inc. <https://www.lightbend.com>
+
+updateOptions                   := updateOptions.value.withLatestSnapshots(false)
+addSbtPlugin("org.playframework" % "sbt-plugin"         % sys.props("project.version"))
+addSbtPlugin("org.playframework" % "sbt-scripted-tools" % sys.props("project.version"))
+
+addSbtPlugin("com.github.sbt"    % "sbt-coffeescript"   % "2.0.1")

dev-mode/sbt-plugin/src/sbt-test/play-sbt-plugin/assets-pipeline/test

@@ -0,0 +1,107 @@
+> clean
+> countFiles "jquery-1.11.3.min.js" 1
+> countFiles "jquery-3.7.1.min.js" 1
+> countFiles "cscript.coffee" 1
+> countFiles "cscript.js" 0
+> assets
+> checkLogPipelineStages
+> countFiles "jquery-1.11.3.min.js" 2
+> countFiles "jquery-3.7.1.min.js" 1
+> countFiles "cscript.coffee" 2
+> countFiles "cscript.js" 2
+$ exists ./public/javascripts/jquery-1.11.3.min.js
+$ exists ./tests/public/javascripts/jquery-3.7.1.min.js
+$ exists ./target/web/public/main/javascripts/jquery-1.11.3.min.js
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/javascripts/jquery-1.11.3.min.js
+-$ exists ./target/web/classes/test/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/javascripts/jquery-3.7.1.min.js
+# Also fall back to check the existence of the folders to stay even more safe in case this project's name changes, etc.
+-$ exists ./target/web/classes/main/META-INF/resources/webjars
+-$ exists ./target/web/classes/test/META-INF/resources/webjars
+# Now check the coffeescript compilation
+$ exists ./app/assets/coffeescripts/cscript.coffee
+$ exists ./target/web/public/main/coffeescripts/cscript.coffee
+$ exists ./target/web/public/main/coffeescripts/cscript.js
+$ exists ./target/web/public/main/coffeescripts/cscript.js.map
+$ exists ./target/web/coffeescript/main/coffeescripts/cscript.js
+$ exists ./target/web/coffeescript/main/coffeescripts/cscript.js.map
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.coffee
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.js.map
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.js
+# Also fall back to check the existence of the folders to stay even more safe in case this project's name changes, etc.
+-$ exists ./target/web/classes/main/META-INF/resources/webjars
+> resetBufferLoggerHelper
+> clean
+
+> countFiles "jquery-1.11.3.min.js" 1
+> countFiles "jquery-3.7.1.min.js" 1
+> countFiles "cscript.coffee" 1
+> countFiles "cscript.js" 0
+> run
+# Trigger "reload":
+> verifyResourceContains / 200
+> playStop
+> checkLogPipelineStages
+> countFiles "jquery-1.11.3.min.js" 2
+> countFiles "jquery-3.7.1.min.js" 1
+> countFiles "cscript.coffee" 2
+> countFiles "cscript.js" 2
+$ exists ./public/javascripts/jquery-1.11.3.min.js
+$ exists ./tests/public/javascripts/jquery-3.7.1.min.js
+$ exists ./target/web/public/main/javascripts/jquery-1.11.3.min.js
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/javascripts/jquery-1.11.3.min.js
+-$ exists ./target/web/classes/test/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/javascripts/jquery-3.7.1.min.js
+# Also fall back to check the existence of the folders to stay even more safe in case this project's name changes, etc.
+-$ exists ./target/web/classes/main/META-INF/resources/webjars
+-$ exists ./target/web/classes/test/META-INF/resources/webjars
+# Now check the coffeescript compilation
+$ exists ./app/assets/coffeescripts/cscript.coffee
+$ exists ./target/web/public/main/coffeescripts/cscript.coffee
+$ exists ./target/web/public/main/coffeescripts/cscript.js
+$ exists ./target/web/public/main/coffeescripts/cscript.js.map
+$ exists ./target/web/coffeescript/main/coffeescripts/cscript.js
+$ exists ./target/web/coffeescript/main/coffeescripts/cscript.js.map
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.coffee
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.js.map
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.js
+# Also fall back to check the existence of the folders to stay even more safe in case this project's name changes, etc.
+-$ exists ./target/web/classes/main/META-INF/resources/webjars
+> resetBufferLoggerHelper
+> clean
+
+# Similiar for test assets
+> countFiles "jquery-1.11.3.min.js" 1
+> countFiles "jquery-3.7.1.min.js" 1
+> countFiles "cscript.coffee" 1
+> countFiles "cscript.js" 0
+> test
+-> checkLogPipelineStages
+> countFiles "jquery-1.11.3.min.js" 1
+> countFiles "jquery-3.7.1.min.js" 2
+> countFiles "cscript.coffee" 1
+> countFiles "cscript.js" 0
+$ exists ./tests/public/javascripts/jquery-3.7.1.min.js
+-$ exists ./target/web/public/main/javascripts/jquery-1.11.3.min.js
+$ exists ./target/web/public/test/public/javascripts/jquery-3.7.1.min.js
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/javascripts/jquery-1.11.3.min.js
+-$ exists ./target/web/classes/test/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/javascripts/jquery-3.7.1.min.js
+# Also fall back to check the existence of the folders to stay even more safe in case this project's name changes, etc.
+-$ exists ./target/web/classes/main/META-INF/resources/webjars
+-$ exists ./target/web/classes/test/META-INF/resources/webjars
+# Now check the coffeescript compilation
+$ exists ./app/assets/coffeescripts/cscript.coffee
+-$ exists ./target/web/public/main/coffeescripts/cscript.coffee
+-$ exists ./target/web/public/main/coffeescripts/cscript.js
+-$ exists ./target/web/public/main/coffeescripts/cscript.js.map
+-$ exists ./target/web/coffeescript/main/coffeescripts/cscript.js
+-$ exists ./target/web/coffeescript/main/coffeescripts/cscript.js.map
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.coffee
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.js.map
+-$ exists ./target/web/classes/main/META-INF/resources/webjars/assets-pipeline/0.1.0-SNAPSHOT/coffeescripts/cscript.js
+# Also fall back to check the existence of the folders to stay even more safe in case this project's name changes, etc.
+-$ exists ./target/web/classes/main/META-INF/resources/webjars
+> clean
+
+# Now let's check if the assets are correctly packaged into the jar files
+> stage
+> checkUnzipListing target/universal/stage/lib/assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-assets.jar expected-assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-assets.jar.txt
+> checkUnzipListing target/universal/stage/lib/assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-sans-externalized.jar expected-assets-pipeline.assets-pipeline-0.1.0-SNAPSHOT-sans-externalized.jar.txt