Merge pull request #10057 from SethTisue/issue-12611

improve escaped-quote handling in sys.process (CommandLineParser)

src/compiler/scala/tools/nsc/settings/MutableSettings.scala

@@ -22,6 +22,7 @@ import scala.collection.mutable.Clearable
 import scala.io.Source
 import scala.reflect.internal.util.{SomeOfNil, StringOps}
 import scala.reflect.{ClassTag, classTag}
+import scala.sys.process.{Parser => CommandLineParser}
 
 /** A mutable Settings object.
  */
@@ -111,7 +112,7 @@ class MutableSettings(val errorFn: String => Unit, val pathFactory: PathFactory)
 
   /** Split the given line into parameters.
    */
-  def splitParams(line: String) = cmd.CommandLineParser.tokenize(line, errorFn)
+  def splitParams(line: String) = CommandLineParser.tokenize(line, errorFn)
 
   /** Returns any unprocessed arguments.
    */

src/compiler/scala/tools/reflect/ToolBoxFactory.scala

@@ -14,12 +14,12 @@ package scala
 package tools
 package reflect
 
-import scala.tools.cmd.CommandLineParser
 import scala.tools.nsc.reporters._
 import scala.tools.nsc.CompilerCommand
 import scala.tools.nsc.io.{AbstractFile, VirtualDirectory}
 import scala.reflect.internal.util.{AbstractFileClassLoader, NoSourceFile}
 import scala.reflect.internal.Flags._
+import scala.sys.process.{Parser => CommandLineParser}
 import java.lang.{Class => jClass}
 import java.lang.System.{lineSeparator => EOL}
 

src/library/scala/sys/process/Parser.scala

@@ -14,17 +14,18 @@ package scala.sys.process
 
 import scala.annotation.tailrec
 
-/** A simple enough command line parser.
+/** A simple enough command line parser using shell quote conventions.
  */
 private[scala] object Parser {
   private final val DQ = '"'
   private final val SQ = '\''
+  private final val EOF = -1
 
   /** Split the line into tokens separated by whitespace or quotes.
    *
    *  @return either an error message or reverse list of tokens
    */
-  private def tokens(in: String) = {
+  def tokenize(line: String, errorFn: String => Unit): List[String] = {
     import Character.isWhitespace
     import java.lang.{StringBuilder => Builder}
     import collection.mutable.ArrayBuffer
@@ -34,66 +35,73 @@ private[scala] object Parser {
     var start = 0
     val qpos = new ArrayBuffer[Int](16)    // positions of paired quotes
 
-    def cur: Int  = if (done) -1 else in.charAt(pos)
+    def cur: Int  = if (done) EOF else line.charAt(pos)
     def bump() = pos += 1
-    def done   = pos >= in.length
+    def done   = pos >= line.length
 
-    def skipToQuote(q: Int) = {
+    // Skip to the next quote as given.
+    def skipToQuote(q: Int): Boolean = {
       var escaped = false
-      def terminal = in.charAt(pos) match {
+      def terminal: Boolean = cur match {
         case _ if escaped => escaped = false ; false
         case '\\'         => escaped = true ; false
-        case `q`          => true
+        case `q` | EOF    => true
         case _            => false
       }
-      while (!done && !terminal) pos += 1
+      while (!terminal) bump()
       !done
     }
-    @tailrec
-    def skipToDelim(): Boolean =
-      cur match {
-        case q @ (DQ | SQ)        => { qpos += pos; bump(); skipToQuote(q) } && { qpos += pos; bump(); skipToDelim() }
-        case -1                   => true
+    // Skip to a word boundary, where words can be quoted and quotes can be escaped
+    def skipToDelim(): Boolean = {
+      var escaped = false
+      def quote() = { qpos += pos ; bump() }
+      @tailrec def advance(): Boolean = cur match {
+        case _ if escaped         => escaped = false ; bump() ; advance()
+        case '\\'                 => escaped = true ; bump() ; advance()
+        case q @ (DQ | SQ)        => { quote() ; skipToQuote(q) } && { quote() ; advance() }
+        case EOF                  => true
         case c if isWhitespace(c) => true
-        case _                    => bump(); skipToDelim()
+        case _                    => bump(); advance()
       }
-    def skipWhitespace() = while (isWhitespace(cur)) pos += 1
+      advance()
+    }
+    def skipWhitespace() = while (isWhitespace(cur)) bump()
     def copyText() = {
       val buf = new Builder
       var p = start
       var i = 0
       while (p < pos) {
         if (i >= qpos.size) {
-          buf.append(in, p, pos)
+          buf.append(line, p, pos)
           p = pos
         } else if (p == qpos(i)) {
-          buf.append(in, qpos(i)+1, qpos(i+1))
+          buf.append(line, qpos(i)+1, qpos(i+1))
           p = qpos(i+1)+1
           i += 2
         } else {
-          buf.append(in, p, qpos(i))
+          buf.append(line, p, qpos(i))
           p = qpos(i)
         }
       }
       buf.toString
     }
     def text() = {
       val res =
-        if (qpos.isEmpty) in.substring(start, pos)
-        else if (qpos(0) == start && qpos(1) == pos) in.substring(start+1, pos-1)
+        if (qpos.isEmpty) line.substring(start, pos)
+        else if (qpos(0) == start && qpos(1) == pos) line.substring(start+1, pos-1)
         else copyText()
       qpos.clear()
       res
     }
-    def badquote = Left("Unmatched quote")
+    def badquote() = errorFn(s"Unmatched quote [${qpos.last}](${line.charAt(qpos.last)})")
 
-    @tailrec def loop(): Either[String, List[String]] = {
+    @tailrec def loop(): List[String] = {
       skipWhitespace()
       start = pos
-      if (done) Right(accum)
-      else if (!skipToDelim()) badquote
+      if (done) accum.reverse
+      else if (!skipToDelim()) { badquote() ; Nil }
       else {
-        accum = text() :: accum
+        accum ::= text()
         loop()
       }
     }
@@ -102,11 +110,5 @@ private[scala] object Parser {
 
   class ParseException(msg: String) extends RuntimeException(msg)
 
-  def tokenize(line: String, errorFn: String => Unit): List[String] =
-    tokens(line) match {
-      case Right(args) => args.reverse
-      case Left(msg)   => errorFn(msg) ; Nil
-    }
-
   def tokenize(line: String): List[String] = tokenize(line, x => throw new ParseException(x))
 }

src/partest/scala/tools/partest/DirectTest.scala

@@ -13,7 +13,7 @@
 package scala.tools.partest
 
 import scala.reflect.internal.util.{BatchSourceFile, SourceFile}
-import scala.tools.cmd.CommandLineParser
+import scala.sys.process.{Parser => CommandLineParser}
 import scala.tools.nsc._
 import scala.tools.nsc.reporters.{ConsoleReporter, Reporter}
 import scala.tools.nsc.settings.ScalaVersion
@@ -44,21 +44,24 @@ abstract class DirectTest {
 
   protected def pathOf(locations: String*) = locations.mkString(sys.props("path.separator"))
 
+  // convenient for test classes not in a subpackage of scala
+  final protected def tokenize(line: String): List[String] = CommandLineParser.tokenize(line)
+
   // override to add additional settings besides -d testOutput.path
   // default is -usejavacp
   def extraSettings: String = "-usejavacp"
   // a default Settings object using only extraSettings
-  def settings: Settings = newSettings(CommandLineParser.tokenize(extraSettings))
+  def settings: Settings = newSettings(tokenize(extraSettings))
   // settings factory using given args and also debug settings
   def newSettings(args: List[String]) = (new Settings).tap { s =>
-    val allArgs = args ++ CommandLineParser.tokenize(debugSettings)
+    val allArgs = args ++ tokenize(debugSettings)
     log(s"newSettings: allArgs = $allArgs")
     val (success, residual) = s.processArguments(allArgs, processAll = false)
     assert(success && residual.isEmpty, s"Bad settings [${args.mkString(",")}], residual [${residual.mkString(",")}]")
   }
   // new compiler using given ad hoc args, -d and extraSettings
   def newCompiler(args: String*): Global = {
-    val settings = newSettings(CommandLineParser.tokenize(s"""-d "${testOutput.path}" ${extraSettings}""") ++ args.toList)
+    val settings = newSettings(tokenize(s"""-d "${testOutput.path}" ${extraSettings}""") ++ args.toList)
     newCompiler(settings)
   }
 

src/partest/scala/tools/partest/ScaladocModelTest.scala

@@ -12,7 +12,7 @@
 
 package scala.tools.partest
 
-import scala.tools.cmd.CommandLineParser
+import scala.sys.process.{Parser => CommandLineParser}
 import scala.tools.nsc._
 import scala.tools.nsc.doc.base.comment._
 import scala.tools.nsc.doc.model._

src/partest/scala/tools/partest/nest/CommandLine.scala

@@ -13,7 +13,7 @@
 package scala.tools.partest.nest
 
 import scala.collection.mutable.ListBuffer
-import scala.tools.cmd.toArgs
+import scala.sys.process.Parser.tokenize
 
 trait CommandLineConfig {
   def enforceArity: Boolean = true
@@ -23,7 +23,7 @@ trait CommandLineConfig {
 /** An instance of a command line, parsed according to a Spec.
  */
 class CommandLine(val spec: Reference, val originalArgs: List[String]) extends CommandLineConfig {
-  def this(spec: Reference, line: String) = this(spec, toArgs(line))
+  def this(spec: Reference, line: String) = this(spec, tokenize(line))
   def this(spec: Reference, args: Array[String]) = this(spec, args.toList)
 
   import spec.{ isUnaryOption, isBinaryOption, isExpandOption }

src/partest/scala/tools/partest/nest/FromString.scala

@@ -13,7 +13,7 @@
 package scala.tools.partest.nest
 
 import scala.reflect.OptManifest
-import scala.tools.cmd.toArgs
+import scala.sys.process.Parser.tokenize
 import scala.tools.nsc.io.Directory
 
 /** A general mechanism for defining how a command line argument
@@ -53,7 +53,7 @@ object FromString {
    *  list "foo", "bar", "baz".
    */
   val ArgumentsFromString: FromString[List[String]] = new FromString[List[String]] {
-    def apply(s: String) = toArgs(s)
+    def apply(s: String) = tokenize(s)
   }
 
   /** Identity.

src/partest/scala/tools/partest/nest/Runner.scala

@@ -470,7 +470,7 @@ class Runner(val testInfo: TestInfo, val suiteRunner: AbstractRunner) {
   def toolArgsFor(files: List[File])(tool: String, split: Boolean = true): List[String] = {
     def argsFor(f: File): List[String] = {
       import scala.jdk.OptionConverters._
-      import scala.tools.cmd.CommandLineParser.tokenize
+      import scala.sys.process.{Parser => CommandLineParser}, CommandLineParser.tokenize
       val max  = 10
       val tag  = s"$tool:"
       val endc = "*" + "/"    // be forgiving of /* scalac: ... */

src/repl-frontend/scala/tools/nsc/interpreter/shell/ILoop.scala

@@ -655,7 +655,7 @@ class ILoop(config: ShellConfig, inOverride: BufferedReader = null,
   }
 
   def loadCommand(arg: String): Result = {
-    import scala.tools.cmd.CommandLineParser
+    import scala.sys.process.{Parser => CommandLineParser}
     def run(file: String, args: List[String], verbose: Boolean) = withFile(file) { f =>
       intp.interpret(s"val args: Array[String] = ${ args.map("\"" + _ + "\"").mkString("Array(", ",", ")") }")
       interpretAllFrom(f, verbose)