cmd/age: fix terminal escape sequences on Windows

If possible, we enable virtual terminal processing, which is necessary
for using terminal escape sequences on instances of the Windows Console.
When enabling virtual terminal processing fails, we completely avoid
using escape sequences to prevent weird characters to be printed to the
console.

Fixes FiloSottile#474

cmd/age/tui.go

@@ -67,6 +67,11 @@ func exit(code int) {
 	os.Exit(code)
 }
 
+// When avoidTerminalEscapeSequences is true, we avoid using escape sequences
+// to prevent weird characters to be printed to the console. This will happen
+// on Windows when virtual terminal processing cannot be enabled.
+var avoidTerminalEscapeSequences bool
+
 // clearLine clears the current line on the terminal, or opens a new line if
 // terminal escape codes don't work.
 func clearLine(out io.Writer) {
@@ -76,6 +81,13 @@ func clearLine(out io.Writer) {
 		EL  = CUI + "K" // Erase in Line
 	)
 
+	// We just open a new line when weird characters may be printed as a result
+	// of using escape codes.
+	if avoidTerminalEscapeSequences {
+		fmt.Fprintf(out, "\r\n")
+		return
+	}
+
 	// First, open a new line, which is guaranteed to work everywhere. Then, try
 	// to erase the line above with escape codes.
 	//

cmd/age/tui_windows.go

@@ -0,0 +1,48 @@
+// Copyright 2022 The age Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/windows"
+)
+
+// Some instances of the Windows Console (e.g., cmd.exe and Windows PowerShell)
+// do not have the virtual terminal processing enabled, which is necessary to
+// make terminal escape sequences work. For this reason the clearLine function
+// may not properly work. Here we enable the virtual terminal processing, if
+// possible.
+//
+// See https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences.
+func init() {
+	const (
+		ENABLE_PROCESSED_OUTPUT            uint32 = 0x1
+		ENABLE_VIRTUAL_TERMINAL_PROCESSING uint32 = 0x4
+	)
+
+	kernel32DLL := windows.NewLazySystemDLL("Kernel32.dll")
+	setConsoleMode := kernel32DLL.NewProc("SetConsoleMode")
+
+	var mode uint32
+	err := syscall.GetConsoleMode(syscall.Stdout, &mode)
+	if err != nil {
+		// TODO: terminal escape sequences may work at this point, even though
+		// we could not get the console mode. It may be worth trying using
+		// them, but more research is needed to avoid the weird characters
+		// printed out.
+		avoidTerminalEscapeSequences = true
+		return
+	}
+
+	mode |= ENABLE_PROCESSED_OUTPUT
+	mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING
+	ret, _, err := setConsoleMode(uintptr(syscall.Stdout), uintptr(mode))
+	// If the SetConsoleMode function fails, the return value is zero.
+	// See https://learn.microsoft.com/en-us/windows/console/setconsolemode#return-value.
+	if ret == 0 {
+		avoidTerminalEscapeSequences = true
+	}
+}