From d19361ac855df3414fbf57fc0c4eda0afefa1526 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Sun, 4 Dec 2022 19:53:36 +0100
Subject: [PATCH 01/13] initial rewrite of addon mechs

---
 .../src/WebLinkProvider.ts                    | 123 ++++++++++++------
 .../src/WebLinksAddon.ts                      |  23 +---
 bin/test_weblinks.sh                          |  22 ++++
 3 files changed, 108 insertions(+), 60 deletions(-)
 create mode 100755 bin/test_weblinks.sh

diff --git a/addons/xterm-addon-web-links/src/WebLinkProvider.ts b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
index fafbb61478..501eba99c7 100644
--- a/addons/xterm-addon-web-links/src/WebLinkProvider.ts
+++ b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
@@ -45,7 +45,10 @@ export class LinkComputer {
   public static computeLink(y: number, regex: RegExp, terminal: Terminal, activate: (event: MouseEvent, uri: string) => void): ILink[] {
     const rex = new RegExp(regex.source, (regex.flags || '') + 'g');
 
-    const [line, startLineIndex] = LinkComputer._translateBufferLineToStringWithWrap(y - 1, false, terminal);
+    const [lines, startLineIndex] = LinkComputer._getFullLineString(y - 1, terminal);
+
+    // TODO: do locally limited search if string is too long
+    const line = lines.join('');
 
     // Don't try if the wrapped line if excessively large as the regex matching will block the main
     // thread.
@@ -58,7 +61,7 @@ export class LinkComputer {
     const result: ILink[] = [];
 
     while ((match = rex.exec(line)) !== null) {
-      const text = match[1];
+      const text = match[0];
       if (!text) {
         // something matched but does not comply with the given matchIndex
         // since this is most likely a bug the regex itself we simply do nothing here
@@ -77,28 +80,39 @@ export class LinkComputer {
         break;
       }
 
-      let endX = stringIndex + text.length;
-      let endY = startLineIndex + 1;
-
-      while (endX > terminal.cols) {
-        endX -= terminal.cols;
-        endY++;
+      // check via URL if the matched text would form a proper url
+      // NOTE: This outsources the ugly url parsing to the browser.
+      // To avoid surprising auto expansion from URL we additionally
+      // check afterwards if the provided string resembles the parsed
+      // one close enough:
+      // - decodeURI  decode path segement back to byte repr
+      //              to detect unicode auto conversion correctly
+      // - append /   also match domain urls w'o any path notion
+      try {
+        const url = new URL(text);
+        const urlText = decodeURI(url.toString());
+        if (text !== urlText && text + '/' !== urlText) {
+          continue;
+        }
+      } catch (e) {
+        continue;
       }
 
-      let startX = stringIndex + 1;
-      let startY = startLineIndex + 1;
-      while (startX > terminal.cols) {
-        startX -= terminal.cols;
-        startY++;
+
+      const [startY, startX] = LinkComputer._mapStringIndexToBuffer(startLineIndex, stringIndex, terminal);
+      const [endY, endX] = LinkComputer._mapStringIndexToBuffer(startLineIndex, stringIndex + text.length, terminal);
+
+      if (startY === -1 || startX === -1 || endY === -1 || endX === -1) {
+        continue;
       }
 
       const range = {
         start: {
-          x: startX,
-          y: startY
+          x: startX + 1,
+          y: startY + 1
         },
         end: {
-          x: endX,
+          x: endX + 1,
           y: endY
         }
       };
@@ -112,39 +126,66 @@ export class LinkComputer {
   /**
    * Gets the entire line for the buffer line
    * @param lineIndex The index of the line being translated.
-   * @param trimRight Whether to trim whitespace to the right.
    */
-  private static _translateBufferLineToStringWithWrap(lineIndex: number, trimRight: boolean, terminal: Terminal): [string, number] {
-    let lineString = '';
-    let lineWrapsToNext: boolean;
-    let prevLinesToWrap: boolean;
+  private static _getFullLineString(lineIndex: number, terminal: Terminal): [string[], number] {
+    let line: any;
 
-    do {
-      const line = terminal.buffer.active.getLine(lineIndex);
-      if (!line) {
-        break;
-      }
+    // expand top
+    let topIdx = lineIndex;
+    while ((line = terminal.buffer.active.getLine(topIdx)) && line.isWrapped) {
+      topIdx--;
+    }
 
-      if (line.isWrapped) {
-        lineIndex--;
-      }
+    // expand bottom
+    let bottomIdx = lineIndex + 1;
+    while ((line = terminal.buffer.active.getLine(bottomIdx)) && line.isWrapped) {
+      bottomIdx++;
+    }
 
-      prevLinesToWrap = line.isWrapped;
-    } while (prevLinesToWrap);
+    const lines: string[] = [];
+    for (let idx = topIdx; idx < bottomIdx; ++idx) {
+      lines.push(terminal.buffer.active.getLine(idx)?.translateToString(true)!);
+    }
 
-    const startLineIndex = lineIndex;
+    return [lines, topIdx];
+  }
 
-    do {
-      const nextLine = terminal.buffer.active.getLine(lineIndex + 1);
-      lineWrapsToNext = nextLine ? nextLine.isWrapped : false;
-      const line = terminal.buffer.active.getLine(lineIndex);
+  /**
+   * Map a string index back to buffer positions.
+   * Returns buffer position as [lineIndex, columnIndex] 0-based,
+   * or [-1, -1] in case the lookup ran into a non-existing line.
+   */
+  private static _mapStringIndexToBuffer(lineIndex: number, stringIndex: number, terminal: Terminal): [number, number] {
+    const buf = terminal.buffer.active;
+    const cell = buf.getNullCell();
+    while (stringIndex) {
+      const line = buf.getLine(lineIndex);
       if (!line) {
-        break;
+        return [-1, -1];
+      }
+      for (let i = 0; i < line.length; ++i) {
+        line.getCell(i, cell);
+        const chars = cell.getChars();
+        const width = cell.getWidth();
+        if (width) {
+          stringIndex -= chars.length || 1;
+        }
+        // look ahead for early wrap around of wide chars
+        if (i === line.length - 1 && chars === '' && width) {
+          const line = buf.getLine(lineIndex + 1);
+          if (line && line.isWrapped) {
+            line.getCell(0, cell);
+            if (cell.getWidth() === 2) {
+              stringIndex += 1;
+            }
+          }
+        }
+        if (stringIndex < 0) {
+          return [lineIndex, i];
+        }
       }
-      lineString += line.translateToString(!lineWrapsToNext && trimRight).substring(0, terminal.cols);
       lineIndex++;
-    } while (lineWrapsToNext);
-
-    return [lineString, startLineIndex];
+    }
+    return [lineIndex, 0];
   }
 }
diff --git a/addons/xterm-addon-web-links/src/WebLinksAddon.ts b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
index 1e3c877d1d..c71753ffac 100644
--- a/addons/xterm-addon-web-links/src/WebLinksAddon.ts
+++ b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
@@ -6,25 +6,10 @@
 import { Terminal, ITerminalAddon, IDisposable } from 'xterm';
 import { ILinkProviderOptions, WebLinkProvider } from './WebLinkProvider';
 
-const protocolClause = '(https?:\\/\\/)';
-const domainCharacterSet = '[\\da-z\\.-]+';
-const negatedDomainCharacterSet = '[^\\da-z\\.-]+';
-const domainBodyClause = '(' + domainCharacterSet + ')';
-const tldClause = '([a-z\\.]{2,18})';
-const ipClause = '((\\d{1,3}\\.){3}\\d{1,3})';
-const localHostClause = '(localhost)';
-const portClause = '(:\\d{1,5})';
-const hostClause = '((' + domainBodyClause + '\\.' + tldClause + ')|' + ipClause + '|' + localHostClause + ')' + portClause + '?';
-const pathCharacterSet = '(\\/[\\/\\w\\.\\-%~:+@]*)*([^:"\'\\s])';
-const pathClause = '(' + pathCharacterSet + ')?';
-const queryStringHashFragmentCharacterSet = '[0-9\\w\\[\\]\\(\\)\\/\\?\\!#@$%&\'*+,:;~\\=\\.\\-]*';
-const queryStringClause = '(\\?' + queryStringHashFragmentCharacterSet + ')?';
-const hashFragmentClause = '(#' + queryStringHashFragmentCharacterSet + ')?';
-const negatedPathCharacterSet = '[^\\/\\w\\.\\-%]+';
-const bodyClause = hostClause + pathClause + queryStringClause + hashFragmentClause;
-const start = '(?:^|' + negatedDomainCharacterSet + ')(';
-const end = ')($|' + negatedPathCharacterSet + ')';
-const strictUrlRegex = new RegExp(start + protocolClause + bodyClause + end);
+// consider everthing starting with http:// or https://
+// up to first whitespace as url
+// gets further narrowed down with URL later on
+const strictUrlRegex = /https?:[/]{2}\S*/;
 
 function handleLink(event: MouseEvent, uri: string): void {
   const newWindow = window.open();
diff --git a/bin/test_weblinks.sh b/bin/test_weblinks.sh
new file mode 100755
index 0000000000..a50db78820
--- /dev/null
+++ b/bin/test_weblinks.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# all half width - only good case
+echo "aaa http://example.com aaa http://example.com aaa"
+
+# full width before - wrong offset
+echo "￥￥￥ http://example.com aaa http://example.com aaa"
+
+# full width between - wrong offset
+echo "aaa http://example.com ￥￥￥ http://example.com aaa"
+
+# full width before and between - error in offsets adding up
+echo "￥￥￥ http://example.com ￥￥￥ http://example.com aaa"
+
+# full width within url - partial wrong match
+echo "aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문"
+
+# full width within and before - partial wrong match + wrong offsets
+echo "￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문"
+
+# not matching at all
+echo "http://test:password@example.com/some_path"

From 39c2e57ad326d39db9ff2022d350a3575a7b7dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Sun, 4 Dec 2022 22:26:29 +0100
Subject: [PATCH 02/13] hack around DOM renderer issue, fix api tests

---
 .../src/WebLinkProvider.ts                    | 19 ++++++-------------
 .../src/WebLinksAddon.ts                      |  7 ++++---
 .../test/WebLinksAddon.api.ts                 |  4 ++++
 src/browser/renderer/dom/DomRenderer.ts       |  2 ++
 .../renderer/dom/DomRendererRowFactory.ts     |  5 +++++
 5 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/addons/xterm-addon-web-links/src/WebLinkProvider.ts b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
index 501eba99c7..0dd4c9bec2 100644
--- a/addons/xterm-addon-web-links/src/WebLinkProvider.ts
+++ b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
@@ -46,13 +46,12 @@ export class LinkComputer {
     const rex = new RegExp(regex.source, (regex.flags || '') + 'g');
 
     const [lines, startLineIndex] = LinkComputer._getFullLineString(y - 1, terminal);
-
-    // TODO: do locally limited search if string is too long
     const line = lines.join('');
 
     // Don't try if the wrapped line if excessively large as the regex matching will block the main
     // thread.
     if (line.length > 1024) {
+      // TODO: more sophisticated handling with individual line introspection?
       return [];
     }
 
@@ -62,12 +61,6 @@ export class LinkComputer {
 
     while ((match = rex.exec(line)) !== null) {
       const text = match[0];
-      if (!text) {
-        // something matched but does not comply with the given matchIndex
-        // since this is most likely a bug the regex itself we simply do nothing here
-        console.log('match found without corresponding matchIndex');
-        break;
-      }
 
       // Get index, match.index is for the outer match which includes negated chars
       // therefore we cannot use match.index directly, instead we search the position
@@ -99,8 +92,8 @@ export class LinkComputer {
       }
 
 
-      const [startY, startX] = LinkComputer._mapStringIndexToBuffer(startLineIndex, stringIndex, terminal);
-      const [endY, endX] = LinkComputer._mapStringIndexToBuffer(startLineIndex, stringIndex + text.length, terminal);
+      const [startY, startX] = LinkComputer._mapStrIdx(startLineIndex, stringIndex, terminal);
+      const [endY, endX] = LinkComputer._mapStrIdx(startLineIndex, stringIndex + text.length, terminal);
 
       if (startY === -1 || startX === -1 || endY === -1 || endX === -1) {
         continue;
@@ -112,8 +105,8 @@ export class LinkComputer {
           y: startY + 1
         },
         end: {
-          x: endX + 1,
-          y: endY
+          x: endX,
+          y: endY + 1
         }
       };
 
@@ -155,7 +148,7 @@ export class LinkComputer {
    * Returns buffer position as [lineIndex, columnIndex] 0-based,
    * or [-1, -1] in case the lookup ran into a non-existing line.
    */
-  private static _mapStringIndexToBuffer(lineIndex: number, stringIndex: number, terminal: Terminal): [number, number] {
+  private static _mapStrIdx(lineIndex: number, stringIndex: number, terminal: Terminal): [number, number] {
     const buf = terminal.buffer.active;
     const cell = buf.getNullCell();
     while (stringIndex) {
diff --git a/addons/xterm-addon-web-links/src/WebLinksAddon.ts b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
index c71753ffac..ccf45e7407 100644
--- a/addons/xterm-addon-web-links/src/WebLinksAddon.ts
+++ b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
@@ -7,9 +7,10 @@ import { Terminal, ITerminalAddon, IDisposable } from 'xterm';
 import { ILinkProviderOptions, WebLinkProvider } from './WebLinkProvider';
 
 // consider everthing starting with http:// or https://
-// up to first whitespace as url
-// gets further narrowed down with URL later on
-const strictUrlRegex = /https?:[/]{2}\S*/;
+// up to first whitespace, `"` or `'` as url
+// NOTE: The repeated end clause is needed to not match a dangling `:`
+// resembling the old (...)*([^:"\'\\s]) final path clause
+const strictUrlRegex = /https?:[/]{2}[^\s^"^']*[^\s^"^'^:]/;
 
 function handleLink(event: MouseEvent, uri: string): void {
   const newWindow = window.open();
diff --git a/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts b/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
index bc97808567..9bcda0a52c 100644
--- a/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
+++ b/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
@@ -35,6 +35,10 @@ describe('WebLinksAddon', () => {
   it('.io', async function(): Promise<any> {
     await testHostName('foo.io');
   });
+
+  it.skip('correct buffer offsets', async () => {
+    // TODO: test strings in test_weblinks.sh automatically
+  });
 });
 
 async function testHostName(hostname: string): Promise<void> {
diff --git a/src/browser/renderer/dom/DomRenderer.ts b/src/browser/renderer/dom/DomRenderer.ts
index 72f7e254b6..094e6a8e3f 100644
--- a/src/browser/renderer/dom/DomRenderer.ts
+++ b/src/browser/renderer/dom/DomRenderer.ts
@@ -376,6 +376,8 @@ export class DomRenderer extends Disposable implements IRenderer {
   }
 
   private _setCellUnderline(x: number, x2: number, y: number, y2: number, cols: number, enabled: boolean): void {
+    // FIXME: offset calculation is wrong (temp. fixed by adding empty spans for wide chars
+    // to fullfill column to element index identity assumption)
     while (x !== x2 || y !== y2) {
       const row = this._rowElements[y];
       if (!row) {
diff --git a/src/browser/renderer/dom/DomRendererRowFactory.ts b/src/browser/renderer/dom/DomRendererRowFactory.ts
index 14b26c9293..e4722186bf 100644
--- a/src/browser/renderer/dom/DomRendererRowFactory.ts
+++ b/src/browser/renderer/dom/DomRendererRowFactory.ts
@@ -32,6 +32,7 @@ export class DomRendererRowFactory {
   private _selectionStart: [number, number] | undefined;
   private _selectionEnd: [number, number] | undefined;
   private _columnSelectMode: boolean = false;
+  private _nullSpan: HTMLSpanElement;
 
   constructor(
     private readonly _document: Document,
@@ -42,6 +43,8 @@ export class DomRendererRowFactory {
     @IDecorationService private readonly _decorationService: IDecorationService,
     @IThemeService private readonly _themeService: IThemeService
   ) {
+    this._nullSpan = this._document.createElement('span');
+    this._nullSpan.style.width = `0`;
   }
 
   public handleSelectionChanged(start: [number, number] | undefined, end: [number, number] | undefined, columnSelectMode: boolean): void {
@@ -75,6 +78,8 @@ export class DomRendererRowFactory {
 
       // The character to the left is a wide character, drawing is owned by the char at x-1
       if (width === 0) {
+        // hack: fix underline bug for wide chars by appending an empty span
+        fragment.appendChild(this._nullSpan.cloneNode());
         continue;
       }
 

From de23616589a1db455b4d3d1a8456654f36bc0e4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 5 Dec 2022 13:45:47 +0100
Subject: [PATCH 03/13] better line expansion

---
 .../src/WebLinkProvider.ts                    | 65 +++++++++++--------
 .../src/WebLinksAddon.ts                      |  3 +-
 bin/test_weblinks.sh                          |  3 +
 3 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/addons/xterm-addon-web-links/src/WebLinkProvider.ts b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
index 0dd4c9bec2..541f614fa3 100644
--- a/addons/xterm-addon-web-links/src/WebLinkProvider.ts
+++ b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
@@ -45,21 +45,14 @@ export class LinkComputer {
   public static computeLink(y: number, regex: RegExp, terminal: Terminal, activate: (event: MouseEvent, uri: string) => void): ILink[] {
     const rex = new RegExp(regex.source, (regex.flags || '') + 'g');
 
-    const [lines, startLineIndex] = LinkComputer._getFullLineString(y - 1, terminal);
+    const [lines, startLineIndex] = LinkComputer._getWindowedLineStrings(y - 1, terminal);
     const line = lines.join('');
 
-    // Don't try if the wrapped line if excessively large as the regex matching will block the main
-    // thread.
-    if (line.length > 1024) {
-      // TODO: more sophisticated handling with individual line introspection?
-      return [];
-    }
-
     let match;
     let stringIndex = -1;
     const result: ILink[] = [];
 
-    while ((match = rex.exec(line)) !== null) {
+    while (match = rex.exec(line)) {
       const text = match[0];
 
       // Get index, match.index is for the outer match which includes negated chars
@@ -117,29 +110,49 @@ export class LinkComputer {
   }
 
   /**
-   * Gets the entire line for the buffer line
-   * @param lineIndex The index of the line being translated.
+   * Get wrapped content lines for the current line index.
+   * The top/bottom line expansion stops at whitespaces or length > 2048.
+   * Returns an array with line strings and the top line index.
    */
-  private static _getFullLineString(lineIndex: number, terminal: Terminal): [string[], number] {
+  private static _getWindowedLineStrings(lineIndex: number, terminal: Terminal): [string[], number] {
     let line: any;
-
-    // expand top
     let topIdx = lineIndex;
-    while ((line = terminal.buffer.active.getLine(topIdx)) && line.isWrapped) {
-      topIdx--;
-    }
+    let bottomIdx = lineIndex;
+    let length = 0;
+    let content = '';
+    const lines: string[] = [];
 
-    // expand bottom
-    let bottomIdx = lineIndex + 1;
-    while ((line = terminal.buffer.active.getLine(bottomIdx)) && line.isWrapped) {
-      bottomIdx++;
-    }
+    if ((line = terminal.buffer.active.getLine(lineIndex))) {
+      const currentContent = line.translateToString(true);
+
+      // expand top, stop on whitespaces or length > 2048
+      if (line.isWrapped && currentContent[0] !== ' ') {
+        length = 0;
+        while ((line = terminal.buffer.active.getLine(--topIdx)) && length < 2048) {
+          content = line.translateToString(true);
+          length += content.length;
+          lines.push(content);
+          if (!line.isWrapped || content.indexOf(' ') !== -1) {
+            break;
+          }
+        }
+        lines.reverse();
+      }
 
-    const lines: string[] = [];
-    for (let idx = topIdx; idx < bottomIdx; ++idx) {
-      lines.push(terminal.buffer.active.getLine(idx)?.translateToString(true)!);
+      // append current line
+      lines.push(currentContent);
+
+      // expand bottom, stop on whitespaces or length > 2048
+      length = 0;
+      while ((line = terminal.buffer.active.getLine(++bottomIdx)) && line.isWrapped && length < 2048) {
+        content = line.translateToString(true);
+        length += content.length;
+        lines.push(content);
+        if (content.indexOf(' ') !== -1) {
+          break;
+        }
+      }
     }
-
     return [lines, topIdx];
   }
 
diff --git a/addons/xterm-addon-web-links/src/WebLinksAddon.ts b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
index ccf45e7407..44dabd452e 100644
--- a/addons/xterm-addon-web-links/src/WebLinksAddon.ts
+++ b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
@@ -10,7 +10,8 @@ import { ILinkProviderOptions, WebLinkProvider } from './WebLinkProvider';
 // up to first whitespace, `"` or `'` as url
 // NOTE: The repeated end clause is needed to not match a dangling `:`
 // resembling the old (...)*([^:"\'\\s]) final path clause
-const strictUrlRegex = /https?:[/]{2}[^\s^"^']*[^\s^"^'^:]/;
+// also exclude final interpunction like ,.!?
+const strictUrlRegex = /https?:[/]{2}[^\s^"^']*[^\s^"^'^:^,^.^!^?]/;
 
 function handleLink(event: MouseEvent, uri: string): void {
   const newWindow = window.open();
diff --git a/bin/test_weblinks.sh b/bin/test_weblinks.sh
index a50db78820..4488b98399 100755
--- a/bin/test_weblinks.sh
+++ b/bin/test_weblinks.sh
@@ -20,3 +20,6 @@ echo "￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko
 
 # not matching at all
 echo "http://test:password@example.com/some_path"
+
+# overly long text with urls with final interpunction
+echo "Lorem ipsum dolor sit amet, consetetur sadipscing elitr https://ko.wikipedia.org/wiki/위키백과:대문, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat: http://test:password@example.com/some_path."
\ No newline at end of file

From 3bf72cdefc35852b39ee46f958032069f3f480c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 5 Dec 2022 15:50:29 +0100
Subject: [PATCH 04/13] DOM renderer fix with much better performance

---
 bin/test_weblinks.sh                          |  6 +--
 src/browser/renderer/dom/DomRenderer.ts       | 15 ++++--
 .../dom/DomRendererRowFactory.test.ts         | 50 ++++++++++---------
 .../renderer/dom/DomRendererRowFactory.ts     | 28 +++++++----
 4 files changed, 60 insertions(+), 39 deletions(-)

diff --git a/bin/test_weblinks.sh b/bin/test_weblinks.sh
index 4488b98399..c06354132b 100755
--- a/bin/test_weblinks.sh
+++ b/bin/test_weblinks.sh
@@ -13,13 +13,13 @@ echo "aaa http://example.com ￥￥￥ http://example.com aaa"
 echo "￥￥￥ http://example.com ￥￥￥ http://example.com aaa"
 
 # full width within url - partial wrong match
-echo "aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문"
+echo "aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa"
 
 # full width within and before - partial wrong match + wrong offsets
-echo "￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문"
+echo "￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 ￥￥￥"
 
 # not matching at all
-echo "http://test:password@example.com/some_path"
+echo "http://test:password@example.com/some_path aaa"
 
 # overly long text with urls with final interpunction
 echo "Lorem ipsum dolor sit amet, consetetur sadipscing elitr https://ko.wikipedia.org/wiki/위키백과:대문, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat: http://test:password@example.com/some_path."
\ No newline at end of file
diff --git a/src/browser/renderer/dom/DomRenderer.ts b/src/browser/renderer/dom/DomRenderer.ts
index 094e6a8e3f..259f1607af 100644
--- a/src/browser/renderer/dom/DomRenderer.ts
+++ b/src/browser/renderer/dom/DomRenderer.ts
@@ -38,6 +38,7 @@ export class DomRenderer extends Disposable implements IRenderer {
   private _rowContainer: HTMLElement;
   private _rowElements: HTMLElement[] = [];
   private _selectionContainer: HTMLElement;
+  private _cellToRowElements: Uint16Array[] = [];
 
   public dimensions: IRenderDimensions;
 
@@ -359,7 +360,10 @@ export class DomRenderer extends Disposable implements IRenderer {
       const row = y + this._bufferService.buffer.ydisp;
       const lineData = this._bufferService.buffer.lines.get(row);
       const cursorStyle = this._optionsService.rawOptions.cursorStyle;
-      rowElement.replaceChildren(this._rowFactory.createRow(lineData!, row, row === cursorAbsoluteY, cursorStyle, cursorX, cursorBlink, this.dimensions.css.cell.width, this._bufferService.cols));
+      if (!this._cellToRowElements[y] || this._cellToRowElements[y].length !== this._bufferService.cols) {
+        this._cellToRowElements[y] = new Uint16Array(this._bufferService.cols);
+      }
+      rowElement.replaceChildren(this._rowFactory.createRow(lineData!, row, row === cursorAbsoluteY, cursorStyle, cursorX, cursorBlink, this.dimensions.css.cell.width, this._bufferService.cols, this._cellToRowElements[y]));
     }
   }
 
@@ -376,8 +380,13 @@ export class DomRenderer extends Disposable implements IRenderer {
   }
 
   private _setCellUnderline(x: number, x2: number, y: number, y2: number, cols: number, enabled: boolean): void {
-    // FIXME: offset calculation is wrong (temp. fixed by adding empty spans for wide chars
-    // to fullfill column to element index identity assumption)
+    x = this._cellToRowElements[y][x];
+    x2 = this._cellToRowElements[y2][x2];
+
+    if (x === -1 || x2 === -1) {
+      return;
+    }
+
     while (x !== x2 || y !== y2) {
       const row = this._rowElements[y];
       if (!row) {
diff --git a/src/browser/renderer/dom/DomRendererRowFactory.test.ts b/src/browser/renderer/dom/DomRendererRowFactory.test.ts
index fbb4853634..7d4adf85ad 100644
--- a/src/browser/renderer/dom/DomRendererRowFactory.test.ts
+++ b/src/browser/renderer/dom/DomRendererRowFactory.test.ts
@@ -14,6 +14,8 @@ import { MockCoreService, MockDecorationService, MockOptionsService } from 'comm
 import { css } from 'common/Color';
 import { MockCharacterJoinerService, MockCoreBrowserService, MockThemeService } from 'browser/TestUtils.test';
 
+const EMPTY_ELEM_MAPPING = new Uint16Array(1000);
+
 describe('DomRendererRowFactory', () => {
   let dom: jsdom.JSDOM;
   let rowFactory: DomRendererRowFactory;
@@ -35,7 +37,7 @@ describe('DomRendererRowFactory', () => {
 
   describe('createRow', () => {
     it('should not create anything for an empty row', () => {
-      const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+      const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
       assert.equal(getFragmentHtml(fragment),
         ''
       );
@@ -45,7 +47,7 @@ describe('DomRendererRowFactory', () => {
       lineData.setCell(0, CellData.fromCharData([DEFAULT_ATTR, '語', 2, '語'.charCodeAt(0)]));
       // There should be no element for the following "empty" cell
       lineData.setCell(1, CellData.fromCharData([DEFAULT_ATTR, '', 0, 0]));
-      const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+      const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
       assert.equal(getFragmentHtml(fragment),
         '<span style="width: 10px;">語</span>'
       );
@@ -53,7 +55,7 @@ describe('DomRendererRowFactory', () => {
 
     it('should add class for cursor and cursor style', () => {
       for (const style of ['block', 'bar', 'underline']) {
-        const fragment = rowFactory.createRow(lineData, 0, true, style, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, true, style, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           `<span class="xterm-cursor xterm-cursor-${style}"> </span>`
         );
@@ -61,7 +63,7 @@ describe('DomRendererRowFactory', () => {
     });
 
     it('should add class for cursor blink', () => {
-      const fragment = rowFactory.createRow(lineData, 0, true, 'block', 0, true, 5, 20);
+      const fragment = rowFactory.createRow(lineData, 0, true, 'block', 0, true, 5, 20, EMPTY_ELEM_MAPPING);
       assert.equal(getFragmentHtml(fragment),
         `<span class="xterm-cursor xterm-cursor-blink xterm-cursor-block"> </span>`
       );
@@ -70,7 +72,7 @@ describe('DomRendererRowFactory', () => {
     it('should not render cells that go beyond the terminal\'s columns', () => {
       lineData.setCell(0, CellData.fromCharData([DEFAULT_ATTR, 'a', 1, 'a'.charCodeAt(0)]));
       lineData.setCell(1, CellData.fromCharData([DEFAULT_ATTR, 'b', 1, 'b'.charCodeAt(0)]));
-      const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 1);
+      const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 1, EMPTY_ELEM_MAPPING);
       assert.equal(getFragmentHtml(fragment),
         '<span>a</span>'
       );
@@ -81,7 +83,7 @@ describe('DomRendererRowFactory', () => {
         const cell = CellData.fromCharData([0, 'a', 1, 'a'.charCodeAt(0)]);
         cell.fg = DEFAULT_ATTR_DATA.fg | FgFlags.BOLD;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-bold">a</span>'
         );
@@ -91,7 +93,7 @@ describe('DomRendererRowFactory', () => {
         const cell = CellData.fromCharData([0, 'a', 1, 'a'.charCodeAt(0)]);
         cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.ITALIC;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-italic">a</span>'
         );
@@ -101,7 +103,7 @@ describe('DomRendererRowFactory', () => {
         const cell = CellData.fromCharData([0, 'a', 1, 'a'.charCodeAt(0)]);
         cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.DIM;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-dim">a</span>'
         );
@@ -114,7 +116,7 @@ describe('DomRendererRowFactory', () => {
           cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.HAS_EXTENDED;
           cell.extended.underlineStyle = UnderlineStyle.SINGLE;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             '<span class="xterm-underline-1">a</span>'
           );
@@ -125,7 +127,7 @@ describe('DomRendererRowFactory', () => {
           cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.HAS_EXTENDED;
           cell.extended.underlineStyle = UnderlineStyle.DOUBLE;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             '<span class="xterm-underline-2">a</span>'
           );
@@ -136,7 +138,7 @@ describe('DomRendererRowFactory', () => {
           cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.HAS_EXTENDED;
           cell.extended.underlineStyle = UnderlineStyle.CURLY;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             '<span class="xterm-underline-3">a</span>'
           );
@@ -147,7 +149,7 @@ describe('DomRendererRowFactory', () => {
           cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.HAS_EXTENDED;
           cell.extended.underlineStyle = UnderlineStyle.DOTTED;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             '<span class="xterm-underline-4">a</span>'
           );
@@ -158,7 +160,7 @@ describe('DomRendererRowFactory', () => {
           cell.bg = DEFAULT_ATTR_DATA.bg | BgFlags.HAS_EXTENDED;
           cell.extended.underlineStyle = UnderlineStyle.DASHED;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             '<span class="xterm-underline-5">a</span>'
           );
@@ -169,7 +171,7 @@ describe('DomRendererRowFactory', () => {
         const cell = CellData.fromCharData([0, 'a', 1, 'a'.charCodeAt(0)]);
         cell.fg = DEFAULT_ATTR_DATA.fg | FgFlags.STRIKETHROUGH;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-strikethrough">a</span>'
         );
@@ -182,7 +184,7 @@ describe('DomRendererRowFactory', () => {
           cell.fg &= ~Attributes.PCOLOR_MASK;
           cell.fg |= i;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             `<span class="xterm-fg-${i}">a</span>`
           );
@@ -196,7 +198,7 @@ describe('DomRendererRowFactory', () => {
           cell.bg &= ~Attributes.PCOLOR_MASK;
           cell.bg |= i;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             `<span class="xterm-bg-${i}">a</span>`
           );
@@ -208,7 +210,7 @@ describe('DomRendererRowFactory', () => {
         cell.fg |= Attributes.CM_P16 | 2 | FgFlags.INVERSE;
         cell.bg |= Attributes.CM_P16 | 1;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-bg-2 xterm-fg-1">a</span>'
         );
@@ -219,7 +221,7 @@ describe('DomRendererRowFactory', () => {
         cell.fg |= FgFlags.INVERSE;
         cell.bg |= Attributes.CM_P16 | 1;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-bg-257 xterm-fg-1">a</span>'
         );
@@ -229,7 +231,7 @@ describe('DomRendererRowFactory', () => {
         const cell = CellData.fromCharData([0, 'a', 1, 'a'.charCodeAt(0)]);
         cell.fg |= Attributes.CM_P16 | 1 | FgFlags.INVERSE;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-bg-1 xterm-fg-257">a</span>'
         );
@@ -242,7 +244,7 @@ describe('DomRendererRowFactory', () => {
           cell.fg &= ~Attributes.PCOLOR_MASK;
           cell.fg |= i;
           lineData.setCell(0, cell);
-          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+          const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
           assert.equal(getFragmentHtml(fragment),
             `<span class="xterm-bold xterm-fg-${i + 8}">a</span>`
           );
@@ -254,7 +256,7 @@ describe('DomRendererRowFactory', () => {
         cell.fg |= Attributes.CM_RGB | 1 << 16 | 2 << 8 | 3;
         cell.bg |= Attributes.CM_RGB | 4 << 16 | 5 << 8 | 6;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span style="background-color:#040506;color:#010203;">a</span>'
         );
@@ -265,7 +267,7 @@ describe('DomRendererRowFactory', () => {
         cell.fg |= Attributes.CM_RGB | 1 << 16 | 2 << 8 | 3 | FgFlags.INVERSE;
         cell.bg |= Attributes.CM_RGB | 4 << 16 | 5 << 8 | 6;
         lineData.setCell(0, cell);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span style="background-color:#010203;color:#040506;">a</span>'
         );
@@ -277,7 +279,7 @@ describe('DomRendererRowFactory', () => {
         lineData.setCell(0, CellData.fromCharData([DEFAULT_ATTR, 'a', 1, 'a'.charCodeAt(0)]));
         lineData.setCell(1, CellData.fromCharData([DEFAULT_ATTR, 'b', 1, 'b'.charCodeAt(0)]));
         rowFactory.handleSelectionChanged([1, 0], [2, 0], false);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span>a</span><span class="xterm-decoration-top">b</span>'
         );
@@ -285,7 +287,7 @@ describe('DomRendererRowFactory', () => {
       it('should force whitespace cells to be rendered above the background', () => {
         lineData.setCell(1, CellData.fromCharData([DEFAULT_ATTR, 'a', 1, 'a'.charCodeAt(0)]));
         rowFactory.handleSelectionChanged([0, 0], [2, 0], false);
-        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20);
+        const fragment = rowFactory.createRow(lineData, 0, false, undefined, 0, false, 5, 20, EMPTY_ELEM_MAPPING);
         assert.equal(getFragmentHtml(fragment),
           '<span class="xterm-decoration-top"> </span><span class="xterm-decoration-top">a</span>'
         );
diff --git a/src/browser/renderer/dom/DomRendererRowFactory.ts b/src/browser/renderer/dom/DomRendererRowFactory.ts
index e4722186bf..b39c62c0e2 100644
--- a/src/browser/renderer/dom/DomRendererRowFactory.ts
+++ b/src/browser/renderer/dom/DomRendererRowFactory.ts
@@ -32,7 +32,6 @@ export class DomRendererRowFactory {
   private _selectionStart: [number, number] | undefined;
   private _selectionEnd: [number, number] | undefined;
   private _columnSelectMode: boolean = false;
-  private _nullSpan: HTMLSpanElement;
 
   constructor(
     private readonly _document: Document,
@@ -42,10 +41,7 @@ export class DomRendererRowFactory {
     @ICoreService private readonly _coreService: ICoreService,
     @IDecorationService private readonly _decorationService: IDecorationService,
     @IThemeService private readonly _themeService: IThemeService
-  ) {
-    this._nullSpan = this._document.createElement('span');
-    this._nullSpan.style.width = `0`;
-  }
+  ) {}
 
   public handleSelectionChanged(start: [number, number] | undefined, end: [number, number] | undefined, columnSelectMode: boolean): void {
     this._selectionStart = start;
@@ -53,7 +49,11 @@ export class DomRendererRowFactory {
     this._columnSelectMode = columnSelectMode;
   }
 
-  public createRow(lineData: IBufferLine, row: number, isCursorRow: boolean, cursorStyle: string | undefined, cursorX: number, cursorBlink: boolean, cellWidth: number, cols: number): DocumentFragment {
+  public createRow(lineData: IBufferLine, row: number, isCursorRow: boolean, cursorStyle: string | undefined, cursorX: number, cursorBlink: boolean, cellWidth: number, cols: number, cellMap: Uint16Array): DocumentFragment {
+    // NOTE: `cellMap` maps cell positions to a span element index in a row.
+    // All positions should be updated, even skipped ones after wide chars or left overs at the end,
+    // otherwise the mouse hover logic might mark the wrong elements as underlined.
+
     const fragment = this._document.createDocumentFragment();
 
     const joinedRanges = this._characterJoinerService.getJoinedCharacters(row);
@@ -71,15 +71,17 @@ export class DomRendererRowFactory {
     }
 
     const colors = this._themeService.colors;
+    let elemIndex = -1;
 
-    for (let x = 0; x < lineLength; x++) {
+    let x = 0;
+    for (; x < lineLength; x++) {
       lineData.loadCell(x, this._workCell);
       let width = this._workCell.getWidth();
 
       // The character to the left is a wide character, drawing is owned by the char at x-1
+      // still have to update cellMap with current element index
       if (width === 0) {
-        // hack: fix underline bug for wide chars by appending an empty span
-        fragment.appendChild(this._nullSpan.cloneNode());
+        cellMap[x] = elemIndex;
         continue;
       }
 
@@ -306,9 +308,17 @@ export class DomRendererRowFactory {
       }
 
       fragment.appendChild(charElement);
+      cellMap[x] = ++elemIndex;
 
       x = lastCharX;
     }
+
+    // since the loop above might exit early not handling all cells,
+    // also set remaining cell positions to last element index
+    if (x < cols - 1) {
+      cellMap.subarray(x + 1).fill(elemIndex);
+    }
+
     return fragment;
   }
 

From 46bd97247d9be732c0b540773ce18988f4ccaaaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 5 Dec 2022 18:28:42 +0100
Subject: [PATCH 05/13] disabled faulty code in linkifier2

---
 src/browser/Linkifier2.ts | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/src/browser/Linkifier2.ts b/src/browser/Linkifier2.ts
index 236efb2285..c601246854 100644
--- a/src/browser/Linkifier2.ts
+++ b/src/browser/Linkifier2.ts
@@ -157,13 +157,15 @@ export class Linkifier2 extends Disposable implements ILinkifier2 {
           // If all providers have responded, remove lower priority links that intersect ranges of
           // higher priority links
           if (this._activeProviderReplies?.size === this._linkProviders.length) {
-            this._removeIntersectingLinks(position.y, this._activeProviderReplies);
+            // FIXME: commented out due to bug below
+            //this._removeIntersectingLinks(position.y, this._activeProviderReplies);
           }
         });
       }
     }
   }
 
+  // FIXME: What is this supposed to do? Currently it removes wrongly a second link on a wrapped line...
   private _removeIntersectingLinks(y: number, replies: Map<Number, ILinkWithState[] | undefined>): void {
     const occupiedCells = new Set<number>();
     for (let i = 0; i < replies.size; i++) {
@@ -367,18 +369,10 @@ export class Linkifier2 extends Disposable implements ILinkifier2 {
    * @param position
    */
   private _linkAtPosition(link: ILink, position: IBufferCellPosition): boolean {
-    const sameLine = link.range.start.y === link.range.end.y;
-    const wrappedFromLeft = link.range.start.y < position.y;
-    const wrappedToRight = link.range.end.y > position.y;
-
-    // If the start and end have the same y, then the position must be between start and end x
-    // If not, then handle each case seperately, depending on which way it wraps
-    return ((sameLine && link.range.start.x <= position.x && link.range.end.x >= position.x) ||
-      (wrappedFromLeft && link.range.end.x >= position.x) ||
-      (wrappedToRight && link.range.start.x <= position.x) ||
-      (wrappedFromLeft && wrappedToRight)) &&
-      link.range.start.y <= position.y &&
-      link.range.end.y >= position.y;
+    const lower = link.range.start.y * this._bufferService.cols + link.range.start.x;
+    const upper = link.range.end.y * this._bufferService.cols + link.range.end.x;
+    const current = position.y * this._bufferService.cols + position.x;
+    return (lower <= current && current <= upper);
   }
 
   /**

From 42bacc3283d606d9c54562f9f3354e38710a7080 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 5 Dec 2022 18:30:25 +0100
Subject: [PATCH 06/13] disabled faulty code in linkifier2

---
 src/browser/Linkifier2.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/browser/Linkifier2.ts b/src/browser/Linkifier2.ts
index c601246854..5543ba808c 100644
--- a/src/browser/Linkifier2.ts
+++ b/src/browser/Linkifier2.ts
@@ -158,7 +158,7 @@ export class Linkifier2 extends Disposable implements ILinkifier2 {
           // higher priority links
           if (this._activeProviderReplies?.size === this._linkProviders.length) {
             // FIXME: commented out due to bug below
-            //this._removeIntersectingLinks(position.y, this._activeProviderReplies);
+            // this._removeIntersectingLinks(position.y, this._activeProviderReplies);
           }
         });
       }

From fd44d76262b9e34b4048dab7aafceb39c9d1846c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 5 Dec 2022 22:49:25 +0100
Subject: [PATCH 07/13] api tests for uri and ranges

---
 .../test/WebLinksAddon.api.ts                 | 57 ++++++++++++++++++-
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts b/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
index 9bcda0a52c..44d09c9586 100644
--- a/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
+++ b/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
@@ -36,8 +36,41 @@ describe('WebLinksAddon', () => {
     await testHostName('foo.io');
   });
 
-  it.skip('correct buffer offsets', async () => {
-    // TODO: test strings in test_weblinks.sh automatically
+  describe('correct buffer offsets & uri', () => {
+    it('all half width', async () => {
+      setupCustom();
+      await writeSync(page, 'aaa http://example.com aaa http://example.com aaa');
+      await resetAndHover(5, 1);
+      await evalData('http://example.com', { start: { x: 5, y: 1 }, end: { x: 22, y: 1 } });
+      await resetAndHover(1, 2);
+      await evalData('http://example.com', { start: { x: 28, y: 1 }, end: { x: 5, y: 2 } });
+    });
+    it('url after full width', async () => {
+      setupCustom();
+      await writeSync(page, '￥￥￥ http://example.com ￥￥￥ http://example.com aaa');
+      await resetAndHover(8, 1);
+      await evalData('http://example.com', { start: { x: 8, y: 1 }, end: { x: 25, y: 1 } });
+      await resetAndHover(1, 2);
+      await evalData('http://example.com', { start: { x: 34, y: 1 }, end: { x: 11, y: 2 } });
+    });
+    it('full width within url and before', async () => {
+      setupCustom();
+      await writeSync(page, '￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 ￥￥￥');
+      await resetAndHover(8, 1);
+      await evalData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 8, y: 1 }, end: { x: 11, y: 2 } });
+      await resetAndHover(1, 2);
+      await evalData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 8, y: 1 }, end: { x: 11, y: 2 } });
+      await resetAndHover(17, 2);
+      await evalData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 17, y: 2 }, end: { x: 19, y: 3 } });
+    });
+    it('name + password url after full width and combining', async () => {
+      setupCustom();
+      await writeSync(page, '￥￥￥cafe\u0301 http://test:password@example.com/some_path');
+      await resetAndHover(12, 1);
+      await evalData('http://test:password@example.com/some_path', { start: { x: 12, y: 1 }, end: { x: 13, y: 2 } });
+      await resetAndHover(13, 2);
+      await evalData('http://test:password@example.com/some_path', { start: { x: 12, y: 1 }, end: { x: 13, y: 2 } });
+    });
   });
 });
 
@@ -68,3 +101,23 @@ async function pollForLinkAtCell(col: number, row: number, value: string): Promi
   await pollFor(page, `document.querySelectorAll('${rowSelector} > span[style]').length >= ${value.length}`, true, async () => page.hover(`${rowSelector} > :nth-child(${col})`));
   assert.equal(await page.evaluate(`Array.prototype.reduce.call(document.querySelectorAll('${rowSelector} > span[style]'), (a, b) => a + b.textContent, '');`), value);
 }
+
+async function setupCustom(): Promise<void> {
+  await openTerminal(page, { cols: 40 });
+  await page.evaluate(`window._customLinkData = [];
+window._linkaddon = new window.WebLinksAddon();
+window._linkaddon._options.hover = (event, uri, range) => { window._customLinkData.push([uri, range]); };
+window.term.loadAddon(window._linkaddon);`);
+}
+
+async function resetAndHover(col: number, row: number): Promise<void> {
+  await page.evaluate(`window._customLinkData = [];`);
+  const rowSelector = `.xterm-rows > :nth-child(${row})`;
+  await page.hover(`${rowSelector} > :nth-child(${col})`);
+}
+
+async function evalData(uri: string, range: any): Promise<void> {
+  const data: any[] = await page.evaluate(`window._customLinkData[0]`);
+  assert.equal(data[0], uri);
+  assert.deepEqual(data[1], range);
+}

From becec91e98cd2fe3c4bf4f6141ef47f551220af2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Thu, 8 Dec 2022 17:35:45 +0100
Subject: [PATCH 08/13] better regexp, fix offset issue in #4294

---
 .../src/WebLinksAddon.ts                      | 11 ++++++--
 bin/test_weblinks.sh                          | 25 +++++++++++++------
 src/browser/renderer/dom/DomRenderer.ts       |  4 +--
 .../dom/DomRendererRowFactory.test.ts         |  2 +-
 .../renderer/dom/DomRendererRowFactory.ts     |  4 +--
 5 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/addons/xterm-addon-web-links/src/WebLinksAddon.ts b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
index 44dabd452e..45a1b294fc 100644
--- a/addons/xterm-addon-web-links/src/WebLinksAddon.ts
+++ b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
@@ -10,8 +10,15 @@ import { ILinkProviderOptions, WebLinkProvider } from './WebLinkProvider';
 // up to first whitespace, `"` or `'` as url
 // NOTE: The repeated end clause is needed to not match a dangling `:`
 // resembling the old (...)*([^:"\'\\s]) final path clause
-// also exclude final interpunction like ,.!?
-const strictUrlRegex = /https?:[/]{2}[^\s^"^']*[^\s^"^'^:^,^.^!^?]/;
+// additionally exclude early + final:
+// - unsafe from rfc3986: !*'()
+// - unsafe chars from rfc1738: {}|\^~[]` (minus [] as we need them for ipv6 adresses)
+// also exclude as finals:
+// - final interpunction like ,.!?
+// - any sort of brackets <>()[]{} (not spec conform, but often used to enclose urls)
+// - unsafe chars from rfc1738: {}|\^~[]`
+const strictUrlRegex = /https?:[/]{2}[^\s"'!*(){}|\\\^~<>`]*[^\s"':,.!?{}|\\\^~\[\]`()<>]/;
+
 
 function handleLink(event: MouseEvent, uri: string): void {
   const newWindow = window.open();
diff --git a/bin/test_weblinks.sh b/bin/test_weblinks.sh
index c06354132b..1389eceeb1 100755
--- a/bin/test_weblinks.sh
+++ b/bin/test_weblinks.sh
@@ -1,25 +1,34 @@
 #!/bin/bash
 
-# all half width - only good case
+# all half width
 echo "aaa http://example.com aaa http://example.com aaa"
 
-# full width before - wrong offset
+# full width before
 echo "￥￥￥ http://example.com aaa http://example.com aaa"
 
-# full width between - wrong offset
+# full width between
 echo "aaa http://example.com ￥￥￥ http://example.com aaa"
 
-# full width before and between - error in offsets adding up
+# full width before and between
 echo "￥￥￥ http://example.com ￥￥￥ http://example.com aaa"
 
-# full width within url - partial wrong match
+# full width within url
 echo "aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa"
 
-# full width within and before - partial wrong match + wrong offsets
+# full width within and before
 echo "￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 ￥￥￥"
 
-# not matching at all
+# username + password scheme
 echo "http://test:password@example.com/some_path aaa"
 
 # overly long text with urls with final interpunction
-echo "Lorem ipsum dolor sit amet, consetetur sadipscing elitr https://ko.wikipedia.org/wiki/위키백과:대문, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat: http://test:password@example.com/some_path."
\ No newline at end of file
+echo "Lorem ipsum dolor sit amet, consetetur sadipscing elitr https://ko.wikipedia.org/wiki/위키백과:대문, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat: http://test:password@example.com/some_path."
+
+# bracket enclosed urls
+echo "[http://example.de]"
+echo "(http://example.de)"
+echo "<http://example.de>"
+echo "{http://example.de}"
+
+# ipv6 scheme
+echo "ipv6 https://[::1]/with/some?vars=and&a#hash"
\ No newline at end of file
diff --git a/src/browser/renderer/dom/DomRenderer.ts b/src/browser/renderer/dom/DomRenderer.ts
index 259f1607af..700a62b63f 100644
--- a/src/browser/renderer/dom/DomRenderer.ts
+++ b/src/browser/renderer/dom/DomRenderer.ts
@@ -38,7 +38,7 @@ export class DomRenderer extends Disposable implements IRenderer {
   private _rowContainer: HTMLElement;
   private _rowElements: HTMLElement[] = [];
   private _selectionContainer: HTMLElement;
-  private _cellToRowElements: Uint16Array[] = [];
+  private _cellToRowElements: Int16Array[] = [];
 
   public dimensions: IRenderDimensions;
 
@@ -361,7 +361,7 @@ export class DomRenderer extends Disposable implements IRenderer {
       const lineData = this._bufferService.buffer.lines.get(row);
       const cursorStyle = this._optionsService.rawOptions.cursorStyle;
       if (!this._cellToRowElements[y] || this._cellToRowElements[y].length !== this._bufferService.cols) {
-        this._cellToRowElements[y] = new Uint16Array(this._bufferService.cols);
+        this._cellToRowElements[y] = new Int16Array(this._bufferService.cols);
       }
       rowElement.replaceChildren(this._rowFactory.createRow(lineData!, row, row === cursorAbsoluteY, cursorStyle, cursorX, cursorBlink, this.dimensions.css.cell.width, this._bufferService.cols, this._cellToRowElements[y]));
     }
diff --git a/src/browser/renderer/dom/DomRendererRowFactory.test.ts b/src/browser/renderer/dom/DomRendererRowFactory.test.ts
index 7d4adf85ad..5969abff21 100644
--- a/src/browser/renderer/dom/DomRendererRowFactory.test.ts
+++ b/src/browser/renderer/dom/DomRendererRowFactory.test.ts
@@ -14,7 +14,7 @@ import { MockCoreService, MockDecorationService, MockOptionsService } from 'comm
 import { css } from 'common/Color';
 import { MockCharacterJoinerService, MockCoreBrowserService, MockThemeService } from 'browser/TestUtils.test';
 
-const EMPTY_ELEM_MAPPING = new Uint16Array(1000);
+const EMPTY_ELEM_MAPPING = new Int16Array(1000);
 
 describe('DomRendererRowFactory', () => {
   let dom: jsdom.JSDOM;
diff --git a/src/browser/renderer/dom/DomRendererRowFactory.ts b/src/browser/renderer/dom/DomRendererRowFactory.ts
index b39c62c0e2..cc64a4383d 100644
--- a/src/browser/renderer/dom/DomRendererRowFactory.ts
+++ b/src/browser/renderer/dom/DomRendererRowFactory.ts
@@ -49,7 +49,7 @@ export class DomRendererRowFactory {
     this._columnSelectMode = columnSelectMode;
   }
 
-  public createRow(lineData: IBufferLine, row: number, isCursorRow: boolean, cursorStyle: string | undefined, cursorX: number, cursorBlink: boolean, cellWidth: number, cols: number, cellMap: Uint16Array): DocumentFragment {
+  public createRow(lineData: IBufferLine, row: number, isCursorRow: boolean, cursorStyle: string | undefined, cursorX: number, cursorBlink: boolean, cellWidth: number, cols: number, cellMap: Int16Array): DocumentFragment {
     // NOTE: `cellMap` maps cell positions to a span element index in a row.
     // All positions should be updated, even skipped ones after wide chars or left overs at the end,
     // otherwise the mouse hover logic might mark the wrong elements as underlined.
@@ -316,7 +316,7 @@ export class DomRendererRowFactory {
     // since the loop above might exit early not handling all cells,
     // also set remaining cell positions to last element index
     if (x < cols - 1) {
-      cellMap.subarray(x + 1).fill(elemIndex);
+      cellMap.subarray(x).fill(++elemIndex);
     }
 
     return fragment;

From e70138b63c6688cb83acf5f4c5390deac1a75714 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Thu, 8 Dec 2022 17:46:45 +0100
Subject: [PATCH 09/13] revert temp changes in linkifier2

---
 src/browser/Linkifier2.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/browser/Linkifier2.ts b/src/browser/Linkifier2.ts
index 5543ba808c..837c7241fe 100644
--- a/src/browser/Linkifier2.ts
+++ b/src/browser/Linkifier2.ts
@@ -157,15 +157,13 @@ export class Linkifier2 extends Disposable implements ILinkifier2 {
           // If all providers have responded, remove lower priority links that intersect ranges of
           // higher priority links
           if (this._activeProviderReplies?.size === this._linkProviders.length) {
-            // FIXME: commented out due to bug below
-            // this._removeIntersectingLinks(position.y, this._activeProviderReplies);
+            this._removeIntersectingLinks(position.y, this._activeProviderReplies);
           }
         });
       }
     }
   }
 
-  // FIXME: What is this supposed to do? Currently it removes wrongly a second link on a wrapped line...
   private _removeIntersectingLinks(y: number, replies: Map<Number, ILinkWithState[] | undefined>): void {
     const occupiedCells = new Set<number>();
     for (let i = 0; i < replies.size; i++) {

From f69f9deccacb77c82e6545a8a10c42988e4f05f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Thu, 8 Dec 2022 17:50:55 +0100
Subject: [PATCH 10/13] allow ~ within urls

---
 addons/xterm-addon-web-links/src/WebLinksAddon.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/addons/xterm-addon-web-links/src/WebLinksAddon.ts b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
index 45a1b294fc..5c956cec36 100644
--- a/addons/xterm-addon-web-links/src/WebLinksAddon.ts
+++ b/addons/xterm-addon-web-links/src/WebLinksAddon.ts
@@ -12,12 +12,12 @@ import { ILinkProviderOptions, WebLinkProvider } from './WebLinkProvider';
 // resembling the old (...)*([^:"\'\\s]) final path clause
 // additionally exclude early + final:
 // - unsafe from rfc3986: !*'()
-// - unsafe chars from rfc1738: {}|\^~[]` (minus [] as we need them for ipv6 adresses)
+// - unsafe chars from rfc1738: {}|\^~[]` (minus [] as we need them for ipv6 adresses, also allow ~)
 // also exclude as finals:
 // - final interpunction like ,.!?
 // - any sort of brackets <>()[]{} (not spec conform, but often used to enclose urls)
 // - unsafe chars from rfc1738: {}|\^~[]`
-const strictUrlRegex = /https?:[/]{2}[^\s"'!*(){}|\\\^~<>`]*[^\s"':,.!?{}|\\\^~\[\]`()<>]/;
+const strictUrlRegex = /https?:[/]{2}[^\s"'!*(){}|\\\^<>`]*[^\s"':,.!?{}|\\\^~\[\]`()<>]/;
 
 
 function handleLink(event: MouseEvent, uri: string): void {

From e66fd5038348f9d8c9b7b521252ec21bc65afb3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Fri, 9 Dec 2022 14:37:31 +0100
Subject: [PATCH 11/13] cleanup & more comments

---
 .../src/WebLinkProvider.ts                    | 57 ++++++++++---------
 1 file changed, 29 insertions(+), 28 deletions(-)

diff --git a/addons/xterm-addon-web-links/src/WebLinkProvider.ts b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
index 541f614fa3..1c9486f5b4 100644
--- a/addons/xterm-addon-web-links/src/WebLinkProvider.ts
+++ b/addons/xterm-addon-web-links/src/WebLinkProvider.ts
@@ -3,7 +3,7 @@
  * @license MIT
  */
 
-import { ILinkProvider, ILink, Terminal, IViewportRange } from 'xterm';
+import { ILinkProvider, ILink, Terminal, IViewportRange, IBufferLine } from 'xterm';
 
 export interface ILinkProviderOptions {
   hover?(event: MouseEvent, text: string, location: IViewportRange): void;
@@ -49,23 +49,11 @@ export class LinkComputer {
     const line = lines.join('');
 
     let match;
-    let stringIndex = -1;
     const result: ILink[] = [];
 
     while (match = rex.exec(line)) {
       const text = match[0];
 
-      // Get index, match.index is for the outer match which includes negated chars
-      // therefore we cannot use match.index directly, instead we search the position
-      // of the match group in text again
-      // also correct regex and string search offsets for the next loop run
-      stringIndex = line.indexOf(text, stringIndex + 1);
-      rex.lastIndex = stringIndex + text.length;
-      if (stringIndex < 0) {
-        // invalid stringIndex (should not have happened)
-        break;
-      }
-
       // check via URL if the matched text would form a proper url
       // NOTE: This outsources the ugly url parsing to the browser.
       // To avoid surprising auto expansion from URL we additionally
@@ -84,14 +72,16 @@ export class LinkComputer {
         continue;
       }
 
-
-      const [startY, startX] = LinkComputer._mapStrIdx(startLineIndex, stringIndex, terminal);
-      const [endY, endX] = LinkComputer._mapStrIdx(startLineIndex, stringIndex + text.length, terminal);
+      // map string positions back to buffer positions
+      // values are 0-based right side excluding
+      const [startY, startX] = LinkComputer._mapStrIdx(terminal, startLineIndex, 0, match.index);
+      const [endY, endX] = LinkComputer._mapStrIdx(terminal, startY, startX, text.length);
 
       if (startY === -1 || startX === -1 || endY === -1 || endX === -1) {
         continue;
       }
 
+      // range expects values 1-based right side including, thus +1 except for endX
       const range = {
         start: {
           x: startX + 1,
@@ -113,9 +103,13 @@ export class LinkComputer {
    * Get wrapped content lines for the current line index.
    * The top/bottom line expansion stops at whitespaces or length > 2048.
    * Returns an array with line strings and the top line index.
+   *
+   * NOTE: We pull line strings with trimRight=true on purpose to make sure
+   * to correctly match urls with early wrapped wide chars. This corrupts the string index
+   * for 1:1 backmapping to buffer positions, thus needs an additional correction in _mapStrIdx.
    */
   private static _getWindowedLineStrings(lineIndex: number, terminal: Terminal): [string[], number] {
-    let line: any;
+    let line: IBufferLine | undefined;
     let topIdx = lineIndex;
     let bottomIdx = lineIndex;
     let length = 0;
@@ -161,28 +155,34 @@ export class LinkComputer {
    * Returns buffer position as [lineIndex, columnIndex] 0-based,
    * or [-1, -1] in case the lookup ran into a non-existing line.
    */
-  private static _mapStrIdx(lineIndex: number, stringIndex: number, terminal: Terminal): [number, number] {
+  private static _mapStrIdx(terminal: Terminal, lineIndex: number, rowIndex: number, stringIndex: number): [number, number] {
     const buf = terminal.buffer.active;
     const cell = buf.getNullCell();
+    let start = rowIndex;
     while (stringIndex) {
       const line = buf.getLine(lineIndex);
       if (!line) {
         return [-1, -1];
       }
-      for (let i = 0; i < line.length; ++i) {
+      for (let i = start; i < line.length; ++i) {
         line.getCell(i, cell);
         const chars = cell.getChars();
         const width = cell.getWidth();
         if (width) {
           stringIndex -= chars.length || 1;
-        }
-        // look ahead for early wrap around of wide chars
-        if (i === line.length - 1 && chars === '' && width) {
-          const line = buf.getLine(lineIndex + 1);
-          if (line && line.isWrapped) {
-            line.getCell(0, cell);
-            if (cell.getWidth() === 2) {
-              stringIndex += 1;
+
+          // correct stringIndex for early wrapped wide chars:
+          // - currently only happens at last cell
+          // - cells to the right are reset with chars='' and width=1 in InputHandler.print
+          // - follow-up line must be wrapped and contain wide char at first cell
+          // --> if all these conditions are met, correct stringIndex by +1
+          if (i === line.length - 1 && chars === '') {
+            const line = buf.getLine(lineIndex + 1);
+            if (line && line.isWrapped) {
+              line.getCell(0, cell);
+              if (cell.getWidth() === 2) {
+                stringIndex += 1;
+              }
             }
           }
         }
@@ -191,7 +191,8 @@ export class LinkComputer {
         }
       }
       lineIndex++;
+      start = 0;
     }
-    return [lineIndex, 0];
+    return [lineIndex, start];
   }
 }

From 9411acbc73b432816dbd6aa811c5268be3b881c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 19 Dec 2022 15:33:10 +0100
Subject: [PATCH 12/13] move link tests to demo

---
 bin/test_weblinks.sh | 34 ----------------------------------
 demo/client.ts       | 23 +++++++++++++++++++++++
 demo/index.html      |  3 +++
 3 files changed, 26 insertions(+), 34 deletions(-)
 delete mode 100755 bin/test_weblinks.sh

diff --git a/bin/test_weblinks.sh b/bin/test_weblinks.sh
deleted file mode 100755
index 1389eceeb1..0000000000
--- a/bin/test_weblinks.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# all half width
-echo "aaa http://example.com aaa http://example.com aaa"
-
-# full width before
-echo "￥￥￥ http://example.com aaa http://example.com aaa"
-
-# full width between
-echo "aaa http://example.com ￥￥￥ http://example.com aaa"
-
-# full width before and between
-echo "￥￥￥ http://example.com ￥￥￥ http://example.com aaa"
-
-# full width within url
-echo "aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa"
-
-# full width within and before
-echo "￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 ￥￥￥"
-
-# username + password scheme
-echo "http://test:password@example.com/some_path aaa"
-
-# overly long text with urls with final interpunction
-echo "Lorem ipsum dolor sit amet, consetetur sadipscing elitr https://ko.wikipedia.org/wiki/위키백과:대문, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat: http://test:password@example.com/some_path."
-
-# bracket enclosed urls
-echo "[http://example.de]"
-echo "(http://example.de)"
-echo "<http://example.de>"
-echo "{http://example.de}"
-
-# ipv6 scheme
-echo "ipv6 https://[::1]/with/some?vars=and&a#hash"
\ No newline at end of file
diff --git a/demo/client.ts b/demo/client.ts
index 759b0658c5..5af6fa8f15 100644
--- a/demo/client.ts
+++ b/demo/client.ts
@@ -228,6 +228,7 @@ if (document.location.pathname === '/test') {
   document.getElementById('sgr-test').addEventListener('click', sgrTest);
   document.getElementById('add-decoration').addEventListener('click', addDecoration);
   document.getElementById('add-overview-ruler').addEventListener('click', addOverviewRuler);
+  document.getElementById('weblinks-test').addEventListener('click', testWeblinks);
   addVtButtons();
 }
 
@@ -1122,3 +1123,25 @@ function addVtButtons(): void {
 
   document.querySelector('#vt-container').appendChild(vtFragment);
 }
+
+function testWeblinks(): void {
+  const linkExamples = `
+aaa http://example.com aaa http://example.com aaa
+￥￥￥ http://example.com aaa http://example.com aaa
+aaa http://example.com ￥￥￥ http://example.com aaa
+￥￥￥ http://example.com ￥￥￥ http://example.com aaa
+aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 aaa
+￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 ￥￥￥
+aaa http://test:password@example.com/some_path aaa
+brackets enclosed:
+aaa [http://example.de] aaa
+aaa (http://example.de) aaa
+aaa <http://example.de> aaa
+aaa {http://example.de} aaa
+ipv6 https://[::1]/with/some?vars=and&a#hash aaa
+stop at final '.': This is a sentence with an url to http://example.com.
+stop at final '?': Is this the right url http://example.com/?
+stop at final '?': Maybe this one http://example.com/with?arguments=false?
+  `;
+  term.write(linkExamples.split('\n').join('\r\n'));
+}
diff --git a/demo/index.html b/demo/index.html
index 5662f361e5..42f8462ef4 100644
--- a/demo/index.html
+++ b/demo/index.html
@@ -87,6 +87,9 @@ <h3>Test</h3>
               <dt>Decorations</dt>
               <dd><button id="add-decoration" title="Add a decoration to the terminal">Decoration</button></dd>
               <dd><button id="add-overview-ruler" title="Add an overview ruler to the terminal">Add Overview Ruler</button></dd>
+
+              <dt>Weblinks Addon</dt>
+              <dd><button id="weblinks-test" title="Various url conditions from demo data, hover&click to test">Test URLs</button></dd>
             </dl>
           </div>
         </div>

From 1757f2733c9adad0b4cb91a3c55aa6e80c5a6128 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= <jerch@rockborn.de>
Date: Mon, 19 Dec 2022 15:49:10 +0100
Subject: [PATCH 13/13] explicit type for link test data

---
 .../test/WebLinksAddon.api.ts                 | 48 ++++++++++++-------
 1 file changed, 32 insertions(+), 16 deletions(-)

diff --git a/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts b/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
index 44d09c9586..8b6008a5c0 100644
--- a/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
+++ b/addons/xterm-addon-web-links/test/WebLinksAddon.api.ts
@@ -14,6 +14,22 @@ let page: Page;
 const width = 800;
 const height = 600;
 
+
+interface ILinkStateData {
+  uri?: string;
+  range?: {
+    start: {
+      x: number;
+      y: number;
+    };
+    end: {
+      x: number;
+      y: number;
+    };
+  };
+}
+
+
 describe('WebLinksAddon', () => {
   before(async function(): Promise<any> {
     browser = await launchBrowser();
@@ -41,35 +57,35 @@ describe('WebLinksAddon', () => {
       setupCustom();
       await writeSync(page, 'aaa http://example.com aaa http://example.com aaa');
       await resetAndHover(5, 1);
-      await evalData('http://example.com', { start: { x: 5, y: 1 }, end: { x: 22, y: 1 } });
+      await evalLinkStateData('http://example.com', { start: { x: 5, y: 1 }, end: { x: 22, y: 1 } });
       await resetAndHover(1, 2);
-      await evalData('http://example.com', { start: { x: 28, y: 1 }, end: { x: 5, y: 2 } });
+      await evalLinkStateData('http://example.com', { start: { x: 28, y: 1 }, end: { x: 5, y: 2 } });
     });
     it('url after full width', async () => {
       setupCustom();
       await writeSync(page, '￥￥￥ http://example.com ￥￥￥ http://example.com aaa');
       await resetAndHover(8, 1);
-      await evalData('http://example.com', { start: { x: 8, y: 1 }, end: { x: 25, y: 1 } });
+      await evalLinkStateData('http://example.com', { start: { x: 8, y: 1 }, end: { x: 25, y: 1 } });
       await resetAndHover(1, 2);
-      await evalData('http://example.com', { start: { x: 34, y: 1 }, end: { x: 11, y: 2 } });
+      await evalLinkStateData('http://example.com', { start: { x: 34, y: 1 }, end: { x: 11, y: 2 } });
     });
     it('full width within url and before', async () => {
       setupCustom();
       await writeSync(page, '￥￥￥ https://ko.wikipedia.org/wiki/위키백과:대문 aaa https://ko.wikipedia.org/wiki/위키백과:대문 ￥￥￥');
       await resetAndHover(8, 1);
-      await evalData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 8, y: 1 }, end: { x: 11, y: 2 } });
+      await evalLinkStateData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 8, y: 1 }, end: { x: 11, y: 2 } });
       await resetAndHover(1, 2);
-      await evalData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 8, y: 1 }, end: { x: 11, y: 2 } });
+      await evalLinkStateData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 8, y: 1 }, end: { x: 11, y: 2 } });
       await resetAndHover(17, 2);
-      await evalData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 17, y: 2 }, end: { x: 19, y: 3 } });
+      await evalLinkStateData('https://ko.wikipedia.org/wiki/위키백과:대문', { start: { x: 17, y: 2 }, end: { x: 19, y: 3 } });
     });
     it('name + password url after full width and combining', async () => {
       setupCustom();
       await writeSync(page, '￥￥￥cafe\u0301 http://test:password@example.com/some_path');
       await resetAndHover(12, 1);
-      await evalData('http://test:password@example.com/some_path', { start: { x: 12, y: 1 }, end: { x: 13, y: 2 } });
+      await evalLinkStateData('http://test:password@example.com/some_path', { start: { x: 12, y: 1 }, end: { x: 13, y: 2 } });
       await resetAndHover(13, 2);
-      await evalData('http://test:password@example.com/some_path', { start: { x: 12, y: 1 }, end: { x: 13, y: 2 } });
+      await evalLinkStateData('http://test:password@example.com/some_path', { start: { x: 12, y: 1 }, end: { x: 13, y: 2 } });
     });
   });
 });
@@ -104,20 +120,20 @@ async function pollForLinkAtCell(col: number, row: number, value: string): Promi
 
 async function setupCustom(): Promise<void> {
   await openTerminal(page, { cols: 40 });
-  await page.evaluate(`window._customLinkData = [];
+  await page.evaluate(`window._linkStateData = {};
 window._linkaddon = new window.WebLinksAddon();
-window._linkaddon._options.hover = (event, uri, range) => { window._customLinkData.push([uri, range]); };
+window._linkaddon._options.hover = (event, uri, range) => { window._linkStateData = { uri, range }; };
 window.term.loadAddon(window._linkaddon);`);
 }
 
 async function resetAndHover(col: number, row: number): Promise<void> {
-  await page.evaluate(`window._customLinkData = [];`);
+  await page.evaluate(`window._linkStateData = {};`);
   const rowSelector = `.xterm-rows > :nth-child(${row})`;
   await page.hover(`${rowSelector} > :nth-child(${col})`);
 }
 
-async function evalData(uri: string, range: any): Promise<void> {
-  const data: any[] = await page.evaluate(`window._customLinkData[0]`);
-  assert.equal(data[0], uri);
-  assert.deepEqual(data[1], range);
+async function evalLinkStateData(uri: string, range: any): Promise<void> {
+  const data: ILinkStateData = await page.evaluate(`window._linkStateData`);
+  assert.equal(data.uri, uri);
+  assert.deepEqual(data.range, range);
 }