Improve token representation in AST (#932)

Previously the AST did not hold the token's source end offset in many
cases. This change adds token end offset by offsetting the end by the text
length from the token start.

common/ast/ast_test.go

@@ -72,7 +72,11 @@ func TestASTCopy(t *testing.T) {
 		if err != nil {
 			t.Errorf("ast.ToAST() failed: %v", err)
 		}
-		if !reflect.DeepEqual(checked, checkedRoundtrip) {
+		same := reflect.DeepEqual(checked.Expr(), checkedRoundtrip.Expr()) &&
+			reflect.DeepEqual(checked.ReferenceMap(), checkedRoundtrip.ReferenceMap()) &&
+			reflect.DeepEqual(checked.TypeMap(), checkedRoundtrip.TypeMap()) &&
+			reflect.DeepEqual(checked.SourceInfo().MacroCalls(), checkedRoundtrip.SourceInfo().MacroCalls())
+		if !same {
 			t.Errorf("Roundtrip got %v, wanted %v", checkedRoundtrip, checked)
 		}
 	}

parser/helper.go

@@ -140,15 +140,12 @@ func (p *parserHelper) id(ctx any) int64 {
 	var offset ast.OffsetRange
 	switch c := ctx.(type) {
 	case antlr.ParserRuleContext:
-		start, stop := c.GetStart(), c.GetStop()
-		if stop == nil {
-			stop = start
-		}
+		start := c.GetStart()
 		offset.Start = p.sourceInfo.ComputeOffset(int32(start.GetLine()), int32(start.GetColumn()))
-		offset.Stop = p.sourceInfo.ComputeOffset(int32(stop.GetLine()), int32(stop.GetColumn()))
+		offset.Stop = offset.Start + int32(len(c.GetText()))
 	case antlr.Token:
 		offset.Start = p.sourceInfo.ComputeOffset(int32(c.GetLine()), int32(c.GetColumn()))
-		offset.Stop = offset.Start
+		offset.Stop = offset.Start + int32(len(c.GetText()))
 	case common.Location:
 		offset.Start = p.sourceInfo.ComputeOffset(int32(c.Line()), int32(c.Column()))
 		offset.Stop = offset.Start