From e876db00e4bdcad98a5950c40ca6e28365c8f9b0 Mon Sep 17 00:00:00 2001
From: NeeEoo <NeeEoo@users.noreply.github.com>
Date: Wed, 24 Apr 2024 19:33:11 +0200
Subject: [PATCH] Added hex, octal, better unicode escape parsing

Fixed printer so it now prints printable strings
---
 hscript/Expr.hx               |   1 +
 hscript/Parser.hx             | 222 ++++++++++++++++++++++------------
 hscript/Printer.hx            |  50 +++++++-
 tests/src/Util.hx             |  23 ++++
 tests/src/tests/FinalCase.hx  |   2 +
 tests/src/tests/StringCase.hx |  46 ++++---
 tests/src/tests/TestCase.hx   |  14 +++
 7 files changed, 267 insertions(+), 91 deletions(-)

diff --git a/hscript/Expr.hx b/hscript/Expr.hx
index 1b9d085..144a5fa 100644
--- a/hscript/Expr.hx
+++ b/hscript/Expr.hx
@@ -165,6 +165,7 @@ enum Error
 	ECustom( msg : String );
 	EInvalidClass( className : String);
 	EAlreadyExistingClass( className : String);
+	EInvalidEscape( s : String );
 }
 
 enum ModuleDecl {
diff --git a/hscript/Parser.hx b/hscript/Parser.hx
index 58f2429..31e6e7c 100644
--- a/hscript/Parser.hx
+++ b/hscript/Parser.hx
@@ -38,7 +38,7 @@ enum Token {
 	TBkOpen;
 	TBkClose;
 	TQuestion;
-	TDoubleDot;
+	TColon;
 	TMeta( s : String );
 	TPrepro( s : String );
 }
@@ -344,7 +344,7 @@ class Parser {
 					unexpected(tk);
 					break;
 			}
-			ensure(TDoubleDot);
+			ensure(TColon);
 			fl.push(new ObjectField(id, parseExpr()));
 			tk = token();
 			switch( tk ) {
@@ -389,7 +389,7 @@ class Parser {
 			switch( tk ) {
 				case TPClose:
 					return parseExprNext(mk(EParent(e),p1,tokenMax));
-				case TDoubleDot:
+				case TColon:
 					var t = parseType();
 					tk = token();
 					switch( tk ) {
@@ -412,37 +412,37 @@ class Parser {
 			return unexpected(tk);
 		case TBrOpen:
 			tk = token();
-			switch( tk ) {
-			case TBrClose:
-				return parseExprNext(mk(EObject([]),p1));
-			case TId(_):
-				var tk2 = token();
-				push(tk2);
-				push(tk);
-				switch( tk2 ) {
-					case TDoubleDot:
-						return parseExprNext(parseObject(p1));
-					default:
-				}
-			case TConst(c):
-				if( allowJSON ) {
-					switch( c ) {
-						case CString(_):
-							var tk2 = token();
-							push(tk2);
-							push(tk);
-							switch( tk2 ) {
-								case TDoubleDot:
-									return parseExprNext(parseObject(p1));
-								default:
-							}
+				switch( tk ) {
+				case TBrClose:
+					return parseExprNext(mk(EObject([]),p1));
+				case TId(_):
+					var tk2 = token();
+					push(tk2);
+					push(tk);
+					switch( tk2 ) {
+						case TColon:
+							return parseExprNext(parseObject(p1));
 						default:
-							push(tk);
 					}
-				} else
+				case TConst(c):
+					if( allowJSON ) { // Json string keys
+						switch( c ) {
+							case CString(_):
+								var tk2 = token();
+								push(tk2);
+								push(tk);
+								switch( tk2 ) {
+									case TColon:
+										return parseExprNext(parseObject(p1));
+									default:
+								}
+							default:
+								push(tk);
+						}
+					} else
+						push(tk);
+				default:
 					push(tk);
-			default:
-				push(tk);
 			}
 			var a = new Array();
 			while( true ) {
@@ -511,7 +511,7 @@ class Parser {
 	function parseLambda( args : Array<Argument>, pmin ) {
 		while( true ) {
 			var id = getIdent();
-			var t = maybe(TDoubleDot) ? parseType() : null;
+			var t = maybe(TColon) ? parseType() : null;
 			args.push(new Argument(id, t));
 			var tk = token();
 			switch( tk ) {
@@ -726,7 +726,7 @@ class Parser {
 			var tk = token();
 			var t = null;
 			nextType = null;
-			if( tk == TDoubleDot && allowTypes ) {
+			if( tk == TColon && allowTypes ) {
 				t = parseType();
 				tk = token();
 
@@ -961,7 +961,7 @@ class Parser {
 			ensureToken(TId("catch"));
 			ensure(TPOpen);
 			var vname = getIdent();
-			ensure(TDoubleDot);
+			ensure(TColon);
 			var t = null;
 			if( allowTypes )
 				t = parseType();
@@ -988,7 +988,7 @@ class Parser {
 							switch( tk ) {
 								case TComma | TOp("|"):
 									// next expr
-								case TDoubleDot:
+								case TColon:
 									break;
 								default:
 									unexpected(tk);
@@ -1017,7 +1017,7 @@ class Parser {
 							mk(EBlock(exprs), pmin(exprs[0]), pmax(exprs[exprs.length - 1]));
 					case TId("default"):
 						if( def != null ) unexpected(tk);
-						ensure(TDoubleDot);
+						ensure(TColon);
 						var exprs = [];
 						while( true ) {
 							tk = token();
@@ -1093,7 +1093,7 @@ class Parser {
 				return parseExprNext(mk(EArray(e1,e2),pmin(e1)));
 			case TQuestion:
 				var e2 = parseExpr();
-				ensure(TDoubleDot);
+				ensure(TColon);
 				var e3 = parseExpr();
 				return mk(ETernary(e1,e2,e3),pmin(e1),pmax(e3));
 			default:
@@ -1125,7 +1125,7 @@ class Parser {
 				args.push(arg);
 				if( opt ) arg.opt = true;
 				if( allowTypes ) {
-					if( maybe(TDoubleDot) )
+					if( maybe(TColon) )
 						arg.t = parseType();
 					if( maybe(TOp("="))) {
 						arg.value = parseExpr();
@@ -1152,7 +1152,7 @@ class Parser {
 		var ret = null;
 		if( allowTypes ) {
 			var tk = token();
-			if( tk != TDoubleDot )
+			if( tk != TColon )
 				push(tk);
 			else
 				ret = parseType();
@@ -1228,7 +1228,7 @@ class Parser {
 				}
 
 				switch [a, b] {
-					case [TPClose, _] | [TId(_), TDoubleDot]:
+					case [TPClose, _] | [TId(_), TColon]:
 
 						var args = [for (arg in parseFunctionArgs()) {
 							switch arg.value {
@@ -1268,20 +1268,20 @@ class Parser {
 						case TBrClose: break;
 						case TId("var"):
 							var name = getIdent();
-							ensure(TDoubleDot);
+							ensure(TColon);
 							fields.push( { name : name, t : parseType(), meta : meta } );
 							meta = null;
 							ensure(TSemicolon);
 						case TId("final"):
 							var name = getIdent();
-							ensure(TDoubleDot);
+							ensure(TColon);
 							if( meta == null ) meta = [];
 							meta.push({ name : ":final", params : [] });
 							fields.push( { name : name, t : parseType(), meta : meta } );
 							meta = null;
 							ensure(TSemicolon);
 						case TId(name):
-							ensure(TDoubleDot);
+							ensure(TColon);
 							fields.push( { name : name, t : parseType(), meta : meta } );
 							t = token();
 							switch( t ) {
@@ -1517,7 +1517,7 @@ class Parser {
 						set = getIdent();
 						ensure(TPClose);
 					}
-					var type = maybe(TDoubleDot) ? parseType() : null;
+					var type = maybe(TColon) ? parseType() : null;
 					var expr = maybe(TOp("=")) ? parseExpr() : null;
 
 					if( expr != null ) {
@@ -1555,6 +1555,15 @@ class Parser {
 		return StringTools.fastCodeAt(input, readPos++);
 	}
 
+	static function convertHex(c:Int) {
+		return switch( c ) {
+			case 48,49,50,51,52,53,54,55,56,57: c - 48; // 0-9
+			case 65,66,67,68,69,70: c - 55; // A-F
+			case 97,98,99,100,101,102: c - 87; // a-f
+			default: -1;
+		}
+	}
+
 	function readString( until ) {
 		var c = 0;
 		var b = new StringBuf();
@@ -1564,6 +1573,18 @@ class Parser {
 		#if hscriptPos
 		var p1 = readPos - 1;
 		#end
+		var ce = ""; // current escape
+		var lc = 0; // last char
+		function readEscape() {
+			var c = lc = readChar();
+			if( StringTools.isEof(c) ) {
+				line = old;
+				error(EUnterminatedString, p1, p1);
+				return -1;
+			}
+			ce += String.fromCharCode(c);
+			return c;
+		}
 		while( true ) {
 			var c = readChar();
 			if( StringTools.isEof(c) ) {
@@ -1572,43 +1593,96 @@ class Parser {
 				break;
 			}
 			if( esc ) {
+				ce = "\\" + String.fromCharCode(c);
 				esc = false;
 				switch( c ) {
-				case 'n'.code: b.addChar('\n'.code);
-				case 'r'.code: b.addChar('\r'.code);
-				case 't'.code: b.addChar('\t'.code);
-				case "'".code, '"'.code, '\\'.code: b.addChar(c);
-				case '/'.code: if( allowJSON ) b.addChar(c) else invalidChar(c);
-				case "u".code:
-					if( !allowJSON ) invalidChar(c);
-					var k = 0;
-					for( i in 0...4 ) {
-						k <<= 4;
-						var char = readChar();
-						switch( char ) {
-						case 48,49,50,51,52,53,54,55,56,57: // 0-9
-							k += char - 48;
-						case 65,66,67,68,69,70: // A-F
-							k += char - 55;
-						case 97,98,99,100,101,102: // a-f
-							k += char - 87;
-						default:
-							if( StringTools.isEof(char) ) {
-								line = old;
-								error(EUnterminatedString, p1, p1);
+					case 'n'.code: b.addChar('\n'.code);
+					case 'r'.code: b.addChar('\r'.code);
+					case 't'.code: b.addChar('\t'.code);
+					case "'".code, '"'.code, '\\'.code: b.addChar(c);
+					case '/'.code: if( allowJSON ) b.addChar(c) else invalidChar(c);
+					case '0'.code | '1'.code | '2'.code | '3'.code | '4'.code | '5'.code | '6'.code | '7'.code: // Octal
+						var n = c - '0'.code;
+						var i = 0;
+						for( i in 0...2 ) { // 2 since we already read the first digit
+							var char = readEscape();
+							if(char == -1) break;
+
+							n = n * 8;
+							if( char >= '0'.code && char <= '7'.code ) {
+								n += char - '0'.code;
+							} else {
+								error(EInvalidEscape(ce), p1, p1);
+								break;
 							}
-							invalidChar(char);
 						}
-					}
-					b.addChar(k);
-				default: invalidChar(c);
+						b.addChar(n);
+					case "x".code:
+						if( !allowJSON ) invalidChar(c);
+						var k = 0;
+						for( i in 0...2 ) {
+							var char = readEscape();
+							if(char == -1) break;
+
+							k <<= 4;
+							var v = convertHex(char);
+							if( v == -1 ) {
+								error(EInvalidEscape(ce), p1, p1);
+							} else {
+								k += v;
+							}
+						}
+						b.addChar(k);
+					case "u".code:
+						if( !allowJSON ) invalidChar(c);
+						if( readEscape() == '{'.code ) { // unicode
+							var k = 0;
+							var i = 0;
+							while( true ) {
+								var char = readEscape();
+								if(char == -1 || char == '}'.code) break;
+								if( i > 6 ) {
+									error(EInvalidEscape(ce), p1, p1);
+									break;
+								}
+
+								k <<= 4;
+								var v = convertHex(char);
+								if( v == -1 ) {
+									error(EInvalidEscape(ce), p1, p1);
+								} else {
+									k += v;
+								}
+								i++;
+							}
+							b.addChar(k);
+							continue;
+						} else {
+							readPos--;
+						}
+						var k = 0;
+						for( i in 0...4 ) {
+							var char = readEscape();
+							if(char == -1) break;
+
+							k <<= 4;
+							var v = convertHex(char);
+							if( v == -1 ) {
+								error(EInvalidEscape(ce), p1, p1);
+							} else {
+								k += v;
+							}
+						}
+						b.addChar(k);
+					default:
+						error(EInvalidEscape(ce), p1, p1);
 				}
-			} else if( c == 92 )
+			} else if( c == "\\".code )
 				esc = true;
 			else if( c == until )
 				break;
 			else {
-				if( c == 10 ) line++;
+				if( c == '\n'.code ) line++;
 				b.addChar(c);
 			}
 		}
@@ -1839,7 +1913,7 @@ class Parser {
 
 				this.char = char;
 				return TQuestion;
-			case ":".code: return TDoubleDot;
+			case ":".code: return TColon;
 			case '='.code:
 				char = readChar();
 				if( char == '='.code )
@@ -2097,7 +2171,7 @@ class Parser {
 		case TBkOpen: "[";
 		case TBkClose: "]";
 		case TQuestion: "?";
-		case TDoubleDot: ":";
+		case TColon: ":";
 		case TMeta(id): "@" + id;
 		case TPrepro(id): "#" + id;
 		}
diff --git a/hscript/Printer.hx b/hscript/Printer.hx
index b014bf2..281dd46 100644
--- a/hscript/Printer.hx
+++ b/hscript/Printer.hx
@@ -152,6 +152,50 @@ class Printer {
 		return false;
 	}
 
+	static inline function isPrintable( c : Int ) {
+		return c >= 32 && c <= 126;
+	}
+
+	static inline function hex( c : Int, ?len : Int = 2 ) {
+		return StringTools.hex(c, len).toLowerCase();
+	}
+
+	public static function getEscapedString( s : String ) {
+		var buf = new StringBuf();
+		#if target.unicode
+		var s = new UnicodeString(s);
+		#end
+		for( i in 0...s.length ) {
+			#if target.unicode
+			var c:Null<Int> = s.charCodeAt(i);
+			#else
+			var c:Null<Int> = StringTools.unsafeCodeAt(s, i);
+			#end
+			switch( c ) {
+				case '"'.code: buf.add('\\"');
+				case '\\'.code: buf.add('\\\\');
+				case '\n'.code: buf.add('\\n');
+				case '\r'.code: buf.add('\\r');
+				case '\t'.code: buf.add('\\t');
+				default:
+					if(c == null) continue;
+					if(isPrintable(c))
+						buf.addChar(c);
+					else {
+						if(c > 0xFF) {
+							buf.add("\\u{");
+							buf.add(hex(c, null));
+							buf.add("}");
+						} else {
+							buf.add("\\x");
+							buf.add(hex((c & 0xFF)));
+						}
+					}
+			}
+		}
+		return buf.toString();
+	}
+
 	function expr( e : Expr ) {
 		if( e == null ) {
 			add("??NULL??");
@@ -185,7 +229,10 @@ class Printer {
 			switch( c ) {
 			case CInt(i): add(i);
 			case CFloat(f): add(f);
-			case CString(s): add('"'); add(s.split('"').join('\\"').split("\n").join("\\n").split("\r").join("\\r").split("\t").join("\\t")); add('"');
+			case CString(s):
+				add('"');
+				add(getEscapedString(s));
+				add('"');
 			}
 		case EIdent(v):
 			add(v);
@@ -456,6 +503,7 @@ class Printer {
 			case ECustom(msg): msg;
 			case EInvalidClass(cla): "Invalid class: " + cla + " was not found.";
 			case EAlreadyExistingClass(cla): 'Custom Class named $cla already exists.';
+			case EInvalidEscape(s): "Invalid escape sequence: " + s;
 		};
 		#if hscriptPos
 		return e.origin + ":" + e.line + ": " + message;
diff --git a/tests/src/Util.hx b/tests/src/Util.hx
index eea5ee0..e04b7f4 100644
--- a/tests/src/Util.hx
+++ b/tests/src/Util.hx
@@ -1,4 +1,5 @@
 import hscript.Expr.Error;
+import hscript.Printer;
 
 using StringTools;
 
@@ -37,6 +38,28 @@ class Util {
 		}
 	}
 
+	public static function assertEqPrintable(value:Dynamic, expected:Dynamic, message:String, ?pos:haxe.PosInfos) {
+		var passed = value == expected;
+		if (Std.isOfType(value, Array) && Std.isOfType(expected, Array)) {
+			if (deepCompareArrays(value, expected))
+				passed = true;
+		}
+		else if (Std.isOfType(value, Enum) && Std.isOfType(expected, Enum)) {
+			if(Type.enumEq(value, expected))
+				passed = true;
+		}
+
+		if (passed) {
+			passedTestUnits++;
+			return true;
+		} else {
+			Sys.println("Assertion failed: " + message + " Expected: " + Printer.getEscapedString(expected) + " Got: " + Printer.getEscapedString(value));
+			Sys.println("> At " + pos.fileName + ":" + pos.lineNumber);
+			failedTestUnits++;
+			return false;
+		}
+	}
+
 	public static function assertNeq(value:Dynamic, expected:Dynamic, message:String, ?pos:haxe.PosInfos) {
 		var passed = value != expected;
 		if (Std.isOfType(value, Array) && Std.isOfType(expected, Array)) {
diff --git a/tests/src/tests/FinalCase.hx b/tests/src/tests/FinalCase.hx
index 4ebd600..581f803 100644
--- a/tests/src/tests/FinalCase.hx
+++ b/tests/src/tests/FinalCase.hx
@@ -51,6 +51,8 @@ class FinalCase extends TestCase {
 		assertEq("TestEnum.B", TestEnum.B);
 		assertEq("TestEnum.C(1, 2)", TestEnum.C(1, 2));
 		assertEq("TestEnum.D(1, 2, 3)", TestEnum.D(1, 2, 3));
+
+		// Test EOF with preprocessor
 	}
 
 	function assertDisplay(script:String, expected:Dynamic) {
diff --git a/tests/src/tests/StringCase.hx b/tests/src/tests/StringCase.hx
index ba8f5e6..a8e24eb 100644
--- a/tests/src/tests/StringCase.hx
+++ b/tests/src/tests/StringCase.hx
@@ -91,22 +91,6 @@ class StringCase extends TestCase {
 		assertEq('("a" > "b")', ("a" > "b"));
 		assertEq('("a" >= "b")', ("a" >= "b"));
 
-		Util.runKnownBug("String interpolation doesnt work", () -> {
-			assertEq("'${5}'", '${5}');
-			assertEq("'${5},${({})}'", '${5},${({})}');
-			assertEq("'${5},${\"Hello\"}'", '${5},${"Hello"}');
-			assertEq("'${5},${'Hello'}'", '${5},${'Hello'}');
-			assertEq("'$${5}'", '$${5}');
-			assertEq("'$${({})}'", '$${({})}');
-		});
-
-		assertEq('"$${5}"', "${5}");
-		assertEq('"$${5},$${({})}"', "${5},${({})}");
-		assertEq('"$${5},$${\\"Hello\\"}"', "${5},${\"Hello\"}");
-		assertEq('"$${5},$${\'Hello\'}"', "${5},${'Hello'}");
-
-
-
 		assertEq('"foo".toUpperCase()', "foo".toUpperCase());
 		assertEq('"_bar".toUpperCase()', "_bar".toUpperCase());
 		assertEq('"123b".toUpperCase()', "123b".toUpperCase());
@@ -119,6 +103,36 @@ class StringCase extends TestCase {
 		assertEq('"".toLowerCase()', "".toLowerCase());
 		assertEq('"a".toLowerCase()', "a".toLowerCase());
 
+		assertEqPrintable("' \\t\\n\\x0b\\x0c\\r16'", " \t\n\x0b\x0c\r16");
+		assertEqPrintable("' \\t\\n\\x0b\\x0c\\r0xa'", " \t\n\x0b\x0c\r0xa");
+		assertEqPrintable("' \\t\\n\\x0b\\x0c\\r1.6'", ' \t\n\x0b\x0c\r1.6');
+
+		assertEq('"$${5}"', "${5}");
+		assertEq('"$${5},$${({})}"', "${5},${({})}");
+		assertEq('"$${5},$${\\"Hello\\"}"', "${5},${\"Hello\"}");
+		assertEq('"$${5},$${\'Hello\'}"', "${5},${'Hello'}");
+
+		Util.runKnownBug("String interpolation doesnt work", () -> {
+			assertEq("'${5}'", '${5}');
+			assertEq("'${5},${({})}'", '${5},${({})}');
+			assertEq("'${5},${\"Hello\"}'", '${5},${"Hello"}');
+			assertEq("'${5},${'Hello'}'", '${5},${'Hello'}');
+			assertEq("'$${5}'", '$${5}');
+			assertEq("'$${({})}'", '$${({})}');
+		});
+
+		assertEqPrintable("'The \\x54\\141b\\tch\\141r\\141ct\\145r:\\n'", 'The \x54\141b\tch\141r\141ct\145r:\n');
+		assertEqPrintable("'\\t\\\"\\101scii\\\"\\n'", '\t\"\101scii\"\n');
+		assertEqPrintable("'\\tcontains\\n'", '\tcontains\n');
+		assertEqPrintable("'\\tspecial\\\\backslash\\\\codes,\\n'", '\tspecial\\backslash\\codes,\n');
+		assertEqPrintable("'\\tdouble quotes \\\"like this\\\",\\n'", '\tdouble quotes \"like this\",\n');
+		assertEqPrintable("'\\tsingle quotes \\'and this\\',\\n'", '\tsingle quotes \'and this\',\n');
+		assertEqPrintable("'\\tASCII bell\\x07and others.\\n'", '\tASCII bell\x07and others.\n');
+		assertEqPrintable("'Unicode samples: Greek \\u03B1 (alpha), smiley \\u263A,\\n'", 'Unicode samples: Greek \u03B1 (alpha), smiley \u263A,\n');
+		assertEqPrintable("'regional indicators \\u{1F1FA}\\u{1F1F8}, and musical note \\u{1F3B5}.'", 'regional indicators \u{1F1FA}\u{1F1F8}, and musical note \u{1F3B5}.');
+		assertEqPrintable("'\\u{10FFFF}\\u{1F1FA}\\u{3042}\\u{3B1}\\u{F1}\\u{A}'", '\u{10FFFF}\u{1F1FA}\u{3042}\u{3B1}\u{F1}\u{A}');
+
+		trace(hscript.Printer.getEscapedString("\u{10FFFF}\u{1F1FA}\u{3042}\u{3B1}\u{F1}\u{A}"));
 	}
 
 	override function teardown() {
diff --git a/tests/src/tests/TestCase.hx b/tests/src/tests/TestCase.hx
index 50d71f1..54eddb9 100644
--- a/tests/src/tests/TestCase.hx
+++ b/tests/src/tests/TestCase.hx
@@ -19,6 +19,20 @@ class TestCase extends HScriptRunner {
 		return true;
 	}
 
+	public function assertEqPrintable(script:String, expected:Dynamic, ?message:String, ?vars:Dynamic, ?pos:haxe.PosInfos) {
+		if(message == null)
+			message = script;
+		var result = if(vars != null)
+			executeWithVars(script, vars);
+		else
+			execute(script);
+		if(!Util.assertEqPrintable(result, expected, message, pos)) {
+			Sys.println("> " + Printer.convertExprToString(lastExpr));
+			return false;
+		}
+		return true;
+	}
+
 	public function assertNeq(script:String, expected:Dynamic, ?message:String, ?vars:Dynamic, ?pos:haxe.PosInfos) {
 		if(message == null)
 			message = script;