From 0b57e70b88f99728b3ff916f24e6258a4dacdbb8 Mon Sep 17 00:00:00 2001
From: darkuss <darkussdelta@gmail.com>
Date: Tue, 29 Sep 2026 03:40:27 +0200
Subject: [PATCH] Add tokenizer tests for processing instructions

---
 tokenizer/README.md  |   1 +
 tokenizer/test2.test |  20 ++--
 tokenizer/test3.test | 276 ++++++++++++++++++++++++++++++-------------
 3 files changed, 204 insertions(+), 93 deletions(-)

diff --git a/tokenizer/README.md b/tokenizer/README.md
index 66b81e8f..996d1e6f 100644
--- a/tokenizer/README.md
+++ b/tokenizer/README.md
@@ -76,6 +76,7 @@ tokens are:
     ["StartTag", name, {attributes}]
     ["EndTag", name]
     ["Comment", data]
+    ["ProcessingInstruction", data]
     ["Character", data]
 
 `public_id` and `system_id` are either strings or `null`. `correctness`
diff --git a/tokenizer/test2.test b/tokenizer/test2.test
index c29e4c31..edbec4b5 100644
--- a/tokenizer/test2.test
+++ b/tokenizer/test2.test
@@ -183,19 +183,17 @@
     { "code": "invalid-first-character-of-tag-name", "line": 1, "col": 3 }
 ]},
 
-{"description":"Simili processing instruction",
+{"description":"Processing instruction",
 "input":"<?namespace>",
-"output":[["Comment", "?namespace"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
+"output":[["ProcessingInstruction", "namespace", ""]]},
 
-{"description":"A bogus comment stops at >, even if preceded by two dashes",
-"input":"<?foo-->",
-"output":[["Comment", "?foo--"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
+{"description":"A processing instruction delimited by ?> doesn't contain the delimiting ?",
+"input":"<?foo bar?>",
+"output":[["ProcessingInstruction", "foo", "bar"]]},
+
+{"description":"A processing instruction's data delimited by ?> contains the correct amount of ?",
+"input":"<?foo bar??>",
+"output":[["ProcessingInstruction", "foo", "bar?"]]},
 
 {"description":"Unescaped <",
 "input":"foo < bar",
diff --git a/tokenizer/test3.test b/tokenizer/test3.test
index 901a581e..7a6a1b90 100644
--- a/tokenizer/test3.test
+++ b/tokenizer/test3.test
@@ -8954,16 +8954,184 @@
 
 {"description":"<?",
 "input":"<?",
-"output":[["Comment", "?"]],
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 3 }
+]},
+
+{"description":"<?a",
+"input":"<?a",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?b",
+"input":"<?b",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?y",
+"input":"<?y",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?z",
+"input":"<?z",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?A",
+"input":"<?A",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?B",
+"input":"<?B",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?Y",
+"input":"<?Y",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?Z",
+"input":"<?Z",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?a a",
+"input":"<?a a",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?b b",
+"input":"<?b b",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?y y",
+"input":"<?y y",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?z z",
+"input":"<?z z",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?A A",
+"input":"<?A A",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?B B",
+"input":"<?B B",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?Y Y",
+"input":"<?Y Y",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?Z Z?",
+"input":"<?Z Z?",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?a a?",
+"input":"<?a a",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?b b?",
+"input":"<?b b",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?y y?",
+"input":"<?y y",
+"output":[],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?z z?",
+"input":"<?z z",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?A A?",
+"input":"<?A A",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?B B?",
+"input":"<?B B",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?Y Y?",
+"input":"<?Y Y",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
+]},
+
+{"description":"<?Z Z?",
+"input":"<?Z Z",
+"output":[],
+"errors":[
+    { "code": "eof-in-processing-instruction", "line": 1, "col": 4 }
 ]},
 
 {"description":"<?\\u0000",
 "input":"<?\u0000",
 "output":[["Comment", "?\uFFFD"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 },
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 },
     { "code": "unexpected-null-character", "line": 1, "col": 3 }
 ]},
 
@@ -8971,21 +9139,21 @@
 "input":"<?\u0009",
 "output":[["Comment", "?\u0009"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?\\u000A",
 "input":"<?\u000A",
 "output":[["Comment", "?\u000A"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?\\u000B",
 "input":"<?\u000B",
 "output":[["Comment", "?\u000B"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 },
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 },
     { "code": "control-character-in-input-stream", "line": 1, "col": 3 }
 ]},
 
@@ -8993,21 +9161,21 @@
 "input":"<?\u000C",
 "output":[["Comment", "?\u000C"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<? ",
 "input":"<? ",
 "output":[["Comment", "? "]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<? \\u0000",
 "input":"<? \u0000",
 "output":[["Comment", "? \uFFFD"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 },
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 },
     { "code": "unexpected-null-character", "line": 1, "col": 4 }
 ]},
 
@@ -9015,175 +9183,119 @@
 "input":"<?!",
 "output":[["Comment", "?!"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?\"",
 "input":"<?\"",
 "output":[["Comment", "?\""]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?&",
 "input":"<?&",
 "output":[["Comment", "?&"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?'",
 "input":"<?'",
 "output":[["Comment", "?'"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?-",
 "input":"<?-",
 "output":[["Comment", "?-"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?/",
 "input":"<?/",
 "output":[["Comment", "?/"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?0",
 "input":"<?0",
 "output":[["Comment", "?0"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?1",
 "input":"<?1",
 "output":[["Comment", "?1"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?9",
 "input":"<?9",
 "output":[["Comment", "?9"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?<",
 "input":"<?<",
 "output":[["Comment", "?<"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?=",
 "input":"<?=",
 "output":[["Comment", "?="]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?>",
 "input":"<?>",
 "output":[["Comment", "?"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<??",
 "input":"<??",
 "output":[["Comment", "??"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?@",
 "input":"<?@",
 "output":[["Comment", "?@"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?A",
-"input":"<?A",
-"output":[["Comment", "?A"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?B",
-"input":"<?B",
-"output":[["Comment", "?B"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?Y",
-"input":"<?Y",
-"output":[["Comment", "?Y"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?Z",
-"input":"<?Z",
-"output":[["Comment", "?Z"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?`",
 "input":"<?`",
 "output":[["Comment", "?`"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?a",
-"input":"<?a",
-"output":[["Comment", "?a"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?b",
-"input":"<?b",
-"output":[["Comment", "?b"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?y",
-"input":"<?y",
-"output":[["Comment", "?y"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
-]},
-
-{"description":"<?z",
-"input":"<?z",
-"output":[["Comment", "?z"]],
-"errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?{",
 "input":"<?{",
 "output":[["Comment", "?{"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<?\\uDBC0\\uDC00",
 "input":"<?\uDBC0\uDC00",
 "output":[["Comment", "?\uDBC0\uDC00"]],
 "errors":[
-    { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 }
+    { "code": "invalid-first-character-of-processing-instruction-target", "line": 1, "col": 3 }
 ]},
 
 {"description":"<@",