= bytes 0-2 → offset 0 + // "hello" = bytes 3-7 → offset 3 (right after '>') + //
= bytes 8-11 → offset 8 + let entries = tokenize_bytes("hello
"); + assert_eq!( + entries, + vec![(start("p"), 0), (chars("hello"), 3), (end("p"), 8),] + ); + } + + #[test] + fn check_byte_offsets_multibyte_text() { + //= bytes 0-2 → offset 0 + // "é" = bytes 3-4 (é = 2 UTF-8 bytes) → offset 3 + //
= bytes 5-8 → offset 5 + let entries = tokenize_bytes("é
"); + assert_eq!( + entries, + vec![(start("p"), 0), (chars("é"), 3), (end("p"), 5),] + ); + } + + #[test] + fn check_byte_offsets_sequential_siblings() { + //= bytes 10-12 → offset 10 + // "Y" = bytes 13 → offset 13 + //
= bytes 14-17 → offset 14 + let entries = tokenize_bytes("Y
"); + assert_eq!( + entries, + vec![ + (start("h1"), 0), + (chars("X"), 4), + (end("h1"), 5), + (start("p"), 10), + (chars("Y"), 13), + (end("p"), 14), + ] + ); + } + + #[test] + fn check_byte_offsets_entity_text_chunks() { + //= bytes 0-2 → offset 0 + // "a" = byte 3 + // "&" = bytes 4-8, decoded to "&" + // "b" = byte 9 + let entries = tokenize_raw_bytes("
a&b
"); + assert_eq!( + entries, + vec![ + (start("p"), 0), + (chars("a"), 3), + (chars("&"), 4), + (chars("b"), 9), + (end("p"), 10), + ] + ); + } +} diff --git a/html5ever/src/tree_builder/mod.rs b/html5ever/src/tree_builder/mod.rs index 3fcfaec3..0be385ed 100644 --- a/html5ever/src/tree_builder/mod.rs +++ b/html5ever/src/tree_builder/mod.rs @@ -474,6 +474,11 @@ where { type Handle = Handle; + #[cfg(feature = "source-positions")] + fn set_current_byte(&self, byte_offset: u64) { + self.sink.set_current_byte(byte_offset); + } + fn process_token(&self, token: tokenizer::Token, line_number: u64) -> TokenSinkResultstarts at byte 0 + //
hello
" = 12 chars + 2 for "") + //hello
= 12 bytes, = 4 bytes →hello
= 0,
= 8,hello
café
next" + // 'é' = 2 bytes, so: + //= byte 0 + //
= byte 3+5 = byte 8 ("café" = c(1)+a(1)+f(1)+é(2) = 5 bytes) + // = byte 8 + 4 = byte 12 ("" = 4 bytes) + let result = parse("café
next"); + let elems = result.content_elements(); + + assert_eq!(elems.len(), 2, "expected p and span, got: {:?}", elems); + assert_eq!(elems[0], ("p".to_string(), 0)); + assert_eq!(elems[1], ("span".to_string(), 12)); + } +} diff --git a/rcdom/tests/xhtml-self-closing-integration.rs b/rcdom/tests/xhtml-self-closing-integration.rs new file mode 100644 index 00000000..c49aaf69 --- /dev/null +++ b/rcdom/tests/xhtml-self-closing-integration.rs @@ -0,0 +1,123 @@ +// Copyright 2014-2026 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0visible
"); + let names = element_names(&dom.document); + + assert!( + names.contains(&"body".to_string()), + "body element should be present; got: {:?}", + names + ); + assert!( + names.contains(&"p".to_string()), + "p element inside body should be present; got: {:?}", + names + ); + + let text = text_of(&dom, "p"); + assert_eq!( + text.as_deref(), + Some("visible"), + "text should be 'visible', got: {:?}", + text + ); + } + + #[test] + fn self_closing_style_does_not_swallow_body() { + let dom = parse("
content
"); + let names = element_names(&dom.document); + + assert!( + names.contains(&"p".to_string()), + "p element should not be swallowed by ; got: {:?}", + names + ); + } + + #[test] + fn normal_closed_title_still_captures_rcdata_text() { + // A properly-closed= bytes 0-2 → offset 0 - // "hello" = bytes 3-7 → offset 3 (right after '>') - //
= bytes 8-11 → offset 8 let entries = tokenize_bytes("hello
"); assert_eq!( entries, @@ -2683,24 +2665,14 @@ mod test_source_positions { #[test] fn check_byte_offsets_multibyte_text() { - //= bytes 0-2 → offset 0 - // "é" = bytes 3-4 (é = 2 UTF-8 bytes) → offset 3 - //
= bytes 5-8 → offset 5 let entries = tokenize_bytes("é
"); assert_eq!( entries, vec![(start("p"), 0), (chars("é"), 3), (end("p"), 5),] ); } - #[test] fn check_byte_offsets_sequential_siblings() { - //= bytes 10-12 → offset 10 - // "Y" = bytes 13 → offset 13 - //
= bytes 14-17 → offset 14 let entries = tokenize_bytes("Y
"); assert_eq!( entries, @@ -2717,10 +2689,6 @@ mod test_source_positions { #[test] fn check_byte_offsets_entity_text_chunks() { - //= bytes 0-2 → offset 0 - // "a" = byte 3 - // "&" = bytes 4-8, decoded to "&" - // "b" = byte 9 let entries = tokenize_raw_bytes("
a&b
"); assert_eq!( entries, diff --git a/html5ever/src/tree_builder/mod.rs b/html5ever/src/tree_builder/mod.rs index 0be385ed..bf28847a 100644 --- a/html5ever/src/tree_builder/mod.rs +++ b/html5ever/src/tree_builder/mod.rs @@ -681,12 +681,9 @@ where /// The generic raw text / RCDATA parsing algorithm. /// Insert a RCDATA/RAWTEXT element and switch the tokenizer to raw-text mode. /// - /// XHTML allows self-closing syntax (`starts at byte 0 - //
hello
" = 12 chars + 2 for "") - //hello
= 12 bytes, = 4 bytes →hello
= 0,
= 8,hello
hi
"); + let elems = result.content_elements(); + + assert_eq!( + elems.len(), + 4, + "expected html, head, body, p, got: {:?}", + elems + ); + assert_eq!(elems[0], ("html".to_string(), 0)); + assert_eq!(elems[1], ("head".to_string(), 6)); + assert_eq!(elems[2], ("body".to_string(), 19)); + assert_eq!(elems[3], ("p".to_string(), 25)); } #[test] + /// should start at byte 12, and not 13 due to é being 2 bytes. fn multibyte_content_does_not_shift_subsequent_offsets() { - // "café
next" - // 'é' = 2 bytes, so: - //= byte 0 - //
= byte 3+5 = byte 8 ("café" = c(1)+a(1)+f(1)+é(2) = 5 bytes) - // = byte 8 + 4 = byte 12 ("" = 4 bytes) let result = parse("café
next"); let elems = result.content_elements(); - assert_eq!(elems.len(), 2, "expected p and span, got: {:?}", elems); - assert_eq!(elems[0], ("p".to_string(), 0)); - assert_eq!(elems[1], ("span".to_string(), 12)); + assert_eq!( + elems.len(), + 5, + "expected html, head, body, p and span, got: {:?}", + elems + ); + assert_eq!(elems[0], ("html".to_string(), 0)); + assert_eq!(elems[1], ("head".to_string(), 0)); + assert_eq!(elems[2], ("body".to_string(), 0)); + assert_eq!(elems[3], ("p".to_string(), 0)); + assert_eq!(elems[4], ("span".to_string(), 12)); } } diff --git a/rcdom/tests/xhtml-self-closing-integration.rs b/rcdom/tests/xhtml-self-closing-integration.rs index c49aaf69..ae3a5d4a 100644 --- a/rcdom/tests/xhtml-self-closing-integration.rs +++ b/rcdom/tests/xhtml-self-closing-integration.rs @@ -1,19 +1,9 @@ -// Copyright 2014-2026 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0visible
"); let names = element_names(&dom.document); @@ -109,8 +96,6 @@ mod xhtml_self_closing { #[test] fn normal_closed_title_still_captures_rcdata_text() { - // A properly-closed