-
-
Notifications
You must be signed in to change notification settings - Fork 897
/
html5.rb
473 lines (443 loc) · 19 KB
/
html5.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
# coding: utf-8
# frozen_string_literal: true
#
# Copyright 2013-2021 Sam Ruby, Stephen Checkoway
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
require_relative 'html5/document'
require_relative 'html5/document_fragment'
require_relative 'html5/node'
module Nokogiri
# @since v1.12.0
# @note HTML5 functionality is not available when running JRuby.
# Parse an HTML5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
def self.HTML5(input, url = nil, encoding = nil, **options, &block)
Nokogiri::HTML5::Document.parse(input, url, encoding, **options, &block)
end
# == Usage
#
# Parse an HTML5 document:
#
# doc = Nokogiri.HTML5(string)
#
# Parse an HTML5 fragment:
#
# fragment = Nokogiri::HTML5.fragment(string)
#
# == Parsing options
#
# The document and fragment parsing methods support options that are different from Nokogiri's.
#
# - <tt>Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})</tt>
# - <tt>Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})</tt>
# - <tt>Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})</tt>
# - <tt>Nokogiri::HTML5.fragment(html, encoding = nil, options = {})</tt>
# - <tt>Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})</tt>
#
# The three currently supported options are +:max_errors+, +:max_tree_depth+ and
# +:max_attributes+, described below.
#
# === Error reporting
#
# Nokogumbo contains an experimental parse error reporting facility. By default, no parse errors
# are reported but this can be configured by passing the +:max_errors+ option to {HTML5.parse} or
# {HTML5.fragment}.
#
# For example, this script:
#
# doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
# doc.errors.each do |err|
# puts(err)
# end
#
# Emits:
#
# 1:1: ERROR: Expected a doctype token
# <span/>Hi there!</span foo=bar />
# ^
# 1:1: ERROR: Start tag of nonvoid HTML element ends with '/>', use '>'.
# <span/>Hi there!</span foo=bar />
# ^
# 1:17: ERROR: End tag ends with '/>', use '>'.
# <span/>Hi there!</span foo=bar />
# ^
# 1:17: ERROR: End tag contains attributes.
# <span/>Hi there!</span foo=bar />
# ^
#
# Using <tt>max_errors: -1</tt> results in an unlimited number of errors being returned.
#
# The errors returned by {HTML5::Document#errors} are instances of {Nokogiri::XML::SyntaxError}.
#
# The {https://html.spec.whatwg.org/multipage/parsing.html#parse-errors HTML standard} defines a
# number of standard parse error codes. These error codes only cover the "tokenization" stage of
# parsing HTML. The parse errors in the "tree construction" stage do not have standardized error
# codes (yet).
#
# As a convenience to Nokogumbo users, the defined error codes are available via
# {Nokogiri::XML::SyntaxError#str1} method.
#
# doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
# doc.errors.each do |err|
# puts("#{err.line}:#{err.column}: #{err.str1}")
# end
# # => 1:1: generic-parser
# # 1:1: non-void-html-element-start-tag-with-trailing-solidus
# # 1:17: end-tag-with-trailing-solidus
# # 1:17: end-tag-with-attributes
#
# Note that the first error is +generic-parser+ because it's an error from the tree construction
# stage and doesn't have a standardized error code.
#
# For the purposes of semantic versioning, the error messages, error locations, and error codes
# are not part of Nokogumbo's public API. That is, these are subject to change without Nokogumbo's
# major version number changing. These may be stabilized in the future.
#
# === Maximum tree depth
#
# The maximum depth of the DOM tree parsed by the various parsing methods is configurable by the
# +:max_tree_depth+ option. If the depth of the tree would exceed this limit, then an
# {::ArgumentError} is thrown.
#
# This limit (which defaults to <tt>Nokogumbo::DEFAULT_MAX_TREE_DEPTH = 400</tt>) can be removed
# by giving the option <tt>max_tree_depth: -1</tt>.
#
# html = '<!DOCTYPE html>' + '<div>' * 1000
# doc = Nokogiri.HTML5(html)
# # raises ArgumentError: Document tree depth limit exceeded
# doc = Nokogiri.HTML5(html, max_tree_depth: -1)
#
# === Attribute limit per element
#
# The maximum number of attributes per DOM element is configurable by the +:max_attributes+
# option. If a given element would exceed this limit, then an {::ArgumentError} is thrown.
#
# This limit (which defaults to <tt>Nokogumbo::DEFAULT_MAX_ATTRIBUTES = 400</tt>) can be removed
# by giving the option <tt>max_attributes: -1</tt>.
#
# html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
# # "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
# doc = Nokogiri.HTML5(html)
# # raises ArgumentError: Attributes per element limit exceeded
# doc = Nokogiri.HTML5(html, max_attributes: -1)
#
# == HTML Serialization
#
# After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization
# methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will
# serialize a given node and its children. (This is the equivalent of JavaScript's
# +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given
# node. (This is the equivalent of JavaScript's +Element.innerHTML+.)
#
# doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>")
# puts doc.serialize
# # => <!DOCTYPE html><html><head></head><body><span>Hello world!</span></body></html>
#
# Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be
# serialized and then re-parsed, resulting in a different DOM. Mostly, this happens with DOMs
# produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and
# re-parsing.
#
# In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by
# the parser.
#
# doc = Nokogiri::HTML5(<<-EOF)
# <!DOCTYPE html>
# <pre>
# Content</pre>
# EOF
# puts doc.at('/html/body/pre').serialize
# # => <pre>Content</pre>
#
# In this case, the original HTML is semantically equivalent to the serialized version. If the
# +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be
# stripped on the first parse and the second newline will be stripped on the second, leading to
# semantically different DOMs. Passing the parameter <tt>preserve_newline: true</tt> will cause
# two or more newlines to be preserved. (A single leading newline will still be removed.)
#
# doc = Nokogiri::HTML5(<<-EOF)
# <!DOCTYPE html>
# <listing>
#
# Content</listing>
# EOF
# puts doc.at('/html/body/listing').serialize(preserve_newline: true)
# # => <listing>
# #
# # Content</listing>
#
# == Encodings
#
# Nokogumbo always parses HTML using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the
# encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is
# most useful when the input comes not from a string but from an IO object.
#
# When serializing a document or node, the encoding of the output string can be specified via the
# +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded
# as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric
# entities}.
#
# frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>')
# html = frag.serialize(encoding: 'US-ASCII')
# puts html
# # => <span>아는 길도 물어가라</span>
# frag = Nokogiri::HTML5.fragment(html)
# puts frag.serialize
# # => <span>아는 길도 물어가라</span>
#
# (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that
# can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only
# encoding I'm aware of that has this bug is <tt>'ISO-2022-JP'</tt>. We recommend avoiding this
# encoding.)
#
# == Notes
#
# * The {Nokogiri::HTML5.fragment} function takes a string and parses it
# as a HTML5 document. The +<html>+, +<head>+, and +<body>+ elements are
# removed from this document, and any children of these elements that remain
# are returned as a {Nokogiri::HTML5::DocumentFragment}.
#
# * The {Nokogiri::HTML5.parse} function takes a string and passes it to the
# <code>gumbo_parse_with_options</code> method, using the default options.
# The resulting Gumbo parse tree is then walked.
#
# * Instead of uppercase element names, lowercase element names are produced.
#
# * Instead of returning +unknown+ as the element name for unknown tags, the
# original tag name is returned verbatim.
#
# @since v1.12.0
# @note HTML5 functionality is not available when running JRuby.
module HTML5
# HTML uses the XHTML namespace.
HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml'.freeze
MATHML_NAMESPACE = 'http://www.w3.org/1998/Math/MathML'.freeze
SVG_NAMESPACE = 'http://www.w3.org/2000/svg'.freeze
XLINK_NAMESPACE = 'http://www.w3.org/1999/xlink'.freeze
XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace'.freeze
XMLNS_NAMESPACE = 'http://www.w3.org/2000/xmlns/'.freeze
# Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
def self.parse(string, url = nil, encoding = nil, **options, &block)
Document.parse(string, url, encoding, **options, &block)
end
# Parse a fragment from +string+. Convenience method for
# {Nokogiri::HTML5::DocumentFragment.parse}.
def self.fragment(string, encoding = nil, **options)
DocumentFragment.parse(string, encoding, options)
end
# Fetch and parse a HTML document from the web, following redirects,
# handling https, and determining the character encoding using HTML5
# rules. +uri+ may be a +String+ or a +URI+. +options+ contains
# http headers and special options. Everything which is not a
# special option is considered a header. Special options include:
# * :follow_limit => number of redirects which are followed
# * :basic_auth => [username, password]
def self.get(uri, options={})
warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.",
uplevel: 1, category: :deprecated)
get_impl(uri, options)
end
private
def self.get_impl(uri, options={})
headers = options.clone
headers = {:follow_limit => headers} if Numeric === headers # deprecated
limit=headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
require 'net/http'
uri = URI(uri) unless URI === uri
http = Net::HTTP.new(uri.host, uri.port)
# TLS / SSL support
http.use_ssl = true if uri.scheme == 'https'
# Pass through Net::HTTP override values, which currently include:
# :ca_file, :ca_path, :cert, :cert_store, :ciphers,
# :close_on_empty_response, :continue_timeout, :key, :open_timeout,
# :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
# :verify_callback, :verify_depth, :verify_mode
options.each do |key, value|
http.send "#{key}=", headers.delete(key) if http.respond_to? "#{key}="
end
request = Net::HTTP::Get.new(uri.request_uri)
# basic authentication
auth = headers.delete(:basic_auth)
auth ||= [uri.user, uri.password] if uri.user && uri.password
request.basic_auth auth.first, auth.last if auth
# remaining options are treated as headers
headers.each {|key, value| request[key.to_s] = value.to_s}
response = http.request(request)
case response
when Net::HTTPSuccess
doc = parse(reencode(response.body, response['content-type']), options)
doc.instance_variable_set('@response', response)
doc.class.send(:attr_reader, :response)
doc
when Net::HTTPRedirection
response.value if limit <= 1
location = URI.join(uri, response['location'])
get_impl(location, options.merge(:follow_limit => limit-1))
else
response.value
end
end
def self.read_and_encode(string, encoding)
# Read the string with the given encoding.
if string.respond_to?(:read)
if encoding.nil?
string = string.read
else
string = string.read(encoding: encoding)
end
else
# Otherwise the string has the given encoding.
string = string.to_s
if encoding
string = string.dup
string.force_encoding(encoding)
end
end
# convert to UTF-8
if string.encoding != Encoding::UTF_8
string = reencode(string)
end
string
end
# Charset sniffing is a complex and controversial topic that understandably isn't done _by
# default_ by the Ruby Net::HTTP library. This being said, it is a very real problem for
# consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
# the Gumbo parser *only* supports utf-8.
#
# Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection. Following
# this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
# the HTML5 standard.
#
# http://bugs.ruby-lang.org/issues/2567
# http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
#
def self.reencode(body, content_type=nil)
if body.encoding == Encoding::ASCII_8BIT
encoding = nil
# look for a Byte Order Mark (BOM)
initial_bytes = body[0..2].bytes
if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
encoding = Encoding::UTF_8
elsif initial_bytes[0..1] == [0xFE, 0xFF]
encoding = Encoding::UTF_16BE
elsif initial_bytes[0..1] == [0xFF, 0xFE]
encoding = Encoding::UTF_16LE
end
# look for a charset in a content-encoding header
if content_type
encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
end
# look for a charset in a meta tag in the first 1024 bytes
if not encoding
data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, '')
data.scan(/<meta.*?>/m).each do |meta|
encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
end
end
# if all else fails, default to the official default encoding for HTML
encoding ||= Encoding::ISO_8859_1
# change the encoding to match the detected or inferred encoding
body = body.dup
begin
body.force_encoding(encoding)
rescue ArgumentError
body.force_encoding(Encoding::ISO_8859_1)
end
end
body.encode(Encoding::UTF_8)
end
def self.serialize_node_internal(current_node, io, encoding, options)
case current_node.type
when XML::Node::ELEMENT_NODE
ns = current_node.namespace
ns_uri = ns.nil? ? nil : ns.href
# XXX(sfc): attach namespaces to all nodes, even html?
if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE
tagname = current_node.name
else
tagname = "#{ns.prefix}:#{current_node.name}"
end
io << '<' << tagname
current_node.attribute_nodes.each do |attr|
attr_ns = attr.namespace
if attr_ns.nil?
attr_name = attr.name
else
ns_uri = attr_ns.href
if ns_uri == XML_NAMESPACE
attr_name = 'xml:' + attr.name.sub(/^[^:]*:/, '')
elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, '') == 'xmlns'
attr_name = 'xmlns'
elsif ns_uri == XMLNS_NAMESPACE
attr_name = 'xmlns:' + attr.name.sub(/^[^:]*:/, '')
elsif ns_uri == XLINK_NAMESPACE
attr_name = 'xlink:' + attr.name.sub(/^[^:]*:/, '')
else
attr_name = "#{attr_ns.prefix}:#{attr.name}"
end
end
io << ' ' << attr_name << '="' << escape_text(attr.content, encoding, true) << '"'
end
io << '>'
if !%w[area base basefont bgsound br col embed frame hr img input keygen
link meta param source track wbr].include?(current_node.name)
io << "\n" if options[:preserve_newline] && prepend_newline?(current_node)
current_node.children.each do |child|
# XXX(sfc): Templates handled specially?
serialize_node_internal(child, io, encoding, options)
end
io << '</' << tagname << '>'
end
when XML::Node::TEXT_NODE
parent = current_node.parent
if parent.element? && %w[style script xmp iframe noembed noframes plaintext noscript].include?(parent.name)
io << current_node.content
else
io << escape_text(current_node.content, encoding, false)
end
when XML::Node::CDATA_SECTION_NODE
io << '<![CDATA[' << current_node.content << ']]>'
when XML::Node::COMMENT_NODE
io << '<!--' << current_node.content << '-->'
when XML::Node::PI_NODE
io << '<?' << current_node.content << '>'
when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE
io << '<!DOCTYPE ' << current_node.name << '>'
when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE
current_node.children.each do |child|
serialize_node_internal(child, io, encoding, options)
end
else
raise "Unexpected node '#{current_node.name}' of type #{current_node.type}"
end
end
def self.escape_text(text, encoding, attribute_mode)
if attribute_mode
text = text.gsub(/[&\u00a0"]/,
'&' => '&', "\u00a0" => ' ', '"' => '"')
else
text = text.gsub(/[&\u00a0<>]/,
'&' => '&', "\u00a0" => ' ', '<' => '<', '>' => '>')
end
# Not part of the standard
text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" })
end
def self.prepend_newline?(node)
return false unless %w[pre textarea listing].include?(node.name) && !node.children.empty?
first_child = node.children[0]
first_child.text? && first_child.content.start_with?("\n")
end
end
end
require_relative 'gumbo'