|
17 | 17 | from dataclasses import replace |
18 | 18 | from datetime import datetime, timezone |
19 | 19 | from email.utils import parsedate_to_datetime |
20 | | -from html import escape |
| 20 | +from html import escape, unescape |
21 | 21 | from itertools import chain, count, zip_longest |
22 | | -from urllib.parse import urlsplit |
| 22 | +from urllib.parse import urljoin, urlsplit |
23 | 23 |
|
24 | 24 | from . import __version__ |
25 | 25 | from .extract import Page, extract_text, normalize |
|
32 | 32 | # Indexed as they are. Some documentation sites send Markdown to anything that isn't |
33 | 33 | # a browser, and it's already clean text. |
34 | 34 | _TEXT_TYPES = ("text/plain", "text/markdown", "text/x-markdown") |
| 35 | +# <meta http-equiv="refresh" content="0; url=...">: a page that only exists to |
| 36 | +# send the reader somewhere else. Browsers follow it, so markfind does too. |
| 37 | +_META_REFRESH = re.compile( |
| 38 | + r"""<meta\s[^>]*http-equiv\s*=\s*["']?refresh["']?[^>]*>""", re.IGNORECASE |
| 39 | +) |
| 40 | +_REFRESH_TARGET = re.compile( |
| 41 | + r"""content\s*=\s*["']?\s*(\d+)\s*;\s*url\s*=\s*['"]?([^'">\s]+)""", re.IGNORECASE |
| 42 | +) |
| 43 | +# A longer wait is a page refreshing itself, not one that moved. |
| 44 | +MAX_REFRESH_DELAY = 5 |
| 45 | +REFRESH_HOPS = 3 |
| 46 | + |
35 | 47 | # How an HTML document starts, after any BOM and whitespace. |
36 | 48 | _HTML_START = re.compile(rb"<(?:!doctype\s+html|html|head|body)(?:[\s>/]|$)", re.IGNORECASE) |
37 | 49 |
|
@@ -204,17 +216,36 @@ def _ssl_context() -> ssl.SSLContext | None: |
204 | 216 | return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) |
205 | 217 |
|
206 | 218 |
|
| 219 | +def meta_refresh(html: str, base_url: str) -> str | None: |
| 220 | + """Where a page's meta refresh sends the reader, or None. |
| 221 | +
|
| 222 | + Only an immediate one counts: a longer delay is a page set to reload |
| 223 | + itself, which is not a move. |
| 224 | + """ |
| 225 | + for tag in _META_REFRESH.findall(html): |
| 226 | + found = _REFRESH_TARGET.search(tag) |
| 227 | + if found and int(found.group(1)) <= MAX_REFRESH_DELAY: |
| 228 | + target = urljoin(base_url, unescape(found.group(2))) |
| 229 | + if target != base_url and urlsplit(target).scheme in ("http", "https"): |
| 230 | + return target |
| 231 | + return None |
| 232 | + |
| 233 | + |
207 | 234 | def fetch_page( |
208 | 235 | url: str, |
209 | 236 | timeout: float = 10.0, |
210 | 237 | max_bytes: int = MAX_BYTES, |
211 | 238 | validators: tuple[str, str] | None = None, |
| 239 | + hops: int = REFRESH_HOPS, |
212 | 240 | ) -> Page: |
213 | 241 | """Download a page and extract its text. |
214 | 242 |
|
215 | 243 | ``validators`` is the (ETag, Last-Modified) pair from the last fetch. Given |
216 | 244 | one, the request asks the server to answer only if the page changed, and |
217 | 245 | NotModified is raised when it says it has not. |
| 246 | +
|
| 247 | + A page that only redirects, through ``<meta http-equiv="refresh">``, is |
| 248 | + followed for up to ``hops`` steps. |
218 | 249 | """ |
219 | 250 | headers = { |
220 | 251 | "User-Agent": USER_AGENT, |
@@ -276,6 +307,14 @@ def fetch_page( |
276 | 307 | body = raw.decode("utf-8", errors="replace") |
277 | 308 | body = body.removeprefix("\ufeff") # a UTF-8 byte-order mark isn't text |
278 | 309 |
|
| 310 | + if content_type == "text/html" and hops > 0: |
| 311 | + moved = meta_refresh(body, final_url) |
| 312 | + if moved: |
| 313 | + # The validators belong to the page that redirected, not to this |
| 314 | + # one, so they are dropped: there is nothing here to revalidate. |
| 315 | + page = fetch_page(moved, timeout, max_bytes, hops=hops - 1) |
| 316 | + return replace(page, etag="", last_modified="") |
| 317 | + |
279 | 318 | if content_type == "text/plain": |
280 | 319 | text = normalize(body) |
281 | 320 | page = Page(title="", text=text, html=f"<pre>{escape(body.strip())}</pre>" if text else "") |
|
0 commit comments