Skip to content

Commit b5cc32d

Browse files
authored
Merge pull request #77 from Alcides1123/follow-meta-refresh
Follow pages that redirect with a meta refresh
2 parents 63d5248 + 041de6f commit b5cc32d

5 files changed

Lines changed: 116 additions & 2 deletions

File tree

‎CHANGES.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@
22

33
## Unreleased
44

5+
- Fixed: a bookmark pointing at a page that only redirects, through
6+
`<meta http-equiv="refresh">`, was indexed as the stub. One real example
7+
indexed 28 characters of "Click here to be redirected." instead of the
8+
7,000-word article it pointed to. Such pages are now followed, up to three
9+
hops; a page merely set to reload itself is left alone. Run
10+
`markfind fetch --all` to re-index bookmarks affected by this (#76).
11+
512
- `markfind topics` no longer crawls on a large collection. Past 3,000 indexed
613
pages it finds the topics from a sample of that size and then places every
714
page against them, which took 10,000 bookmarks from about 9 minutes down to

‎docs/cli.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,10 @@ What gets indexed:
193193
- HTML pages (`text/html`, `application/xhtml+xml`) are reduced to their main
194194
text. See [text extraction](how-it-works.md#text-extraction).
195195
- `text/plain` and Markdown pages are indexed as they are.
196+
- A page that only redirects, with `<meta http-equiv="refresh">` and no real
197+
content of its own, is followed to where it points, for up to three hops. A
198+
refresh more than five seconds away is a page reloading itself, and is left
199+
alone.
196200
- PDFs are indexed when the optional `pypdf` package is installed
197201
(`pipx inject markfind pypdf`, or `python -m pip install pypdf`):
198202
- their text and title are read from the first 300 pages;

‎docs/how-it-works.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,14 @@ validators are worth keeping: Wikipedia and example.com sent only
137137
Bookmarks whose last fetch failed never take part, since there is nothing good
138138
to revalidate against, and neither do pages whose server offered no validator.
139139

140+
Some pages answer 200 and then send the reader somewhere else with
141+
`<meta http-equiv="refresh" content="0; url=...">`, usually as the `<noscript>`
142+
fallback behind a script redirect. urllib does not treat that as a redirect, so
143+
`fetch_page()` looks for one itself and follows it for up to three hops, as a
144+
browser would. Only an immediate refresh counts: beyond `MAX_REFRESH_DELAY` (5
145+
seconds) it is a page reloading itself. The target's `ETag` and `Last-Modified`
146+
are dropped, since they describe a different URL from the bookmark's.
147+
140148
Servers don't always say what they're sending. `sniff_content_type()`
141149
checks the start of the body when the `Content-Type` header is missing (Python
142150
would otherwise report `text/plain`) or says `text/plain`: a body that opens

‎src/markfind/fetch.py‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
from dataclasses import replace
1818
from datetime import datetime, timezone
1919
from email.utils import parsedate_to_datetime
20-
from html import escape
20+
from html import escape, unescape
2121
from itertools import chain, count, zip_longest
22-
from urllib.parse import urlsplit
22+
from urllib.parse import urljoin, urlsplit
2323

2424
from . import __version__
2525
from .extract import Page, extract_text, normalize
@@ -32,6 +32,18 @@
3232
# Indexed as they are. Some documentation sites send Markdown to anything that isn't
3333
# a browser, and it's already clean text.
3434
_TEXT_TYPES = ("text/plain", "text/markdown", "text/x-markdown")
35+
# <meta http-equiv="refresh" content="0; url=...">: a page that only exists to
36+
# send the reader somewhere else. Browsers follow it, so markfind does too.
37+
_META_REFRESH = re.compile(
38+
r"""<meta\s[^>]*http-equiv\s*=\s*["']?refresh["']?[^>]*>""", re.IGNORECASE
39+
)
40+
_REFRESH_TARGET = re.compile(
41+
r"""content\s*=\s*["']?\s*(\d+)\s*;\s*url\s*=\s*['"]?([^'">\s]+)""", re.IGNORECASE
42+
)
43+
# A longer wait is a page refreshing itself, not one that moved.
44+
MAX_REFRESH_DELAY = 5
45+
REFRESH_HOPS = 3
46+
3547
# How an HTML document starts, after any BOM and whitespace.
3648
_HTML_START = re.compile(rb"<(?:!doctype\s+html|html|head|body)(?:[\s>/]|$)", re.IGNORECASE)
3749

@@ -204,17 +216,36 @@ def _ssl_context() -> ssl.SSLContext | None:
204216
return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
205217

206218

219+
def meta_refresh(html: str, base_url: str) -> str | None:
220+
"""Where a page's meta refresh sends the reader, or None.
221+
222+
Only an immediate one counts: a longer delay is a page set to reload
223+
itself, which is not a move.
224+
"""
225+
for tag in _META_REFRESH.findall(html):
226+
found = _REFRESH_TARGET.search(tag)
227+
if found and int(found.group(1)) <= MAX_REFRESH_DELAY:
228+
target = urljoin(base_url, unescape(found.group(2)))
229+
if target != base_url and urlsplit(target).scheme in ("http", "https"):
230+
return target
231+
return None
232+
233+
207234
def fetch_page(
208235
url: str,
209236
timeout: float = 10.0,
210237
max_bytes: int = MAX_BYTES,
211238
validators: tuple[str, str] | None = None,
239+
hops: int = REFRESH_HOPS,
212240
) -> Page:
213241
"""Download a page and extract its text.
214242
215243
``validators`` is the (ETag, Last-Modified) pair from the last fetch. Given
216244
one, the request asks the server to answer only if the page changed, and
217245
NotModified is raised when it says it has not.
246+
247+
A page that only redirects, through ``<meta http-equiv="refresh">``, is
248+
followed for up to ``hops`` steps.
218249
"""
219250
headers = {
220251
"User-Agent": USER_AGENT,
@@ -276,6 +307,14 @@ def fetch_page(
276307
body = raw.decode("utf-8", errors="replace")
277308
body = body.removeprefix("\ufeff") # a UTF-8 byte-order mark isn't text
278309

310+
if content_type == "text/html" and hops > 0:
311+
moved = meta_refresh(body, final_url)
312+
if moved:
313+
# The validators belong to the page that redirected, not to this
314+
# one, so they are dropped: there is nothing here to revalidate.
315+
page = fetch_page(moved, timeout, max_bytes, hops=hops - 1)
316+
return replace(page, etag="", last_modified="")
317+
279318
if content_type == "text/plain":
280319
text = normalize(body)
281320
page = Page(title="", text=text, html=f"<pre>{escape(body.strip())}</pre>" if text else "")

‎tests/test_fetch_page.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,36 @@
2626
"/no-type-pdf": (None, b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n1 0 obj"),
2727
"/no-type-binary": (None, b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"),
2828
"/plain-html": ("text/plain", HTML),
29+
# Pages whose only job is to send the reader somewhere else.
30+
"/moved": ("text/html", b'<meta http-equiv="refresh" content="0; url=/html">Click here.'),
31+
"/moved-noscript": ( # where the real ones put it, behind a script redirect
32+
"text/html",
33+
b"<!doctype html><title>Redirect</title><noscript>"
34+
b'<meta http-equiv="refresh" content="0; url=/html"></noscript>'
35+
b'<p><a href="/html">Click here</a> to be redirected.</p>',
36+
),
37+
"/moved-quoted": (
38+
"text/html",
39+
b"<meta http-equiv=refresh content='0; url=&#47;html'>Click here.",
40+
),
41+
"/moved-slowly": ( # a page reloading itself, not one that moved
42+
"text/html",
43+
b'<meta http-equiv="refresh" content="300; url=/html">'
44+
b"<main><p>" + b"Live scores and updates. " * 20 + b"</p></main>",
45+
),
46+
"/moved-to-itself": (
47+
"text/html",
48+
b'<meta http-equiv="refresh" content="0; url=/moved-to-itself">'
49+
b"<main><p>" + b"This page refreshes itself forever. " * 12 + b"</p></main>",
50+
),
51+
"/hop1": ("text/html", b'<meta http-equiv="refresh" content="0; url=/hop2">one'),
52+
"/hop2": ("text/html", b'<meta http-equiv="refresh" content="0; url=/hop3">two'),
53+
"/hop3": ("text/html", b'<meta http-equiv="refresh" content="0; url=/hop4">three'),
54+
"/hop4": (
55+
"text/html",
56+
b'<meta http-equiv="refresh" content="0; url=/html">'
57+
b"<main><p>" + b"Still hopping along. " * 20 + b"</p></main>",
58+
),
2959
"/markdown": ("text/markdown; charset=utf-8", b"# Volumes\n\nPersistent data stores."),
3060
"/x-markdown": ("text/x-markdown", b"# Volumes\n\nPersistent data stores."),
3161
"/plain-text": ("text/plain; charset=utf-8", b"<b> is how you write bold in HTML."),
@@ -78,6 +108,32 @@ def test_html_is_extracted_whatever_the_label(base_url, path):
78108
assert page.text == "Welcome back"
79109

80110

111+
@pytest.mark.parametrize("path", ["/moved", "/moved-noscript", "/moved-quoted"])
112+
def test_a_redirecting_page_is_followed(base_url, path):
113+
# Without this markfind would index "Click here to be redirected."
114+
page = fetch_page(base_url + path)
115+
assert page.title == "Redirector" and page.text == "Welcome back"
116+
117+
118+
def test_a_page_that_reloads_itself_is_not_a_redirect(base_url):
119+
assert "Live scores" in fetch_page(base_url + "/moved-slowly").text
120+
121+
122+
def test_a_refresh_to_itself_is_not_followed(base_url):
123+
assert "refreshes itself" in fetch_page(base_url + "/moved-to-itself").text
124+
125+
126+
def test_redirects_stop_after_a_few_hops(base_url):
127+
assert "Still hopping" in fetch_page(base_url + "/hop1").text
128+
assert fetch_page(base_url + "/hop1", hops=0).text == "one"
129+
130+
131+
def test_a_followed_redirect_keeps_no_validators(base_url):
132+
# They belong to the page that redirected, so they cannot revalidate this one.
133+
page = fetch_page(base_url + "/moved")
134+
assert page.etag == "" and page.last_modified == ""
135+
136+
81137
def test_html_after_bom_and_whitespace(base_url):
82138
assert fetch_page(base_url + "/no-type-html-bom").text == "Hello there"
83139

0 commit comments

Comments
 (0)