From 3963da7cf4bc5cea979c7d491e1f8a005ad375fe Mon Sep 17 00:00:00 2001 From: 23241a6749 Date: Mon, 13 Jul 2026 11:49:58 +0000 Subject: [PATCH 1/2] fix(http): sanitize non-ASCII in URL before urllib.request to prevent latin-1 encode crash --- skills/last30days/scripts/lib/http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.py index b525c25b..c22065de 100644 --- a/skills/last30days/scripts/lib/http.py +++ b/skills/last30days/scripts/lib/http.py @@ -13,7 +13,7 @@ from contextvars import ContextVar, copy_context from pathlib import Path from typing import Any, Dict, Optional, Union -from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit, quote from . import health from . import log as _log @@ -556,6 +556,9 @@ def request( if filtered: separator = "&" if ("?" in url) else "?" url = f"{url}{separator}{urlencode(filtered)}" + # Encode any non-ASCII characters to prevent UnicodeEncodeError from + # http.client.HTTPConnection.putrequest (which uses latin-1 internally). + url = quote(url, safe='/:@!$&\'()*+,;=-._~%?#[]=+') fixture_request = _fixture_request(method, url, json_data, raw) fixture_redactions = _fixture_redactions(url, headers, json_data) From 055f47582ff8e926f4f9f631d28dbd0e617c1d58 Mon Sep 17 00:00:00 2001 From: 23241a6749 Date: Tue, 14 Jul 2026 12:48:02 +0000 Subject: [PATCH 2/2] fix: guard all .get(key, 0) patterns that can return None, preventing TypeError on arithmetic/sort/sum/max dict.get(key, default) only substitutes the default when the key is absent. A key that exists with value None passes through, producing None instead of the intended default. This causes TypeError in: - arithmetic (reactions + comments) - list.sort() with key function - max() comparisons - sum() over generator expressions Fixes 17 occurrences across 8 files: - github.py: item.get("comments", 0) used in reactions + comments - briefing.py: f.get("engagement_score", 0) used in max()/sum() - youtube_yt.py: .get("views", 0) inside sort key - instagram.py: same sort-key pattern - tiktok.py: same sort-key pattern - hackernews.py: .get("points", 0) in sort key - signals.py: .get("views", 0) in engagement floor check - store.py: .get("engagement_score", 0) / .get("relevance_score", 0) in store_sightings (same root cause as PR #796) --- skills/last30days/scripts/briefing.py | 10 +++++----- skills/last30days/scripts/lib/github.py | 8 ++++---- skills/last30days/scripts/lib/hackernews.py | 2 +- skills/last30days/scripts/lib/http.py | 12 +++++++++++- skills/last30days/scripts/lib/instagram.py | 2 +- skills/last30days/scripts/lib/signals.py | 4 ++-- skills/last30days/scripts/lib/tiktok.py | 2 +- skills/last30days/scripts/lib/youtube_yt.py | 2 +- skills/last30days/scripts/store.py | 4 ++-- 9 files changed, 28 insertions(+), 18 deletions(-) diff --git a/skills/last30days/scripts/briefing.py b/skills/last30days/scripts/briefing.py index 1fe1abf3..f19d4b25 100644 --- a/skills/last30days/scripts/briefing.py +++ b/skills/last30days/scripts/briefing.py @@ -85,7 +85,7 @@ def generate_daily(since: str = None) -> dict: # Extract top finding by engagement if findings: - top = max(findings, key=lambda f: f.get("engagement_score", 0)) + top = max(findings, key=lambda f: f.get("engagement_score") or 0) topic_data["top_finding"] = { "title": top.get("source_title", ""), "source": top.get("source", ""), @@ -110,7 +110,7 @@ def generate_daily(since: str = None) -> dict: top_overall = None if all_findings: - top_overall = max(all_findings, key=lambda f: f.get("engagement_score", 0)) + top_overall = max(all_findings, key=lambda f: f.get("engagement_score") or 0) result = { "status": "ok", @@ -172,8 +172,8 @@ def generate_weekly() -> dict: finally: conn.close() - this_engagement = sum(f.get("engagement_score", 0) for f in this_week) - last_engagement = sum(f.get("engagement_score", 0) for f in last_week) + this_engagement = sum(f.get("engagement_score") or 0 for f in this_week) + last_engagement = sum(f.get("engagement_score") or 0 for f in last_week) # Trend calculation if last_engagement > 0: @@ -194,7 +194,7 @@ def generate_weekly() -> dict: # engagement too). "top_findings": sorted( this_week, - key=lambda f: f.get("engagement_score", 0), + key=lambda f: f.get("engagement_score") or 0, reverse=True, )[:5], }) diff --git a/skills/last30days/scripts/lib/github.py b/skills/last30days/scripts/lib/github.py index 1d6465a6..a4979b2a 100644 --- a/skills/last30days/scripts/lib/github.py +++ b/skills/last30days/scripts/lib/github.py @@ -269,7 +269,7 @@ def parse_github_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: title = item.get("title", "") body_text = item.get("body") or "" reactions_total = item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0 - comment_count = item.get("comments", 0) + comment_count = item.get("comments") or 0 labels = [ lbl.get("name", "") for lbl in (item.get("labels") or []) if isinstance(lbl, dict) @@ -488,7 +488,7 @@ def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]: result["top_feature_request"] = { "title": item.get("title", ""), "reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0, - "comments": item.get("comments", 0), + "comments": item.get("comments") or 0, "url": item.get("html_url", ""), } elif feat_data and feat_data.get("total_count", 0) == 0: @@ -501,7 +501,7 @@ def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]: result["top_feature_request"] = { "title": item.get("title", ""), "reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0, - "comments": item.get("comments", 0), + "comments": item.get("comments") or 0, "url": item.get("html_url", ""), } @@ -514,7 +514,7 @@ def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]: result["top_complaint"] = { "title": item.get("title", ""), "reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0, - "comments": item.get("comments", 0), + "comments": item.get("comments") or 0, "url": item.get("html_url", ""), } diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.py index 6dce6164..53bb7b31 100644 --- a/skills/last30days/scripts/lib/hackernews.py +++ b/skills/last30days/scripts/lib/hackernews.py @@ -370,7 +370,7 @@ def enrich_top_stories( # Sort by points to enrich the most popular stories by_points = sorted( range(len(items)), - key=lambda i: items[i].get("engagement", {}).get("points", 0), + key=lambda i: items[i].get("engagement", {}).get("points") or 0, reverse=True, ) to_enrich = by_points[:limit] diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.py index c22065de..12095f0e 100644 --- a/skills/last30days/scripts/lib/http.py +++ b/skills/last30days/scripts/lib/http.py @@ -558,7 +558,17 @@ def request( url = f"{url}{separator}{urlencode(filtered)}" # Encode any non-ASCII characters to prevent UnicodeEncodeError from # http.client.HTTPConnection.putrequest (which uses latin-1 internally). - url = quote(url, safe='/:@!$&\'()*+,;=-._~%?#[]=+') + # Only encode path, query, and fragment — not the hostname (netloc), which + # needs IDNA encoding instead of percent-encoding for non-ASCII domains. + parts = urlsplit(url) + safe = '/:@!$&\'()*+,;=-._~%?#[]=+' + url = urlunsplit(( + parts.scheme, + parts.netloc, + quote(parts.path, safe=safe), + quote(parts.query, safe=safe), + quote(parts.fragment, safe=safe), + )) fixture_request = _fixture_request(method, url, json_data, raw) fixture_redactions = _fixture_redactions(url, headers, json_data) diff --git a/skills/last30days/scripts/lib/instagram.py b/skills/last30days/scripts/lib/instagram.py index 8292df27..7825349a 100644 --- a/skills/last30days/scripts/lib/instagram.py +++ b/skills/last30days/scripts/lib/instagram.py @@ -494,7 +494,7 @@ def search_and_enrich( items.append(item) # Sort merged results by views descending - items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True) + items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True) if not items: return {"items": [], "error": last_error} diff --git a/skills/last30days/scripts/lib/signals.py b/skills/last30days/scripts/lib/signals.py index 08bfe1d0..740984d6 100644 --- a/skills/last30days/scripts/lib/signals.py +++ b/skills/last30days/scripts/lib/signals.py @@ -48,7 +48,7 @@ def local_relevance( # often have titles that don't keyword-match the query (e.g., "YE - FATHER # (feat. TRAVIS SCOTT)" doesn't match "kanye west"). The engagement signals # say "this is important" even when text overlap is weak. - if item.source == "youtube" and item.engagement.get("views", 0) > 100_000: + if item.source == "youtube" and (item.engagement.get("views") or 0) > 100_000: score = max(score, 0.3) # Project-mode GitHub floor: items fetched via --github-repo are explicitly @@ -311,7 +311,7 @@ def _passes_engagement_floor(item: schema.SourceItem, sole_source: bool) -> bool return True if sole_source: return True - views = item.engagement.get("views", 0) if item.engagement else 0 + views = item.engagement.get("views") or 0 if item.engagement else 0 return views >= _VIDEO_ENGAGEMENT_FLOOR_VIEWS diff --git a/skills/last30days/scripts/lib/tiktok.py b/skills/last30days/scripts/lib/tiktok.py index ef4c9a76..3cf9d47a 100644 --- a/skills/last30days/scripts/lib/tiktok.py +++ b/skills/last30days/scripts/lib/tiktok.py @@ -446,7 +446,7 @@ def search_and_enrich( items.append(item) # Sort merged results by views descending - items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True) + items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True) if not items: return {"items": [], "error": last_error} diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index 966d9f0c..95d905cd 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -975,7 +975,7 @@ def search_and_transcribe( items.append(item) # Sort merged results by views descending - items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True) + items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True) if not items: return search_result diff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.py index 233289d3..8f61ea46 100644 --- a/skills/last30days/scripts/store.py +++ b/skills/last30days/scripts/store.py @@ -584,8 +584,8 @@ def _record_sightings( finding.get("source", "unknown"), url, finding.get("source_title") or finding.get("title", ""), - finding.get("engagement_score", 0), - finding.get("relevance_score", 0), + finding.get("engagement_score") if finding.get("engagement_score") is not None else 0, + finding.get("relevance_score") if finding.get("relevance_score") is not None else 0, )) if not sighting_rows: