diff --git a/skills/last30days/scripts/briefing.py b/skills/last30days/scripts/briefing.py index 1fe1abf3..f19d4b25 100644 --- a/skills/last30days/scripts/briefing.py +++ b/skills/last30days/scripts/briefing.py @@ -85,7 +85,7 @@ def generate_daily(since: str = None) -> dict: # Extract top finding by engagement if findings: - top = max(findings, key=lambda f: f.get("engagement_score", 0)) + top = max(findings, key=lambda f: f.get("engagement_score") or 0) topic_data["top_finding"] = { "title": top.get("source_title", ""), "source": top.get("source", ""), @@ -110,7 +110,7 @@ def generate_daily(since: str = None) -> dict: top_overall = None if all_findings: - top_overall = max(all_findings, key=lambda f: f.get("engagement_score", 0)) + top_overall = max(all_findings, key=lambda f: f.get("engagement_score") or 0) result = { "status": "ok", @@ -172,8 +172,8 @@ def generate_weekly() -> dict: finally: conn.close() - this_engagement = sum(f.get("engagement_score", 0) for f in this_week) - last_engagement = sum(f.get("engagement_score", 0) for f in last_week) + this_engagement = sum(f.get("engagement_score") or 0 for f in this_week) + last_engagement = sum(f.get("engagement_score") or 0 for f in last_week) # Trend calculation if last_engagement > 0: @@ -194,7 +194,7 @@ def generate_weekly() -> dict: # engagement too). "top_findings": sorted( this_week, - key=lambda f: f.get("engagement_score", 0), + key=lambda f: f.get("engagement_score") or 0, reverse=True, )[:5], }) diff --git a/skills/last30days/scripts/lib/github.py b/skills/last30days/scripts/lib/github.py index 1d6465a6..a4979b2a 100644 --- a/skills/last30days/scripts/lib/github.py +++ b/skills/last30days/scripts/lib/github.py @@ -269,7 +269,7 @@ def parse_github_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: title = item.get("title", "") body_text = item.get("body") or "" reactions_total = item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0 - comment_count = item.get("comments", 0) + comment_count = item.get("comments") or 0 labels = [ lbl.get("name", "") for lbl in (item.get("labels") or []) if isinstance(lbl, dict) @@ -488,7 +488,7 @@ def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]: result["top_feature_request"] = { "title": item.get("title", ""), "reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0, - "comments": item.get("comments", 0), + "comments": item.get("comments") or 0, "url": item.get("html_url", ""), } elif feat_data and feat_data.get("total_count", 0) == 0: @@ -501,7 +501,7 @@ def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]: result["top_feature_request"] = { "title": item.get("title", ""), "reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0, - "comments": item.get("comments", 0), + "comments": item.get("comments") or 0, "url": item.get("html_url", ""), } @@ -514,7 +514,7 @@ def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]: result["top_complaint"] = { "title": item.get("title", ""), "reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0, - "comments": item.get("comments", 0), + "comments": item.get("comments") or 0, "url": item.get("html_url", ""), } diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.py index 6dce6164..53bb7b31 100644 --- a/skills/last30days/scripts/lib/hackernews.py +++ b/skills/last30days/scripts/lib/hackernews.py @@ -370,7 +370,7 @@ def enrich_top_stories( # Sort by points to enrich the most popular stories by_points = sorted( range(len(items)), - key=lambda i: items[i].get("engagement", {}).get("points", 0), + key=lambda i: items[i].get("engagement", {}).get("points") or 0, reverse=True, ) to_enrich = by_points[:limit] diff --git a/skills/last30days/scripts/lib/http.py b/skills/last30days/scripts/lib/http.py index b525c25b..12095f0e 100644 --- a/skills/last30days/scripts/lib/http.py +++ b/skills/last30days/scripts/lib/http.py @@ -13,7 +13,7 @@ from contextvars import ContextVar, copy_context from pathlib import Path from typing import Any, Dict, Optional, Union -from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit, quote from . import health from . import log as _log @@ -556,6 +556,19 @@ def request( if filtered: separator = "&" if ("?" in url) else "?" url = f"{url}{separator}{urlencode(filtered)}" + # Encode any non-ASCII characters to prevent UnicodeEncodeError from + # http.client.HTTPConnection.putrequest (which uses latin-1 internally). + # Only encode path, query, and fragment — not the hostname (netloc), which + # needs IDNA encoding instead of percent-encoding for non-ASCII domains. + parts = urlsplit(url) + safe = '/:@!$&\'()*+,;=-._~%?#[]=+' + url = urlunsplit(( + parts.scheme, + parts.netloc, + quote(parts.path, safe=safe), + quote(parts.query, safe=safe), + quote(parts.fragment, safe=safe), + )) fixture_request = _fixture_request(method, url, json_data, raw) fixture_redactions = _fixture_redactions(url, headers, json_data) diff --git a/skills/last30days/scripts/lib/instagram.py b/skills/last30days/scripts/lib/instagram.py index 8292df27..7825349a 100644 --- a/skills/last30days/scripts/lib/instagram.py +++ b/skills/last30days/scripts/lib/instagram.py @@ -494,7 +494,7 @@ def search_and_enrich( items.append(item) # Sort merged results by views descending - items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True) + items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True) if not items: return {"items": [], "error": last_error} diff --git a/skills/last30days/scripts/lib/signals.py b/skills/last30days/scripts/lib/signals.py index 08bfe1d0..740984d6 100644 --- a/skills/last30days/scripts/lib/signals.py +++ b/skills/last30days/scripts/lib/signals.py @@ -48,7 +48,7 @@ def local_relevance( # often have titles that don't keyword-match the query (e.g., "YE - FATHER # (feat. TRAVIS SCOTT)" doesn't match "kanye west"). The engagement signals # say "this is important" even when text overlap is weak. - if item.source == "youtube" and item.engagement.get("views", 0) > 100_000: + if item.source == "youtube" and (item.engagement.get("views") or 0) > 100_000: score = max(score, 0.3) # Project-mode GitHub floor: items fetched via --github-repo are explicitly @@ -311,7 +311,7 @@ def _passes_engagement_floor(item: schema.SourceItem, sole_source: bool) -> bool return True if sole_source: return True - views = item.engagement.get("views", 0) if item.engagement else 0 + views = item.engagement.get("views") or 0 if item.engagement else 0 return views >= _VIDEO_ENGAGEMENT_FLOOR_VIEWS diff --git a/skills/last30days/scripts/lib/tiktok.py b/skills/last30days/scripts/lib/tiktok.py index ef4c9a76..3cf9d47a 100644 --- a/skills/last30days/scripts/lib/tiktok.py +++ b/skills/last30days/scripts/lib/tiktok.py @@ -446,7 +446,7 @@ def search_and_enrich( items.append(item) # Sort merged results by views descending - items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True) + items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True) if not items: return {"items": [], "error": last_error} diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index 966d9f0c..95d905cd 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -975,7 +975,7 @@ def search_and_transcribe( items.append(item) # Sort merged results by views descending - items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True) + items.sort(key=lambda x: x.get("engagement", {}).get("views") or 0, reverse=True) if not items: return search_result diff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.py index 233289d3..8f61ea46 100644 --- a/skills/last30days/scripts/store.py +++ b/skills/last30days/scripts/store.py @@ -584,8 +584,8 @@ def _record_sightings( finding.get("source", "unknown"), url, finding.get("source_title") or finding.get("title", ""), - finding.get("engagement_score", 0), - finding.get("relevance_score", 0), + finding.get("engagement_score") if finding.get("engagement_score") is not None else 0, + finding.get("relevance_score") if finding.get("relevance_score") is not None else 0, )) if not sighting_rows: