fix(#30): detect CF challenge pages and retry with premium_proxy

When Cloudflare serves a JS challenge interstitial (HTTP 200 but no
real content), the existing retry logic was never triggered because
status was 200. Crawl would silently return 0 campaigns and fail the
sanity check.

Fix: isCFChallengePage() detects pages that contain the CF challenge
platform script AND are small (<200KB). Real Kickstarter pages are
500KB+. Detected challenge pages are treated as retryable errors,
which will eventually escalate to premium_proxy on the 4th attempt.

Also adds a parser-level warning log when 0 campaigns are found in a
large page, to surface structural changes earlier.

Adds unit tests for isCFChallengePage.

Author: Jing-yilin

backend/internal/service/kickstarter_parser.go

@@ -3,6 +3,7 @@ package service
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"regexp"
 	"strconv"
 	"strings"
@@ -45,6 +46,10 @@ func parseDiscoverPageHTML(html string) ([]model.Campaign, error) {
 		campaigns = parseFromHTMLStructure(doc)
 	}
 
+	if len(campaigns) == 0 && len(html) > 50_000 {
+		log.Printf("parser: 0 campaigns from %d-byte page — possible HTML structure change or blocked response", len(html))
+	}
+
 	return campaigns, nil
 }
 

backend/internal/service/scrapingbee_client.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -109,6 +110,15 @@ func (c *ScrapingBeeClient) FetchUsage(ctx context.Context) (*UsageResult, error
 	return &usage, nil
 }
 
+// isCFChallengePage returns true when the response is a Cloudflare interstitial
+// challenge rather than real page content. CF challenge pages are small (<200KB)
+// and always contain the challenge-platform main.js script. Real Kickstarter
+// discover pages are 500KB+ and do not require client-side JS challenge resolution.
+func isCFChallengePage(html string) bool {
+	const cfMarker = "challenge-platform/scripts/jsd/main.js"
+	return strings.Contains(html, cfMarker) && len(html) < 200_000
+}
+
 func (c *ScrapingBeeClient) doRequest(ctx context.Context, targetURL string, useAI bool, aiQuery, aiSelector string, sessionID int) (string, error) {
 	if err := c.rateLimiter.Acquire(ctx); err != nil {
 		return "", fmt.Errorf("rate limiter: %w", err)
@@ -195,10 +205,20 @@ func (c *ScrapingBeeClient) doRequest(ctx context.Context, targetURL string, use
 			return "", fmt.Errorf("read response: %w", err)
 		}
 
+		// Detect Cloudflare JS challenge pages (HTTP 200 but no real content).
+		// CF challenge pages are small and contain the challenge-platform script.
+		// Real Kickstarter pages are 500KB+ and contain project data.
+		bodyStr := string(body)
+		if isCFChallengePage(bodyStr) {
+			lastErr = fmt.Errorf("cloudflare challenge page detected (size=%d)", len(body))
+			log.Printf("ScrapingBee: CF challenge at %s (size=%d, premium=%v), retrying", targetURL, len(body), premiumProxy)
+			continue
+		}
+
 		credits := resp.Header.Get("Spb-Cost")
-		log.Printf("ScrapingBee success: url=%s, credits=%s, useAI=%v, premium=%v", targetURL, credits, useAI, premiumProxy)
+		log.Printf("ScrapingBee success: url=%s, credits=%s, size=%d, useAI=%v, premium=%v", targetURL, credits, len(body), useAI, premiumProxy)
 
-		return string(body), nil
+		return bodyStr, nil
 	}
 
 	return "", fmt.Errorf("failed after 4 attempts: %w", lastErr)

backend/internal/service/scrapingbee_client_test.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestIsCFChallengePage(t *testing.T) {
+	cfMarker := "challenge-platform/scripts/jsd/main.js"
+
+	tests := []struct {
+		name     string
+		html     string
+		expected bool
+	}{
+		{
+			name:     "pure CF challenge page (small + marker)",
+			html:     strings.Repeat("x", 10_000) + cfMarker,
+			expected: true,
+		},
+		{
+			name:     "real page with CF bot-management script injected (large)",
+			html:     strings.Repeat("x", 250_000) + cfMarker,
+			expected: false,
+		},
+		{
+			name:     "normal page without CF marker",
+			html:     strings.Repeat("x", 800_000),
+			expected: false,
+		},
+		{
+			name:     "empty page",
+			html:     "",
+			expected: false,
+		},
+		{
+			name:     "small page without CF marker",
+			html:     "<html><body>Not found</body></html>",
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := isCFChallengePage(tt.html)
+			if got != tt.expected {
+				t.Errorf("isCFChallengePage() = %v, want %v (html size=%d)", got, tt.expected, len(tt.html))
+			}
+		})
+	}
+}