diff --git a/package-lock.json b/package-lock.json index f59b00d..aae2aae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "skillui", - "version": "1.2.8", + "version": "1.3.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "skillui", - "version": "1.2.8", + "version": "1.3.3", "license": "MIT", "dependencies": { "@babel/parser": "^7.24.0", diff --git a/src/cli.ts b/src/cli.ts index ad0ddfc..7792ff7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -19,10 +19,35 @@ import { showUltraPlaywrightError, showResults, runInteractivePrompts, + promptLoginCredentials, + showLoginSuccess, + showLoginFailed, } from './ui.js'; +import { detectLoginPage, detectLoginPageWithPlaywright, performLogin, extractCookieHeader, StorageState } from './login.js'; +import { loadPlaywright } from './playwright-loader.js'; const program = new Command(); +// Ensure Ctrl+C kills the process even when Playwright browsers are running. +// Playwright child processes (Chromium) can keep the event loop alive and +// prevent a clean process.exit(). We use SIGKILL as a fallback. +process.on('SIGINT', () => { + process.stderr.write('\n Interrupted — shutting down...\n'); + // Give 3s for graceful cleanup, then force-kill + const forceTimer = setTimeout(() => { + process.kill(process.pid, 'SIGKILL'); + }, 3000); + forceTimer.unref(); + process.exit(130); +}); +process.on('SIGTERM', () => { + const forceTimer = setTimeout(() => { + process.kill(process.pid, 'SIGKILL'); + }, 3000); + forceTimer.unref(); + process.exit(143); +}); + program .name('skillui') .description('Reverse-engineer design systems from any project. Pure static analysis — no AI, no API keys.') @@ -36,6 +61,9 @@ program .option('--format ', 'Output format: design-md | skill | both', 'both') .option('--mode ', 'Extraction mode: default | ultra', 'default') .option('--screens ', 'Ultra mode: max pages to crawl (default: 5)', '5') + .option('--user ', 'Login username or email (for authenticated sites)') + .option('--pass ', 'Login password (for authenticated sites)') + .option('--login', 'Force login prompt (skip auto-detection)') .action(async (opts: CLIOptions) => { // Always show the logo on every command await showLogo(); @@ -63,6 +91,7 @@ program try { let profile: DesignProfile; let screenshotPath: string | null = null; + let storageState: StorageState | null = null; const outputDir = path.resolve(opts.out); fs.mkdirSync(outputDir, { recursive: true }); @@ -101,10 +130,76 @@ program const skillDir = path.join(outputDir, `${safeName}-design`); fs.mkdirSync(path.join(skillDir, 'screenshots'), { recursive: true }); + // ── Login detection + auth (BEFORE spinner) ────────────────── + // Support SKILLUI_USER / SKILLUI_PASSWORD env vars (avoids exposing creds in ps) + const authUser = opts.user || process.env.SKILLUI_USER; + const authPass = opts.pass || process.env.SKILLUI_PASSWORD; + const preAuthCreds = authUser && authPass ? { username: authUser, password: authPass } : undefined; + + if (opts.login) { + // --login flag: skip detection, prompt immediately + const creds = preAuthCreds || await promptLoginCredentials(); + if (creds) { + const spLogin = startSpinner('Authenticating...'); + storageState = await performLogin(opts.url!, creds); + if (storageState) { + succeedSpinner(spLogin, 'Login', 'authenticated successfully'); + showLoginSuccess(opts.url!); + } else { + failSpinner(spLogin, 'Login', 'authentication failed — continuing without auth'); + showLoginFailed(); + } + } + } else { + // Auto-detect login page + let loginUrl: string | null = null; + + const spDetect = startSpinner('Checking for login page...'); + try { + // Pass 1: HTTP fetch + const checkRes = await fetch(opts.url!, { + headers: { 'User-Agent': 'Mozilla/5.0 (compatible; skillui/1.0)', 'Accept': 'text/html' }, + redirect: 'follow', + signal: AbortSignal.timeout(10000), + }); + if (checkRes.ok) { + const checkHtml = await checkRes.text(); + if (detectLoginPage(checkHtml, checkRes.url)) { + loginUrl = checkRes.url; + } + } + } catch { /* pre-flight failed */ } + + // Pass 2: Playwright-based SPA detection + if (!loginUrl && loadPlaywright()) { + loginUrl = await detectLoginPageWithPlaywright(opts.url!); + } + + if (loginUrl) { + succeedSpinner(spDetect, 'Login detected', loginUrl); + // Prompt for credentials (no spinner running — input fields visible) + const creds = preAuthCreds || await promptLoginCredentials(); + if (creds) { + const spLogin = startSpinner('Authenticating...'); + storageState = await performLogin(loginUrl, creds); + if (storageState) { + succeedSpinner(spLogin, 'Login', 'authenticated successfully'); + showLoginSuccess(opts.url!); + } else { + failSpinner(spLogin, 'Login', 'authentication failed — continuing without auth'); + showLoginFailed(); + } + } + } else { + succeedSpinner(spDetect, 'Login check', 'no login page detected'); + } + } + + // ── Extraction (with auth if available) ────────────────────── const sp1 = startSpinner('Fetching HTML + CSS...'); let urlResult: Awaited>; try { - urlResult = await runUrlMode(opts.url!, opts.name, skillDir); + urlResult = await runUrlMode(opts.url!, opts.name, skillDir, storageState); const { cssColorCount, cssFontCount, computedColorCount, hadPlaywright } = urlResult; const detail = hadPlaywright ? `${cssColorCount} CSS colors · ${computedColorCount} computed · ${cssFontCount} fonts` @@ -120,6 +215,7 @@ program } profile = urlResult.profile; screenshotPath = urlResult.screenshotPath; + storageState = urlResult.storageState || storageState; } // ── Ultra mode (URL only) ────────────────────────────────────── @@ -140,7 +236,9 @@ program } else { const spAnim = startSpinner('Capturing scroll journey + animations...'); try { - const ultraResult = await runUltraMode(opts.url!, profile, skillDir, { screens: ultraScreens }); + const ultraResult = await runUltraMode(opts.url!, profile, skillDir, { screens: ultraScreens }, storageState, (step) => { + spAnim.text = step; + }); ultraAnimations = ultraResult.animations; const kf = ultraAnimations.keyframes.length; const sf = ultraAnimations.scrollFrames.length; diff --git a/src/extractors/tokens/computed.ts b/src/extractors/tokens/computed.ts index 74020e9..e3c462e 100644 --- a/src/extractors/tokens/computed.ts +++ b/src/extractors/tokens/computed.ts @@ -1,11 +1,11 @@ -import { RawTokens } from '../../types'; +import { RawTokens, StorageState } from '../../types'; import { loadPlaywright } from '../../playwright-loader'; /** * URL mode: Extract computed styles from live DOM using Playwright. * Playwright is an optional peer dependency. */ -export async function extractComputedTokens(url: string, maxPages = 5): Promise { +export async function extractComputedTokens(url: string, maxPages = 5, storageState?: StorageState | null): Promise { const tokens: RawTokens = { colors: [], fonts: [], @@ -36,6 +36,7 @@ export async function extractComputedTokens(url: string, maxPages = 5): Promise< const context = await browser.newContext({ viewport: { width: 1440, height: 900 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ...(storageState ? { storageState } : {}), }); try { @@ -239,7 +240,16 @@ export async function extractComputedTokens(url: string, maxPages = 5): Promise< return ''; } }) - .filter(href => href.startsWith(origin) && !href.includes('#')); + .filter(href => { + if (!href.startsWith(origin)) return false; + // Allow hash routes (#/path) but skip same-page anchors + const hashIdx = href.indexOf('#'); + if (hashIdx !== -1) { + const hash = href.slice(hashIdx); + if (!hash.startsWith('#/')) return false; + } + return true; + }); }, currentUrl); for (const link of links.slice(0, 10)) { diff --git a/src/extractors/tokens/http-css.ts b/src/extractors/tokens/http-css.ts index 6c1b2f3..b28127f 100644 --- a/src/extractors/tokens/http-css.ts +++ b/src/extractors/tokens/http-css.ts @@ -13,7 +13,7 @@ export interface HttpExtractionResult { components: ComponentInfo[]; } -export async function extractHttpCSSTokens(url: string, maxPages = 3): Promise { +export async function extractHttpCSSTokens(url: string, maxPages = 3, cookies?: string): Promise { const tokens: RawTokens = { colors: [], fonts: [], @@ -53,7 +53,7 @@ export async function extractHttpCSSTokens(url: string, maxPages = 3): Promise]+href\s*=\s*["']([^"'#]+)["']/gi); + const hrefMatches = html.matchAll(/]+href\s*=\s*["']([^"']+)["']/gi); for (const m of hrefMatches) { try { const resolved = new URL(m[1], baseUrl).href; - if (resolved.startsWith(origin) && !resolved.includes('#') && !links.includes(resolved)) { + // Allow hash routes (#/path) but skip same-page anchors + const hashIdx = resolved.indexOf('#'); + const hasBlockingHash = hashIdx !== -1 && !resolved.slice(hashIdx).startsWith('#/'); + if (resolved.startsWith(origin) && !hasBlockingHash && !links.includes(resolved)) { links.push(resolved); } } catch { @@ -1596,17 +1599,20 @@ function buildDarkModePairs(tokens: RawTokens): void { // ── Utility Helpers ─────────────────────────────────────────────────── -async function fetchText(url: string): Promise { +async function fetchText(url: string, cookies?: string): Promise { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 15000); + const headers: Record = { + 'User-Agent': 'Mozilla/5.0 (compatible; skillui/1.0; +https://github.com/amaanbuilds/skillui)', + 'Accept': 'text/html,text/css,application/xhtml+xml,*/*', + }; + if (cookies) headers['Cookie'] = cookies; + const response = await fetch(url, { signal: controller.signal, - headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; skillui/1.0; +https://github.com/amaanbuilds/skillui)', - 'Accept': 'text/html,text/css,application/xhtml+xml,*/*', - }, + headers, redirect: 'follow', }); diff --git a/src/extractors/ultra/animations.ts b/src/extractors/ultra/animations.ts index c301469..e854c54 100644 --- a/src/extractors/ultra/animations.ts +++ b/src/extractors/ultra/animations.ts @@ -1,5 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; +import { StorageState } from '../../types'; import { loadPlaywright } from '../../playwright-loader'; import { FullAnimationResult, @@ -31,7 +32,9 @@ import { */ export async function captureAnimations( url: string, - skillDir: string + skillDir: string, + storageState?: StorageState | null, + onProgress?: (step: string) => void ): Promise { const empty: FullAnimationResult = { keyframes: [], @@ -46,6 +49,7 @@ export async function captureAnimations( lottieCount: 0, }; + const log = onProgress || (() => {}); const playwright = loadPlaywright(); if (!playwright) return empty; @@ -58,6 +62,7 @@ export async function captureAnimations( viewport: { width: 1440, height: 900 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ...(storageState ? { storageState } : {}), }); const page = await context.newPage(); @@ -73,6 +78,7 @@ export async function captureAnimations( await page.waitForTimeout(2000); // ── Phase 1: Extract CSS keyframes from document.styleSheets ────────── + log('Animations — extracting keyframes + libraries...'); const keyframesRaw = await page.evaluate(() => { const result: Array<{ name: string; @@ -288,6 +294,7 @@ export async function captureAnimations( const libraries: DetectedLibrary[] = librariesRaw as DetectedLibrary[]; // ── Phase 5: Video detection + first-frame capture ──────────────────── + log('Animations — detecting videos + scroll patterns...'); const videosRaw = await page.evaluate(() => { return Array.from(document.querySelectorAll('video')).map((v, i) => ({ index: i + 1, @@ -474,6 +481,7 @@ export async function captureAnimations( }); // ── Phase 9: Scroll Journey Screenshots ─────────────────────────────── + log('Animations — capturing scroll journey screenshots...'); const scrollFrames: ScrollFrame[] = []; const scrollPercents = [0, 17, 33, 50, 67, 83, 100]; diff --git a/src/extractors/ultra/components-dom.ts b/src/extractors/ultra/components-dom.ts index e8fa5d3..3d62da8 100644 --- a/src/extractors/ultra/components-dom.ts +++ b/src/extractors/ultra/components-dom.ts @@ -1,3 +1,4 @@ +import { StorageState } from '../../types'; import { DOMComponent } from '../../types-ultra'; import { loadPlaywright } from '../../playwright-loader'; @@ -9,9 +10,18 @@ import { loadPlaywright } from '../../playwright-loader'; * - Groups by normalized class fingerprint * - Extracts representative HTML snippet * + * Accepts a single URL or array of URLs — visits each page, merges + * and deduplicates components across all pages. + * * Requires Playwright (optional peer dependency). */ -export async function detectDOMComponents(url: string): Promise { +export async function detectDOMComponents( + urls: string | string[], + storageState?: StorageState | null, + onProgress?: (step: string) => void +): Promise { + const urlList = Array.isArray(urls) ? urls : [urls]; + const log = onProgress || (() => {}); const playwright = loadPlaywright(); if (!playwright) return []; @@ -21,9 +31,41 @@ export async function detectDOMComponents(url: string): Promise viewport: { width: 1440, height: 900 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ...(storageState ? { storageState } : {}), }); - const page = await context.newPage(); + const allComponents: DOMComponent[] = []; + + for (let pi = 0; pi < urlList.length; pi++) { + const url = urlList[pi]; + log(`Components — page ${pi + 1}/${urlList.length}: ${new URL(url).pathname}`); + const pageComponents = await extractComponentsFromPage(context, url); + allComponents.push(...pageComponents); + } + + // Merge: deduplicate by pattern, sum instance counts + const merged = new Map(); + for (const comp of allComponents) { + const existing = merged.get(comp.pattern); + if (existing) { + existing.instances += comp.instances; + } else { + merged.set(comp.pattern, { ...comp }); + } + } + + return Array.from(merged.values()) + .sort((a, b) => b.instances - a.instances) + .slice(0, 20); + } finally { + await browser.close().catch(() => {}); + } +} + +async function extractComponentsFromPage(context: any, url: string): Promise { + let page: any; + try { + page = await context.newPage(); await page.goto(url, { waitUntil: 'networkidle', timeout: 25000 }); await page.waitForTimeout(1000); @@ -139,12 +181,10 @@ export async function detectDOMComponents(url: string): Promise return results.sort((a, b) => b.instances - a.instances); }); - await page.close(); - await browser.close(); - return components; } catch { - await browser.close().catch(() => {}); return []; + } finally { + await page?.close().catch(() => {}); } } diff --git a/src/extractors/ultra/interactions.ts b/src/extractors/ultra/interactions.ts index 18748aa..136f0e2 100644 --- a/src/extractors/ultra/interactions.ts +++ b/src/extractors/ultra/interactions.ts @@ -1,5 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; +import { StorageState } from '../../types'; import { InteractionRecord, StyleDiff, StyleSnapshot } from '../../types-ultra'; import { loadPlaywright } from '../../playwright-loader'; @@ -35,13 +36,20 @@ const INTERACTIVE_SELECTORS: Array<{ * - Simulate hover → capture screenshot + diff computed styles * - Simulate focus → capture screenshot + diff computed styles * + * Accepts a single URL or array of URLs — visits each page and captures + * interactions from all of them. + * * Saves screenshots to screens/states/ * Returns InteractionRecord[] for INTERACTIONS.md generation. */ export async function captureInteractions( - url: string, - skillDir: string + urls: string | string[], + skillDir: string, + storageState?: StorageState | null, + onProgress?: (step: string) => void ): Promise { + const urlList = Array.isArray(urls) ? urls : [urls]; + const log = onProgress || (() => {}); const playwright = loadPlaywright(); if (!playwright) return []; @@ -56,9 +64,33 @@ export async function captureInteractions( viewport: { width: 1440, height: 900 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ...(storageState ? { storageState } : {}), }); - const page = await context.newPage(); + for (let pi = 0; pi < urlList.length; pi++) { + const url = urlList[pi]; + log(`Interactions — page ${pi + 1}/${urlList.length}: ${new URL(url).pathname}`); + const pageRecords = await captureInteractionsFromPage(context, url, statesDir, records.length); + records.push(...pageRecords); + } + } finally { + await browser.close().catch(() => {}); + } + + return records; +} + +async function captureInteractionsFromPage( + context: any, + url: string, + statesDir: string, + indexOffset: number +): Promise { + const records: InteractionRecord[] = []; + let page: any; + + try { + page = await context.newPage(); await page.goto(url, { waitUntil: 'networkidle', timeout: 25000 }); await page.waitForTimeout(1500); @@ -66,7 +98,7 @@ export async function captureInteractions( try { // Find up to 3 visible elements of this type const elements = await page.locator(selector).all(); - const visible = []; + const visible: any[] = []; for (const el of elements) { try { const box = await el.boundingBox(); @@ -79,7 +111,8 @@ export async function captureInteractions( for (let i = 0; i < visible.length; i++) { const el = visible[i]; - const prefix = `${type}-${i + 1}`; + const globalIdx = indexOffset + records.length + 1; + const prefix = `${type}-${globalIdx}`; try { // Get label text @@ -133,16 +166,12 @@ export async function captureInteractions( : []; // Only record if there are actual visual changes - if ( - hoverChanges.length > 0 || - focusChanges.length > 0 || - defaultFile - ) { + if (hoverChanges.length > 0 || focusChanges.length > 0) { records.push({ componentType: type, label: label || type, selector: `${selector}:nth-of-type(${i + 1})`, - index: i + 1, + index: globalIdx, screenshots: { default: `screens/states/${defaultFile}`, hover: hoverFile ? `screens/states/${hoverFile}` : undefined, @@ -158,9 +187,10 @@ export async function captureInteractions( } catch { /* selector failed — skip */ } } - await page.close(); + } catch { + // Page failed — return what we have } finally { - await browser.close().catch(() => {}); + await page?.close().catch(() => {}); } return records; diff --git a/src/extractors/ultra/layout.ts b/src/extractors/ultra/layout.ts index e47cd11..71cca06 100644 --- a/src/extractors/ultra/layout.ts +++ b/src/extractors/ultra/layout.ts @@ -1,3 +1,4 @@ +import { StorageState } from '../../types'; import { LayoutRecord } from '../../types-ultra'; import { loadPlaywright } from '../../playwright-loader'; @@ -9,10 +10,19 @@ import { loadPlaywright } from '../../playwright-loader'; * - section-level wrappers * - nav/header/footer * + * Accepts a single URL or array of URLs — visits each page, merges + * and deduplicates layout records across all pages. + * * Returns LayoutRecord[] for LAYOUT.md generation. * Requires Playwright (optional peer dependency). */ -export async function extractLayouts(url: string): Promise { +export async function extractLayouts( + urls: string | string[], + storageState?: StorageState | null, + onProgress?: (step: string) => void +): Promise { + const urlList = Array.isArray(urls) ? urls : [urls]; + const log = onProgress || (() => {}); const playwright = loadPlaywright(); if (!playwright) return []; @@ -22,9 +32,40 @@ export async function extractLayouts(url: string): Promise { viewport: { width: 1440, height: 900 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ...(storageState ? { storageState } : {}), }); - const page = await context.newPage(); + const allRecords: LayoutRecord[] = []; + + for (let pi = 0; pi < urlList.length; pi++) { + const url = urlList[pi]; + log(`Layout — page ${pi + 1}/${urlList.length}: ${new URL(url).pathname}`); + const pageRecords = await extractLayoutsFromPage(context, url); + allRecords.push(...pageRecords); + } + + // Deduplicate by selector — keep first occurrence + const seen = new Set(); + const deduped: LayoutRecord[] = []; + for (const rec of allRecords) { + if (!seen.has(rec.selector)) { + seen.add(rec.selector); + deduped.push(rec); + } + } + + return deduped + .sort((a, b) => a.depth - b.depth) + .slice(0, 40); + } finally { + await browser.close().catch(() => {}); + } +} + +async function extractLayoutsFromPage(context: any, url: string): Promise { + let page: any; + try { + page = await context.newPage(); await page.goto(url, { waitUntil: 'networkidle', timeout: 25000 }); await page.waitForTimeout(1000); @@ -117,11 +158,10 @@ export async function extractLayouts(url: string): Promise { return results.sort((a, b) => a.depth - b.depth).slice(0, 40); }); - await page.close(); - await browser.close(); return records; } catch { - await browser.close().catch(() => {}); return []; + } finally { + await page?.close().catch(() => {}); } } diff --git a/src/extractors/ultra/pages.ts b/src/extractors/ultra/pages.ts index 49c91e9..c0b28cd 100644 --- a/src/extractors/ultra/pages.ts +++ b/src/extractors/ultra/pages.ts @@ -1,23 +1,31 @@ import * as fs from 'fs'; import * as path from 'path'; +import { StorageState } from '../../types'; import { PageScreenshot, SectionScreenshot } from '../../types-ultra'; import { loadPlaywright } from '../../playwright-loader'; /** * Ultra mode — Page & Section Screenshots * - * 1. Crawl up to `maxPages` internal links from the origin URL + * 1. Crawl internal links from the origin URL (both and SPA navigation) * 2. Take a full-page screenshot for each (screens/pages/[slug].png) - * 3. Detect major sections (section, article, main > div, height > 300px) - * and clip one screenshot per section (screens/sections/[page]-section-N.png) + * 3. Detect major sections and clip one screenshot per section + * + * SPA support: + * - Clicks navigable non-link elements (cards, tabs, sidebar items) to discover routes + * - Opens navigation drawers/sidebars to find hidden links + * - Supports hash-based routing (#/path) * * Requires Playwright (optional peer dependency). */ export async function capturePageScreenshots( originUrl: string, skillDir: string, - maxPages: number + maxPages: number, + storageState?: StorageState | null, + onProgress?: (step: string) => void ): Promise<{ pages: PageScreenshot[]; sections: SectionScreenshot[] }> { + const log = onProgress || (() => {}); const playwright = loadPlaywright(); if (!playwright) return { pages: [], sections: [] }; @@ -34,12 +42,14 @@ export async function capturePageScreenshots( viewport: { width: 1440, height: 900 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ...(storageState ? { storageState } : {}), }); try { const origin = new URL(originUrl).origin; const visited = new Set(); const queue: string[] = [originUrl]; + let spaDiscoveryDone = false; while (queue.length > 0 && visited.size < maxPages) { const url = queue.shift()!; @@ -50,8 +60,10 @@ export async function capturePageScreenshots( const slug = urlToSlug(url, origin); const pageFile = path.join(pagesDir, `${slug}.png`); + let page: any; try { - const page = await context.newPage(); + log(`Page ${visited.size}/${maxPages} — ${slug}`); + page = await context.newPage(); await page.goto(url, { waitUntil: 'networkidle', timeout: 25000 }); await page.waitForTimeout(1500); @@ -150,38 +162,55 @@ export async function capturePageScreenshots( } } - // Discover internal links for queue + // ── Discover internal links (with hash route support) ────────── if (visited.size < maxPages) { - const links = await page.evaluate((origin: string) => { + const currentPathname = new URL(url).pathname; + const links = await page.evaluate(({ origin, currentPathname }: { origin: string; currentPathname: string }) => { return Array.from(document.querySelectorAll('a[href]')) .map((a) => (a as HTMLAnchorElement).href) .filter((href) => { try { const u = new URL(href); - return ( - u.origin === origin && - !u.pathname.match(/\.(pdf|zip|png|jpg|svg|ico|css|js|xml|json|txt)$/i) && - !u.hash - ); + if (u.origin !== origin) return false; + if (u.pathname.match(/\.(pdf|zip|png|jpg|svg|ico|css|js|xml|json|txt)$/i)) return false; + // Allow hash routes (#/path) but skip same-page anchors (#section) + if (u.hash) { + if (u.hash.startsWith('#/')) return true; + if (u.pathname === currentPathname) return false; + } + return true; } catch { return false; } }) .slice(0, 20); - }, origin); + }, { origin, currentPathname }); for (const link of links) { const norm = normalizeUrl(link); - if (!visited.has(norm) && !queue.includes(link)) { + if (!visited.has(norm) && !queue.some(q => normalizeUrl(q) === norm)) { queue.push(link); } } } + // ── SPA: click-based route discovery (once per crawl) ────────── + if (!spaDiscoveryDone && visited.size < maxPages) { + spaDiscoveryDone = true; + log('SPA route discovery — clicking navigable elements...'); + const spaRoutes = await discoverSPARoutes(context, url, origin, visited, queue, maxPages, log); + for (const route of spaRoutes) { + const norm = normalizeUrl(route); + if (!visited.has(norm) && !queue.some(q => normalizeUrl(q) === norm)) { + queue.push(route); + } + } + } + await page.close(); } catch (err) { // Page failed — continue with next - try { await context.pages()[0]?.close(); } catch {} + await page?.close().catch(() => {}); } } } finally { @@ -191,10 +220,211 @@ export async function capturePageScreenshots( return { pages, sections }; } +// ── SPA Route Discovery ──────────────────────────────────────────────── + +/** Combined selector for non-link clickable elements that might navigate */ +const SPA_NAV_SELECTOR = [ + // Vuetify + '.v-card--link', + '.v-list-item--link', + '.v-tab', + '.v-list-item[tabindex]', + // Material / Angular + 'mat-list-item[routerlink]', + 'mat-tab', + // Ant Design + '.ant-menu-item', + '.ant-card[tabindex]', + // Generic patterns + '[role="link"]:not(a)', + '[role="menuitem"]:not(a)', + '[role="tab"]:not(a)', + '[tabindex="0"]:not(a):not(button):not(input):not(textarea):not(select)', +].join(', '); + +/** Selectors for drawer/sidebar toggle buttons */ +const DRAWER_TRIGGERS = [ + '.v-app-bar__nav-icon', + 'button[aria-label*="menu" i]', + 'button[aria-label*="Menu"]', + 'button[aria-label*="navigation" i]', + '.mdi-menu', + 'button.hamburger', + '[class*="hamburger"]', + '[class*="menu-toggle"]', + '[class*="nav-toggle"]', + '[class*="sidebar-toggle"]', +]; + +/** + * Click navigable non-link elements to discover SPA routes. + * Reuses the same page, navigating back to origin between attempts. + */ +async function discoverSPARoutes( + context: any, + pageUrl: string, + origin: string, + visited: Set, + existingQueue: string[], + maxPages: number, + log: (step: string) => void = () => {} +): Promise { + const discovered: string[] = []; + const page = await context.newPage(); + + try { + await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: 25000 }); + await page.waitForTimeout(1500); + + // Step 1: Open any closed navigation drawers to reveal hidden links + log('SPA — checking drawer/sidebar for links...'); + const drawerLinks = await discoverDrawerLinks(page, origin); + for (const link of drawerLinks) { + const norm = normalizeUrl(link); + if (!visited.has(norm) && !existingQueue.some((q: string) => normalizeUrl(q) === norm)) { + discovered.push(link); + } + } + if (drawerLinks.length > 0) { + log(`SPA — drawer: ${drawerLinks.length} links found`); + } + + // Step 2: Find clickable non-link navigation elements on the main page + await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: 15000 }); + await page.waitForTimeout(1000); + + const elementCount = await page.locator(SPA_NAV_SELECTOR).count().catch(() => 0); + + if (elementCount > 0) { + log(`SPA — testing ${elementCount} clickable elements...`); + } + + const maxTest = Math.min(elementCount, 12); + for (let i = 0; i < maxTest; i++) { + if (discovered.length + visited.size >= maxPages) break; + + log(`SPA — clicking element ${i + 1}/${maxTest}...`); + + try { + // Reset to origin before each click + await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: 15000 }); + await page.waitForTimeout(800); + + const beforeUrl = page.url(); + const el = page.locator(SPA_NAV_SELECTOR).nth(i); + + if (!await el.isVisible({ timeout: 2000 })) continue; + + await el.click({ timeout: 3000 }); + + // Wait for URL change + try { + await page.waitForURL((url: URL) => url.toString() !== beforeUrl, { timeout: 5000 }); + } catch { + // URL didn't change — not navigation (modal, expand, etc.) + continue; + } + await page.waitForTimeout(500); + + const afterUrl = page.url(); + if (afterUrl !== beforeUrl) { + try { + const u = new URL(afterUrl); + if (u.origin !== origin) continue; + } catch { continue; } + + const norm = normalizeUrl(afterUrl); + if ( + !visited.has(norm) && + !existingQueue.some((q: string) => normalizeUrl(q) === norm) && + !discovered.some(d => normalizeUrl(d) === norm) + ) { + discovered.push(afterUrl); + log(`SPA — found route: ${afterUrl}`); + } + } + } catch { + // Click failed — skip element + } + } + + if (discovered.length > 0) { + log(`SPA discovery done — ${discovered.length} new routes`); + } else if (elementCount > 0) { + log('SPA discovery done — no new routes'); + } + } catch { + // SPA discovery failed entirely — non-fatal + } finally { + await page.close().catch(() => {}); + } + + return discovered; +} + +/** + * Open navigation drawers/sidebars and extract links from within. + */ +async function discoverDrawerLinks(page: any, origin: string): Promise { + const links: string[] = []; + + // Try each drawer trigger + for (const sel of DRAWER_TRIGGERS) { + try { + const trigger = page.locator(sel).first(); + if (await trigger.isVisible({ timeout: 500 })) { + await trigger.click({ timeout: 2000 }); + await page.waitForTimeout(800); + break; // Only need to open one drawer + } + } catch {} + } + + // Extract links from opened drawer/sidebar + const drawerLinks: string[] = await page.evaluate((origin: string) => { + const DRAWER_LINK_SELECTORS = [ + 'nav a[href]', + '.v-navigation-drawer a[href]', + '[role="navigation"] a[href]', + '.sidebar a[href]', + '[class*="drawer"] a[href]', + '[class*="sidebar"] a[href]', + '[class*="nav-menu"] a[href]', + ]; + + const found: string[] = []; + const seen = new Set(); + + for (const sel of DRAWER_LINK_SELECTORS) { + document.querySelectorAll(sel).forEach(el => { + const href = (el as HTMLAnchorElement).href; + if (!href || seen.has(href)) return; + seen.add(href); + try { + const u = new URL(href); + if (u.origin === origin && !u.pathname.match(/\.(pdf|zip|png|jpg|svg|ico|css|js)$/i)) { + found.push(href); + } + } catch {} + }); + } + + return found; + }, origin); + + links.push(...drawerLinks); + return links; +} + +// ── Helpers ──────────────────────────────────────────────────────────── + function normalizeUrl(url: string): string { try { const u = new URL(url); - return `${u.origin}${u.pathname}`.replace(/\/$/, ''); + const pathname = u.pathname.replace(/\/$/, ''); + // Include hash for SPA routing (#/path) + const hash = u.hash.startsWith('#/') ? u.hash : ''; + return `${u.origin}${pathname}${hash}`; } catch { return url; } @@ -203,11 +433,17 @@ function normalizeUrl(url: string): string { function urlToSlug(url: string, origin: string): string { try { const u = new URL(url); - const rel = u.pathname.replace(/^\//, '').replace(/\/$/, '') || 'home'; + let rel = u.pathname.replace(/^\//, '').replace(/\/$/, '') || ''; + // Include hash route in slug + if (u.hash.startsWith('#/')) { + const hashPath = u.hash.slice(2).replace(/^\//, '').replace(/\/$/, ''); + rel = rel ? `${rel}--${hashPath}` : hashPath; + } + rel = rel || 'home'; return rel .replace(/[^a-zA-Z0-9/]/g, '-') - .replace(/\//g, '--') .replace(/-{2,}/g, '-') + .replace(/\//g, '--') .slice(0, 60) || 'home'; } catch { return 'home'; diff --git a/src/login.ts b/src/login.ts new file mode 100644 index 0000000..d773b52 --- /dev/null +++ b/src/login.ts @@ -0,0 +1,429 @@ +import { loadPlaywright } from './playwright-loader'; +import { StorageState } from './types'; + +/** + * Login page detection + Playwright-based authentication. + * + * Flow: + * 1. detectLoginPage() checks HTML/URL for login form signals + * 2. performLogin() drives a headless browser through the form + * 3. extractCookieHeader() turns Playwright storageState into an HTTP Cookie header + */ + +// ── Detection ──────────────────────────────────────────────────────────── + +const LOGIN_URL_PATTERNS = /\/(login|signin|sign-in|sign_in|auth|sso|authenticate|session\/new|account\/login)\b/i; + +const LOGIN_TEXT_PATTERNS = /\b(sign\s*in|log\s*in|log\s*on|enter\s*your\s*password|forgot\s*password|remember\s*me)\b/i; + +/** + * Returns true when the HTML + URL look like a login / sign-in page. + * + * Heuristic: + * - URL path matches common auth routes, OR + * - HTML contains a
with a password field AND login-related text + */ +export function detectLoginPage(html: string, url: string): boolean { + const hasPasswordField = /]*type\s*=\s*["']password["']/i.test(html); + + // Strong signal: URL is an auth route AND page has a password field + if (LOGIN_URL_PATTERNS.test(url) && hasPasswordField) return true; + + // Medium signal: password field + login-related text + if (hasPasswordField && LOGIN_TEXT_PATTERNS.test(html)) return true; + + // Weak signal: URL is an auth route even without password field (SSO pages, etc.) + // Only flag this if the page also lacks substantial content (< 5 KB stripped) + if (LOGIN_URL_PATTERNS.test(url)) { + const stripped = html.replace(/<[^>]+>/g, '').trim(); + if (stripped.length < 5000) return true; + } + + return false; +} + +/** + * Playwright-based login detection for SPAs. + * + * SPAs render login forms via JavaScript — the raw HTML from fetch() is an + * empty shell. This function loads the page in a headless browser, waits for + * JS to render, then checks the live DOM for login form elements. + * + * Also detects: + * - Auth provider widgets (Clerk, Auth0, Firebase, Supabase, Cognito) + * - Login forms inside iframes + * - OAuth/SSO redirect pages + * - Pages with minimal content (protected SPA that hasn't loaded) + * + * Returns the final URL (after any client-side redirects) if login detected, + * or null if the page is NOT a login page. + */ +export async function detectLoginPageWithPlaywright(url: string): Promise { + const playwright = loadPlaywright(); + if (!playwright) return null; + + const browser = await playwright.chromium.launch({ headless: true }); + try { + const context = await browser.newContext({ + viewport: { width: 1440, height: 900 }, + userAgent: + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }); + + const page = await context.newPage(); + await page.goto(url, { waitUntil: 'networkidle', timeout: 20000 }); + await page.waitForTimeout(3000); // extra wait for auth provider widgets + + const finalUrl = page.url(); + + // If the page redirected to a known auth provider domain, that's a login + if (/\b(clerk\.|auth0\.com|accounts\.google|login\.microsoftonline|cognito|supabase\.co\/auth|firebase)\b/i.test(finalUrl)) { + await browser.close(); + return finalUrl; + } + + // Check the rendered DOM for login signals + const isLogin = await page.evaluate(() => { + // ── Standard password field in main DOM ───────────────────────── + const hasPassword = document.querySelector('input[type="password"]') !== null; + + // ── Auth provider custom elements ─────────────────────────────── + const authProviderElements = [ + // Clerk + 'clerk-sign-in', 'clerk-sign-up', '[data-clerk]', + '.cl-signIn-root', '.cl-component', '.cl-rootBox', + // Auth0 Lock + '.auth0-lock', '.auth0-lock-widget', '#auth0-lock-container-1', + // Firebase UI + '.firebaseui-container', '.firebaseui-auth-container', + // Supabase + '[data-supabase-auth]', '.supabase-auth-ui', + // Generic OAuth buttons + '[data-provider="google"]', '[data-provider="github"]', + 'button[class*="oauth"]', 'button[class*="social-login"]', + 'a[href*="oauth"]', 'a[href*="/auth/"]', + ]; + const hasAuthProvider = authProviderElements.some( + sel => { try { return document.querySelector(sel) !== null; } catch { return false; } } + ); + + // ── Login text in rendered content ────────────────────────────── + const bodyText = document.body?.innerText || ''; + const hasLoginText = /\b(sign\s*in|log\s*in|log\s*on|forgot\s*password|remember\s*me|create\s*account|don'?t have an account|sign\s*up|welcome\s*back|enter\s*your\s*(email|credentials))\b/i.test(bodyText); + + // ── Username/email field ──────────────────────────────────────── + const hasUsernameField = document.querySelector( + 'input[type="email"], input[name="email"], input[name="username"], input[autocomplete="username"], input[autocomplete="email"]' + ) !== null; + + // ── Login form ────────────────────────────────────────────────── + const hasLoginForm = document.querySelector('form') !== null; + + // ── Password field inside iframes ─────────────────────────────── + let hasPasswordInIframe = false; + try { + const iframes = Array.from(document.querySelectorAll('iframe')); + for (const iframe of iframes) { + try { + const iframeDoc = iframe.contentDocument; + if (iframeDoc?.querySelector('input[type="password"]')) { + hasPasswordInIframe = true; + break; + } + } catch { /* cross-origin iframe */ } + } + } catch {} + + // ── Minimal content signal (protected SPA shell) ──────────────── + // If the visible text is very short, the page likely hasn't loaded + // real content because it's waiting for authentication + const visibleText = bodyText.replace(/\s+/g, ' ').trim(); + const isMinimalContent = visibleText.length < 200; + + // ── Decision logic ────────────────────────────────────────────── + // Strong: password field + anything else + if (hasPassword && (hasLoginText || hasLoginForm || hasUsernameField)) return true; + // Strong: auth provider widget detected + if (hasAuthProvider) return true; + // Strong: password in iframe (auth provider) + if (hasPasswordInIframe) return true; + // Medium: username/email field + login text (no password yet — multi-step) + if (hasUsernameField && hasLoginText) return true; + // Medium: login text + minimal content (SPA hasn't loaded behind auth) + if (hasLoginText && isMinimalContent) return true; + // Weak: just a password field alone + if (hasPassword) return true; + + return false; + }); + + // Also check URL-based signals on the final URL (after SPA routing) + if (!isLogin && LOGIN_URL_PATTERNS.test(finalUrl)) { + await browser.close(); + return finalUrl; + } + + // ── Gateway page detection ──────────────────────────────────────── + // Some sites show a welcome/splash page with a LOGIN button that + // navigates to the actual login form. If the current page has no + // login form but has a "Login" / "Sign in" button, click it and + // check the resulting page. + if (!isLogin) { + const LOGIN_BUTTON_SELECTORS = [ + 'button:has-text("Login")', + 'button:has-text("Log in")', + 'button:has-text("Sign in")', + 'a:has-text("Login")', + 'a:has-text("Log in")', + 'a:has-text("Sign in")', + '[role="button"]:has-text("Login")', + '[role="button"]:has-text("Log in")', + '[role="button"]:has-text("Sign in")', + ]; + + for (const sel of LOGIN_BUTTON_SELECTORS) { + try { + const btn = page.locator(sel).first(); + if (await btn.isVisible({ timeout: 500 })) { + await btn.click(); + await page.waitForTimeout(3000); + + const afterClickUrl = page.url(); + + // Check if we landed on a login page after clicking + if (LOGIN_URL_PATTERNS.test(afterClickUrl)) { + await browser.close(); + return afterClickUrl; + } + + // Check the new DOM for login signals + const hasLoginNow = await page.evaluate(() => { + const hasPassword = document.querySelector('input[type="password"]') !== null; + const hasUsername = document.querySelector( + 'input[type="email"], input[name="email"], input[name="username"], input[autocomplete="username"]' + ) !== null; + return hasPassword || hasUsername; + }); + + if (hasLoginNow) { + await browser.close(); + return afterClickUrl; + } + break; // only try one button + } + } catch { /* selector not found or click failed */ } + } + } + + await browser.close(); + return isLogin ? finalUrl : null; + } catch { + await browser.close().catch(() => {}); + return null; + } +} + +// ── Authentication ─────────────────────────────────────────────────────── + +// Re-export for backwards compatibility +export type { StorageState } from './types'; + +const USERNAME_SELECTORS = [ + 'input[type="email"]', + 'input[name="email"]', + 'input[name="username"]', + 'input[name="user"]', + 'input[name="login"]', + 'input[name="identifier"]', + 'input[id*="email" i]', + 'input[id*="user" i]', + 'input[autocomplete="username"]', + 'input[autocomplete="email"]', + // Fallback: first visible text input that is NOT the password field + 'input[type="text"]', +]; + +const SUBMIT_SELECTORS = [ + 'button[type="submit"]', + 'input[type="submit"]', + 'button:has-text("Sign in")', + 'button:has-text("Log in")', + 'button:has-text("Login")', + 'button:has-text("Continue")', + 'button:has-text("Submit")', + 'button:has-text("Next")', + '[role="button"]:has-text("Sign in")', + '[role="button"]:has-text("Log in")', +]; + +/** + * Drive a headless Playwright browser through a login form. + * + * Steps: + * 1. Navigate to loginUrl + * 2. Fill username/email field + * 3. Fill password field + * 4. Submit the form + * 5. Wait for navigation away from the login page + * 6. Return the storage state (cookies + localStorage) or null on failure + * + * Handles multi-step logins (email first → password second) by checking + * whether the password field is visible before and after filling username. + */ +export async function performLogin( + loginUrl: string, + credentials: { username: string; password: string } +): Promise { + const playwright = loadPlaywright(); + if (!playwright) return null; + + const browser = await playwright.chromium.launch({ headless: true }); + try { + const context = await browser.newContext({ + viewport: { width: 1440, height: 900 }, + userAgent: + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }); + + const page = await context.newPage(); + await page.goto(loginUrl, { waitUntil: 'networkidle', timeout: 30000 }); + await page.waitForTimeout(1500); + + // ── Step 1: Fill username / email ────────────────────────────────── + let filledUsername = false; + for (const sel of USERNAME_SELECTORS) { + try { + const field = page.locator(sel).first(); + if (await field.isVisible({ timeout: 500 })) { + await field.fill(credentials.username); + filledUsername = true; + break; + } + } catch { /* selector not found */ } + } + + if (!filledUsername) { + await browser.close(); + return null; + } + + // ── Step 2: Check if password field is visible now ───────────────── + const passwordField = page.locator('input[type="password"]').first(); + let passwordVisible = false; + try { + passwordVisible = await passwordField.isVisible({ timeout: 500 }); + } catch { /* not visible yet */ } + + // Multi-step login: submit username first, wait for password field + if (!passwordVisible) { + // Click "Next" / "Continue" / submit to advance to password step + let advanced = false; + for (const sel of SUBMIT_SELECTORS) { + try { + const btn = page.locator(sel).first(); + if (await btn.isVisible({ timeout: 500 })) { + await btn.click(); + advanced = true; + break; + } + } catch { /* skip */ } + } + if (!advanced) { + // Try pressing Enter on the username field + for (const sel of USERNAME_SELECTORS) { + try { + const field = page.locator(sel).first(); + if (await field.isVisible({ timeout: 300 })) { + await field.press('Enter'); + break; + } + } catch { /* skip */ } + } + } + + // Wait for password field to appear + try { + await page.locator('input[type="password"]').first().waitFor({ state: 'visible', timeout: 8000 }); + } catch { + // Password field never appeared — cannot proceed + await browser.close(); + return null; + } + } + + // ── Step 3: Fill password ───────────────────────────────────────── + try { + await page.locator('input[type="password"]').first().fill(credentials.password); + } catch { + await browser.close(); + return null; + } + + // ── Step 4: Submit ──────────────────────────────────────────────── + let submitted = false; + for (const sel of SUBMIT_SELECTORS) { + try { + const btn = page.locator(sel).first(); + if (await btn.isVisible({ timeout: 500 })) { + await btn.click(); + submitted = true; + break; + } + } catch { /* skip */ } + } + if (!submitted) { + await page.locator('input[type="password"]').first().press('Enter'); + } + + // ── Step 5: Wait for navigation ─────────────────────────────────── + try { + await page.waitForURL((url: URL) => !LOGIN_URL_PATTERNS.test(url.pathname), { timeout: 15000 }); + } catch { + // Fallback: just wait a few seconds + await page.waitForTimeout(4000); + } + await page.waitForTimeout(1500); + + // ── Step 6: Verify login succeeded ──────────────────────────────── + const finalUrl = page.url(); + const finalHtml = await page.content(); + if (detectLoginPage(finalHtml, finalUrl)) { + // Still on login page → credentials likely wrong + await browser.close(); + return null; + } + + // ── Step 7: Capture storage state ───────────────────────────────── + const storageState = await context.storageState() as StorageState; + await browser.close(); + return storageState; + } catch { + await browser.close().catch(() => {}); + return null; + } +} + +// ── Cookie Extraction ──────────────────────────────────────────────────── + +/** + * Build an HTTP `Cookie` header string from Playwright storage state, + * filtered to cookies that match the target URL's domain. + */ +export function extractCookieHeader(storageState: StorageState | null, url: string): string { + if (!storageState?.cookies?.length) return ''; + + let hostname: string; + try { + hostname = new URL(url).hostname; + } catch { + return ''; + } + + return storageState.cookies + .filter(c => { + const cookieDomain = c.domain.replace(/^\./, ''); + return hostname === cookieDomain || hostname.endsWith('.' + cookieDomain); + }) + .map(c => `${c.name}=${c.value}`) + .join('; '); +} diff --git a/src/modes/ultra.ts b/src/modes/ultra.ts index 5885d2f..d4b39bc 100644 --- a/src/modes/ultra.ts +++ b/src/modes/ultra.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import { DesignProfile } from '../types'; +import { DesignProfile, StorageState } from '../types'; import { UltraOptions, UltraResult, FullAnimationResult } from '../types-ultra'; import { capturePageScreenshots } from '../extractors/ultra/pages'; import { captureInteractions } from '../extractors/ultra/interactions'; @@ -36,8 +36,11 @@ export async function runUltraMode( url: string, profile: DesignProfile, skillDir: string, - opts: UltraOptions + opts: UltraOptions, + storageState?: StorageState | null, + onProgress?: (step: string) => void ): Promise { + const log = onProgress || (() => {}); // Ensure all output directories exist fs.mkdirSync(path.join(skillDir, 'screens', 'pages'), { recursive: true }); fs.mkdirSync(path.join(skillDir, 'screens', 'sections'), { recursive: true }); @@ -49,8 +52,7 @@ export async function runUltraMode( const hasPlaywright = loadPlaywright() !== null; if (!hasPlaywright) { - process.stdout.write('\n ⚠ Playwright not found — ultra visual features skipped\n'); - process.stdout.write(' Fix: npm install -g playwright && npx playwright install chromium\n\n'); + log('Playwright not found — ultra visual features skipped'); writeTokensJson(profile, skillDir); writeStubs(skillDir); const emptyAnim = emptyAnimResult(); @@ -58,21 +60,34 @@ export async function runUltraMode( } // ── Step 1: Animation extraction (scroll journey + keyframes + libraries) ── - const animations = await captureAnimations(url, skillDir); + log('Step 1/6 — Scroll journey + animations...'); + const animations = await captureAnimations(url, skillDir, storageState, log); // ── Step 2: Multi-page screenshots + section clips ───────────────────── - const { pages, sections } = await capturePageScreenshots(url, skillDir, opts.screens); + log('Step 2/6 — Page screenshots + SPA discovery...'); + const { pages, sections } = await capturePageScreenshots(url, skillDir, opts.screens, storageState, log); - // ── Step 3: Micro-interactions ──────────────────────────────────────── - const interactions = await captureInteractions(url, skillDir); + // Collect all discovered URLs for cross-page extraction + const discoveredUrls = pages.length > 0 + ? pages.map(p => p.url) + : [url]; - // ── Step 4: Layout extraction ────────────────────────────────────────── - const layouts = await extractLayouts(url); + const urlCount = discoveredUrls.length; - // ── Step 5: DOM component detection ─────────────────────────────────── - const domComponents = await detectDOMComponents(url); + // ── Step 3: Micro-interactions (across all discovered pages) ────────── + log(`Step 3/6 — Interactions (${urlCount} pages)...`); + const interactions = await captureInteractions(discoveredUrls, skillDir, storageState, log); + + // ── Step 4: Layout extraction (across all discovered pages) ─────────── + log(`Step 4/6 — Layout extraction (${urlCount} pages)...`); + const layouts = await extractLayouts(discoveredUrls, storageState, log); + + // ── Step 5: DOM component detection (across all discovered pages) ───── + log(`Step 5/6 — Component detection (${urlCount} pages)...`); + const domComponents = await detectDOMComponents(discoveredUrls, storageState, log); // ── Step 6: Write all reference files ───────────────────────────────── + log('Step 6/6 — Writing reference files...'); const refsDir = path.join(skillDir, 'references'); @@ -102,11 +117,6 @@ export async function runUltraMode( // Ultra screenshot index writeScreensIndex(pages, sections, animations, skillDir); - console.log(' ✓'); - - // ── Print ultra summary ──────────────────────────────────────────────── - printUltraSummary(animations); - return { pageScreenshots: pages, sectionScreenshots: sections, interactions, layouts, domComponents, animations }; } @@ -188,20 +198,6 @@ function generateVisualGuideMd( // ── Helpers ─────────────────────────────────────────────────────────── -function printUltraSummary(anim: FullAnimationResult): void { - const libs = anim.libraries.map(l => l.name).join(', ') || 'none'; - console.log(''); - console.log(` Animation Stack: ${libs}`); - if (anim.webglDetected) console.log(` WebGL/3D: detected (${anim.canvasCount} canvas elements)`); - if (anim.videos.length > 0) { - const bg = anim.videos.filter(v => v.role === 'background').length; - console.log(` Video: ${anim.videos.length} elements (${bg} background)`); - } - if (anim.lottieCount > 0) console.log(` Lottie: ${anim.lottieCount} players`); - if (anim.keyframes.length > 0) console.log(` Keyframes: ${anim.keyframes.length} extracted`); - if (anim.scrollPatterns.length > 0) console.log(` Scroll patterns: ${anim.scrollPatterns.length} types`); -} - function writeScreensIndex( pages: import('../types-ultra').PageScreenshot[], sections: import('../types-ultra').SectionScreenshot[], diff --git a/src/modes/url.ts b/src/modes/url.ts index e6a4ccf..21e881f 100644 --- a/src/modes/url.ts +++ b/src/modes/url.ts @@ -4,20 +4,25 @@ import { normalize } from '../normalizer'; import { captureScreenshot } from '../screenshot'; import { DesignProfile, RawTokens, ComponentInfo } from '../types'; import { loadPlaywright } from '../playwright-loader'; +import { extractCookieHeader, StorageState } from '../login'; /** * URL mode: crawl a live website and extract design tokens. * * Strategy: - * 1. ALWAYS fetch HTML + linked CSS via HTTP (no Playwright needed) - * 2. If Playwright is available, ALSO extract computed styles from live DOM - * 3. Merge both token sets for maximum coverage + * 1. Accept pre-resolved auth state (login detection happens in cli.ts) + * 2. ALWAYS fetch HTML + linked CSS via HTTP (no Playwright needed) + * 3. If Playwright is available, ALSO extract computed styles from live DOM + * 4. Merge both token sets for maximum coverage */ -export async function runUrlMode(url: string, nameOverride?: string, skillDir?: string): Promise<{ profile: DesignProfile; screenshotPath: string | null; cssColorCount: number; cssFontCount: number; computedColorCount: number; hadPlaywright: boolean }> { +export async function runUrlMode(url: string, nameOverride?: string, skillDir?: string, preAuthState?: StorageState | null): Promise<{ profile: DesignProfile; screenshotPath: string | null; cssColorCount: number; cssFontCount: number; computedColorCount: number; hadPlaywright: boolean; storageState: StorageState | null }> { const projectName = nameOverride || deriveUrlName(url); - // Step 1: HTTP-based extraction (always works, no Playwright) - const httpResult = await extractHttpCSSTokens(url, 5); + const storageState: StorageState | null = preAuthState || null; + const cookies: string | undefined = storageState ? extractCookieHeader(storageState, url) || undefined : undefined; + + // Step 1: HTTP-based extraction (with cookies if authenticated) + const httpResult = await extractHttpCSSTokens(url, 5, cookies); const httpTokens = httpResult.tokens; const httpComponents = httpResult.components; const cssColorCount = httpTokens.colors.length; @@ -31,7 +36,7 @@ export async function runUrlMode(url: string, nameOverride?: string, skillDir?: if (hadPlaywright) { try { - computedTokens = await extractComputedTokens(url, 3); + computedTokens = await extractComputedTokens(url, 3, storageState); computedColorCount = computedTokens.colors.length; } catch { /* Playwright extraction failed, use CSS-only */ } } @@ -56,7 +61,7 @@ export async function runUrlMode(url: string, nameOverride?: string, skillDir?: screenshotPath = await captureScreenshot(url, skillDir); } - return { profile, screenshotPath, cssColorCount, cssFontCount, computedColorCount, hadPlaywright }; + return { profile, screenshotPath, cssColorCount, cssFontCount, computedColorCount, hadPlaywright, storageState }; } function mergeUrlTokens(http: RawTokens, computed: RawTokens | null): RawTokens { diff --git a/src/types.ts b/src/types.ts index ca93366..bff0675 100644 --- a/src/types.ts +++ b/src/types.ts @@ -248,6 +248,24 @@ export interface MotionTokens { // ── CLI Options ────────────────────────────────────────────────────── +/** Playwright storage-state shape (cookies + origins with localStorage). */ +export interface StorageState { + cookies: Array<{ + name: string; + value: string; + domain: string; + path: string; + expires: number; + httpOnly: boolean; + secure: boolean; + sameSite: 'Strict' | 'Lax' | 'None'; + }>; + origins: Array<{ + origin: string; + localStorage: Array<{ name: string; value: string }>; + }>; +} + export interface CLIOptions { dir?: string; repo?: string; @@ -258,4 +276,7 @@ export interface CLIOptions { format: 'design-md' | 'skill' | 'both'; mode: 'default' | 'ultra'; screens: string; + user?: string; + pass?: string; + login?: boolean; } diff --git a/src/ui.ts b/src/ui.ts index 65a48e6..195e252 100644 --- a/src/ui.ts +++ b/src/ui.ts @@ -426,3 +426,48 @@ export async function runInteractivePrompts(): Promise { + const prompts = (await import('prompts')).default; + + console.log(''); + console.log(boxen( + chalk.yellow.bold(' Login page detected\n\n') + + chalk.white(' The target URL requires authentication.\n') + + chalk.dim(' Enter credentials to log in and continue extraction.'), + { + title: chalk.bold.yellow(' Authentication Required '), + borderStyle: 'round', + borderColor: 'yellow', + width: 76, + padding: { top: 0, bottom: 0, left: 0, right: 1 }, + } + )); + console.log(''); + + const answers = await prompts([ + { + type: 'text', + name: 'username', + message: chalk.white('Username or email:'), + }, + { + type: 'password', + name: 'password', + message: chalk.white('Password:'), + }, + ], { onCancel: () => process.exit(0) }); + + if (!answers.username || !answers.password) return null; + return { username: answers.username, password: answers.password }; +} + +export function showLoginSuccess(url: string): void { + console.log(' ' + chalk.green('\u2713') + ' ' + chalk.white('Authenticated') + ' ' + chalk.dim(url)); +} + +export function showLoginFailed(): void { + console.log(' ' + chalk.red('\u2717') + ' ' + chalk.white('Login failed') + ' ' + chalk.dim('Check credentials and try again')); +}