dashboard.py

# dashboard.py
import streamlit as st
import plotly.express as px
import pandas as pd

from src.config import TIMEZONE, validate_tiktok_url
from src.database import load_comments, save_comments, init_db
from src.scraper import TikTokScraper
from src.preprocessing import (
    preprocess_items, clean_text, extract_video_id_from_url, extract_account_from_url
)
from src.insights import enrich_analysis
from src.trend import (
    prepare_time, get_comment_volume, get_peak_hours,
    get_top_keywords_by_period, detect_spikes, get_sentiment_trend, auto_freq
)

st.set_page_config(page_title="TikTok Comment Intelligence", layout="wide", page_icon="📈")
st.title("📈 TikTok Comment Intelligence Dashboard")

# ====================== Sidebar: Scraper ======================
st.sidebar.header("🔗 Input Scraper")
st.sidebar.markdown("Masukkan URL video TikTok (satu per baris).")

urls_input = st.sidebar.text_area(
    "Daftar URL Video:",
    value="https://www.tiktok.com/@metro_tv/video/7611919246016204053",
    height=150
)

if st.sidebar.button("🚀 Mulai Scrape & Analisis", type="primary"):
    urls = [u.strip() for u in urls_input.split("\n") if u.strip()]
    valid_urls = [u for u in urls if validate_tiktok_url(u)]
    invalid_count = len(urls) - len(valid_urls)

    if invalid_count > 0:
        st.sidebar.warning(f"⚠️ {invalid_count} URL tidak valid dilewati.")

    if valid_urls:
        with st.spinner("Scraping & analisis berjalan... (lihat terminal untuk log)"):
            try:
                init_db()
                scraper = TikTokScraper()
                all_items = []

                # Scrape per URL agar setiap komentar tertag video & akun yang benar
                for url in valid_urls:
                    raw_df = scraper.scrape(post_urls=[url])
                    if raw_df.empty:
                        continue

                    # Tag setiap item dengan video_id dan postURL (agar auto-fill)
                    vid = extract_video_id_from_url(url)
                    acc = extract_account_from_url(url)

                    for item in raw_df.to_dict("records"):
                        if not item.get("videoId"):
                            item["videoId"] = vid
                        if not item.get("postURL"):
                            item["postURL"] = url
                        if not item.get("authorName"):
                            item["authorName"] = acc

                    all_items.extend(raw_df.to_dict("records"))

                if all_items:
                    df_new = preprocess_items(all_items)
                    if not df_new.empty:
                        inserted = save_comments(enrich_analysis(df_new))
                        st.success(f"✅ +{inserted} komentar baru tersimpan.")
                        st.cache_data.clear()
                        st.rerun()
                    else:
                        st.warning("Tidak ada komentar valid dari URL tersebut.")
                else:
                    st.error("Gagal scraping. Cek token / kuota Apify Anda.")
            except Exception as e:
                st.error(f"Error: {e}")
    else:
        st.sidebar.error("Masukkan minimal 1 URL video TikTok yang valid.")

st.sidebar.divider()

# ====================== Sidebar: Pengaturan ======================
st.sidebar.header("⚙️ Pengaturan Grafik")

freq_choice = st.sidebar.selectbox(
    "Periode Analisis",
    options=["auto", "hour", "day", "week", "month"],
    format_func=lambda x: {
        "auto": "Otomatis (Rekomendasi)", "hour": "Per Jam", "day": "Per Hari",
        "week": "Per Minggu", "month": "Per Bulan"
    }[x]
)

top_n = st.sidebar.slider("Jumlah Top Keywords", 3, 20, 10)

# ====================== Load Data ======================
@st.cache_data(ttl=60)
def load_data():
    df = load_comments()
    if df.empty:
        return df
    if "clean_text" not in df.columns or df["clean_text"].isna().all():
        df["clean_text"] = df["text"].apply(clean_text)
    if "sentiment" not in df.columns:
        df = enrich_analysis(df)
    return prepare_time(df, time_col="created_at", timezone=TIMEZONE)

df_all = load_data()

if df_all.empty:
    st.info("Database kosong. Masukkan URL di sidebar lalu klik **Mulai Scrape & Analisis**.")
    st.stop()

# ====================== Filter Akun ======================
st.sidebar.divider()
st.sidebar.header("🎯 Filter Data")

# Pastikan kolom video_account ada (backward compat dengan data lama)
if "video_account" not in df_all.columns:
    df_all["video_account"] = None

df_all["video_account"] = df_all["video_account"].fillna("tanpa-akun")

accounts = sorted(df_all["video_account"].unique())
selected_accounts = st.sidebar.multiselect(
    "Filter Akun",
    options=accounts,
    default=accounts,
    help="Pilih akun TikTok yang ingin dianalisis"
)

df = df_all[df_all["video_account"].isin(selected_accounts)] if selected_accounts else df_all

freq_used = auto_freq(df) if freq_choice == "auto" else freq_choice

# ====================== Warning Data Kecil ======================
if len(df) < 50:
    st.warning(
        f"⚠️ Data masih sedikit ({len(df)} komentar dari {len(selected_accounts)} akun). "
        "Insight statistik (spike, jam aktif, trend) belum stabil. "
        "Tambahkan lebih banyak video untuk hasil yang bermakna."
    )

# ====================== Metrik ======================
st.subheader("Ringkasan Data")
col1, col2, col3, col4 = st.columns(4)

col1.metric("Total Komentar", f"{len(df):,}")
col2.metric("Unique Username", f"{df['username'].nunique():,}")
col3.metric("Perlu Moderasi", f"{int(df['needs_moderation'].sum()):,}")
col4.metric("Rata-rata Risk Score", f"{df['risk_score'].mean():.2f}")

st.download_button(
    "⬇️ Download Data (CSV)",
    data=df.drop(columns=["raw_json", "raw_item"], errors="ignore").to_csv(index=False).encode("utf-8"),
    file_name="tiktok_comments_export.csv"
)

st.divider()

# ====================== Volume Trend ======================
st.subheader("📈 Volume Komentar")

volume = get_comment_volume(df, freq=freq_used)
volume = detect_spikes(volume, window=7, threshold=2.0, min_count=3)

if not volume.empty:
    fig_volume = px.line(volume, x="created_at", y="total_comments", markers=True,
                         title=f"Trend Volume Komentar ({freq_used})")
    st.plotly_chart(fig_volume, use_container_width=True)

    if "is_spike" in volume.columns:
        spikes = volume[volume["is_spike"] == True]
        if not spikes.empty:
            with st.expander("🚨 Terdeteksi Lonjakan Komentar (Spike)"):
                st.dataframe(spikes)
        else:
            st.info("Tidak ada lonjakan signifikan (atau data belum cukup untuk deteksi spike).")
else:
    st.info("Data waktu belum cukup untuk trend volume.")

st.divider()

# ====================== Peak Hours ======================
st.subheader("🕒 Jam Aktif Audiens")
peak_hours = get_peak_hours(df)
if not peak_hours.empty:
    fig_peak = px.bar(peak_hours.sort_values("hour"), x="hour", y="total_comments",
                      title="Distribusi Komentar per Jam",
                      labels={"hour": "Jam (0-23)", "total_comments": "Jumlah Komentar"})
    st.plotly_chart(fig_peak, use_container_width=True)

st.divider()

# ====================== Perbandingan Akun ======================
st.subheader("🎬 Perbandingan Akun & Video")

comp = (
    df.groupby(["video_account", "video_id"])
    .agg(
        total_komentar=("text", "count"),
        unique_user=("username", "nunique"),
        positif=("sentiment", lambda s: int((s == "positive").sum())),
        negatif=("sentiment", lambda s: int((s == "negative").sum())),
        netral=("sentiment", lambda s: int((s == "neutral").sum())),
        moderasi=("needs_moderation", "sum"),
        rata_risk=("risk_score", "mean"),
    )
    .reset_index()
    .sort_values("total_komentar", ascending=False)
)

# Rename kolom agar lebih manusiawi
comp = comp.rename(columns={
    "video_account": "Akun",
    "video_id": "Video ID",
    "total_komentar": "Jumlah Komentar",
    "unique_user": "User Unik",
    "positif": "Positif",
    "negatif": "Negatif",
    "netral": "Netral",
    "moderasi": "Perlu Moderasi",
    "rata_risk": "Rata-rata Risk"
})

comp["Rata-rata Risk"] = comp["Rata-rata Risk"].round(2)

st.dataframe(comp, use_container_width=True)

st.divider()

# ====================== Distribusi Sentimen ======================
st.subheader("😊 Distribusi Sentimen")
sent_counts = df["sentiment"].value_counts().reset_index()
sent_counts.columns = ["sentiment", "jumlah"]
if not sent_counts.empty:
    fig_sent = px.pie(sent_counts, names="sentiment", values="jumlah", title="Proporsi Sentimen")
    st.plotly_chart(fig_sent, use_container_width=True)

st.divider()

# ====================== Top Keywords ======================
st.subheader("🔥 Top Keywords")
keywords = get_top_keywords_by_period(df, period_col="date", text_col="clean_text",
                                      top_n=top_n, use_bigrams=True)
if not keywords.empty:
    top_overall = (keywords.groupby("keyword")["count"].sum()
                   .reset_index().sort_values("count", ascending=False).head(top_n))
    fig_kw = px.bar(top_overall, x="count", y="keyword", orientation="h",
                    title=f"Top {top_n} Kata/Frasa",
                    labels={"count": "Frekuensi", "keyword": "Kata Kunci"})
    fig_kw.update_layout(yaxis={"categoryorder": "total ascending"})
    st.plotly_chart(fig_kw, use_container_width=True)

    with st.expander("Lihat Detail Top Keywords per Hari"):
        st.dataframe(keywords)
else:
    st.info("Belum ada keyword yang bisa dianalisis.")

st.divider()

# ====================== Tabel Komentar ======================
st.subheader("📋 Database Komentar Terbaru")
display_cols = ["created_at_local", "video_account", "username", "text",
                "sentiment", "intent", "risk_score", "needs_moderation"]
display_df = df[[c for c in display_cols if c in df.columns]].copy()
if "created_at_local" in display_df.columns:
    display_df = display_df.sort_values("created_at_local", ascending=False)

# Rename kolom untuk tampilan
rename_map = {
    "created_at_local": "Waktu",
    "video_account": "Akun",
    "username": "User",
    "text": "Komentar",
    "sentiment": "Sentimen",
    "intent": "Niat",
    "risk_score": "Risk",
    "needs_moderation": "Moderasi"
}
display_df = display_df.rename(columns={k: v for k, v in rename_map.items() if k in display_df.columns})

st.dataframe(display_df, use_container_width=True)