acquire-lead/collectors/douyin/comments.py

"""步骤：评论区滚动采集与入库。"""

from __future__ import annotations

import re

from playwright.sync_api import Page

from config import settings
from collectors.douyin.common import SkipVideoError, human_pause
from collectors.douyin.panel import (
    _click_side_panel_tab,
    _is_side_panel_open,
    open_comment_panel,
)
from collectors.douyin.profile import (
    _extract_douyin_id,
    _normalize_user_key,
    _open_user_profile,
    _to_abs_url,
)
from db.repository import save_comments_batch
from db.session import session_scope

# 评论旁「作者xxx」状态 tag，不应写入 content
_AUTHOR_TAG_RE = re.compile(
    r"(?:\s*作者(?:回复过|赞过|置顶|喜欢过)\s*)+$"
)
_AUTHOR_TAG_ONLY_RE = re.compile(
    r"^作者(?:回复过|赞过|置顶|喜欢过)?$"
)


def _clean_comment_content(raw: str) -> str:
    """去掉末尾/整段的作者互动 tag，只保留真实评论文案。"""
    text = (raw or "").strip()
    if not text:
        return ""
    text = _AUTHOR_TAG_RE.sub("", text).strip()
    if _AUTHOR_TAG_ONLY_RE.match(text):
        return ""
    return text


def _find_comment_scroll_box(page: Page) -> dict | None:
    """定位评论区可滚动容器及其中心坐标。"""
    return page.evaluate(
        """() => {
            const list = document.querySelector('[data-e2e="comment-list"]');
            if (!list) return null;
            let el = list;
            let scrollEl = null;
            while (el && el !== document.documentElement) {
                const s = getComputedStyle(el);
                const oy = s.overflowY;
                if ((oy === 'auto' || oy === 'scroll' || oy === 'overlay')
                    && el.scrollHeight > el.clientHeight + 5) {
                    scrollEl = el;
                    break;
                }
                el = el.parentElement;
            }
            const target = scrollEl || list;
            const r = target.getBoundingClientRect();
            if (r.width <= 0 || r.height <= 0) return null;
            return {
                x: Math.round(r.x + r.width / 2),
                y: Math.round(r.y + Math.min(r.height / 2, 280)),
                scrollTop: target.scrollTop || 0,
                scrollHeight: target.scrollHeight || 0,
                clientHeight: target.clientHeight || 0,
            };
        }"""
    )


def _scroll_comment_panel_once(page: Page) -> None:
    """
    滚动评论区一次（虚拟列表：真正滚的是可滚动父容器 + 滚轮）。
    绝不点击「展开N条回复」。
    """
    box = _find_comment_scroll_box(page)
    page.evaluate(
        """() => {
            const list = document.querySelector('[data-e2e="comment-list"]');
            if (!list) return;
            let el = list;
            let scrollEl = null;
            while (el && el !== document.documentElement) {
                const s = getComputedStyle(el);
                const oy = s.overflowY;
                if ((oy === 'auto' || oy === 'scroll' || oy === 'overlay')
                    && el.scrollHeight > el.clientHeight + 5) {
                    scrollEl = el;
                    break;
                }
                el = el.parentElement;
            }
            const target = scrollEl || list;
            target.scrollTop = Math.min(
                target.scrollTop + Math.max(240, Math.floor(target.clientHeight * 0.85)),
                target.scrollHeight
            );
            const items = Array.from(list.querySelectorAll('[data-e2e="comment-item"]'));
            const last = items[items.length - 1];
            if (last) last.scrollIntoView({ block: 'end', inline: 'nearest' });
        }"""
    )
    if box and box.get("x") and box.get("y"):
        try:
            page.mouse.move(box["x"], box["y"])
            page.mouse.wheel(0, 900)
        except Exception:
            pass
    human_pause(0.9, 1.5)


def _comment_row_key(row: dict) -> str:
    return (
        f"{row.get('href') or ''}|"
        f"{row.get('content') or ''}|"
        f"{1 if row.get('is_reply') else 0}"
    )


def _collect_comments_with_scroll(page: Page, limit: int) -> list[dict]:
    """
    边滚动边采集：抖音评论是虚拟列表，DOM 里通常只有几条，
    必须滚动后继续读，凑满 limit 条一级评论；已展开回复一并收（不计数）。
    """
    collected: list[dict] = []
    seen: set[str] = set()
    top_count = 0
    stagnant = 0

    for round_i in range(25):
        batch = _list_visible_comments(page, max(limit * 2, 20))
        added_top = 0
        last_top_accepted = False

        for row in batch:
            key = _comment_row_key(row)
            if row.get("is_reply"):
                if key in seen or not last_top_accepted:
                    continue
                seen.add(key)
                collected.append(row)
                continue

            # 一级评论
            if key in seen:
                last_top_accepted = True
                continue
            if top_count >= limit:
                last_top_accepted = False
                continue
            seen.add(key)
            collected.append(row)
            top_count += 1
            added_top += 1
            last_top_accepted = True

        print(
            f"[评论滚动] 第 {round_i + 1} 轮：本轮新增一级 {added_top}，"
            f"累计一级 {top_count}/{limit}，合计条目 {len(collected)}"
        )
        if top_count >= limit:
            break

        if added_top == 0:
            stagnant += 1
        else:
            stagnant = 0
        if stagnant >= 4:
            print(f"[评论滚动] 连续无新增，停止。已采一级 {top_count}/{limit}")
            break

        _scroll_comment_panel_once(page)

    for i, row in enumerate(collected, start=1):
        row["index"] = i
    return collected


def _list_visible_comments(page: Page, limit: int) -> list[dict]:
    """
    读取当前 DOM 中可见评论（虚拟列表仅含屏上几条）。
    - 取最多 limit 条一级评论
    - 若某条下已展开可见的回复，按顺序一并采集（不点「展开」）
    """
    comments = page.evaluate(
        """(limit) => {
            const list = document.querySelector('[data-e2e="comment-list"]');
            if (!list) return [];

            const skipExact = new Set([
                '分享', '回复', '作者', '...', '', '收起', '展开',
                '作者回复过', '作者赞过', '作者置顶', '作者喜欢过',
            ]);

            // 评论旁状态 tag（整行都是 tag，或夹在正文末尾）
            const authorTagExact = /^(作者(回复过|赞过|置顶|喜欢过)?)$/;
            const authorTagSuffix = /\\s*作者(回复过|赞过|置顶|喜欢过)\\s*$/;

            const isVisible = (el) => {
                const r = el.getBoundingClientRect();
                const s = getComputedStyle(el);
                return r.width > 0 && r.height > 0
                    && s.display !== 'none' && s.visibility !== 'hidden';
            };

            const cleanContent = (raw) => {
                let s = String(raw || '').trim();
                // 反复剥离末尾作者互动 tag，避免「正文作者回复过」入库
                for (let i = 0; i < 3; i++) {
                    const next = s.replace(authorTagSuffix, '').trim();
                    if (next === s) break;
                    s = next;
                }
                if (authorTagExact.test(s) || skipExact.has(s)) return '';
                return s;
            };

            const isMetaOrNoise = (line, nickname) => {
                if (!line || skipExact.has(line)) return true;
                if (nickname && line === nickname) return true;
                if (authorTagExact.test(line)) return true;
                // 短行且含「作者」：作者 / 作者赞 等徽章，不当正文
                if (line.includes('作者') && line.length <= 6
                    && !/[？?！!。，,、]/.test(line)) return true;
                if (/^\\d+$/.test(line)) return true;
                if (/^展开\\d*条?回复/.test(line)) return true;
                if (/^收起/.test(line)) return true;
                return false;
            };

            const isTimeLine = (line) => (
                /刚刚|\\d+\\s*(秒|分钟|小时|天|周)前/.test(line)
                || (/·/.test(line) && /前/.test(line))
            );

            const pickContent = (textRoot, nickname) => {
                // 优先从独立正文节点取，减少把昵称/徽章/时间拼进 content
                const contentCandidates = [
                    textRoot.querySelector('[data-e2e="comment-content"]'),
                    textRoot.querySelector('[class*="CommentContent"]'),
                    textRoot.querySelector('span[class*="comment"]'),
                ].filter(Boolean);

                let content = '';
                for (const el of contentCandidates) {
                    const t = cleanContent(el.innerText || '');
                    if (t && !isMetaOrNoise(t, nickname) && !isTimeLine(t)) {
                        content = t;
                        break;
                    }
                }

                const lines = (textRoot.innerText || '')
                    .split('\\n')
                    .map(s => s.trim())
                    .filter(Boolean);
                let commentTime = '';
                for (const line of lines) {
                    if (isMetaOrNoise(line, nickname)) continue;
                    if (isTimeLine(line)) {
                        if (!commentTime) commentTime = line;
                        continue;
                    }
                    if (!content) {
                        const cleaned = cleanContent(line);
                        if (cleaned) content = cleaned;
                    }
                }
                return { content: cleanContent(content), comment_time: commentTime };
            };

            const results = [];
            const seen = new Set();
            const push = (row) => {
                if (!row || !row.content || !row.href) return;
                const key = `${row.href}|${row.content}|${row.is_reply ? 1 : 0}`;
                if (seen.has(key)) return;
                seen.add(key);
                results.push(row);
            };

            const items = Array.from(list.querySelectorAll('[data-e2e="comment-item"]'));
            let topLevelCount = 0;

            for (const item of items) {
                if (topLevelCount >= limit) break;
                if (!isVisible(item)) continue;
                if (item.parentElement && item.parentElement.closest('[data-e2e="comment-item"]')) {
                    continue;
                }

                // 头像在左侧容器，正文在 info-wrap —— 必须分开取
                const avatarA = item.querySelector('.comment-item-avatar a[href*="/user/"]')
                    || Array.from(item.querySelectorAll('a[href*="/user/"]')).find(a => {
                        const r = a.getBoundingClientRect();
                        return r.width >= 28 && r.width <= 56 && r.height >= 28 && r.height <= 56;
                    });
                if (!avatarA) continue;

                const href = avatarA.getAttribute('href') || '';
                if (!href || href.includes('/user/self')) continue;

                const infoWrap = item.querySelector('.comment-item-info-wrap');
                // 正文不在 info-wrap 内，而在其兄弟节点；用右侧整块 F7ubq_7y / 整条 item
                const textRoot = item.querySelector('.F7ubq_7y') || item;

                let nickname = '';
                const nickRoot = infoWrap || textRoot;
                for (const a of nickRoot.querySelectorAll('a[href*="/user/"]')) {
                    const t = (a.innerText || '').trim().replace(/作者/g, '').trim();
                    if (t && t !== '...') {
                        nickname = t;
                        break;
                    }
                }
                const { content, comment_time } = pickContent(textRoot, nickname);
                if (!content) continue;

                const ar = avatarA.getBoundingClientRect();
                const main = {
                    nickname,
                    content,
                    href,
                    comment_time,
                    is_reply: false,
                    avatar_x: Math.round(ar.x + ar.width / 2),
                    avatar_y: Math.round(ar.y + ar.height / 2),
                };
                push(main);
                topLevelCount += 1;

                const mainX = main.avatar_x;
                const replyAvatars = Array.from(item.querySelectorAll('a[href*="/user/"]'))
                    .filter(a => {
                        if (a === avatarA) return false;
                        if (a.closest('button.comment-reply-expand-btn')) return false;
                        const r = a.getBoundingClientRect();
                        if (!isVisible(a)) return false;
                        if (r.width < 28 || r.width > 56 || r.height < 28 || r.height > 56) return false;
                        return (r.x + r.width / 2) > mainX + 24;
                    })
                    .sort((a, b) => a.getBoundingClientRect().y - b.getBoundingClientRect().y);

                for (const a of replyAvatars) {
                    let replyRoot = a.parentElement;
                    for (let i = 0; i < 8 && replyRoot && replyRoot !== item; i++) {
                        const t = (replyRoot.innerText || '');
                        if ((/前/.test(t) || t.includes('回复')) && t.length < 500) break;
                        replyRoot = replyRoot.parentElement;
                    }
                    if (!replyRoot || replyRoot === item) replyRoot = a.parentElement;

                    let replyNick = '';
                    for (const link of replyRoot.querySelectorAll('a[href*="/user/"]')) {
                        const t = (link.innerText || '').trim().replace(/作者/g, '').trim();
                        if (t && t !== '...') {
                            replyNick = t;
                            break;
                        }
                    }
                    const parsed = pickContent(replyRoot, replyNick);
                    if (!parsed.content) continue;
                    const rr = a.getBoundingClientRect();
                    push({
                        nickname: replyNick,
                        content: parsed.content,
                        href: a.getAttribute('href') || '',
                        comment_time: parsed.comment_time,
                        is_reply: true,
                        avatar_x: Math.round(rr.x + rr.width / 2),
                        avatar_y: Math.round(rr.y + rr.height / 2),
                    });
                }
            }

            return results.map((row, index) => ({
                index: index + 1,
                nickname: row.nickname,
                content: row.content,
                href: row.href,
                comment_time: row.comment_time,
                is_reply: row.is_reply,
                avatar_x: row.avatar_x,
                avatar_y: row.avatar_y,
            }));
        }""",
        limit,
    )
    return comments or []

def collect_comments_and_save(
    page: Page,
    video_meta: dict,
    limit: int = settings.COMMENT_LIMIT,
) -> None:
    """
    在「评论」tab 下滚动采集前 limit 条一级评论并入库。
    若右侧栏不在评论 tab，则切过去（不再点评论图标，避免误关）。
    """
    video_pk = video_meta.get("db_id")
    if not video_pk:
        raise RuntimeError("缺少 videos.id（db_id），无法入库评论")

    print("[评论采集] 确认在评论 tab...")
    if not _is_side_panel_open(page):
        open_comment_panel(page)
    if not _click_side_panel_tab(page, "评论"):
        raise RuntimeError("无法切换到评论栏")

    try:
        page.wait_for_selector('[data-e2e="comment-item"]', timeout=10_000)
    except Exception as exc:
        raise SkipVideoError("评论数为 0，跳过该视频") from exc

    comments = _collect_comments_with_scroll(page, limit)
    top_n = sum(1 for c in comments if not c.get("is_reply"))
    reply_n = sum(1 for c in comments if c.get("is_reply"))

    if top_n < limit:
        raise SkipVideoError(f"一级评论仅 {top_n} 条，不足 {limit}，跳过该视频")

    print(
        f"\n按顺序处理：一级评论 {top_n} 条（目标 {limit}，回复不计数），"
        f"已展开回复 {reply_n} 条，合计入库候选 {len(comments)} 条："
    )

    id_cache: dict[str, str] = {}
    rows_for_db: list[dict] = []

    for item in comments:
        idx = item.get("index")
        nickname = item.get("nickname") or ""
        content = _clean_comment_content(item.get("content") or "")
        href = item.get("href") or ""
        user_key = _normalize_user_key(href)
        user_link = _to_abs_url(href) if href else ""
        comment_time = item.get("comment_time") or ""
        is_reply = bool(item.get("is_reply"))
        douyin_id = ""

        if not content:
            print("-" * 40)
            print(f"[{idx}] 跳过: 评论内容为空（已剔除作者互动 tag）")
            continue

        print("-" * 40)
        print(f"[{idx}] 类型:       {'回复' if is_reply else '一级评论'}")
        print(f"[{idx}] 视频ID:     {video_meta.get('video_id', '')} (db_id={video_pk})")
        print(f"[{idx}] 评论昵称:   {nickname}")
        print(f"[{idx}] 评论内容:   {content}")
        print(f"[{idx}] 评论时间:   {comment_time or '(未获取)'}")
        print(f"[{idx}] 用户主页:   {user_link or '(未找到)'}")

        if user_key and user_key in id_cache:
            douyin_id = id_cache[user_key]
            print(f"[{idx}] 抖音号:     {douyin_id} (缓存)")
        elif user_key and "/user/" in user_link:
            human_pause(1.0, 2.0)
            profile = None
            try:
                profile = _open_user_profile(page, href)
                opened = profile.url or ""
                if user_key not in opened and "user" not in opened:
                    raise RuntimeError(f"主页跳转异常: expect {user_key}, got {opened}")
                human_pause(1.8, 3.0)
                douyin_id = _extract_douyin_id(profile)
                if nickname:
                    page_text = profile.evaluate("() => document.body.innerText || ''") or ""
                    if nickname not in page_text:
                        print(
                            f"[{idx}] 警告: 主页未出现评论昵称「{nickname}」，"
                            f"仍按主页抖音号入库: {douyin_id}"
                        )
                id_cache[user_key] = douyin_id
                print(f"[{idx}] 抖音号:     {douyin_id}")
                print(f"[{idx}] 主页URL:    {opened}")
            except Exception as exc:
                print(f"[{idx}] 抖音号:     (读取失败: {exc})")
                douyin_id = ""
            finally:
                if profile is not None and profile != page:
                    try:
                        profile.close()
                    except Exception:
                        pass
                try:
                    page.bring_to_front()
                except Exception:
                    pass
                human_pause(0.8, 1.5)
        else:
            print(f"[{idx}] 抖音号:     (未找到用户主页链接)")

        if not douyin_id:
            print(f"[{idx}] 跳过入库: 无可靠抖音号")
            continue

        rows_for_db.append(
            {
                "nick_name": nickname,
                "content": content,
                "comment_time": comment_time or None,
                "user_link": user_link or None,
                "douyin_id": douyin_id,
                "is_reply": is_reply,
            }
        )

    print("-" * 40)
    print(f"评论采集结束，成功获取评论者抖音号 {len(id_cache)} 个。")

    with session_scope() as session:
        result = save_comments_batch(session, int(video_pk), rows_for_db)
    print(
        f"[评论入库完成] videos.id={result['video_pk']} "
        f"线索处理={result['leads']} "
        f"新增评论={result['comments']} "
        f"跳过={result['skipped_comments']} "
        f"(无抖音号={result.get('no_douyin_id', 0)})"
    )