"""步骤:评论区滚动采集与入库。""" from __future__ import annotations import re from playwright.sync_api import Page from config import settings from collectors.douyin.common import SkipVideoError, human_pause from collectors.douyin.panel import ( _click_side_panel_tab, _is_side_panel_open, open_comment_panel, ) from collectors.douyin.profile import ( _extract_douyin_id, _normalize_user_key, _open_user_profile, _to_abs_url, ) from db.repository import save_comments_batch from db.session import session_scope # 评论旁「作者xxx」状态 tag,不应写入 content _AUTHOR_TAG_RE = re.compile( r"(?:\s*作者(?:回复过|赞过|置顶|喜欢过)\s*)+$" ) _AUTHOR_TAG_ONLY_RE = re.compile( r"^作者(?:回复过|赞过|置顶|喜欢过)?$" ) def _clean_comment_content(raw: str) -> str: """去掉末尾/整段的作者互动 tag,只保留真实评论文案。""" text = (raw or "").strip() if not text: return "" text = _AUTHOR_TAG_RE.sub("", text).strip() if _AUTHOR_TAG_ONLY_RE.match(text): return "" return text def _find_comment_scroll_box(page: Page) -> dict | None: """定位评论区可滚动容器及其中心坐标。""" return page.evaluate( """() => { const list = document.querySelector('[data-e2e="comment-list"]'); if (!list) return null; let el = list; let scrollEl = null; while (el && el !== document.documentElement) { const s = getComputedStyle(el); const oy = s.overflowY; if ((oy === 'auto' || oy === 'scroll' || oy === 'overlay') && el.scrollHeight > el.clientHeight + 5) { scrollEl = el; break; } el = el.parentElement; } const target = scrollEl || list; const r = target.getBoundingClientRect(); if (r.width <= 0 || r.height <= 0) return null; return { x: Math.round(r.x + r.width / 2), y: Math.round(r.y + Math.min(r.height / 2, 280)), scrollTop: target.scrollTop || 0, scrollHeight: target.scrollHeight || 0, clientHeight: target.clientHeight || 0, }; }""" ) def _scroll_comment_panel_once(page: Page) -> None: """ 滚动评论区一次(虚拟列表:真正滚的是可滚动父容器 + 滚轮)。 绝不点击「展开N条回复」。 """ box = _find_comment_scroll_box(page) page.evaluate( """() => { const list = document.querySelector('[data-e2e="comment-list"]'); if (!list) return; let el = list; let scrollEl = null; while (el && el !== document.documentElement) { const s = getComputedStyle(el); const oy = s.overflowY; if ((oy === 'auto' || oy === 'scroll' || oy === 'overlay') && el.scrollHeight > el.clientHeight + 5) { scrollEl = el; break; } el = el.parentElement; } const target = scrollEl || list; target.scrollTop = Math.min( target.scrollTop + Math.max(240, Math.floor(target.clientHeight * 0.85)), target.scrollHeight ); const items = Array.from(list.querySelectorAll('[data-e2e="comment-item"]')); const last = items[items.length - 1]; if (last) last.scrollIntoView({ block: 'end', inline: 'nearest' }); }""" ) if box and box.get("x") and box.get("y"): try: page.mouse.move(box["x"], box["y"]) page.mouse.wheel(0, 900) except Exception: pass human_pause(0.9, 1.5) def _comment_row_key(row: dict) -> str: return ( f"{row.get('href') or ''}|" f"{row.get('content') or ''}|" f"{1 if row.get('is_reply') else 0}" ) def _collect_comments_with_scroll(page: Page, limit: int) -> list[dict]: """ 边滚动边采集:抖音评论是虚拟列表,DOM 里通常只有几条, 必须滚动后继续读,凑满 limit 条一级评论;已展开回复一并收(不计数)。 """ collected: list[dict] = [] seen: set[str] = set() top_count = 0 stagnant = 0 for round_i in range(25): batch = _list_visible_comments(page, max(limit * 2, 20)) added_top = 0 last_top_accepted = False for row in batch: key = _comment_row_key(row) if row.get("is_reply"): if key in seen or not last_top_accepted: continue seen.add(key) collected.append(row) continue # 一级评论 if key in seen: last_top_accepted = True continue if top_count >= limit: last_top_accepted = False continue seen.add(key) collected.append(row) top_count += 1 added_top += 1 last_top_accepted = True print( f"[评论滚动] 第 {round_i + 1} 轮:本轮新增一级 {added_top}," f"累计一级 {top_count}/{limit},合计条目 {len(collected)}" ) if top_count >= limit: break if added_top == 0: stagnant += 1 else: stagnant = 0 if stagnant >= 4: print(f"[评论滚动] 连续无新增,停止。已采一级 {top_count}/{limit}") break _scroll_comment_panel_once(page) for i, row in enumerate(collected, start=1): row["index"] = i return collected def _list_visible_comments(page: Page, limit: int) -> list[dict]: """ 读取当前 DOM 中可见评论(虚拟列表仅含屏上几条)。 - 取最多 limit 条一级评论 - 若某条下已展开可见的回复,按顺序一并采集(不点「展开」) """ comments = page.evaluate( """(limit) => { const list = document.querySelector('[data-e2e="comment-list"]'); if (!list) return []; const skipExact = new Set([ '分享', '回复', '作者', '...', '', '收起', '展开', '作者回复过', '作者赞过', '作者置顶', '作者喜欢过', ]); // 评论旁状态 tag(整行都是 tag,或夹在正文末尾) const authorTagExact = /^(作者(回复过|赞过|置顶|喜欢过)?)$/; const authorTagSuffix = /\\s*作者(回复过|赞过|置顶|喜欢过)\\s*$/; const isVisible = (el) => { const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden'; }; const cleanContent = (raw) => { let s = String(raw || '').trim(); // 反复剥离末尾作者互动 tag,避免「正文作者回复过」入库 for (let i = 0; i < 3; i++) { const next = s.replace(authorTagSuffix, '').trim(); if (next === s) break; s = next; } if (authorTagExact.test(s) || skipExact.has(s)) return ''; return s; }; const isMetaOrNoise = (line, nickname) => { if (!line || skipExact.has(line)) return true; if (nickname && line === nickname) return true; if (authorTagExact.test(line)) return true; // 短行且含「作者」:作者 / 作者赞 等徽章,不当正文 if (line.includes('作者') && line.length <= 6 && !/[??!!。,,、]/.test(line)) return true; if (/^\\d+$/.test(line)) return true; if (/^展开\\d*条?回复/.test(line)) return true; if (/^收起/.test(line)) return true; return false; }; const isTimeLine = (line) => ( /刚刚|\\d+\\s*(秒|分钟|小时|天|周)前/.test(line) || (/·/.test(line) && /前/.test(line)) ); const pickContent = (textRoot, nickname) => { // 优先从独立正文节点取,减少把昵称/徽章/时间拼进 content const contentCandidates = [ textRoot.querySelector('[data-e2e="comment-content"]'), textRoot.querySelector('[class*="CommentContent"]'), textRoot.querySelector('span[class*="comment"]'), ].filter(Boolean); let content = ''; for (const el of contentCandidates) { const t = cleanContent(el.innerText || ''); if (t && !isMetaOrNoise(t, nickname) && !isTimeLine(t)) { content = t; break; } } const lines = (textRoot.innerText || '') .split('\\n') .map(s => s.trim()) .filter(Boolean); let commentTime = ''; for (const line of lines) { if (isMetaOrNoise(line, nickname)) continue; if (isTimeLine(line)) { if (!commentTime) commentTime = line; continue; } if (!content) { const cleaned = cleanContent(line); if (cleaned) content = cleaned; } } return { content: cleanContent(content), comment_time: commentTime }; }; const results = []; const seen = new Set(); const push = (row) => { if (!row || !row.content || !row.href) return; const key = `${row.href}|${row.content}|${row.is_reply ? 1 : 0}`; if (seen.has(key)) return; seen.add(key); results.push(row); }; const items = Array.from(list.querySelectorAll('[data-e2e="comment-item"]')); let topLevelCount = 0; for (const item of items) { if (topLevelCount >= limit) break; if (!isVisible(item)) continue; if (item.parentElement && item.parentElement.closest('[data-e2e="comment-item"]')) { continue; } // 头像在左侧容器,正文在 info-wrap —— 必须分开取 const avatarA = item.querySelector('.comment-item-avatar a[href*="/user/"]') || Array.from(item.querySelectorAll('a[href*="/user/"]')).find(a => { const r = a.getBoundingClientRect(); return r.width >= 28 && r.width <= 56 && r.height >= 28 && r.height <= 56; }); if (!avatarA) continue; const href = avatarA.getAttribute('href') || ''; if (!href || href.includes('/user/self')) continue; const infoWrap = item.querySelector('.comment-item-info-wrap'); // 正文不在 info-wrap 内,而在其兄弟节点;用右侧整块 F7ubq_7y / 整条 item const textRoot = item.querySelector('.F7ubq_7y') || item; let nickname = ''; const nickRoot = infoWrap || textRoot; for (const a of nickRoot.querySelectorAll('a[href*="/user/"]')) { const t = (a.innerText || '').trim().replace(/作者/g, '').trim(); if (t && t !== '...') { nickname = t; break; } } const { content, comment_time } = pickContent(textRoot, nickname); if (!content) continue; const ar = avatarA.getBoundingClientRect(); const main = { nickname, content, href, comment_time, is_reply: false, avatar_x: Math.round(ar.x + ar.width / 2), avatar_y: Math.round(ar.y + ar.height / 2), }; push(main); topLevelCount += 1; const mainX = main.avatar_x; const replyAvatars = Array.from(item.querySelectorAll('a[href*="/user/"]')) .filter(a => { if (a === avatarA) return false; if (a.closest('button.comment-reply-expand-btn')) return false; const r = a.getBoundingClientRect(); if (!isVisible(a)) return false; if (r.width < 28 || r.width > 56 || r.height < 28 || r.height > 56) return false; return (r.x + r.width / 2) > mainX + 24; }) .sort((a, b) => a.getBoundingClientRect().y - b.getBoundingClientRect().y); for (const a of replyAvatars) { let replyRoot = a.parentElement; for (let i = 0; i < 8 && replyRoot && replyRoot !== item; i++) { const t = (replyRoot.innerText || ''); if ((/前/.test(t) || t.includes('回复')) && t.length < 500) break; replyRoot = replyRoot.parentElement; } if (!replyRoot || replyRoot === item) replyRoot = a.parentElement; let replyNick = ''; for (const link of replyRoot.querySelectorAll('a[href*="/user/"]')) { const t = (link.innerText || '').trim().replace(/作者/g, '').trim(); if (t && t !== '...') { replyNick = t; break; } } const parsed = pickContent(replyRoot, replyNick); if (!parsed.content) continue; const rr = a.getBoundingClientRect(); push({ nickname: replyNick, content: parsed.content, href: a.getAttribute('href') || '', comment_time: parsed.comment_time, is_reply: true, avatar_x: Math.round(rr.x + rr.width / 2), avatar_y: Math.round(rr.y + rr.height / 2), }); } } return results.map((row, index) => ({ index: index + 1, nickname: row.nickname, content: row.content, href: row.href, comment_time: row.comment_time, is_reply: row.is_reply, avatar_x: row.avatar_x, avatar_y: row.avatar_y, })); }""", limit, ) return comments or [] def collect_comments_and_save( page: Page, video_meta: dict, limit: int = settings.COMMENT_LIMIT, ) -> None: """ 在「评论」tab 下滚动采集前 limit 条一级评论并入库。 若右侧栏不在评论 tab,则切过去(不再点评论图标,避免误关)。 """ video_pk = video_meta.get("db_id") if not video_pk: raise RuntimeError("缺少 videos.id(db_id),无法入库评论") print("[评论采集] 确认在评论 tab...") if not _is_side_panel_open(page): open_comment_panel(page) if not _click_side_panel_tab(page, "评论"): raise RuntimeError("无法切换到评论栏") try: page.wait_for_selector('[data-e2e="comment-item"]', timeout=10_000) except Exception as exc: raise SkipVideoError("评论数为 0,跳过该视频") from exc comments = _collect_comments_with_scroll(page, limit) top_n = sum(1 for c in comments if not c.get("is_reply")) reply_n = sum(1 for c in comments if c.get("is_reply")) if top_n < limit: raise SkipVideoError(f"一级评论仅 {top_n} 条,不足 {limit},跳过该视频") print( f"\n按顺序处理:一级评论 {top_n} 条(目标 {limit},回复不计数)," f"已展开回复 {reply_n} 条,合计入库候选 {len(comments)} 条:" ) id_cache: dict[str, str] = {} rows_for_db: list[dict] = [] for item in comments: idx = item.get("index") nickname = item.get("nickname") or "" content = _clean_comment_content(item.get("content") or "") href = item.get("href") or "" user_key = _normalize_user_key(href) user_link = _to_abs_url(href) if href else "" comment_time = item.get("comment_time") or "" is_reply = bool(item.get("is_reply")) douyin_id = "" if not content: print("-" * 40) print(f"[{idx}] 跳过: 评论内容为空(已剔除作者互动 tag)") continue print("-" * 40) print(f"[{idx}] 类型: {'回复' if is_reply else '一级评论'}") print(f"[{idx}] 视频ID: {video_meta.get('video_id', '')} (db_id={video_pk})") print(f"[{idx}] 评论昵称: {nickname}") print(f"[{idx}] 评论内容: {content}") print(f"[{idx}] 评论时间: {comment_time or '(未获取)'}") print(f"[{idx}] 用户主页: {user_link or '(未找到)'}") if user_key and user_key in id_cache: douyin_id = id_cache[user_key] print(f"[{idx}] 抖音号: {douyin_id} (缓存)") elif user_key and "/user/" in user_link: human_pause(1.0, 2.0) profile = None try: profile = _open_user_profile(page, href) opened = profile.url or "" if user_key not in opened and "user" not in opened: raise RuntimeError(f"主页跳转异常: expect {user_key}, got {opened}") human_pause(1.8, 3.0) douyin_id = _extract_douyin_id(profile) if nickname: page_text = profile.evaluate("() => document.body.innerText || ''") or "" if nickname not in page_text: print( f"[{idx}] 警告: 主页未出现评论昵称「{nickname}」," f"仍按主页抖音号入库: {douyin_id}" ) id_cache[user_key] = douyin_id print(f"[{idx}] 抖音号: {douyin_id}") print(f"[{idx}] 主页URL: {opened}") except Exception as exc: print(f"[{idx}] 抖音号: (读取失败: {exc})") douyin_id = "" finally: if profile is not None and profile != page: try: profile.close() except Exception: pass try: page.bring_to_front() except Exception: pass human_pause(0.8, 1.5) else: print(f"[{idx}] 抖音号: (未找到用户主页链接)") if not douyin_id: print(f"[{idx}] 跳过入库: 无可靠抖音号") continue rows_for_db.append( { "nick_name": nickname, "content": content, "comment_time": comment_time or None, "user_link": user_link or None, "douyin_id": douyin_id, "is_reply": is_reply, } ) print("-" * 40) print(f"评论采集结束,成功获取评论者抖音号 {len(id_cache)} 个。") with session_scope() as session: result = save_comments_batch(session, int(video_pk), rows_for_db) print( f"[评论入库完成] videos.id={result['video_pk']} " f"线索处理={result['leads']} " f"新增评论={result['comments']} " f"跳过={result['skipped_comments']} " f"(无抖音号={result.get('no_douyin_id', 0)})" )