|
|
@@ -0,0 +1,528 @@
|
|
|
+"""步骤:评论区滚动采集与入库。"""
|
|
|
+
|
|
|
+from __future__ import annotations
|
|
|
+
|
|
|
+import re
|
|
|
+
|
|
|
+from playwright.sync_api import Page
|
|
|
+
|
|
|
+from config import settings
|
|
|
+from collectors.douyin.common import SkipVideoError, human_pause
|
|
|
+from collectors.douyin.panel import (
|
|
|
+ _click_side_panel_tab,
|
|
|
+ _is_side_panel_open,
|
|
|
+ open_comment_panel,
|
|
|
+)
|
|
|
+from collectors.douyin.profile import (
|
|
|
+ _extract_douyin_id,
|
|
|
+ _normalize_user_key,
|
|
|
+ _open_user_profile,
|
|
|
+ _to_abs_url,
|
|
|
+)
|
|
|
+from db.repository import save_comments_batch
|
|
|
+from db.session import session_scope
|
|
|
+
|
|
|
+# 评论旁「作者xxx」状态 tag,不应写入 content
|
|
|
+_AUTHOR_TAG_RE = re.compile(
|
|
|
+ r"(?:\s*作者(?:回复过|赞过|置顶|喜欢过)\s*)+$"
|
|
|
+)
|
|
|
+_AUTHOR_TAG_ONLY_RE = re.compile(
|
|
|
+ r"^作者(?:回复过|赞过|置顶|喜欢过)?$"
|
|
|
+)
|
|
|
+
|
|
|
+
|
|
|
+def _clean_comment_content(raw: str) -> str:
|
|
|
+ """去掉末尾/整段的作者互动 tag,只保留真实评论文案。"""
|
|
|
+ text = (raw or "").strip()
|
|
|
+ if not text:
|
|
|
+ return ""
|
|
|
+ text = _AUTHOR_TAG_RE.sub("", text).strip()
|
|
|
+ if _AUTHOR_TAG_ONLY_RE.match(text):
|
|
|
+ return ""
|
|
|
+ return text
|
|
|
+
|
|
|
+
|
|
|
+def _find_comment_scroll_box(page: Page) -> dict | None:
|
|
|
+ """定位评论区可滚动容器及其中心坐标。"""
|
|
|
+ return page.evaluate(
|
|
|
+ """() => {
|
|
|
+ const list = document.querySelector('[data-e2e="comment-list"]');
|
|
|
+ if (!list) return null;
|
|
|
+ let el = list;
|
|
|
+ let scrollEl = null;
|
|
|
+ while (el && el !== document.documentElement) {
|
|
|
+ const s = getComputedStyle(el);
|
|
|
+ const oy = s.overflowY;
|
|
|
+ if ((oy === 'auto' || oy === 'scroll' || oy === 'overlay')
|
|
|
+ && el.scrollHeight > el.clientHeight + 5) {
|
|
|
+ scrollEl = el;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ el = el.parentElement;
|
|
|
+ }
|
|
|
+ const target = scrollEl || list;
|
|
|
+ const r = target.getBoundingClientRect();
|
|
|
+ if (r.width <= 0 || r.height <= 0) return null;
|
|
|
+ return {
|
|
|
+ x: Math.round(r.x + r.width / 2),
|
|
|
+ y: Math.round(r.y + Math.min(r.height / 2, 280)),
|
|
|
+ scrollTop: target.scrollTop || 0,
|
|
|
+ scrollHeight: target.scrollHeight || 0,
|
|
|
+ clientHeight: target.clientHeight || 0,
|
|
|
+ };
|
|
|
+ }"""
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def _scroll_comment_panel_once(page: Page) -> None:
|
|
|
+ """
|
|
|
+ 滚动评论区一次(虚拟列表:真正滚的是可滚动父容器 + 滚轮)。
|
|
|
+ 绝不点击「展开N条回复」。
|
|
|
+ """
|
|
|
+ box = _find_comment_scroll_box(page)
|
|
|
+ page.evaluate(
|
|
|
+ """() => {
|
|
|
+ const list = document.querySelector('[data-e2e="comment-list"]');
|
|
|
+ if (!list) return;
|
|
|
+ let el = list;
|
|
|
+ let scrollEl = null;
|
|
|
+ while (el && el !== document.documentElement) {
|
|
|
+ const s = getComputedStyle(el);
|
|
|
+ const oy = s.overflowY;
|
|
|
+ if ((oy === 'auto' || oy === 'scroll' || oy === 'overlay')
|
|
|
+ && el.scrollHeight > el.clientHeight + 5) {
|
|
|
+ scrollEl = el;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ el = el.parentElement;
|
|
|
+ }
|
|
|
+ const target = scrollEl || list;
|
|
|
+ target.scrollTop = Math.min(
|
|
|
+ target.scrollTop + Math.max(240, Math.floor(target.clientHeight * 0.85)),
|
|
|
+ target.scrollHeight
|
|
|
+ );
|
|
|
+ const items = Array.from(list.querySelectorAll('[data-e2e="comment-item"]'));
|
|
|
+ const last = items[items.length - 1];
|
|
|
+ if (last) last.scrollIntoView({ block: 'end', inline: 'nearest' });
|
|
|
+ }"""
|
|
|
+ )
|
|
|
+ if box and box.get("x") and box.get("y"):
|
|
|
+ try:
|
|
|
+ page.mouse.move(box["x"], box["y"])
|
|
|
+ page.mouse.wheel(0, 900)
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+ human_pause(0.9, 1.5)
|
|
|
+
|
|
|
+
|
|
|
+def _comment_row_key(row: dict) -> str:
|
|
|
+ return (
|
|
|
+ f"{row.get('href') or ''}|"
|
|
|
+ f"{row.get('content') or ''}|"
|
|
|
+ f"{1 if row.get('is_reply') else 0}"
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def _collect_comments_with_scroll(page: Page, limit: int) -> list[dict]:
|
|
|
+ """
|
|
|
+ 边滚动边采集:抖音评论是虚拟列表,DOM 里通常只有几条,
|
|
|
+ 必须滚动后继续读,凑满 limit 条一级评论;已展开回复一并收(不计数)。
|
|
|
+ """
|
|
|
+ collected: list[dict] = []
|
|
|
+ seen: set[str] = set()
|
|
|
+ top_count = 0
|
|
|
+ stagnant = 0
|
|
|
+
|
|
|
+ for round_i in range(25):
|
|
|
+ batch = _list_visible_comments(page, max(limit * 2, 20))
|
|
|
+ added_top = 0
|
|
|
+ last_top_accepted = False
|
|
|
+
|
|
|
+ for row in batch:
|
|
|
+ key = _comment_row_key(row)
|
|
|
+ if row.get("is_reply"):
|
|
|
+ if key in seen or not last_top_accepted:
|
|
|
+ continue
|
|
|
+ seen.add(key)
|
|
|
+ collected.append(row)
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 一级评论
|
|
|
+ if key in seen:
|
|
|
+ last_top_accepted = True
|
|
|
+ continue
|
|
|
+ if top_count >= limit:
|
|
|
+ last_top_accepted = False
|
|
|
+ continue
|
|
|
+ seen.add(key)
|
|
|
+ collected.append(row)
|
|
|
+ top_count += 1
|
|
|
+ added_top += 1
|
|
|
+ last_top_accepted = True
|
|
|
+
|
|
|
+ print(
|
|
|
+ f"[评论滚动] 第 {round_i + 1} 轮:本轮新增一级 {added_top},"
|
|
|
+ f"累计一级 {top_count}/{limit},合计条目 {len(collected)}"
|
|
|
+ )
|
|
|
+ if top_count >= limit:
|
|
|
+ break
|
|
|
+
|
|
|
+ if added_top == 0:
|
|
|
+ stagnant += 1
|
|
|
+ else:
|
|
|
+ stagnant = 0
|
|
|
+ if stagnant >= 4:
|
|
|
+ print(f"[评论滚动] 连续无新增,停止。已采一级 {top_count}/{limit}")
|
|
|
+ break
|
|
|
+
|
|
|
+ _scroll_comment_panel_once(page)
|
|
|
+
|
|
|
+ for i, row in enumerate(collected, start=1):
|
|
|
+ row["index"] = i
|
|
|
+ return collected
|
|
|
+
|
|
|
+
|
|
|
+def _list_visible_comments(page: Page, limit: int) -> list[dict]:
|
|
|
+ """
|
|
|
+ 读取当前 DOM 中可见评论(虚拟列表仅含屏上几条)。
|
|
|
+ - 取最多 limit 条一级评论
|
|
|
+ - 若某条下已展开可见的回复,按顺序一并采集(不点「展开」)
|
|
|
+ """
|
|
|
+ comments = page.evaluate(
|
|
|
+ """(limit) => {
|
|
|
+ const list = document.querySelector('[data-e2e="comment-list"]');
|
|
|
+ if (!list) return [];
|
|
|
+
|
|
|
+ const skipExact = new Set([
|
|
|
+ '分享', '回复', '作者', '...', '', '收起', '展开',
|
|
|
+ '作者回复过', '作者赞过', '作者置顶', '作者喜欢过',
|
|
|
+ ]);
|
|
|
+
|
|
|
+ // 评论旁状态 tag(整行都是 tag,或夹在正文末尾)
|
|
|
+ const authorTagExact = /^(作者(回复过|赞过|置顶|喜欢过)?)$/;
|
|
|
+ const authorTagSuffix = /\\s*作者(回复过|赞过|置顶|喜欢过)\\s*$/;
|
|
|
+
|
|
|
+ const isVisible = (el) => {
|
|
|
+ const r = el.getBoundingClientRect();
|
|
|
+ const s = getComputedStyle(el);
|
|
|
+ return r.width > 0 && r.height > 0
|
|
|
+ && s.display !== 'none' && s.visibility !== 'hidden';
|
|
|
+ };
|
|
|
+
|
|
|
+ const cleanContent = (raw) => {
|
|
|
+ let s = String(raw || '').trim();
|
|
|
+ // 反复剥离末尾作者互动 tag,避免「正文作者回复过」入库
|
|
|
+ for (let i = 0; i < 3; i++) {
|
|
|
+ const next = s.replace(authorTagSuffix, '').trim();
|
|
|
+ if (next === s) break;
|
|
|
+ s = next;
|
|
|
+ }
|
|
|
+ if (authorTagExact.test(s) || skipExact.has(s)) return '';
|
|
|
+ return s;
|
|
|
+ };
|
|
|
+
|
|
|
+ const isMetaOrNoise = (line, nickname) => {
|
|
|
+ if (!line || skipExact.has(line)) return true;
|
|
|
+ if (nickname && line === nickname) return true;
|
|
|
+ if (authorTagExact.test(line)) return true;
|
|
|
+ // 短行且含「作者」:作者 / 作者赞 等徽章,不当正文
|
|
|
+ if (line.includes('作者') && line.length <= 6
|
|
|
+ && !/[??!!。,,、]/.test(line)) return true;
|
|
|
+ if (/^\\d+$/.test(line)) return true;
|
|
|
+ if (/^展开\\d*条?回复/.test(line)) return true;
|
|
|
+ if (/^收起/.test(line)) return true;
|
|
|
+ return false;
|
|
|
+ };
|
|
|
+
|
|
|
+ const isTimeLine = (line) => (
|
|
|
+ /刚刚|\\d+\\s*(秒|分钟|小时|天|周)前/.test(line)
|
|
|
+ || (/·/.test(line) && /前/.test(line))
|
|
|
+ );
|
|
|
+
|
|
|
+ const pickContent = (textRoot, nickname) => {
|
|
|
+ // 优先从独立正文节点取,减少把昵称/徽章/时间拼进 content
|
|
|
+ const contentCandidates = [
|
|
|
+ textRoot.querySelector('[data-e2e="comment-content"]'),
|
|
|
+ textRoot.querySelector('[class*="CommentContent"]'),
|
|
|
+ textRoot.querySelector('span[class*="comment"]'),
|
|
|
+ ].filter(Boolean);
|
|
|
+
|
|
|
+ let content = '';
|
|
|
+ for (const el of contentCandidates) {
|
|
|
+ const t = cleanContent(el.innerText || '');
|
|
|
+ if (t && !isMetaOrNoise(t, nickname) && !isTimeLine(t)) {
|
|
|
+ content = t;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const lines = (textRoot.innerText || '')
|
|
|
+ .split('\\n')
|
|
|
+ .map(s => s.trim())
|
|
|
+ .filter(Boolean);
|
|
|
+ let commentTime = '';
|
|
|
+ for (const line of lines) {
|
|
|
+ if (isMetaOrNoise(line, nickname)) continue;
|
|
|
+ if (isTimeLine(line)) {
|
|
|
+ if (!commentTime) commentTime = line;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (!content) {
|
|
|
+ const cleaned = cleanContent(line);
|
|
|
+ if (cleaned) content = cleaned;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return { content: cleanContent(content), comment_time: commentTime };
|
|
|
+ };
|
|
|
+
|
|
|
+ const results = [];
|
|
|
+ const seen = new Set();
|
|
|
+ const push = (row) => {
|
|
|
+ if (!row || !row.content || !row.href) return;
|
|
|
+ const key = `${row.href}|${row.content}|${row.is_reply ? 1 : 0}`;
|
|
|
+ if (seen.has(key)) return;
|
|
|
+ seen.add(key);
|
|
|
+ results.push(row);
|
|
|
+ };
|
|
|
+
|
|
|
+ const items = Array.from(list.querySelectorAll('[data-e2e="comment-item"]'));
|
|
|
+ let topLevelCount = 0;
|
|
|
+
|
|
|
+ for (const item of items) {
|
|
|
+ if (topLevelCount >= limit) break;
|
|
|
+ if (!isVisible(item)) continue;
|
|
|
+ if (item.parentElement && item.parentElement.closest('[data-e2e="comment-item"]')) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 头像在左侧容器,正文在 info-wrap —— 必须分开取
|
|
|
+ const avatarA = item.querySelector('.comment-item-avatar a[href*="/user/"]')
|
|
|
+ || Array.from(item.querySelectorAll('a[href*="/user/"]')).find(a => {
|
|
|
+ const r = a.getBoundingClientRect();
|
|
|
+ return r.width >= 28 && r.width <= 56 && r.height >= 28 && r.height <= 56;
|
|
|
+ });
|
|
|
+ if (!avatarA) continue;
|
|
|
+
|
|
|
+ const href = avatarA.getAttribute('href') || '';
|
|
|
+ if (!href || href.includes('/user/self')) continue;
|
|
|
+
|
|
|
+ const infoWrap = item.querySelector('.comment-item-info-wrap');
|
|
|
+ // 正文不在 info-wrap 内,而在其兄弟节点;用右侧整块 F7ubq_7y / 整条 item
|
|
|
+ const textRoot = item.querySelector('.F7ubq_7y') || item;
|
|
|
+
|
|
|
+ let nickname = '';
|
|
|
+ const nickRoot = infoWrap || textRoot;
|
|
|
+ for (const a of nickRoot.querySelectorAll('a[href*="/user/"]')) {
|
|
|
+ const t = (a.innerText || '').trim().replace(/作者/g, '').trim();
|
|
|
+ if (t && t !== '...') {
|
|
|
+ nickname = t;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ const { content, comment_time } = pickContent(textRoot, nickname);
|
|
|
+ if (!content) continue;
|
|
|
+
|
|
|
+ const ar = avatarA.getBoundingClientRect();
|
|
|
+ const main = {
|
|
|
+ nickname,
|
|
|
+ content,
|
|
|
+ href,
|
|
|
+ comment_time,
|
|
|
+ is_reply: false,
|
|
|
+ avatar_x: Math.round(ar.x + ar.width / 2),
|
|
|
+ avatar_y: Math.round(ar.y + ar.height / 2),
|
|
|
+ };
|
|
|
+ push(main);
|
|
|
+ topLevelCount += 1;
|
|
|
+
|
|
|
+ const mainX = main.avatar_x;
|
|
|
+ const replyAvatars = Array.from(item.querySelectorAll('a[href*="/user/"]'))
|
|
|
+ .filter(a => {
|
|
|
+ if (a === avatarA) return false;
|
|
|
+ if (a.closest('button.comment-reply-expand-btn')) return false;
|
|
|
+ const r = a.getBoundingClientRect();
|
|
|
+ if (!isVisible(a)) return false;
|
|
|
+ if (r.width < 28 || r.width > 56 || r.height < 28 || r.height > 56) return false;
|
|
|
+ return (r.x + r.width / 2) > mainX + 24;
|
|
|
+ })
|
|
|
+ .sort((a, b) => a.getBoundingClientRect().y - b.getBoundingClientRect().y);
|
|
|
+
|
|
|
+ for (const a of replyAvatars) {
|
|
|
+ let replyRoot = a.parentElement;
|
|
|
+ for (let i = 0; i < 8 && replyRoot && replyRoot !== item; i++) {
|
|
|
+ const t = (replyRoot.innerText || '');
|
|
|
+ if ((/前/.test(t) || t.includes('回复')) && t.length < 500) break;
|
|
|
+ replyRoot = replyRoot.parentElement;
|
|
|
+ }
|
|
|
+ if (!replyRoot || replyRoot === item) replyRoot = a.parentElement;
|
|
|
+
|
|
|
+ let replyNick = '';
|
|
|
+ for (const link of replyRoot.querySelectorAll('a[href*="/user/"]')) {
|
|
|
+ const t = (link.innerText || '').trim().replace(/作者/g, '').trim();
|
|
|
+ if (t && t !== '...') {
|
|
|
+ replyNick = t;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ const parsed = pickContent(replyRoot, replyNick);
|
|
|
+ if (!parsed.content) continue;
|
|
|
+ const rr = a.getBoundingClientRect();
|
|
|
+ push({
|
|
|
+ nickname: replyNick,
|
|
|
+ content: parsed.content,
|
|
|
+ href: a.getAttribute('href') || '',
|
|
|
+ comment_time: parsed.comment_time,
|
|
|
+ is_reply: true,
|
|
|
+ avatar_x: Math.round(rr.x + rr.width / 2),
|
|
|
+ avatar_y: Math.round(rr.y + rr.height / 2),
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return results.map((row, index) => ({
|
|
|
+ index: index + 1,
|
|
|
+ nickname: row.nickname,
|
|
|
+ content: row.content,
|
|
|
+ href: row.href,
|
|
|
+ comment_time: row.comment_time,
|
|
|
+ is_reply: row.is_reply,
|
|
|
+ avatar_x: row.avatar_x,
|
|
|
+ avatar_y: row.avatar_y,
|
|
|
+ }));
|
|
|
+ }""",
|
|
|
+ limit,
|
|
|
+ )
|
|
|
+ return comments or []
|
|
|
+
|
|
|
+def collect_comments_and_save(
|
|
|
+ page: Page,
|
|
|
+ video_meta: dict,
|
|
|
+ limit: int = settings.COMMENT_LIMIT,
|
|
|
+) -> None:
|
|
|
+ """
|
|
|
+ 在「评论」tab 下滚动采集前 limit 条一级评论并入库。
|
|
|
+ 若右侧栏不在评论 tab,则切过去(不再点评论图标,避免误关)。
|
|
|
+ """
|
|
|
+ video_pk = video_meta.get("db_id")
|
|
|
+ if not video_pk:
|
|
|
+ raise RuntimeError("缺少 videos.id(db_id),无法入库评论")
|
|
|
+
|
|
|
+ print("[评论采集] 确认在评论 tab...")
|
|
|
+ if not _is_side_panel_open(page):
|
|
|
+ open_comment_panel(page)
|
|
|
+ if not _click_side_panel_tab(page, "评论"):
|
|
|
+ raise RuntimeError("无法切换到评论栏")
|
|
|
+
|
|
|
+ try:
|
|
|
+ page.wait_for_selector('[data-e2e="comment-item"]', timeout=10_000)
|
|
|
+ except Exception as exc:
|
|
|
+ raise SkipVideoError("评论数为 0,跳过该视频") from exc
|
|
|
+
|
|
|
+ comments = _collect_comments_with_scroll(page, limit)
|
|
|
+ top_n = sum(1 for c in comments if not c.get("is_reply"))
|
|
|
+ reply_n = sum(1 for c in comments if c.get("is_reply"))
|
|
|
+
|
|
|
+ if top_n < limit:
|
|
|
+ raise SkipVideoError(f"一级评论仅 {top_n} 条,不足 {limit},跳过该视频")
|
|
|
+
|
|
|
+ print(
|
|
|
+ f"\n按顺序处理:一级评论 {top_n} 条(目标 {limit},回复不计数),"
|
|
|
+ f"已展开回复 {reply_n} 条,合计入库候选 {len(comments)} 条:"
|
|
|
+ )
|
|
|
+
|
|
|
+ id_cache: dict[str, str] = {}
|
|
|
+ rows_for_db: list[dict] = []
|
|
|
+
|
|
|
+ for item in comments:
|
|
|
+ idx = item.get("index")
|
|
|
+ nickname = item.get("nickname") or ""
|
|
|
+ content = _clean_comment_content(item.get("content") or "")
|
|
|
+ href = item.get("href") or ""
|
|
|
+ user_key = _normalize_user_key(href)
|
|
|
+ user_link = _to_abs_url(href) if href else ""
|
|
|
+ comment_time = item.get("comment_time") or ""
|
|
|
+ is_reply = bool(item.get("is_reply"))
|
|
|
+ douyin_id = ""
|
|
|
+
|
|
|
+ if not content:
|
|
|
+ print("-" * 40)
|
|
|
+ print(f"[{idx}] 跳过: 评论内容为空(已剔除作者互动 tag)")
|
|
|
+ continue
|
|
|
+
|
|
|
+ print("-" * 40)
|
|
|
+ print(f"[{idx}] 类型: {'回复' if is_reply else '一级评论'}")
|
|
|
+ print(f"[{idx}] 视频ID: {video_meta.get('video_id', '')} (db_id={video_pk})")
|
|
|
+ print(f"[{idx}] 评论昵称: {nickname}")
|
|
|
+ print(f"[{idx}] 评论内容: {content}")
|
|
|
+ print(f"[{idx}] 评论时间: {comment_time or '(未获取)'}")
|
|
|
+ print(f"[{idx}] 用户主页: {user_link or '(未找到)'}")
|
|
|
+
|
|
|
+ if user_key and user_key in id_cache:
|
|
|
+ douyin_id = id_cache[user_key]
|
|
|
+ print(f"[{idx}] 抖音号: {douyin_id} (缓存)")
|
|
|
+ elif user_key and "/user/" in user_link:
|
|
|
+ human_pause(1.0, 2.0)
|
|
|
+ profile = None
|
|
|
+ try:
|
|
|
+ profile = _open_user_profile(page, href)
|
|
|
+ opened = profile.url or ""
|
|
|
+ if user_key not in opened and "user" not in opened:
|
|
|
+ raise RuntimeError(f"主页跳转异常: expect {user_key}, got {opened}")
|
|
|
+ human_pause(1.8, 3.0)
|
|
|
+ douyin_id = _extract_douyin_id(profile)
|
|
|
+ if nickname:
|
|
|
+ page_text = profile.evaluate("() => document.body.innerText || ''") or ""
|
|
|
+ if nickname not in page_text:
|
|
|
+ print(
|
|
|
+ f"[{idx}] 警告: 主页未出现评论昵称「{nickname}」,"
|
|
|
+ f"仍按主页抖音号入库: {douyin_id}"
|
|
|
+ )
|
|
|
+ id_cache[user_key] = douyin_id
|
|
|
+ print(f"[{idx}] 抖音号: {douyin_id}")
|
|
|
+ print(f"[{idx}] 主页URL: {opened}")
|
|
|
+ except Exception as exc:
|
|
|
+ print(f"[{idx}] 抖音号: (读取失败: {exc})")
|
|
|
+ douyin_id = ""
|
|
|
+ finally:
|
|
|
+ if profile is not None and profile != page:
|
|
|
+ try:
|
|
|
+ profile.close()
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+ try:
|
|
|
+ page.bring_to_front()
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+ human_pause(0.8, 1.5)
|
|
|
+ else:
|
|
|
+ print(f"[{idx}] 抖音号: (未找到用户主页链接)")
|
|
|
+
|
|
|
+ if not douyin_id:
|
|
|
+ print(f"[{idx}] 跳过入库: 无可靠抖音号")
|
|
|
+ continue
|
|
|
+
|
|
|
+ rows_for_db.append(
|
|
|
+ {
|
|
|
+ "nick_name": nickname,
|
|
|
+ "content": content,
|
|
|
+ "comment_time": comment_time or None,
|
|
|
+ "user_link": user_link or None,
|
|
|
+ "douyin_id": douyin_id,
|
|
|
+ "is_reply": is_reply,
|
|
|
+ }
|
|
|
+ )
|
|
|
+
|
|
|
+ print("-" * 40)
|
|
|
+ print(f"评论采集结束,成功获取评论者抖音号 {len(id_cache)} 个。")
|
|
|
+
|
|
|
+ with session_scope() as session:
|
|
|
+ result = save_comments_batch(session, int(video_pk), rows_for_db)
|
|
|
+ print(
|
|
|
+ f"[评论入库完成] videos.id={result['video_pk']} "
|
|
|
+ f"线索处理={result['leads']} "
|
|
|
+ f"新增评论={result['comments']} "
|
|
|
+ f"跳过={result['skipped_comments']} "
|
|
|
+ f"(无抖音号={result.get('no_douyin_id', 0)})"
|
|
|
+ )
|
|
|
+
|
|
|
+
|