104 lines
3.5 KiB
Python
104 lines
3.5 KiB
Python
|
#!/usr/bin/env python
|
|||
|
|
# -*- coding: UTF-8 -*-
|
|||
|
|
"""
|
|||
|
|
@Project ：redbook
|
|||
|
|
@File ：scraper.py
|
|||
|
|
@IDE ：PyCharm
|
|||
|
|
@Author ：rengengchen
|
|||
|
|
@Time ：2024/4/17 11:53
|
|||
|
|
"""
|
|||
|
|
import datetime
|
|||
|
|
import random
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
from loguru import logger
|
|||
|
|
import requests
|
|||
|
|
|
|||
|
|
from config import HEAD, INFO_COLUMNS, RESULT_COLUMNS, RESULT_PATH, ENCODING, KEYWORDS
|
|||
|
|
|
|||
|
|
|
|||
|
|
def request_comment(aweme_id, cursor=0, count=20, comment_id=None, s1=1, s2=3):
|
|||
|
|
is_child = bool(comment_id)
|
|||
|
|
params = {'count': count, 'aid': 6383}
|
|||
|
|
if is_child:
|
|||
|
|
url = 'https://www.douyin.com/aweme/v1/web/comment/list/reply/'
|
|||
|
|
params['item_id'] = aweme_id
|
|||
|
|
params['comment_id'] = comment_id
|
|||
|
|
params['cursor'] = cursor
|
|||
|
|
else:
|
|||
|
|
url = 'https://www.douyin.com/aweme/v1/web/comment/list/'
|
|||
|
|
params['aweme_id'] = aweme_id
|
|||
|
|
params['cursor'] = cursor
|
|||
|
|
# url = 'https://www.douyin.com/aweme/v1/web/comment/list/?aweme_id=7353583858400398642&cursor=0&count=20'
|
|||
|
|
# r = requests.get(url, headers=HEAD)
|
|||
|
|
r = requests.get(url, headers=HEAD, params=params)
|
|||
|
|
respond = r.json()
|
|||
|
|
if respond['status_code'] == 0:
|
|||
|
|
if s2 > s1:
|
|||
|
|
gap = random.uniform(s1, s2)
|
|||
|
|
elif s1 == s2:
|
|||
|
|
gap = s1
|
|||
|
|
else:
|
|||
|
|
gap = 0
|
|||
|
|
time.sleep(gap)
|
|||
|
|
return respond
|
|||
|
|
logger.error(f'fail to request {url}')
|
|||
|
|
logger.error(respond)
|
|||
|
|
raise ConnectionError(f'fail to request {url}, respond: {respond}')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def transform_comment(comment):
|
|||
|
|
dt_object = datetime.datetime.fromtimestamp(int(comment['create_time']))
|
|||
|
|
comment['create_time'] = dt_object.strftime('%Y-%m-%d %H:%M:%S')
|
|||
|
|
comment['uid'] = comment['user']['uid']
|
|||
|
|
comment['short_id'] = comment['user']['short_id']
|
|||
|
|
comment['nickname'] = comment['user']['nickname']
|
|||
|
|
comment['signature'] = comment['user']['signature']
|
|||
|
|
comment1 = {k: comment[k] for k in INFO_COLUMNS}
|
|||
|
|
comment1['ip_label'] = comment.get('ip_label', '')
|
|||
|
|
return comment1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_comment(comments):
|
|||
|
|
comments1 = []
|
|||
|
|
sub_comment_has_more = False
|
|||
|
|
subs = []
|
|||
|
|
for comment in comments:
|
|||
|
|
logger.debug(f'parse comment:\n{comment}')
|
|||
|
|
# 子评论没有这些属性
|
|||
|
|
reply_comment_total = comment.get('reply_comment_total', 0)
|
|||
|
|
if reply_comment_total:
|
|||
|
|
sub_comment_has_more = True
|
|||
|
|
subs.append((comment['aweme_id'], comment['cid']))
|
|||
|
|
have_word = not bool(len(KEYWORDS))
|
|||
|
|
for keyword in KEYWORDS:
|
|||
|
|
have_word = keyword in comment['text']
|
|||
|
|
if have_word:
|
|||
|
|
break
|
|||
|
|
if not have_word:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
comment = transform_comment(comment)
|
|||
|
|
logger.debug(comment)
|
|||
|
|
comments1.append(comment)
|
|||
|
|
with open(f'{RESULT_PATH}/comments.csv', mode='a', encoding=ENCODING, errors='ignore') as f:
|
|||
|
|
for comment in comments1:
|
|||
|
|
f.write(','.join([str(comment[k]) for k in RESULT_COLUMNS]))
|
|||
|
|
f.write('\n')
|
|||
|
|
if sub_comment_has_more:
|
|||
|
|
logger.debug('load sub comment')
|
|||
|
|
for note_id, comment_id in subs:
|
|||
|
|
read_comment(note_id,
|
|||
|
|
comment_id=comment_id,
|
|||
|
|
cursor=0)
|
|||
|
|
return comments1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def read_comment(aweme_id, comment_id=None, cursor=0, s1=1, s2=3):
|
|||
|
|
data = request_comment(aweme_id, cursor=cursor, comment_id=comment_id, s1=s1, s2=s2)
|
|||
|
|
parse_comment(data['comments'])
|
|||
|
|
while data['has_more']:
|
|||
|
|
logger.debug('load next page')
|
|||
|
|
data = request_comment(aweme_id, cursor=data['cursor'], comment_id=comment_id, s1=s1, s2=s2)
|
|||
|
|
parse_comment(data['comments'])
|