const CMD = {} const time_count = 5000; const redis_help = require('../src/use_redis'); const origin_data_controllers = require('../src/data_manager/Controllers/origin_data_controllers'); const pull_log_controllers = require('../src/data_manager/Controllers/pull_log_controllers'); const helper = require('../src/helper'); const axios = require('axios') const HttpClient = require('../src/HttpClient') const config = require('../etc/config.json') var session = null var cur_page = 0 var task_list = [] async function processTask(){ try{ if(cur_page>6){ throw "没有数据了" } let FilterConfig = await redis_help.getKeyValue("FilterConfig") FilterConfig = JSON.parse(FilterConfig) let response = await CMD.get_novel_material_list(cur_page,5000) // console.log("get_novel_material_list:",response) let materialId_list = [] let materialId_data_list = [] for (let index = 0; index < response.result.list.length; index++) { const origin_element = response.result.list[index]; const result = await origin_data_controllers.getOriginData({ materialId:origin_element.materialId}) if(result.success){ await origin_data_controllers.updateOriginData({id:result.data.id}, { kepp_num:origin_element.favoriteCount.count, comment_num:origin_element.commentCount.count, like_num:origin_element.likeCount.count, shared_num:origin_element.shareCount.count }) }else{ if(CMD.isRight(FilterConfig,origin_element)){ materialId_list.push(origin_element.materialId) materialId_data_list[origin_element.materialId] = { kepp_num:origin_element.favoriteCount.count, comment_num:origin_element.commentCount.count, like_num:origin_element.likeCount.count, shared_num:origin_element.shareCount.count } } } } response = await CMD.get_detail(materialId_list) for (let index = 0; index < response.result.list.length; index++) { const element = response.result.list[index]; const info = materialId_data_list[element.materialId] if(element.hasAnchorInfo){ await origin_data_controllers.createOriginData( { video_id:element.awemeId, materialId:element.materialId, video_link:element.awemeUrl, title:element.title, publish_time:new Date(element.publishTime), kepp_num:info.kepp_num, comment_num:info.comment_num, like_num:info.like_num, shared_num:info.shared_num, is_guajian:1, guajian_link:element.landingUrl, status:0 } ) } } // console.log("get_detail:",response) // await pull_log_controllers.createData() }catch(e){ console.log("PULL_DATA: error:",e) }finally{ cur_page++; setTimeout(processTask,time_count) } } CMD.isRight = function(FilterConfig,origin_element){ if(origin_element.likeCount.count < FilterConfig.like_num){// console.log("点赞数正确") return false } if(origin_element.favoriteCount.count < FilterConfig.kepp_num){ // console.log("收藏数正确") return false } if(origin_element.shareCount.count < FilterConfig.shared_num){ // console.log("分享数正确") return false } if(origin_element.commentCount.count < FilterConfig.comment_num){ // console.log("评论") return false } return true } CMD.getToken = async function(){ const response = await session.post(config.pull_data_config.get_token,{ userName:config.pull_data_config.userName, verCode:"1234", password:config.pull_data_config.password, loginType:"OPENAPI" }); return response.data } CMD.get_novel_material_list = async function(page,size) { const now = new Date(); // 当前时间 // const fiveMinutesAgo = new Date(now - 5 * 60 * 1000); // 5分钟前的时间 const fiveMinutesAgo = new Date(now - 200 * 60 * 1000); // 50分钟前的时间 let data = { startTime:fiveMinutesAgo.getTime(), endTime:now.getTime() } let params = `page=${page}&size=${size}&sort=update_time,desc` let url = config.pull_data_config.get_novel_material_list+params console.log(url,data) const response = await session.post(url,data, { headers: { 'token':await redis_help.getKeyValue("pull_data_token") } }); return response.data } CMD.get_detail = async function(materialId_list) { let data = { list:materialId_list } const response = await session.post(config.pull_data_config.get_detail,data, { headers: { 'token':await redis_help.getKeyValue("pull_data_token") } }); return response.data } CMD.init = async function(){ session = axios.create({ baseURL: config.isDebug?config.pull_data_config.debug_host:config.pull_data_config.release_host, headers: { 'Accept': 'application/json, text/plain, */*', } }); redis_help.connect(async ()=>{ let response = await CMD.getToken() console.log("getToken:",response.result.token) await redis_help.setKeyValue("pull_data_token",response.result.token) processTask() }) } CMD.init()