tg_factory/PULL_DATA_FACTORY/PULL_DATA.js

const CMD = {}
const time_count = 5000;
const redis_help = require('../src/use_redis');
const origin_data_controllers = require('../src/data_manager/Controllers/origin_data_controllers');
const pull_log_controllers = require('../src/data_manager/Controllers/pull_log_controllers');
const helper = require('../src/helper');
const axios = require('axios')
const HttpClient = require('../src/HttpClient')
const config = require('../etc/config.json')
var session = null
var cur_page = 0
var task_list = []
async function processTask(){
    try{
        if(cur_page>6){
            throw "没有数据了"
        }
        let FilterConfig = await redis_help.getKeyValue("FilterConfig")
        FilterConfig = JSON.parse(FilterConfig)
        let response = await CMD.get_novel_material_list(cur_page,5000)
        // console.log("get_novel_material_list:",response)
        let materialId_list = []
        let materialId_data_list = []
        for (let index = 0; index < response.result.list.length; index++) {
            const origin_element = response.result.list[index];
            const result = await origin_data_controllers.getOriginData({
                materialId:origin_element.materialId})
            if(result.success){
                await origin_data_controllers.updateOriginData({id:result.data.id},
                    {
                        kepp_num:origin_element.favoriteCount.count,
                        comment_num:origin_element.commentCount.count,
                        like_num:origin_element.likeCount.count,
                        shared_num:origin_element.shareCount.count
                    })
            }else{
                if(CMD.isRight(FilterConfig,origin_element)){
                    materialId_list.push(origin_element.materialId)
                    materialId_data_list[origin_element.materialId] =  {
                        kepp_num:origin_element.favoriteCount.count,
                        comment_num:origin_element.commentCount.count,
                        like_num:origin_element.likeCount.count,
                        shared_num:origin_element.shareCount.count
                    }
                }
            }
        }


        response = await CMD.get_detail(materialId_list)
        for (let index = 0; index < response.result.list.length; index++) {
            const element = response.result.list[index];
            const info = materialId_data_list[element.materialId]
            if(element.hasAnchorInfo){
                await origin_data_controllers.createOriginData(
                    {
                        video_id:element.awemeId,
                        materialId:element.materialId,
                        video_link:element.awemeUrl,
                        title:element.title,
                        publish_time:new Date(element.publishTime),
                        kepp_num:info.kepp_num,
                        comment_num:info.comment_num,
                        like_num:info.like_num,
                        shared_num:info.shared_num,
                        is_guajian:1,
                        guajian_link:element.landingUrl,
                        status:0
                    }
                )
            }
        
        }
        // console.log("get_detail:",response)
        // await pull_log_controllers.createData()
    }catch(e){
        console.log("PULL_DATA: error:",e)
    }finally{
        cur_page++;
        setTimeout(processTask,time_count)
    }

}
CMD.isRight = function(FilterConfig,origin_element){
    if(origin_element.likeCount.count < FilterConfig.like_num){// console.log("点赞数正确")
        return false
    }
   
    if(origin_element.favoriteCount.count < FilterConfig.kepp_num){   // console.log("收藏数正确")
        return false
    }

    if(origin_element.shareCount.count < FilterConfig.shared_num){      // console.log("分享数正确")
        return false
    }

    if(origin_element.commentCount.count < FilterConfig.comment_num){          // console.log("评论")      
        return false
    }
    return true
}

CMD.getToken = async function(){
    const response = await session.post(config.pull_data_config.get_token,{
        userName:config.pull_data_config.userName,
        verCode:"1234",
        password:config.pull_data_config.password,
        loginType:"OPENAPI"
    });
    return response.data
}

CMD.get_novel_material_list = async function(page,size) {
    const now = new Date(); // 当前时间
    // const fiveMinutesAgo = new Date(now - 5 * 60 * 1000); // 5分钟前的时间
    const fiveMinutesAgo = new Date(now - 200 * 60 * 1000); // 50分钟前的时间
    let data = {
        startTime:fiveMinutesAgo.getTime(),
        endTime:now.getTime()
    }
    let params = `page=${page}&size=${size}&sort=update_time,desc`
    let url = config.pull_data_config.get_novel_material_list+params
    console.log(url,data)
    const response = await session.post(url,data, {
        headers: {
            'token':await redis_help.getKeyValue("pull_data_token")
        }
    });
    return response.data
}

CMD.get_detail = async function(materialId_list) {
    let data = {
        list:materialId_list
    }
    const response = await session.post(config.pull_data_config.get_detail,data, {
        headers: {
            'token':await redis_help.getKeyValue("pull_data_token")
        }
    });
    return response.data
}



CMD.init = async function(){
    session = axios.create({
        baseURL: config.isDebug?config.pull_data_config.debug_host:config.pull_data_config.release_host,
        headers: {
          'Accept': 'application/json, text/plain, */*',
        }
    });


    redis_help.connect(async ()=>{
        let response = await CMD.getToken()
        console.log("getToken:",response.result.token)
        await redis_help.setKeyValue("pull_data_token",response.result.token)
        processTask()
    })

}

CMD.init()