|
@@ -10,15 +10,11 @@ const manager = new DailyRecordManager();
|
|
|
const rabbitMq = require('../src/mq/rabbit-mq');
|
|
|
const helper = require('../src/helper');
|
|
|
const PullDataService = require('../src/PullDataService');
|
|
|
-const pull_data_task_item = require('./pull_data_task_item');
|
|
|
var _24HourRanges = []
|
|
|
var jobs = []
|
|
|
var cur_day_data = new Map()
|
|
|
-var pull_data_task_map = []
|
|
|
var cur_timeRange = null
|
|
|
var pullDataService = new PullDataService(redis_help)
|
|
|
-var detail_task = []
|
|
|
-var task_params = []
|
|
|
var cur_page = 0
|
|
|
const messageHandler = async (msg) => {
|
|
|
let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
|
|
@@ -27,17 +23,6 @@ const messageHandler = async (msg) => {
|
|
|
CMD.startScheduler(taskCallback)
|
|
|
};
|
|
|
|
|
|
-const get_pull_data_task = function(){
|
|
|
- for (let index = 0; index < pull_data_task_map.length; index++) {
|
|
|
- const element = pull_data_task_map[index];
|
|
|
- if(!element.isRunTask()){
|
|
|
- return element
|
|
|
- }
|
|
|
- }
|
|
|
- let item = new pull_data_task_item(redis_help)
|
|
|
- pull_data_task_map.push(item)
|
|
|
- return item
|
|
|
-}
|
|
|
const taskCallback = async (context) => {
|
|
|
var { timeRange, executionTime } = context;
|
|
|
|
|
@@ -159,7 +144,7 @@ async function process_material_list(response,page) {
|
|
|
if(response.data.list.length<=0){
|
|
|
throw "没有数据了"
|
|
|
}
|
|
|
- if(cur_timeRange!=null){
|
|
|
+ if(cur_timeRange!=null){ //统计
|
|
|
const today = helper.getLocalDate();
|
|
|
// console.log("response.data:",response.data,today)
|
|
|
if(cur_day_data.get(today)[cur_timeRange.start]){
|
|
@@ -167,6 +152,8 @@ async function process_material_list(response,page) {
|
|
|
console.log("(cur_day_data[today][timeRange.start]:",cur_day_data.get(today)[cur_timeRange.start])
|
|
|
cur_timeRange.count = cur_timeRange.count + response.data.list.length
|
|
|
cur_timeRange.cur_page = page
|
|
|
+ cur_day_data.get(today)[cur_timeRange.start].start_time = cur_timeRange.start_time
|
|
|
+ cur_day_data.get(today)[cur_timeRange.start].pull_time = cur_timeRange.pull_time ||{}
|
|
|
cur_day_data.get(today)[cur_timeRange.start].count = response.data.count
|
|
|
cur_day_data.get(today)[cur_timeRange.start].cur_page = page
|
|
|
}else{
|
|
@@ -174,7 +161,6 @@ async function process_material_list(response,page) {
|
|
|
}
|
|
|
manager.updateRecord(today,cur_day_data.get(today));
|
|
|
}
|
|
|
- console.log("materialId_list:",response.data.list.length)
|
|
|
for (let index = 0; index < response.data.list.length; index++) {
|
|
|
const origin_element = response.data.list[index];
|
|
|
materialId_list.push(origin_element.materialId)
|
|
@@ -186,38 +172,6 @@ async function process_material_list(response,page) {
|
|
|
}
|
|
|
}
|
|
|
let result = await origin_data_controllers.findNonExistentMaterialIds(materialId_list)
|
|
|
- // console.log("nonExistentIds:",result.data.nonExistentIds)
|
|
|
- // for (let index = 0; index < response.data.list.length; index++) {
|
|
|
- // const origin_element = response.data.list[index];
|
|
|
- // const result = await origin_data_controllers.getOriginData({
|
|
|
- // materialId:origin_element.materialId})
|
|
|
- // if(result.success){
|
|
|
- // // await origin_data_controllers.updateOriginData({id:result.data.id},
|
|
|
- // // {
|
|
|
- // // kepp_num:origin_element.favoriteCount.count,
|
|
|
- // // comment_num:origin_element.commentCount.count,
|
|
|
- // // like_num:origin_element.likeCount.count,
|
|
|
- // // shared_num:origin_element.shareCount.count
|
|
|
- // // })
|
|
|
- // }else{
|
|
|
- // if(CMD.isRight(FilterConfig,origin_element)){
|
|
|
- // materialId_list.push(origin_element.materialId)
|
|
|
- // // materialId_data_list[origin_element.materialId] = {
|
|
|
- // // kepp_num:origin_element.favoriteCount.count,
|
|
|
- // // comment_num:origin_element.commentCount.count,
|
|
|
- // // like_num:origin_element.likeCount.count,
|
|
|
- // // shared_num:origin_element.shareCount.count
|
|
|
- // // }
|
|
|
- // console.log("materialId_list:",index,origin_element.materialId)
|
|
|
- // materialId_data_list[origin_element.materialId] = {
|
|
|
- // kepp_num:0,
|
|
|
- // comment_num:0,
|
|
|
- // like_num:0,
|
|
|
- // shared_num:0
|
|
|
- // }
|
|
|
- // }
|
|
|
- // }
|
|
|
- // }
|
|
|
return {materialId_list:result.data.nonExistentIds,materialId_data_list:materialId_data_list,count:response.data.list.length}
|
|
|
}catch(e){
|
|
|
console.log("materialId_list:",e)
|
|
@@ -226,46 +180,7 @@ async function process_material_list(response,page) {
|
|
|
|
|
|
}
|
|
|
|
|
|
-async function finish_material_list(response,page) {
|
|
|
- if(!response.success){
|
|
|
- console.error("finish_material_list:",response)
|
|
|
- }else{
|
|
|
- let detail_item = await process_material_list(response,page,null)
|
|
|
- detail_task.push(detail_item)
|
|
|
- }
|
|
|
-}
|
|
|
-async function process_update_token() {
|
|
|
- try{
|
|
|
- let response = await pullDataService.getToken()
|
|
|
- console.log("token:",response.data.token)
|
|
|
- }catch(e){
|
|
|
- console.error("process_update_token:",e)
|
|
|
- }finally{
|
|
|
- // setTimeout(process_update_token,200)
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-async function process_novel_material_list() {
|
|
|
- try{
|
|
|
- if(task_params.length>0){
|
|
|
- let element = task_params.shift()
|
|
|
- const pull_task_item = get_pull_data_task();
|
|
|
- await pull_task_item.runTask(
|
|
|
- finish_material_list,
|
|
|
- cur_timeRange,
|
|
|
- element.page,
|
|
|
- element.limit
|
|
|
- );
|
|
|
- await new Promise(resolve => setTimeout(resolve, 200));
|
|
|
- }
|
|
|
- }catch(e){
|
|
|
|
|
|
- }finally{
|
|
|
- if(task_params.length>0){
|
|
|
- setTimeout(process_novel_material_list,200)
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
async function processTask(){
|
|
|
try{
|
|
|
let FilterConfig = await redis_help.getKeyValue("FilterConfig")
|
|
@@ -281,13 +196,9 @@ async function processTask(){
|
|
|
cur_page = -1;
|
|
|
throw response
|
|
|
}
|
|
|
- // console.log("get_novel_material_list:",response)
|
|
|
+
|
|
|
let detail_item = await process_material_list(response,cur_page,cur_timeRange)
|
|
|
- // { page: 1, offset: 0, limit: 500 },
|
|
|
- // task_params = helper.getPaginationParams(detail_item.count,500)
|
|
|
- // task_params.shift()
|
|
|
- // detail_task.push(detail_item)
|
|
|
- // console.log("detail_item:",detail_item)
|
|
|
+
|
|
|
if(detail_item.materialId_list.length<=0){
|
|
|
|
|
|
}else{
|
|
@@ -350,7 +261,6 @@ CMD.init = async function(){
|
|
|
await pullDataService.getToken()
|
|
|
// 启动调度器
|
|
|
CMD.startScheduler(taskCallback);
|
|
|
- // process_update_token()
|
|
|
})
|
|
|
}
|
|
|
|