904118851 8 сар өмнө
parent
commit
03e863a3d6

+ 5 - 95
PULL_DATA_FACTORY/PULL_DATA.js

@@ -10,15 +10,11 @@ const manager = new DailyRecordManager();
 const rabbitMq = require('../src/mq/rabbit-mq');
 const helper = require('../src/helper');
 const PullDataService = require('../src/PullDataService');
-const pull_data_task_item = require('./pull_data_task_item');
 var _24HourRanges = []
 var jobs = []
 var cur_day_data = new Map()
-var pull_data_task_map = []
 var cur_timeRange = null
 var pullDataService = new PullDataService(redis_help)
-var detail_task = []
-var task_params = []
 var cur_page = 0
 const messageHandler = async (msg) => {
     let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
@@ -27,17 +23,6 @@ const messageHandler = async (msg) => {
     CMD.startScheduler(taskCallback)
 };
 
-const get_pull_data_task = function(){
-   for (let index = 0; index < pull_data_task_map.length; index++) {
-        const element = pull_data_task_map[index];
-        if(!element.isRunTask()){
-            return element
-        }
-   }
-   let item = new pull_data_task_item(redis_help)
-   pull_data_task_map.push(item)
-   return item
-}
 const taskCallback = async (context) => {
     var { timeRange, executionTime } = context;
     
@@ -159,7 +144,7 @@ async function process_material_list(response,page) {
         if(response.data.list.length<=0){
             throw "没有数据了"
         }
-        if(cur_timeRange!=null){
+        if(cur_timeRange!=null){ //统计
             const today = helper.getLocalDate();
             // console.log("response.data:",response.data,today)
             if(cur_day_data.get(today)[cur_timeRange.start]){
@@ -167,6 +152,8 @@ async function process_material_list(response,page) {
                 console.log("(cur_day_data[today][timeRange.start]:",cur_day_data.get(today)[cur_timeRange.start])
                 cur_timeRange.count = cur_timeRange.count + response.data.list.length
                 cur_timeRange.cur_page = page
+                cur_day_data.get(today)[cur_timeRange.start].start_time = cur_timeRange.start_time
+                cur_day_data.get(today)[cur_timeRange.start].pull_time = cur_timeRange.pull_time ||{}
                 cur_day_data.get(today)[cur_timeRange.start].count = response.data.count
                 cur_day_data.get(today)[cur_timeRange.start].cur_page = page
             }else{
@@ -174,7 +161,6 @@ async function process_material_list(response,page) {
             }
             manager.updateRecord(today,cur_day_data.get(today));
         }
-        console.log("materialId_list:",response.data.list.length)
         for (let index = 0; index < response.data.list.length; index++) {
             const origin_element = response.data.list[index];
             materialId_list.push(origin_element.materialId)
@@ -186,38 +172,6 @@ async function process_material_list(response,page) {
             }
         }
         let result = await origin_data_controllers.findNonExistentMaterialIds(materialId_list)
-        // console.log("nonExistentIds:",result.data.nonExistentIds)
-        // for (let index = 0; index < response.data.list.length; index++) {
-        //     const origin_element = response.data.list[index];
-        //     const result = await origin_data_controllers.getOriginData({
-        //         materialId:origin_element.materialId})
-        //     if(result.success){
-        //         // await origin_data_controllers.updateOriginData({id:result.data.id},
-        //         //     {
-        //         //         kepp_num:origin_element.favoriteCount.count,
-        //         //         comment_num:origin_element.commentCount.count,
-        //         //         like_num:origin_element.likeCount.count,
-        //         //         shared_num:origin_element.shareCount.count
-        //         //     })
-        //     }else{
-        //         if(CMD.isRight(FilterConfig,origin_element)){
-        //             materialId_list.push(origin_element.materialId)
-        //             // materialId_data_list[origin_element.materialId] =  {
-        //             //     kepp_num:origin_element.favoriteCount.count,
-        //             //     comment_num:origin_element.commentCount.count,
-        //             //     like_num:origin_element.likeCount.count,
-        //             //     shared_num:origin_element.shareCount.count
-        //             // }
-        //             console.log("materialId_list:",index,origin_element.materialId)
-        //             materialId_data_list[origin_element.materialId] =  {
-        //                 kepp_num:0,
-        //                 comment_num:0,
-        //                 like_num:0,
-        //                 shared_num:0
-        //             }
-        //         }
-        //     }
-        // }
         return {materialId_list:result.data.nonExistentIds,materialId_data_list:materialId_data_list,count:response.data.list.length}
     }catch(e){
         console.log("materialId_list:",e)
@@ -226,46 +180,7 @@ async function process_material_list(response,page) {
    
 }
 
-async function finish_material_list(response,page) {
-    if(!response.success){
-        console.error("finish_material_list:",response)
-    }else{
-        let detail_item =  await process_material_list(response,page,null)
-        detail_task.push(detail_item)
-    }
-}
-async function process_update_token() {
-    try{
-        let response = await pullDataService.getToken()
-        console.log("token:",response.data.token)
-    }catch(e){
-        console.error("process_update_token:",e)
-    }finally{
-        // setTimeout(process_update_token,200)
-    }
-}
-
-async function process_novel_material_list() {
-    try{
-        if(task_params.length>0){
-            let element = task_params.shift()
-            const pull_task_item = get_pull_data_task();
-            await pull_task_item.runTask(
-                finish_material_list,
-                cur_timeRange,
-                element.page,
-                element.limit
-            );
-            await new Promise(resolve => setTimeout(resolve, 200));
-        }
-    }catch(e){
 
-    }finally{
-        if(task_params.length>0){
-            setTimeout(process_novel_material_list,200)
-        }
-    }
-}
 async function processTask(){
     try{
         let FilterConfig = await redis_help.getKeyValue("FilterConfig")
@@ -281,13 +196,9 @@ async function processTask(){
             cur_page = -1;
             throw response
         }
-        // console.log("get_novel_material_list:",response)
+    
         let detail_item =  await process_material_list(response,cur_page,cur_timeRange)
-        // { page: 1, offset: 0, limit: 500 },
-        // task_params = helper.getPaginationParams(detail_item.count,500)
-        // task_params.shift()
-        // detail_task.push(detail_item)
-        // console.log("detail_item:",detail_item)
+
         if(detail_item.materialId_list.length<=0){
 
         }else{
@@ -350,7 +261,6 @@ CMD.init = async function(){
         await pullDataService.getToken()
          // 启动调度器
         CMD.startScheduler(taskCallback);
-        // process_update_token()
     })
 }