123

PULL_DATA_FACTORY/PULL_DATA.js

@@ -93,49 +93,44 @@ async function startConsumer() {
     }
 }
 
-async function processDetailTask() {
+async function processDetailTask(task_item) {
     try{
-        if(detail_task.length>0){
-            console.log("processDetailTask")
-            let task_item = detail_task.shift()
-            let materialId_list = task_item.materialId_list
-            let materialId_data_list = task_item.materialId_data_list
-            let response = await pullDataService.get_detail(materialId_list)
-            if(!response.success){
-                console.log("get_detail:",response)
-                throw {msg:response,timeRange:cur_timeRange,fun:"get_detail",materialId_list:materialId_list}
-            }
+        let materialId_list = task_item.materialId_list
+        let materialId_data_list = task_item.materialId_data_list
+        let response = await pullDataService.get_detail(materialId_list)
+        if(!response.success){
+            console.log("get_detail:",response)
+            throw {msg:response,timeRange:cur_timeRange,fun:"get_detail",materialId_list:materialId_list}
+        }
 
-            if(cur_timeRange!=null){
-                cur_timeRange.finish_count+=response.data.list.length
-                const today = helper.getLocalDate();
-                cur_day_data.get(today)[cur_timeRange.start].finish_count = cur_timeRange.finish_count
-                await manager.updateRecord(today,cur_day_data.get(today));
-            }
-            for (let index = 0; index < response.data.list.length; index++) {
-                const element = response.data.list[index];
-                const info = materialId_data_list[element.materialId]
-                if(element.hasAnchorInfo){
-                    await origin_data_controllers.createOriginData(
-                        {
-                            video_id:element.awemeId,
-                            materialId:element.materialId,
-                            video_link:element.awemeUrl,
-                            title:CMD.subTitle(element.title),
-                            publish_time:new Date(element.publishTime),
-                            kepp_num:info.kepp_num,
-                            comment_num:info.comment_num,
-                            like_num:info.like_num,
-                            shared_num:info.shared_num,
-                            is_guajian:1,
-                            guajian_link:element.landingUrl,
-                            status:0,
-                            createTime:info.createTime,
-                            updateTime:info.updateTime
-                        }
-                    )
-                }
-            
+        if(cur_timeRange!=null){
+            cur_timeRange.finish_count+=response.data.list.length
+            const today = helper.getLocalDate();
+            cur_day_data.get(today)[cur_timeRange.start].finish_count = cur_timeRange.finish_count
+            await manager.updateRecord(today,cur_day_data.get(today));
+        }
+        for (let index = 0; index < response.data.list.length; index++) {
+            const element = response.data.list[index];
+            const info = materialId_data_list[element.materialId]
+            if(element.hasAnchorInfo){
+                await origin_data_controllers.createOriginData(
+                    {
+                        video_id:element.awemeId,
+                        materialId:element.materialId,
+                        video_link:element.awemeUrl,
+                        title:CMD.subTitle(element.title),
+                        publish_time:new Date(element.publishTime),
+                        kepp_num:info.kepp_num,
+                        comment_num:info.comment_num,
+                        like_num:info.like_num,
+                        shared_num:info.shared_num,
+                        is_guajian:1,
+                        guajian_link:element.landingUrl,
+                        status:0,
+                        createTime:info.createTime,
+                        updateTime:info.updateTime
+                    }
+                )
             }
             console.log(" finish processDetailTask")
         }
@@ -267,13 +262,12 @@ async function processTask(){
         }
         // console.log("get_novel_material_list:",response)
         let detail_item =  await process_material_list(response,cur_page,cur_timeRange)
-        detail_task.push(detail_item)
         // { page: 1, offset: 0, limit: 500 },
         // task_params = helper.getPaginationParams(detail_item.count,500)
         // task_params.shift()
         // detail_task.push(detail_item)
 
-        await processDetailTask()
+        await processDetailTask(detail_item)
 
         console.log("processTask over!")
 
@@ -334,7 +328,6 @@ CMD.init = async function(){
         await pullDataService.getToken()
          // 启动调度器
         CMD.startScheduler(taskCallback);
-        processDetailTask()
         // process_update_token()
     })
 }