123

MAIN_SERVE_FACTORY/MAIN_SERVE_FACTORY.js

@@ -17,6 +17,7 @@ async function startConsumer() {
     }
 }
 
+
 CMD.init = async function(){
     redis_help.connect(()=>{
 

PULL_DATA_FACTORY/PULL_DATA.js

@@ -19,6 +19,7 @@ var cur_timeRange = null
 var pullDataService = new PullDataService(redis_help)
 var detail_task = []
 var task_params = []
+var cur_page = 0
 const messageHandler = async (msg) => {
     let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
     PullDataConfig = JSON.parse(PullDataConfig)
@@ -79,8 +80,9 @@ const taskCallback = async (context) => {
     cur_timeRange = timeRange
     cur_timeRange.finish_count = 0
     process_update_token()
+    cur_page = 0
     // 示例：实际任务逻辑
-    await processTask(timeRange);
+    await processTask();
 };
 
 // 启动消费者
@@ -141,11 +143,11 @@ async function processDetailTask() {
     }catch (error) {
         console.error("processDetailTask:",error)
     }finally{
-        global.setTimeout(processDetailTask,500)
+        // global.setTimeout(processDetailTask,500)
     }
 }
 
-async function process_material_list(response,page,timeRange) {
+async function process_material_list(response,page) {
 
     let FilterConfig = await redis_help.getKeyValue("FilterConfig")
     FilterConfig = JSON.parse(FilterConfig)
@@ -166,16 +168,16 @@ async function process_material_list(response,page,timeRange) {
     if(response.data.list.length<=0){
         throw "没有数据了"
     }
-    if(timeRange!=null){
+    if(cur_timeRange!=null){
         const today = helper.getLocalDate();
         console.log("response.data:",response.data,today)
-        if(cur_day_data.get(today)[timeRange.start]){
+        if(cur_day_data.get(today)[cur_timeRange.start]){
             console.log("response.data.count:",response.data.count)
-            console.log("(cur_day_data[today][timeRange.start]:",cur_day_data.get(today)[timeRange.start])
+            console.log("(cur_day_data[today][timeRange.start]:",cur_day_data.get(today)[cur_timeRange.start])
             cur_timeRange.count = response.data.count
             cur_timeRange.cur_page = page
-            cur_day_data.get(today)[timeRange.start].count = response.data.count
-            cur_day_data.get(today)[timeRange.start].cur_page = page
+            cur_day_data.get(today)[cur_timeRange.start].count = response.data.count
+            cur_day_data.get(today)[cur_timeRange.start].cur_page = page
         }else{
             console.log("cur_day_data空")
         }
@@ -249,23 +251,40 @@ async function process_novel_material_list() {
         }
     }
 }
-async function processTask(timeRange){
+async function processTask(){
     try{
         let FilterConfig = await redis_help.getKeyValue("FilterConfig")
         FilterConfig = JSON.parse(FilterConfig)
-        let response = await pullDataService.get_novel_material_list(cur_timeRange,0,500)
+        let response = await pullDataService.get_novel_material_list(cur_timeRange,cur_page,500)
+
+        if(response.success){
+            if(response.data.list.length<=0){
+                cur_page = -1;
+                throw response
+            }
+        }else{
+            cur_page = -1;
+            throw response
+        }
         // console.log("get_novel_material_list:",response)
-        let detail_item =  await process_material_list(response,0,timeRange)
-        // { page: 1, offset: 0, limit: 500 },
-        task_params = helper.getPaginationParams(detail_item.count,500)
-        task_params.shift()
+        let detail_item =  await process_material_list(response,cur_page,cur_timeRange)
         detail_task.push(detail_item)
-        await process_novel_material_list()     
+        // { page: 1, offset: 0, limit: 500 },
+        // task_params = helper.getPaginationParams(detail_item.count,500)
+        // task_params.shift()
+        // detail_task.push(detail_item)
+
+        await processDetailTask()
+
         console.log("processTask over!")
 
     }catch(e){
         console.error("PULL_DATA: error:",e)
     }finally{
+        if(cur_page!=-1){
+            cur_page++;
+            setTimeout(processTask,1000)
+        }
     }
 
 }

src/PullDataService.js

@@ -65,7 +65,8 @@ class PullDataService {
                 const fiveMinutesAgo = new Date(now - interval_minute * interval_minute * 1000); // 60分钟前的时间
                 let data = {
                     startTime:fiveMinutesAgo.getTime(),
-                    endTime:now.getTime()
+                    endTime:now.getTime(),
+                    count:false
                 }
             
                 if(cur_timeRange!=null){