123

2_PRODUCT_FACTORY/fq_batch_factory.js

@@ -49,40 +49,41 @@ async function processTask(){
             //先到番茄数据里查找
             let fq_book_result = await get_fq_book_tab_controllers.findNonExistentProductIds(ProductIds);
 
-            
-
-            let product_list_result = await video_product_controllers.findNonExistentProductIds(ProductIds);
-            if(product_list_result.success){
-                for (let index = 0; index < result.data.length; index++) {
-                    let filter_item = result.data[index];
-                    if(product_list_result.data.existingProductIds.has(filter_item.book_id)){
-                        let info = FilterList[filter_item.book_id];
-                        let product_item = getProductById(product_list_result.data.existingRecords,filter_item.book_id)
-                        if (!info) {
-                            throw `No filter info found for book_id: ${filter_item.book_id}`
-                        }
-                        if (!product_item) {
-                            throw `product_item is null: ${filter_item.book_id}`
-                        }
-                        info.words = product_item.words
-                        info.genre =  product_item.genre
-                        info.book_id = product_item.product_id
-                        info.book_name = product_item.product_name
-                        await CMD.update_filter_data(FilterConfig,info)
-                        await new Promise(resolve => setTimeout(resolve,100));
-                    } 
+            if(fq_book_result.success){
+                let product_id_list = []
+                let product_key_list = []
+                for (let index = 0; index < fq_book_result.data.existingRecords.length; index++) {
+                    const fq_book_info = fq_book_result.data.existingRecords[index];
+                    product_id_list.push(fq_book_info.book_id)
+                    product_key_list[fq_book_info.book_key] = fq_book_info
                 }
-                let Ids = []
-                for (let index = 0; index < product_list_result.data.nonExistentIds.length; index++) {
-                    const product_id = product_list_result.data.nonExistentIds[index];
-                    // Ids.push(FilterList[product_id].id)
-                    task_queue.push({result:{data:FilterList[product_id]}})
-                }
-                if(Ids.length>0){
-                    console.log("Ids:",Ids)
-                    await filter_data_controllers.updateFilterDataByIds({ids:Ids},{status:1})
+                let product_list_result = await video_product_controllers.findNonExistentProductIds(product_id_list);
+                if(product_list_result.success){
+                    for (let index = 0; index < result.data.length; index++) {
+               
+                        let filter_item = result.data[index];
+
+                        let info = filter_item;
+
+                        let product_item = product_key_list[info.book_id]
+                        
+                        if(product_list_result.data.existingProductIds.has(product_item.book_id)){
+     
+                            if (!info) {
+                                throw `No filter info found for book_id: ${filter_item.book_id}`
+                            }
+                            if (!product_item) {
+                                throw `product_item is null: ${filter_item.book_id}`
+                            }
+                            info.words = product_item.words
+                            info.genre =  product_item.genre
+                            info.book_id = product_item.product_id
+                            info.book_name = product_item.product_name
+                            await CMD.update_filter_data(FilterConfig,info)
+                            await new Promise(resolve => setTimeout(resolve,100));
+                        } 
+                    }
                 }
-              
             }
             
         }

2_PRODUCT_FACTORY/hy_batch_factory.js

@@ -81,12 +81,8 @@ async function processTask(){
                 FilterList[filter_item.book_id] = filter_item
             }
 
-            console.log("ProductIds:",ProductIds)
             let product_list_result = await video_product_controllers.findNonExistentProductIds(ProductIds);
             if(product_list_result.success){
-                
-                console.log("product_list_result:",product_list_result)
-
                 for (let index = 0; index < result.data.length; index++) {
                     let filter_item = result.data[index];
                     if(product_list_result.data.existingProductIds.has(filter_item.book_id)){
@@ -122,7 +118,7 @@ async function processTask(){
                         task_queue.push({result:{data:FilterList[product_id]}})
                     }
                 }
-                console.log("remove_product_list:",remove_product_list)
+                // console.log("remove_product_list:",remove_product_list)
                 // if(Ids.length>0){
                 //     await filter_data_controllers.updateFilterDataByIds({ids:Ids},{status:1})
                 // }

2_PRODUCT_FACTORY/qm_batch_factory.js

@@ -50,7 +50,7 @@ async function processTask(){
                 for (let index = 0; index < result.data.length; index++) {
                     let filter_item = result.data[index];
                     if(product_list_result.data.existingProductIds.has(filter_item.book_id)){
-                        let info = FilterList[filter_item.book_id];
+                        let info = filter_item;
                         let product_item = getProductById(product_list_result.data.existingRecords,filter_item.book_id)
                         if (!info) {
                             throw `No filter info found for book_id: ${filter_item.book_id}`
@@ -69,14 +69,21 @@ async function processTask(){
                     }
                 }
 
+                let remove_product_list = []
+
                 for (let index = 0; index < product_list_result.data.nonExistentIds.length; index++) {
                     const product_id = product_list_result.data.nonExistentIds[index];
-                    task_queue.push({result:{data:FilterList[product_id]}})
-                }
-                if(Ids.length>0){
-                    console.log("Ids:",Ids)
-                    await filter_data_controllers.updateFilterDataByIds({ids:Ids},{status:1})
+                    if(remove_product_list[product_id]==true){
+          
+                    }else{
+                        remove_product_list[product_id] = true
+                        task_queue.push({result:{data:FilterList[product_id]}})
+                    }
                 }
+                // if(Ids.length>0){
+                //     console.log("Ids:",Ids)
+                //     await filter_data_controllers.updateFilterDataByIds({ids:Ids},{status:1})
+                // }
               
             }
             
@@ -97,11 +104,11 @@ async function processCreateTask(){
     try{
         if(task_queue.length>0){
             let task_queue_item = task_queue.pop()
+            result = task_queue_item.result
             let product = await video_product_controllers.getProductData({ book_id: task_queue_item.result.data.book_id });
             let FilterConfig = await redis_help.getKeyValue("FilterConfig")
             FilterConfig = JSON.parse(FilterConfig)
             if(!product.success){
-                result = task_queue_item.result
                 let tg_platform_id = result.data.tg_platform_id
                 if(tg_platform_id==config.platform_qimao){ //七猫
                     let qm_book_data = await require('../src/api/qm/qm_search_book').search_id(result.data.book_id)

PULL_DATA_FACTORY/PULL_DATA.js

@@ -153,6 +153,7 @@ async function process_material_list(response,page) {
                 cur_timeRange.count = cur_timeRange.count + response.data.list.length
                 cur_timeRange.cur_page = page
                 cur_day_data.get(today)[cur_timeRange.start].start_time = cur_timeRange.start_time
+                cur_day_data.get(today)[cur_timeRange.start].pull_day = cur_timeRange.pull_day
                 cur_day_data.get(today)[cur_timeRange.start].pull_time = cur_timeRange.pull_time ||{}
                 cur_day_data.get(today)[cur_timeRange.start].count = response.data.count
                 cur_day_data.get(today)[cur_timeRange.start].cur_page = page

pm2json.json

@@ -16,6 +16,8 @@
         {
             "name": "0_RECEIVE_FACTORY",
             "script": "./0_RECEIVE_FACTORY/RECEIVE_FACTORY.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -28,6 +30,8 @@
         {
             "name": "1_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -40,6 +44,8 @@
         {
             "name": "2_PRODUCT_FACTORY",
             "script": "./2_PRODUCT_FACTORY/PRODUCT_FACTORY.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -52,6 +58,8 @@
         {
             "name": "DZ_FACTORY",
             "script": "./2_PRODUCT_FACTORY/dz_batch_factory.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -64,6 +72,8 @@
         {
             "name": "FQ_FACTORY",
             "script": "./2_PRODUCT_FACTORY/fq_factory.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -76,6 +86,8 @@
         {
             "name": "HY_FACTORY",
             "script": "./2_PRODUCT_FACTORY/hy_batch_factory.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -87,7 +99,9 @@
         },
         {
             "name": "QM_FACTORY",
-            "script": "./2_PRODUCT_FACTORY/qm_factory.js",
+            "script": "./2_PRODUCT_FACTORY/qm_batch_factory.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -100,6 +114,8 @@
         {
             "name": "5_CREATE_LINK_FACTORY",
             "script": "./5_CREATE_LINK_FACTORY/CREATE_LINK_FACTORY.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -112,6 +128,8 @@
         {
             "name": "7_CHECK_AND_CREATE_URL_FACTORY",
             "script": "./7_CHECK_AND_CREATE_URL_FACTORY/CHECK_AND_CREATE_URL_FACTORY.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -124,6 +142,8 @@
         {
             "name": "MESSAGE_DISPATCH_FACTORY",
             "script": "./MESSAGE_DISPATCH/MESSAGE_DISPATCH.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -136,6 +156,8 @@
         {
             "name": "MAIN_SERVE_FACTORY_FACTORY",
             "script": "./MAIN_SERVE_FACTORY/MAIN_SERVE_FACTORY.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
@@ -148,6 +170,8 @@
         {
             "name": "PULL_DATA_FACTORY",
             "script": "./PULL_DATA_FACTORY/PULL_DATA.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "4G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",

src/data_manager/BaseModel.js

@@ -70,9 +70,6 @@ class BaseModel {
             .orderBy('id',orderBy)
             .limit(limit)
             .getQuery();
-        if(orderBy!=""){
-            console.log("sql:",sql,params)
-        }
         const results = await db.query(sql, params);
         return results || null;
     }