123

2_PRODUCT_FACTORY/PRODUCT_FACTORY.js

@@ -212,7 +212,7 @@ function deepCopy3(obj) {
 
 CMD.update_filter_data = async function(FilterConfig,data) {
     let isGenre = CMD.isGenre(FilterConfig,data.genre)
-    console.log("data.genre:",data.genre,data,isGenre)
+    // console.log("data.genre:",data.genre,data,isGenre)
     if(isGenre){
         try{
             await CMD.video_product_material(data)
@@ -227,8 +227,8 @@ CMD.update_filter_data = async function(FilterConfig,data) {
 
         let AppInfoList = CMD.getAppInfoByPlatformId(AppConfig,data.tg_platform_id)
 
-        for (let index = 0; index < AppInfoList.length; index++) {
-            const AppInfo = AppInfoList[index];
+        for (let j = 0; j < AppInfoList.length; j++) {
+            const AppInfo = AppInfoList[j];
             data.app_id = AppInfo.app_id
 
             let PlatformInfo = await redis_help.getKeyValue("PlatformConfig")
@@ -364,28 +364,7 @@ CMD.add_fq_book = async function(data,response){
     // console.log("tab:",tab,data)
     return await get_fq_book_tab_controllers.createData( tab)
 }
-CMD.build_check_url = function(data){
-    let url = 'https://clipvideoup.s6kuwan.com/sapi/check/get_check_url';
-    const formData = new FormData();
-    formData.append('platform_id', data.book_platform);
-    formData.append('cp_spread_id', data.promotion_id);
-    // console.log("formData:", formData);
-    axios.post(url, formData, {
-        headers: {
-            'Content-Type': 'multipart/form-data'
-        }
-    })
-    .then(response => {
-        if(response.data.code==10000){
-            CMD.update_video_applet_product({promotion_id:data.promotion_id,
-                product_id:data.product_id,check_url:response.data.content.check_url
-            })
-        }
-    })
-    .catch(error => {
-        console.error('create_link was an error!', error);
-    });
-}
+
 
 CMD.update_video_applet_product = function(data){
      sql_task_help.update('UPDATE video_applet_product SET status = ?, wait_status = ?  , check_url = ?   WHERE promotion_id = ? and product_id = ?', 

5_CREATE_LINK_FACTORY/hy_create_link.js

@@ -65,7 +65,7 @@ function get_recharge_template(book_id,chongzhi_name,call_back){
 }
 CMD.runTask = function(data,main_info,PlatformInfo,call_back){
   let group_id = main_info.group_id
-  if(group_id==null){
+  if(group_id==null||group_id==""){
     group_id = "35"
   }
   let  tg_link_config = JSON.parse(main_info.tg_link_config)

MESSAGE_DISPATCH/MESSAGE_DISPATCH.js

@@ -96,6 +96,21 @@ CMD.updateBlackBooks = async function(){
       console.error('all_black_books error!', error);
     });
 }
+
+CMD.getPullDataConfig = async function() {
+    const postData = {
+        cmd:"pull_data_config",
+        fun:"get_all_pull_data_list",
+        data:{}
+    };
+    axios.post('http://127.0.0.1:9100/tg/back/api', postData)
+    .then(response => {
+        redis_help.setKeyValue("PullDataConfig",JSON.stringify(response.data.data))
+    })
+    .catch(error => {
+      console.error('getPullDataConfig error!', error);
+    });
+}
 var ws_isOpen = false
 CMD.myListener = {
     onOpen: function (webSocket, response) {
@@ -188,8 +203,15 @@ async function recvMessage(data){
                 }
 
             break;
-            case "updatePullConig":
+            case "getPullDataConfig": //获取拉取数据类配置
+                {
+                    CMD.getPullDataConfig()
+                }
+     
+            break;
+            case "updatePullConig": //同步拉取配置
                 {
+                    await CMD.getPullDataConfig()
                     const postData = {
                         cmd:"filter_task",
                         fun:json_msg.cmd,

PULL_DATA_FACTORY/PULL_DATA.js

@@ -2,10 +2,7 @@ const CMD = {}
 const time_count = 5000;
 const redis_help = require('../src/use_redis');
 const origin_data_controllers = require('../src/data_manager/Controllers/origin_data_controllers');
-const pull_log_controllers = require('../src/data_manager/Controllers/pull_log_controllers');
-const helper = require('../src/helper');
 const axios = require('axios')
-const HttpClient = require('../src/HttpClient')
 const config = require('../etc/config.json');
 const DailyRecordManager = require('./daily_records');
 const schedule = require('node-schedule');
@@ -13,14 +10,60 @@ const manager = new DailyRecordManager();
 const rabbitMq = require('../src/mq/rabbit-mq');
 var session = null
 var cur_page = 0
-var task_list = []
 var _24HourRanges = []
 var jobs = []
-var cur_day_data = []
-
+var cur_day_data = new Map()
+var cur_timeRange = null
 const messageHandler = async (msg) => {
-    // CMD.start_task(msg['data']['data'])
-    console.log("msg:",msg)
+    let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
+    PullDataConfig = JSON.parse(PullDataConfig)
+    _24HourRanges = PullDataConfig
+    CMD.startScheduler(taskCallback)
+};
+
+const taskCallback = async (context) => {
+    var { timeRange, executionTime } = context;
+    
+    console.log('执行任务:', {
+        时间区间: timeRange.name,
+        开始时间: timeRange.start,
+        执行时间: executionTime
+    });
+    const today = new Date().toISOString().slice(0, 10);
+    cur_day_data.forEach((v,k)=>{
+        if(k==today){
+
+        }else{
+            cur_day_data.set(k, [])
+        }
+    })
+    if(!cur_day_data.has(today)){
+        cur_day_data.set(today,[])
+    }
+
+    if(!cur_day_data.get(today)[timeRange.start]){
+        cur_day_data.get(today)[timeRange.start]  = {name:timeRange.name}
+    }
+    console.log("cur_day_data[today]:",cur_day_data.get(today),today)
+    const existingRecord = await manager.getRecord(today);
+
+    if (existingRecord) {
+        // 如果存在今天的记录，则更新
+        const result = await manager.updateRecord(today, cur_day_data.get(today));
+        console.log("更新记录结果:", result);
+    } else {
+        // 如果不存在今天的记录，则创建新记录
+        const result = await manager.createRecord(today, cur_day_data.get(today));
+        console.log("创建新记录结果:", result);
+    }
+    // const result = await manager.createRecord(today,cur_day_data[today]);
+    // if(result.success){
+
+    // }
+    cur_page = 0
+    cur_timeRange = timeRange
+    // 示例：实际任务逻辑
+    await processTask(timeRange);
 };
 
 // 启动消费者
@@ -50,6 +93,10 @@ async function processTask(timeRange){
             cur_page = -1
             throw "没有数据了"
         }
+        if(response.result == undefined){
+            cur_page = -1
+            throw "没有数据了"
+        }
         if(response.result.list == undefined){
             cur_page = -1
             throw "没有数据了"
@@ -59,16 +106,15 @@ async function processTask(timeRange){
             throw "没有数据了"
         }
         if(timeRange!=null){
-            const today = new Date().toISOString().split('T')[0];
-            console.log("response.result:",response.result)
-            if(cur_day_data[today][timeRange.start]){
+            const today = new Date().toISOString().slice(0, 10);
+            console.log("response.result:",response.result,today)
+            if(cur_day_data.get(today)[timeRange.start]){
                 console.log("response.result.count:",response.result.count)
-                console.log("(cur_day_data[today][timeRange.start]:",cur_day_data[today][timeRange.start])
-                cur_day_data[today][timeRange.start].count = response.result.count
+                console.log("(cur_day_data[today][timeRange.start]:",cur_day_data.get(today)[timeRange.start])
+                cur_day_data.get(today)[timeRange.start].count = response.result.count
             }else{
                 console.log("cur_day_data空")
             }
-    
             const updateResult = await manager.updateRecord(today,cur_day_data[today]);
         }
 
@@ -125,7 +171,7 @@ async function processTask(timeRange){
         // console.log("get_detail:",response)
         // await pull_log_controllers.createData()
     }catch(e){
-        console.log("PULL_DATA: error:",e)
+        console.error("PULL_DATA: error:",e)
     }finally{
         if(cur_page!=-1){
             cur_page++;
@@ -172,8 +218,14 @@ CMD.getToken = async function(){
 
 CMD.get_novel_material_list = async function(page,size=500) {
     const now = new Date(); // 当前时间
+
+    let interval_minute = 60;
+    
+    if(cur_timeRange!=null){
+        interval_minute = cur_timeRange.interval_minute
+    }
     // const fiveMinutesAgo = new Date(now - 5 * 60 * 1000); // 5分钟前的时间
-    const fiveMinutesAgo = new Date(now - 60 * 60 * 1000); // 60分钟前的时间
+    const fiveMinutesAgo = new Date(now - interval_minute * interval_minute * 1000); // 60分钟前的时间
     let data = {
         startTime:fiveMinutesAgo.getTime(),
         endTime:now.getTime()
@@ -212,51 +264,14 @@ CMD.init = async function(){
 
     // await startConsumer();
     redis_help.connect(async ()=>{
+        startConsumer()
+        let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
+        PullDataConfig = JSON.parse(PullDataConfig)
         let response = await CMD.getToken()
         await redis_help.setKeyValue("pull_data_token",response.result.token)
         // processTask()
-        _24HourRanges = CMD.generate24HourRanges()
+        _24HourRanges = PullDataConfig
         // 定义任务回调函数
-        const taskCallback = async (context) => {
-            var { timeRange, executionTime } = context;
-            
-            console.log('执行任务:', {
-                时间区间: timeRange.name,
-                开始时间: timeRange.start,
-                执行时间: executionTime
-            });
-            const today = new Date().toISOString().split('T')[0];
-            for (const key in cur_day_data) {
-                if (Object.prototype.hasOwnProperty.call(cur_day_data, key)) {
-                    if(key==today){
-
-                    }else{
-                        cur_day_data[key] = []
-                    }
-                }
-            }
-            if(!cur_day_data[today]){
-                cur_day_data[today] = {}
-            }
-
-            if(!cur_day_data[today][timeRange.start]){
-                cur_day_data[today][timeRange.start]  = {name:timeRange.name}
-            }
-            console.log("cur_day_data[today]:",cur_day_data[today])
-            const result = await manager.createRecord(today,cur_day_data[today]);
-            if(result.success){
-
-            }
-            cur_page = 0
-            // 示例：实际任务逻辑
-            await processTask(timeRange);
-        };
-
-        let key = new Date().toISOString().split('T')[0]
-        let record_res =  await manager.getRecord(key);
-        if(record_res!=null){
-            cur_day_data[key]  = record_res.content
-        }
          // 启动调度器
         CMD.startScheduler(taskCallback);
 

logs/combined.log

@@ -718,3 +718,33 @@
 2024-12-10T19:01:05: 
 2024-12-10T19:01:07: Connected to Redis
 2024-12-10T19:01:07: results: null
+2024-12-10T23:36:51: 
+2024-12-10T23:36:51: > heiyan@1.0.0 start
+2024-12-10T23:36:51: > node tg_factory_main.js
+2024-12-10T23:36:51: 
+2024-12-10T23:36:56: Connected to Redis
+2024-12-10T23:36:56: results: false
+2024-12-10T23:38:23: 
+2024-12-10T23:38:23: > heiyan@1.0.0 start
+2024-12-10T23:38:23: > node tg_factory_main.js
+2024-12-10T23:38:23: 
+2024-12-10T23:38:29: Connected to Redis
+2024-12-10T23:38:29: results: false
+2024-12-10T23:43:19: 
+2024-12-10T23:43:19: > heiyan@1.0.0 start
+2024-12-10T23:43:19: > node tg_factory_main.js
+2024-12-10T23:43:19: 
+2024-12-10T23:43:24: Connected to Redis
+2024-12-10T23:43:24: results: false
+2024-12-10T23:46:32: 
+2024-12-10T23:46:32: > heiyan@1.0.0 start
+2024-12-10T23:46:32: > node tg_factory_main.js
+2024-12-10T23:46:32: 
+2024-12-10T23:46:38: Connected to Redis
+2024-12-10T23:46:38: results: false
+2024-12-10T23:49:06: 
+2024-12-10T23:49:06: > heiyan@1.0.0 start
+2024-12-10T23:49:06: > node tg_factory_main.js
+2024-12-10T23:49:06: 
+2024-12-10T23:49:12: Connected to Redis
+2024-12-10T23:49:12: results: false

logs/out.log

@@ -1180,3 +1180,33 @@
 2024-12-10T19:01:05: 
 2024-12-10T19:01:07: Connected to Redis
 2024-12-10T19:01:07: results: null
+2024-12-10T23:36:51: 
+2024-12-10T23:36:51: > heiyan@1.0.0 start
+2024-12-10T23:36:51: > node tg_factory_main.js
+2024-12-10T23:36:51: 
+2024-12-10T23:36:56: Connected to Redis
+2024-12-10T23:36:56: results: false
+2024-12-10T23:38:23: 
+2024-12-10T23:38:23: > heiyan@1.0.0 start
+2024-12-10T23:38:23: > node tg_factory_main.js
+2024-12-10T23:38:23: 
+2024-12-10T23:38:29: Connected to Redis
+2024-12-10T23:38:29: results: false
+2024-12-10T23:43:19: 
+2024-12-10T23:43:19: > heiyan@1.0.0 start
+2024-12-10T23:43:19: > node tg_factory_main.js
+2024-12-10T23:43:19: 
+2024-12-10T23:43:24: Connected to Redis
+2024-12-10T23:43:24: results: false
+2024-12-10T23:46:32: 
+2024-12-10T23:46:32: > heiyan@1.0.0 start
+2024-12-10T23:46:32: > node tg_factory_main.js
+2024-12-10T23:46:32: 
+2024-12-10T23:46:38: Connected to Redis
+2024-12-10T23:46:38: results: false
+2024-12-10T23:49:06: 
+2024-12-10T23:49:06: > heiyan@1.0.0 start
+2024-12-10T23:49:06: > node tg_factory_main.js
+2024-12-10T23:49:06: 
+2024-12-10T23:49:12: Connected to Redis
+2024-12-10T23:49:12: results: false