123

5_CREATE_LINK_FACTORY/hy_create_link_new.js

@@ -82,7 +82,7 @@ CMD.runTask = async function(data,main_info,PlatformInfo,call_back){
         let link_data  = response.data
         console.log("response:",response)
         if(link_data.code==1){
-            let params = link_data.data.fullPath.split('?')
+            let params = PlatformInfo.mini_program_platform_id==config.wx?link_data.data.wxFullPath.split('?'):link_data.data.fullPath.split('?')
             let start_page = params[0]
             let start_param = params[1]
             const linkId = start_param.match(/linkId=([^&]*)/)[1];

5_CREATE_LINK_FACTORY/hymf_create_link_new.js

@@ -81,7 +81,7 @@ CMD.runTask = async function(data,main_info,PlatformInfo,call_back){
         let link_data  = response.data
         console.log("response:",response)
         if(link_data.code==1){
-            let params = link_data.data.fullPath.split('?')
+            let params = PlatformInfo.mini_program_platform_id==config.wx?link_data.data.wxFullPath.split('?'):link_data.data.fullPath.split('?')
             let start_page = params[0]
             let start_param = params[1]
             const linkId = start_param.match(/linkId=([^&]*)/)[1];

PULL_DATA_FACTORY/PULL_DATA_NEW.js

@@ -7,12 +7,12 @@ const schedule = require('node-schedule');
 const manager = new DailyRecordManager();
 const rabbitMq = require('../src/mq/rabbit-mq');
 const helper = require('../src/helper');
-const PullDataService = require('../src/PullDataService');
+const PullDataServiceNew = require('../src/PullDataServiceNew');
 var _24HourRanges = []
 var jobs = []
 var cur_day_data = new Map()
 var cur_timeRange = null
-var pullDataService = new PullDataService(redis_help)
+var pullDataService = new PullDataServiceNew(redis_help)
 var cur_page = 0
 const messageHandler = async (msg) => {
     let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
@@ -110,65 +110,7 @@ function generate_pull_time_data(list,material_list){
     sendPullDataMQMessage(JSON.stringify({cmd:"on_recv_pull_data",data:pull_data,today:cur_timeRange.today}))
 }
 
-async function processDetailTask(task_item) {
-    try{
-        console.log(" start processDetailTask")
-        let materialId_list = task_item.materialId_list
-        let materialId_data_list = task_item.materialId_data_list
-        let response = await pullDataService.get_detail(task_item.id_list)
-        if(!response.success){
-            console.log("get_detail:",response)
-            throw {msg:response,timeRange:cur_timeRange,fun:"get_detail",materialId_list:materialId_list}
-        }
-
-        if(cur_timeRange!=null){
-            cur_timeRange.finish_count+=materialId_list.length
-            const today = cur_timeRange.today;
-            cur_day_data.get(today)[cur_timeRange.start].finish_count = cur_timeRange.finish_count
-            await manager.updateRecord(today,cur_day_data.get(today));
-        }
-
-        for (let j = 0; j < materialId_list.length; j++) {
-            const materialId = materialId_list[j];
-            for (let index = 0; index < response.data.list.length; index++) {
-                const element = response.data.list[index];
-                if(element.materialId==materialId){
-                    const info = materialId_data_list[element.materialId]
-                    if(element.hasAnchorInfo){
-                        await origin_data_controllers.createOriginData(
-                            {
-                                video_id:element.awemeId,
-                                materialId:element.materialId,
-                                video_link:element.awemeUrl,
-                                title:CMD.subTitle(element.title),
-                                publish_time:new Date(element.publishTime),
-                                kepp_num:info.kepp_num,
-                                comment_num:info.comment_num,
-                                like_num:info.like_num,
-                                shared_num:info.shared_num,
-                                is_guajian:1,
-                                guajian_link:element.landingUrl,
-                                status:0,
-                                createTime_new:new Date(element.createTime),
-                                updateTime_new:new Date(element.updateTime),
-                                create_day:new Date(element.createTime)
-                            }
-                        )
-                    }
-                    break
-                }
-            }
-            
-        }
 
-        generate_pull_time_data(response.data.list,task_item.list)
-        console.log(" finish processDetailTask")
-    }catch (error) {
-        console.error("processDetailTask:",error)
-    }finally{
-        // global.setTimeout(processDetailTask,500)
-    }
-}
 
 async function process_material_list(response,page) {
     try{
@@ -236,20 +178,23 @@ async function processTask(){
         FilterConfig = JSON.parse(FilterConfig)
         let response = await pullDataService.get_novel_material_list(cur_timeRange,cur_page,500)
 
-        if(response.success){
+        if(response.msg=="success"){
             if(response.data.list.length<=0){
                 cur_page = -1;
                 throw response
             }
+            let last_data = response.data.list[response.data.list.length-1]
+            cur_page  = new Date(last_data["updateTime"]);
+            let endTime = new Date(cur_timeRange.pull_day);
+            if(cur_page>=endTime){
+                cur_page = -1;
+                throw response
+            }
         }else{
             cur_page = -1;
             throw response
         }
-    
-        let detail_item =  await process_material_list(response,cur_page,cur_timeRange)
-
-        await processDetailTask(detail_item)
-
+        sendPullDataMQMessage(JSON.stringify({cmd:"on_recv_pull_data",data:response.data.list,today:cur_timeRange.today}))
         // if(detail_item.materialId_list.length<=0){
 
         // }else{
@@ -309,7 +254,6 @@ CMD.init = async function(){
                 cur_day_data.set(key,record_res.content)
             }
         }
-        await pullDataService.getToken()
          // 启动调度器
         CMD.startScheduler(taskCallback);
     })
@@ -546,7 +490,6 @@ CMD.startScheduler = function(taskCallback) {
             try {
                 console.log(`开始执行任务: ${timeRange.name} ${timeRange.start}`);
                
-                await pullDataService.getToken()
 
                 // 执行回调函数
                 await taskCallback({
@@ -575,9 +518,9 @@ CMD.startScheduler = function(taskCallback) {
 async function sendPullDataMQMessage(message,routingKey = "on_recv_pull_data") {
     try {
         if(routingKey!=null){
-            await rabbitMq.producerDirectMsg( message,"exchange_pull_data_system",routingKey);
+            await rabbitMq.producerDirectMsg( message,"exchange_pull_data_system_new",routingKey);
         }else{
-            await rabbitMq.producerDirectMsg( message,"exchange_pull_data_system");
+            await rabbitMq.producerDirectMsg( message,"exchange_pull_data_system_new");
         }
         console.log('消息发送成功');
     } catch (error) {
@@ -585,6 +528,6 @@ async function sendPullDataMQMessage(message,routingKey = "on_recv_pull_data") {
     }
 }
 
-if(!config.isDebug){
+if(config.isDebug){
     CMD.init()
 }

PULL_TIME_DATA/PULL_TIME_DATA_NEW.js

@@ -0,0 +1,169 @@
+const CMD = {}
+const rabbitMq = require('../src/mq/rabbit-mq');
+const helper = require('../src/helper');
+const mysql = require('mysql2/promise');
+const crypto = require('crypto');
+const config = require('../etc/config.json')
+const dbConfig = config.isDebug?config.debug_mysql:config.release_mysql
+
+// 时间戳转MySQL datetime格式
+function timestampToMySQLDatetime(timestamp) {
+    if (!timestamp) return null;
+    // 将毫秒时间戳转换为Date对象
+    const date = new Date(parseInt(timestamp));
+    // 格式化为MySQL datetime格式
+    return date.toISOString().slice(0, 19).replace('T', ' ');
+}
+
+// 创建表的SQL模板（保持不变）
+const createTableSQL = (tableName) => `
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+        id BIGINT NOT NULL AUTO_INCREMENT,
+        materialId VARCHAR(64) DEFAULT NULL,
+        awemeId VARCHAR(64) DEFAULT NULL,
+        title VARCHAR(500) DEFAULT NULL,
+        awemeUrl VARCHAR(500) DEFAULT NULL,
+        publishTime DATETIME DEFAULT NULL,
+        hasAnchorInfo TINYINT(1) DEFAULT NULL,
+        landingUrl VARCHAR(2048) DEFAULT NULL,
+        createTime DATETIME DEFAULT NULL,
+        updateTime DATETIME DEFAULT NULL,
+        created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        likeCount INT(11) DEFAULT NULL,
+        commentCount INT(11) DEFAULT NULL,
+        shareCount INT(11) DEFAULT NULL,
+        favoriteCount INT(11) DEFAULT NULL,
+        PRIMARY KEY (id),
+        KEY idx_material_id (awemeId),
+        KEY idx_aweme_id_like_id (awemeId, likeCount, id)
+    ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='COLUMNAR=1'
+`;
+
+// 生成MD5（保持不变）
+function generateMD5(text) {
+    return crypto.createHash('md5').update(text).digest('hex');
+}
+
+// 确保表存在（保持不变）
+async function ensureTable(connection, tableName) {
+    try {
+        await connection.query(`CREATE DATABASE IF NOT EXISTS ${dbConfig.database}`);
+        await connection.query(`USE ${dbConfig.database}`);
+        const sql = createTableSQL(tableName);
+        await connection.query(sql);
+    } catch (error) {
+        console.error(`创建表失败: ${error.message}`);
+        throw error;
+    }
+}
+
+
+// 处理视频标题（保持不变）
+async function processVideoTitles(connection, titles) {
+    if (!titles || titles.length === 0) return;
+    const titleData = titles.map(title => ({
+        title,
+        md5_tag: generateMD5(title)
+    }));
+
+    try {
+        const values = titleData.map(data => [data.title, data.md5_tag]);
+        const insertSQL = `
+            INSERT IGNORE INTO origin_video_titles 
+            (title, md5_tag) 
+            VALUES ?
+        `;
+        const [result] = await connection.query(insertSQL, [values]);
+        console.log(`处理了 ${titles.length} 个标题，新增 ${result.affectedRows} 条记录`);
+    } catch (error) {
+        console.error('处理视频标题失败:', error);
+        throw error;
+    }
+}
+
+const insertOriginData = async(connection,values) =>{
+    const insertSQL = `
+    INSERT INTO origin_data
+    (video_id, materialId, title, video_link, publish_time, 
+     is_guajian, guajian_link, createTime, create_time,
+     like_num, comment_num, shared_num, kepp_num)
+    VALUES ?
+    ON DUPLICATE KEY UPDATE 
+        video_id = VALUES(video_id)
+    `;
+    
+    await connection.query(insertSQL, [values]);
+};
+// 处理消息的主函数（修改时间戳处理部分）
+const messageHandler = async (msg) => {
+    let message = JSON.parse(msg)
+    if(message.cmd!="on_recv_pull_data"){
+        return
+    }
+    let today = message.today
+    // let date = helper.getLocalDate().replace(/-/g, '');
+    let date = today.replace(/-/g, '');
+    let table_name = `pull_data_${date}`;
+    let connection = null
+    try {
+        connection = await mysql.createConnection({
+            ...dbConfig,
+            multipleStatements: true
+        });
+        
+        await ensureTable(connection, table_name);
+        const data = message.data;
+        await processVideoTitles(connection, data.map(item => item.contentTitle));
+        
+        // 转换时间戳为MySQL datetime格式
+        const values = data.map(item => [
+            item.contentId,
+            item.contentId,
+            item.contentTitle,
+            item.contentUrl,
+            new Date(item.publishTime),  // 转换时间格式
+            item.landingUrlType!=0,
+            decodeURIComponent(item.landingInfo.landingUrl),
+            new Date(item.updateTime),   // 转换时间格式
+            new Date(item.updateTime),    // 转换时间格式
+            item.likeNum,
+            item.commentNum,
+            item.shareNum,
+            item.collectNum
+        ]);
+        
+        const insertSQL = `
+        INSERT INTO ${table_name}
+        (materialId, awemeId, title, awemeUrl, publishTime, 
+         hasAnchorInfo, landingUrl, createTime, updateTime,
+         likeCount, commentCount, shareCount, favoriteCount)
+        VALUES ?
+        `;
+        
+        await insertOriginData(connection,values)
+        await connection.query(insertSQL, [values]);
+
+        
+        console.log(`成功插入 ${values.length} 条记录到表 ${table_name}`);
+    } catch (error) {
+        console.error('处理消息失败:', error);
+    } finally{
+        if(connection!=null){
+            await connection.end();
+        }
+    }
+};
+
+async function startConsumer() {
+    try {
+        await rabbitMq.consumerDirectMsg(messageHandler, "exchange_pull_data_system_new", "on_recv_pull_data");
+    } catch (error) {
+        console.error('启动消费者失败:', error);
+    }
+}
+
+CMD.init = function() {
+    startConsumer();
+}
+CMD.init()
+module.exports = CMD

logs/combined.log

@@ -1,4 +1,17 @@
+<<<<<<< HEAD
 2025-03-30T13:34:46: 
 2025-03-30T13:34:46: > heiyan@1.0.0 start
 2025-03-30T13:34:46: > node tg_factory_main.js
 2025-03-30T13:34:46: 
+=======
+2025-03-31T16:33:10: 
+2025-03-31T16:33:10: > heiyan@1.0.0 start
+2025-03-31T16:33:10: > node tg_factory_main.js
+2025-03-31T16:33:10: 
+2025-03-31T16:33:15: Connected to Redis
+2025-03-31T16:40:27: 
+2025-03-31T16:40:27: > heiyan@1.0.0 start
+2025-03-31T16:40:27: > node tg_factory_main.js
+2025-03-31T16:40:27: 
+2025-03-31T16:40:31: Connected to Redis
+>>>>>>> 598ad81756fb6af3eb5e57791415f9a36f81b297

logs/out.log

@@ -1,4 +1,17 @@
+<<<<<<< HEAD
 2025-03-30T13:34:46: 
 2025-03-30T13:34:46: > heiyan@1.0.0 start
 2025-03-30T13:34:46: > node tg_factory_main.js
 2025-03-30T13:34:46: 
+=======
+2025-03-31T16:33:10: 
+2025-03-31T16:33:10: > heiyan@1.0.0 start
+2025-03-31T16:33:10: > node tg_factory_main.js
+2025-03-31T16:33:10: 
+2025-03-31T16:33:15: Connected to Redis
+2025-03-31T16:40:27: 
+2025-03-31T16:40:27: > heiyan@1.0.0 start
+2025-03-31T16:40:27: > node tg_factory_main.js
+2025-03-31T16:40:27: 
+2025-03-31T16:40:31: Connected to Redis
+>>>>>>> 598ad81756fb6af3eb5e57791415f9a36f81b297

src/PullDataServiceNew.js

@@ -5,44 +5,18 @@ class PullDataServiceNew {
     constructor(redis_help) {
         this.mRedis = redis_help
         this.session = axios.create({
-            baseURL: config.isDebug?config.pull_data_config.debug_host:config.pull_data_config.release_host,
+            baseURL: config.isDebug?config.pull_data_new_config.debug_host:config.pull_data_new_config.release_host,
             headers: {
               'Accept': 'application/json, text/plain, */*',
+              'api-key': config.pull_data_new_config['api-key'],
+              'api-secret': config.pull_data_new_config['api-secret']
             }
         });
         this.token = ""
     }
 
 
-    async getToken() {
-        try {
-            let maxRetries = 3
-            let delay = 1000;
-            for (let i = 0; i < maxRetries; i++) {
-                const response = await this.session.post(config.pull_data_config.get_token,{
-                    userName:config.pull_data_config.userName,
-                    verCode:config.pull_data_config.verCode,
-                    password:config.pull_data_config.release_password,
-                    loginType:config.pull_data_config.loginType
-                });
-
-                if(!response.data.success){
-                    if (i === maxRetries - 1) throw response;
-                    console.log(`Retry getToken ${i + 1} of ${maxRetries}`);
-                    await new Promise(resolve => setTimeout(resolve, delay * (i + 1)));
-                }else{
-                    this.token = response.data.data.token
-                    await this.mRedis.setKeyValue("pull_data_token",this.token)
-                    return response.data;
-                }
-            }
-        } catch (error) {
-            return  {data:null,success:false,msg:error}
-        }
-    }
-
-
-    async get_novel_material_list(cur_timeRange,page,size=500) {
+    async get_novel_material_list(cur_timeRange,n_start_time,size=500) {
         let maxRetries = 3
         let delay = 1000;
 
@@ -78,15 +52,11 @@ class PullDataServiceNew {
                 if(cur_timeRange!=null){
                     cur_timeRange.pull_time = data
                 }
-                let params = `page=${page}&size=${size}&sort=update_time,desc`
-                let url = config.pull_data_config.get_novel_material_list+params
+                let params = `?startTime=${n_start_time==0?data.startTime:n_start_time}&pageSize=500`
+                let url = config.pull_data_new_config.get_novel_material_list+params
                 console.log(url,data)
-                var response = await this.session.post(url,data, {
-                    headers: {
-                        'token':await this.mRedis.getKeyValue("pull_data_token")
-                    }
-                });
-                if(!response.data.success){
+                var response = await this.session.get(url);
+                if(response.data.msg!="success"){
                     throw response.data
                 }else{
                     return response.data;
@@ -101,39 +71,6 @@ class PullDataServiceNew {
      
     }
 
-
-    async get_detail(materialId_list) {
-
-        let maxRetries = 3
-        let delay = 1000;
-        for (let i = 0; i < maxRetries; i++) {
-            try {
-                let data = {
-                    list:materialId_list,
-                    count:false
-                }
-                // console.log("config.pull_data_config.get_detail:",data)
-                const response = await this.session.post(config.pull_data_config.get_detail,data, {
-                    headers: {
-                        'token':await this.mRedis.getKeyValue("pull_data_token")
-                    }
-                });
-                if(!response.data.success){
-                    throw response.data
-                }else{
-                    return response.data;
-                }
-            } catch (error) {
-                if (i === maxRetries - 1) return {data:null,success:false,msg:error};
-                console.log(`Retry get_detail ${i + 1} of ${maxRetries}`);
-                await new Promise(resolve => setTimeout(resolve, delay * (i + 1)));
-            }
- 
-        }
-  
-
-    }
-
 }
 
 module.exports = PullDataServiceNew;