123

1_FILTER_FACTORY/FILTER_FACTORY.js

@@ -1,57 +1,112 @@
 const CMD = {}
-const time_count = 100;
+const time_count = 1000;
 const redis_help = require('../src/use_redis');
 const origin_data_controllers = require('../src/data_manager/Controllers/origin_data_controllers');
 const filter_data_controllers = require('../src/data_manager/Controllers/filter_data_controllers');
 const helper = require('../src/helper');
+async function createFilterData(result,FilterConfig) {
+    let filter_data = filter_and_insert(result.data,FilterConfig)
+    if(filter_data!=null){
+        filter_data.status = 0; //设置筛选的这个状态未定义
+        let res =  await filter_data_controllers.createFilterData({
+            video_id:filter_data.video_id,
+            video_link:filter_data.video_link,
+            title:result.data.title,
+            publish_time:result.data.publish_time,
+            kepp_num:filter_data.kepp_num,
+            comment_num:filter_data.comment_num,
+            like_num:filter_data.like_num,
+            shared_num:filter_data.shared_num,
+            is_guajian:result.data.is_guajian,
+            book_id:filter_data.book_id,
+            book_name:"",
+            genre:0,
+            words:0,
+            guajian_link:result.data.guajian_link,
+            tg_platform_id:filter_data.tg_platform_id,
+            source:filter_data.source,
+            status:0,
+        })
+        console.log("新增一条数据:",res.data.id)
+    }
+}
 
 async function processTask(){
     try{
         let FilterConfig = await redis_help.getKeyValue("FilterConfig")
         FilterConfig = JSON.parse(FilterConfig)
-        const result = await origin_data_controllers.getOriginDataByStatus({status:0})
+
+        const result = await origin_data_controllers.getOriginDataByStatusAndLimit({status:0},500)
         if(result.success){
-            console.log("getOriginDataByStatus:",result.data.video_id)
-            let filter_result = await filter_data_controllers.getFilterData({video_id:result.data.video_id})
+            let origin_data_list = result.data
+            let videoIds = []
+            let videoDataList = []
+            let Ids = []
+            for (let index = 0; index < origin_data_list.length; index++) {
+                const origin_data_item = origin_data_list[index];
+                videoIds.push(origin_data_item.video_id)
+                videoDataList[origin_data_item.video_id] = origin_data_item
+                Ids.push(origin_data_item.id)
+            }
+            let filter_result =  await filter_data_controllers.findNonExistentVideoIDs(videoIds)
             if(filter_result.success){
-                await filter_data_controllers.updateFilterData({id:filter_result.data.id},{
-                    like_num:result.data.like_num,
-                    kepp_num:result.data.kepp_num,
-                    shared_num:result.data.shared_num,
-                    comment_num:result.data.comment_num
-                })
-            }else{
-
-                let filter_data = filter_and_insert(result.data,FilterConfig)
-                if(filter_data!=null){
-                    filter_data.status = 0; //设置筛选的这个状态未定义
-                    let res =  await filter_data_controllers.createFilterData({
-                        video_id:filter_data.video_id,
-                        video_link:filter_data.video_link,
-                        title:result.data.title,
-                        publish_time:result.data.publish_time,
-                        kepp_num:filter_data.kepp_num,
-                        comment_num:filter_data.comment_num,
-                        like_num:filter_data.like_num,
-                        shared_num:filter_data.shared_num,
-                        is_guajian:result.data.is_guajian,
-                        book_id:filter_data.book_id,
-                        book_name:"",
-                        genre:0,
-                        words:0,
-                        guajian_link:result.data.guajian_link,
-                        tg_platform_id:filter_data.tg_platform_id,
-                        source:filter_data.source,
-                        status:0,
-                    })
-                    console.log("新增一条数据:",res.data.id)
+                let nonExistentIds = filter_result.data.nonExistentIds
+                for (let index = 0; index < nonExistentIds.length; index++) {
+                    const video_id = nonExistentIds[index];
+                    let filter_data = videoDataList[video_id]
+                    await createFilterData({data:filter_data},FilterConfig)
+                    await new Promise(resolve => setTimeout(resolve, 100));
                 }
+                console.log("nonExistentIds:",nonExistentIds)
+            }
+            let update_result =  await origin_data_controllers.updateOriginDataByIds({ids:Ids},{status:0})
+            if(update_result.success){
+                console.log(update_result.message)
             }
-
-            origin_data_controllers.updateOriginData({id:result.data.id},{
-                status:1
-            })
         }
+        // const result = await origin_data_controllers.getOriginDataByStatus({status:0})
+        // if(result.success){
+        //     console.log("getOriginDataByStatus:",result.data.video_id)
+        //     let filter_result = await filter_data_controllers.getFilterData({video_id:result.data.video_id})
+        //     if(filter_result.success){
+        //         await filter_data_controllers.updateFilterData({id:filter_result.data.id},{
+        //             like_num:result.data.like_num,
+        //             kepp_num:result.data.kepp_num,
+        //             shared_num:result.data.shared_num,
+        //             comment_num:result.data.comment_num
+        //         })
+        //     }else{
+
+        //         let filter_data = filter_and_insert(result.data,FilterConfig)
+        //         if(filter_data!=null){
+        //             filter_data.status = 0; //设置筛选的这个状态未定义
+        //             let res =  await filter_data_controllers.createFilterData({
+        //                 video_id:filter_data.video_id,
+        //                 video_link:filter_data.video_link,
+        //                 title:result.data.title,
+        //                 publish_time:result.data.publish_time,
+        //                 kepp_num:filter_data.kepp_num,
+        //                 comment_num:filter_data.comment_num,
+        //                 like_num:filter_data.like_num,
+        //                 shared_num:filter_data.shared_num,
+        //                 is_guajian:result.data.is_guajian,
+        //                 book_id:filter_data.book_id,
+        //                 book_name:"",
+        //                 genre:0,
+        //                 words:0,
+        //                 guajian_link:result.data.guajian_link,
+        //                 tg_platform_id:filter_data.tg_platform_id,
+        //                 source:filter_data.source,
+        //                 status:0,
+        //             })
+        //             console.log("新增一条数据:",res.data.id)
+        //         }
+        //     }
+
+        //     origin_data_controllers.updateOriginData({id:result.data.id},{
+        //         status:1
+        //     })
+        // }
     }catch(e){
         console.log("筛选数据错误：",e)
     } finally{

src/data_manager/BaseModel.js

@@ -69,7 +69,7 @@ class BaseModel {
             .limit(limit)
             .getQuery();
         const results = await db.query(sql, params);
-        return results[0] || null;
+        return results || null;
     }
 
     async findAll(conditions = {}, limit = null, offset = 0, orderBy = null) {

src/data_manager/Controllers/filter_data_controllers.js

@@ -1,6 +1,61 @@
 const filter_data_model =  require('../Model/filter_data_model');
 
 class filter_data_controllers {
+
+    async findNonExistentVideoIDs(videoIds) {
+        try {
+            // 参数验证
+            if (!videoIds) {
+                throw new Error('videoIds parameter is required');
+            }
+            if (!Array.isArray(videoIds)) {
+                throw new Error('videoIds must be an array');
+            }
+            if (videoIds.length === 0) {
+                throw new Error('videoIds array cannot be empty');
+            }
+            if (!videoIds.every(id => typeof id === 'string' || typeof id === 'number')) {
+                throw new Error('All video IDs must be strings or numbers');
+            }
+    
+            // 限制最大处理数量为500
+            const maxIds = videoIds.slice(0, 500);
+    
+            // 构建 IN 查询条件
+            const placeholders = maxIds.map(() => '?').join(',');
+            const sql = `SELECT video_id FROM ?? WHERE video_id IN (${placeholders})`;
+            
+            // 执行查询
+            const existingRecords = await filter_data_model.query(sql, [filter_data_model.tableName, ...maxIds]);
+    
+            // 获取已存在的video_id集合
+            const existingVideoIDs = new Set(
+                existingRecords.map(record => record.video_id)
+            );
+    
+            // 找出不存在的video_id
+            const nonExistentVideoIDs = maxIds.filter(
+                id => !existingVideoIDs.has(id)
+            );
+    
+            return {
+                success: true,
+                data: {
+                    total: maxIds.length,
+                    existing: existingVideoIDs.size,
+                    nonExistent: nonExistentVideoIDs.length,
+                    nonExistentIds: nonExistentVideoIDs
+                }
+            };
+    
+        } catch (error) {
+            console.error('Error in findNonExistentVideoIDs:', error);
+            return {
+                success: false,
+                error: error.toString()  // 确保错误消息始终是字符串
+            };
+        }
+    }
     /**
      * 获取过滤数据
      * @param {Object} params - 请求参数

src/data_manager/Controllers/origin_data_controllers.js

@@ -213,6 +213,55 @@ class origin_data_controllers {
     }
 
 
+    async updateOriginDataByIds(params, tableData) {
+        try {
+            // 参数验证
+            if (!params?.ids || !Array.isArray(params.ids) || params.ids.length === 0) {
+                throw new Error('ids must be a non-empty array');
+            }
+    
+            if (!tableData || Object.keys(tableData).length === 0) {
+                throw new Error('No data to update');
+            }
+    
+            const ids = params.ids;
+    
+            // 构建批量更新SQL
+            const updateFields = Object.entries(tableData)
+                .map(([key, value]) => `${key} = ?`)
+                .join(', ');
+                
+            const placeholders = ids.map(() => '?').join(',');
+            
+            const sql = `
+                UPDATE origin_data 
+                SET ${updateFields}
+                WHERE id IN (${placeholders})
+            `;
+    
+            // 构建参数数组：先是set的值，然后是where in的值
+            const values = [
+                ...Object.values(tableData),
+                ...ids
+            ];
+    
+            // 执行更新
+            const result = await origin_data_model.query(sql, values);
+    
+            return {
+                success: true,
+                message: `Successfully updated ${result.affectedRows} records`,
+                data: {
+                    affectedRows: result.affectedRows
+                }
+            };
+    
+        } catch (error) {
+            console.error('Error in updateOriginData:', error);
+            throw error;
+        }
+    }
+
     async updateOriginData(params, tableData) {
         try {
             // 参数验证
@@ -226,17 +275,6 @@ class origin_data_controllers {
 
             const id = params.id;
 
-            // 检查数据是否存在
-            // const existingData = await origin_data_model.findOne({ video_id:video_id });
-            // if (!existingData) {
-            //     throw 'Filter data not found';
-            // }
-
-            // 添加更新时间戳
-            // const dataToUpdtableDataate = {
-            //     ...,
-            // };
-
             // 使用乐观锁更新
             const affectedRows = await origin_data_model.updateOriginData(
                 id,