123

1_FILTER_FACTORY/FILTER_FACTORY.js

@@ -88,7 +88,7 @@ async function processTask(){
         let FilterConfig = await redis_help.getKeyValue("FilterConfig")
         FilterConfig = JSON.parse(FilterConfig)
         let create_day = helper.getLocalDate()
-        const result = await origin_data_controllers.getOriginDataByStatusAndLimitCreateDay({status:0,create_day:create_day},2000)
+        const result = await origin_data_controllers.getOriginDataByStatusAndLimitCreateDay({status:0,create_day:""},2000,` create_day != ${create_day}  AND `)
         // const result = await origin_data_controllers.getOriginDataByStatusAndLimit({status:0},500)
         if(result.success){
             let origin_data_list = result.data

1_FILTER_FACTORY/FILTER_FACTORY0.js

@@ -0,0 +1,498 @@
+const CMD = {}
+const time_count = 1000;
+const redis_help = require('../src/use_redis');
+const origin_data_controllers = require('../src/data_manager/Controllers/origin_data_controllers');
+const filter_data_controllers = require('../src/data_manager/Controllers/filter_data_controllers');
+const video_product_controllers = require('../src/data_manager/Controllers/video_product_controllers');
+const get_fq_mf_book_tab_controllers = require('../src/data_manager/Controllers/get_fq_mf_book_tab_controllers');
+const get_fq_book_tab_controllers = require('../src/data_manager/Controllers/get_fq_book_tab_controllers');
+const helper = require('../src/helper');
+const config = require('../etc/config.json')
+async function createFilterData(result,FilterConfig) {
+    
+    let PlatformConfig = await redis_help.getKeyValue("PlatformConfig")
+
+    PlatformConfig = JSON.parse(PlatformConfig)
+
+    let filter_data = filter_and_insert(result.data,FilterConfig,PlatformConfig)
+    if(filter_data!=null){
+        console.log("filter_data:",filter_data)
+        if(filter_data.book_id==undefined||filter_data.book_id==null){
+            return null
+        }
+        filter_data.status = 0; //设置筛选的这个状态未定义
+        let book_name = ""
+        let genre = 0;
+        let words = 0;
+        let book_id = filter_data.book_id;
+        let status = 0;
+        if(filter_data.tg_platform_id!=config.platform_fanqie&&filter_data.tg_platform_id!=config.platform_fanqiemf){
+            let book_info =  await video_product_controllers.getProductData({book_id:filter_data.book_id})
+            if(book_info.success){
+                book_name = book_info.data.product_name
+                genre = book_info.data.genre
+                words = book_info.data.words
+                status = 1
+            }
+        }else{
+            if(filter_data.tg_platform_id==config.platform_fanqiemf){
+                let fq_book_result = await get_fq_mf_book_tab_controllers.getData({ book_id: book_id });
+                if(fq_book_result.success){
+                    book_id = fq_book_result.data.book_id
+                    book_name = fq_book_result.data.info.book_name
+                    genre = fq_book_result.data.info.genre
+                    words = fq_book_result.data.info.words==undefined?0:fq_book_result.data.info.words
+                    status = 1
+                }
+            }else if(filter_data.tg_platform_id==config.platform_fanqie){
+                let fq_book_result = await get_fq_book_tab_controllers.getData({ book_id: book_id });
+                if(fq_book_result.success){//如果番茄书库里有这本书
+                    book_id = fq_book_result.data.book_id
+                    genre = fq_book_result.data.info.genre
+                    book_name = fq_book_result.data.info.book_name
+                    words = fq_book_result.data.info.words
+                    status = 1
+                }
+            }
+        }
+
+        let res =  await filter_data_controllers.createFilterData({
+            video_id:filter_data.video_id,
+            video_link:filter_data.video_link,
+            title:result.data.title,
+            publish_time:result.data.publish_time,
+            kepp_num:filter_data.kepp_num,
+            comment_num:filter_data.comment_num,
+            like_num:filter_data.like_num,
+            shared_num:filter_data.shared_num,
+            is_guajian:result.data.is_guajian,
+            book_id:book_id,
+            book_name:book_name,
+            genre:genre,
+            words:words,
+            guajian_link:result.data.guajian_link,
+            tg_platform_id:filter_data.tg_platform_id,
+            source:filter_data.source,
+            status:status,
+            material_sync_status:0,
+            create_day: new Date(helper.getLocalDate())
+        })
+        console.log("新增一条数据:",res.data.id)
+    }
+    return filter_data
+}
+
+
+async function processTask(){
+    try{
+        let FilterConfig = await redis_help.getKeyValue("FilterConfig")
+        FilterConfig = JSON.parse(FilterConfig)
+        let create_day = helper.getLocalDate()
+        const result = await origin_data_controllers.getOriginDataByStatusAndLimitCreateDay({status:0,create_day:"2025-05-21"},1000," id % 10 = 0 AND ")
+        // const result = await origin_data_controllers.getOriginDataByStatusAndLimit({status:0},500)
+        if(result.success){
+            let origin_data_list = result.data
+            if(origin_data_list.length<=0){
+                throw "no data!"
+            }
+            let videoIds = []
+            let videoDataList = []
+            let Ids = []
+            let wxIds = []
+            for (let index = 0; index < origin_data_list.length; index++) {
+                const origin_data_item = origin_data_list[index];
+                videoDataList[origin_data_item.video_id] = origin_data_item
+                if(helper.checkChannel(origin_data_item.guajian_link,"https://api.amemv.com/insights/lynx_web/anchor/preview?")){
+                    wxIds.push(origin_data_item.id)
+                }else{
+                    Ids.push(origin_data_item.id)
+                }
+            }
+
+            for (const video_id in videoDataList) {
+                if (Object.prototype.hasOwnProperty.call(videoDataList, video_id)) {
+                    videoIds.push(video_id)
+                }
+            }
+            let filter_result =  await filter_data_controllers.findNonExistentVideoIDs(videoIds)
+            if(filter_result.success){
+                let nonExistentIds = filter_result.data.nonExistentIds
+                for (let index = 0; index < nonExistentIds.length; index++) {
+                    const video_id = nonExistentIds[index];
+                    let filter_data = videoDataList[video_id]
+                    let d = await createFilterData({data:filter_data},FilterConfig)
+                    if(d){
+                        await new Promise(resolve => setTimeout(resolve, 100));
+                    }
+                }
+                console.log("nonExistentIds:",nonExistentIds)
+            }
+            let update_result
+            if(Ids.length>0){
+                update_result =  await origin_data_controllers.updateOriginDataByIds({ids:Ids},{status:1})
+                if(update_result.success){
+                    console.log(update_result.message)
+                }
+            }
+            if(wxIds.length>0){
+                update_result =  await origin_data_controllers.updateOriginDataByIds({ids:wxIds},{status:3,is_wx_guajian:1})
+            }
+            
+        }
+    }catch(e){
+        console.log("筛选数据错误：",e)
+    } finally{
+        global.setTimeout(processTask,time_count)
+    }
+}
+
+CMD.init = function(){
+    redis_help.connect(()=>{
+
+    })
+    processTask()
+}
+function getBookId(url) {
+    try {
+        if (!url.includes('bookId=')) {
+            return null; // 或者返回其他你想要的默认值
+        }
+        const bookId = url.split('bookId=')[1].split('&')[0];
+        return bookId;
+    } catch (error) {
+        return null; // 出现任何错误都返回null
+    }
+}
+
+function getKey(url) {
+    try {
+        if (!url.includes('key=')) {
+            return null; // 或者返回其他你想要的默认值
+        }
+        const bookId = url.split('key=')[1].split('&')[0];
+        return bookId;
+    } catch (error) {
+        return null; // 出现任何错误都返回null
+    }
+}
+
+function getName(url) {
+    try {
+        if (!url.includes('name=')) {
+            return null; // 或者返回其他你想要的默认值
+        }
+        const bookId = url.split('name=')[1].split('&')[0];
+        return bookId;
+    } catch (error) {
+        return null; // 出现任何错误都返回null
+    }
+}
+
+function getId(url) {
+    try {
+        if (!url.includes('id=')) {
+            return null; // 或者返回其他你想要的默认值
+        }
+        const bookId = url.split('id=')[1].split('&')[0];
+        return bookId;
+    } catch (error) {
+        return null; // 出现任何错误都返回null
+    }
+}
+
+function getCBID(url) {
+    try {
+        if (!url.includes('cbid=')) {
+            return null; // 或者返回其他你想要的默认值
+        }
+        const bookId = url.split('cbid=')[1].split('&')[0];
+        return bookId;
+    } catch (error) {
+        return null; // 出现任何错误都返回null
+    }
+}
+
+function get_work_id(url) {
+    try {
+        if (!url.includes('work_id=')) {
+            return null; // 或者返回其他你想要的默认值
+        }
+        const bookId = url.split('work_id=')[1].split('&')[0];
+        return bookId;
+    } catch (error) {
+        return null; // 出现任何错误都返回null
+    }
+}
+
+function filter_and_insert(origin_data,tg_config,PlatformConfig){
+    if(origin_data.guajian_link.length<=0){
+        return null
+    }
+
+    if(helper.checkChannel(origin_data.guajian_link,"wx_gua_jian=1")){
+        PlatformConfig = getWxPlatformConfig(PlatformConfig)
+        let  parent_platform_id = getWxChannel(tg_config.platform_select,origin_data.guajian_link,PlatformConfig)
+        console.log("parent_platform_id:",parent_platform_id)
+        if(parent_platform_id==null){
+            return null
+        }
+        let params = {}
+        let url = origin_data.guajian_link
+        params['bookId'] =  getBookId(url);
+        params['name'] =  getName(url);
+
+        if(parent_platform_id==config.platform_fanqie){
+            params['bookId'] = getKey(url);
+        }else if(parent_platform_id==config.platform_qimao){
+            params['bookId'] = getId(url);
+            if(params['bookId']==undefined||params['bookId']==null){
+                return null
+            }
+            if(params['bookId'].length>6){
+                return null
+            }
+        }else if(parent_platform_id==config.platform_fanqiemf){
+            params['bookId'] = getId(url);
+            if(params['bookId']==undefined||params['bookId']==null){
+                return null
+            }
+            if(params['bookId'].length>6){
+                return null
+            }
+        }else if(parent_platform_id==config.platform_yuewen){
+            params['bookId'] = getCBID(url);
+            if(params['bookId']==undefined||params['bookId']==null){
+                return null
+            }
+        }
+        
+        let obj =  {video_id:origin_data.video_id,book_id:params['bookId'],source:params['name'].substring(0,params['name'].length>64?64:params['name'].length),video_link:origin_data.video_link,
+            like_num:origin_data.like_num,kepp_num:origin_data.kepp_num ,shared_num:origin_data.shared_num,comment_num:origin_data.comment_num,tg_platform_id:parent_platform_id}
+        // console.log("obj:",obj)
+        return obj
+    }
+    let obj = parsePromotion(origin_data.guajian_link,tg_config.source_filter,tg_config.platform_select,PlatformConfig)
+    // console.log("obj::",obj)
+    if(obj==null){
+        console.log("xml[挂件地址] error 2:",obj)
+        return null
+    }
+    console.log(JSON.stringify(obj))
+    // console.log("挂件地址正确")
+    if(origin_data.like_num < tg_config.like_num){
+        return null
+    }
+    // console.log("点赞数正确")
+    if(origin_data.kepp_num < tg_config.kepp_num){
+        return null
+    }
+    // console.log("收藏数正确")
+    if(origin_data.shared_num < tg_config.shared_num){
+        return null
+    }
+    // console.log("分享数正确")
+    if(origin_data.comment_num < tg_config.comment_num){
+        return null
+    }
+    let tg_platform_id = 0
+    if(obj.channel==2){ //"番茄"
+        tg_platform_id = 2
+        obj.book_id = obj.key
+    }else if(obj.channel==1){ //"黑岩"
+        tg_platform_id = 1
+        if(obj.book_id==undefined||obj.book_id==null){
+            return null
+        }
+    }else if(obj.channel==3){ //"七猫"
+        tg_platform_id = 3
+        if(obj.id==undefined||obj.id==null){
+            return null
+        }
+        obj.book_id = obj.id
+    }else if(obj.channel==4){ //"点众"
+        tg_platform_id = 4
+        obj.book_id = obj.book_id
+    }else if(obj.channel==5){ //"书旗-"
+        tg_platform_id = 5
+        obj.book_id = obj.book_id
+    }else if(obj.channel==6){ //"阅文"
+        tg_platform_id = 6
+        obj.book_id = obj.book_id
+    }else if(obj.channel==7){ //"番茄免費"
+        tg_platform_id = 7
+        obj.book_id = obj.code
+    }else if(obj.channel==15){ //"知乎"
+        tg_platform_id = 15
+        obj.book_id = obj.id
+        if(obj.book_id==undefined||obj.book_id==null){
+            return null
+        }
+    }else if(obj.channel==19){ //"知乎"
+        tg_platform_id = 19
+        if(obj.id==undefined||obj.id==null){
+            return null
+        }
+        obj.book_id = obj.id
+    }else if(obj.channel==config.platform_zy){ //"掌阅"
+        tg_platform_id = config.platform_zy
+        if(obj.zy_book_id==undefined||obj.zy_book_id==null){
+            return null
+        }
+        obj.book_id = obj.zy_book_id
+    }
+    // console.log("评论数正确")
+    return {video_id:origin_data.video_id,book_id:obj.book_id,channel:obj.channel,source:obj.source.substring(0,obj.source.length>64?64:obj.source.length),key:obj.key,video_link:origin_data.video_link,
+        like_num:origin_data.like_num,kepp_num:origin_data.kepp_num ,shared_num:origin_data.shared_num,comment_num:origin_data.comment_num,tg_platform_id:tg_platform_id}
+}
+
+function getWxPlatformConfig(PlatformConfig){
+    let list = []
+    for (let index = 0; index < PlatformConfig.length; index++) {
+        const element = PlatformConfig[index];
+        if(element.mini_program_platform_id==config.wx){
+            list.push(element)
+        }
+    }
+    return list
+}
+
+function getDyPlatformConfig(PlatformConfig){
+    let list = []
+    for (let index = 0; index < PlatformConfig.length; index++) {
+        const element = PlatformConfig[index];
+        if(element.mini_program_platform_id==config.dy){
+            list.push(element)
+        }
+    }
+    return list
+}
+
+function getIsOpen(platform_select,tg_platform_id){
+    for (let index = 0; index < platform_select.length; index++) {
+        const id = platform_select[index];
+        if(tg_platform_id==id){
+            return true
+        }
+    }
+    return false
+}
+
+function getWxChannel(platform_select,promotion,PlatformConfig){
+    for (let index = 0; index < PlatformConfig.length; index++) {
+        const platform_info = PlatformConfig[index];
+        let filter_guajian_logic_list = JSON.parse(platform_info.filter_guajian_logic_list)
+        for (let i = 0; i < filter_guajian_logic_list.length; i++) {
+            const logic_item = filter_guajian_logic_list[i];
+            if(getIsOpen(platform_select,platform_info.tg_platform_id)){
+                if(helper.checkChannelListKey(promotion,logic_item)){
+                    return platform_info.parent_platform_id
+                }
+            }
+        }
+    
+    }
+    return null
+}
+
+
+function getChannel(start_page,platform_select,promotion,PlatformConfig){
+    let kv = new Map()
+    PlatformConfig = getDyPlatformConfig(PlatformConfig)
+    for (let index = 0; index < PlatformConfig.length; index++) {
+        const platform_info = PlatformConfig[index];
+        kv.set(platform_info.tg_platform_id,false)
+    }
+
+    for (let index = 0; index < platform_select.length; index++) {
+        const element = platform_select[index];
+        kv.set(element,true)
+    }
+
+    // console.log("PlatformConfig:",PlatformConfig,kv)
+    for (let index = 0; index < PlatformConfig.length; index++) {
+        const platform_info = PlatformConfig[index];
+        let filter_guajian_logic_list = JSON.parse(platform_info.filter_guajian_logic_list)
+        for (let i = 0; i < filter_guajian_logic_list.length; i++) {
+            const logic_item = filter_guajian_logic_list[i];
+            if(kv.get(platform_info.tg_platform_id)==true){
+                console.log("filter_guajian_logic_list:",promotion,logic_item)
+                if(helper.checkChannelListKey(promotion,logic_item)){
+                    return platform_info.tg_platform_id
+                }
+            }
+        }
+
+   
+  
+    }
+    return null
+}
+
+//解析挂件
+function parsePromotion(promotion,source_list,platform_select,PlatformConfig){
+    let list = promotion.split('&')
+    if(list.length!=undefined){
+        if(list.length>2){
+            let params = {}
+            for (let index = 0; index < list.length; index++) {
+                const element = list[index];
+                let obj = parsePromotionItem(element)
+                if(obj!=null){
+                    if(obj.key=="start_page"&&obj.bookId!=null&&obj.bookId!=undefined){
+                        params['bookId'] = obj.bookId
+                    }
+                    params[obj.key] = obj.value
+                }
+            }
+            console.log("params:",params['id'])
+            let channel = getChannel(params['start_page'],platform_select,promotion,PlatformConfig)
+            // console.log("source_list:",source_list,platform_select,channel,params['start_page'],params)
+            if(channel!=null){
+                console.log("channel:",channel,promotion)
+                let item_info = {}
+                item_info['book_id'] = params['bookId'];
+                item_info['channel'] = channel;
+                item_info['id'] = params['id']
+                item_info['cbid'] = params['cbid']
+                item_info['code'] = params['code']
+                item_info['zy_book_id'] = params['zy_book_id']
+                const str = params['x-signature']
+                const regex = /"name":"([^"]+)"/;
+                const match = str.match(regex);
+                let source = ''
+                if (match && match[1]) {
+                    source = match[1]
+                }
+                for (let j = 0; j < source_list.length; j++) {
+                    const source_item = source_list[j];
+                    if(source_item==source){
+                        return null
+                    }
+                }
+                item_info['source'] = source;
+                item_info['key'] = params['key']
+                return item_info
+            }
+            return null
+        }else{
+            return null
+        }
+    }else{
+        return null
+    }
+}
+
+function parsePromotionItem(item){
+    let list = item.split('=')
+    if(list.length>1){
+        let key = list[0]
+        let value = list[1]
+        if(key==="start_page"&&list.length>2){
+            return {key:key,value:value,bookId:list[2]}
+        }
+        return {key:key,value:value}
+    }
+    return null
+}
+
+CMD.init()

1_FILTER_FACTORY/FILTER_FACTORY1.js

@@ -87,7 +87,6 @@ async function processTask(){
     try{
         let FilterConfig = await redis_help.getKeyValue("FilterConfig")
         FilterConfig = JSON.parse(FilterConfig)
-
         const result = await origin_data_controllers.getOriginDataByStatusAndLimitCreateDay({status:0,create_day:"2025-05-21"},1000," id % 10 = 1 AND ")
         if(result.success){
             let origin_data_list = result.data

filter_pm2.json

@@ -1,15 +1,15 @@
 {
     "apps": [
         {
-            "name": "1_FILTER_FACTORY",
-            "script": "./1_FILTER_FACTORY/FILTER_FACTORY.js",
+            "name": "0_FILTER_FACTORY",
+            "script": "./1_FILTER_FACTORY/FILTER_FACTORY0.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
             "args": ["--type=data"],
             "cwd": "/home/tg_factory",
             "log_date_format": "YYYY-MM-DD HH:mm:ss",
-            "error_file": "./logs/FILTER_FACTORY/1_FILTER_FACTORY_error.log",
-            "out_file": "./logs/FILTER_FACTORY/1_FILTER_FACTORY_out.log",
+            "error_file": "./logs/FILTER_FACTORY/1_FILTER_FACTORY0_error.log",
+            "out_file": "./logs/FILTER_FACTORY/1_FILTER_FACTORY0_out.log",
             "merge_logs": true,
             "exec_mode": "fork",
             "autorestart": true
@@ -29,7 +29,21 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "2_FILTER_FACTORY",
+            "script": "./1_FILTER_FACTORY/FILTER_FACTORY2.js",
+            "node_args": "--max-old-space-size=8192 --expose-gc",
+            "max_memory_restart": "2G",
+            "args": ["--type=data"],
+            "cwd": "/home/tg_factory",
+            "log_date_format": "YYYY-MM-DD HH:mm:ss",
+            "error_file": "./logs/FILTER_FACTORY/1_FILTER_FACTORY2_error.log",
+            "out_file": "./logs/FILTER_FACTORY/1_FILTER_FACTORY2_out.log",
+            "merge_logs": true,
+            "exec_mode": "fork",
+            "autorestart": true
+        },
+        {
+            "name": "3_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY3.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
@@ -43,7 +57,7 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "4_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY4.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
@@ -57,7 +71,7 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "5_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY5.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
@@ -71,7 +85,7 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "6_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY6.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
@@ -85,7 +99,7 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "7_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY7.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
@@ -99,7 +113,7 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "8_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY8.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",
@@ -113,7 +127,7 @@
             "autorestart": true
         },
         {
-            "name": "1_FILTER_FACTORY",
+            "name": "9_FILTER_FACTORY",
             "script": "./1_FILTER_FACTORY/FILTER_FACTORY9.js",
             "node_args": "--max-old-space-size=8192 --expose-gc",
             "max_memory_restart": "2G",

src/data_manager/Controllers/origin_data_controllers.js

@@ -132,7 +132,15 @@ class origin_data_controllers {
             }
 
             const status = params.status;
-            const filterData = await origin_data_model.findDataByLimitCreateDay({ status: status ,create_day:params.create_day },limit,yu);
+            let filterData
+            // if(params.create_day==""){
+            //     filterData = await origin_data_model.findDataByLimitCreateDay({ status: status  },limit,yu);
+            // }else{
+            //     filterData = await origin_data_model.findDataByLimitCreateDay({ status: status ,create_day:params.create_day },limit,yu);
+            // }
+
+            filterData = await origin_data_model.findDataByLimitCreateDay({ status: status  },limit,yu);
+        
 
             if (!filterData) {
                 return {