PULL_DATA.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527
  1. const CMD = {}
  2. const time_count = 1000;
  3. const redis_help = require('../src/use_redis');
  4. const origin_data_controllers = require('../src/data_manager/Controllers/origin_data_controllers');
  5. const axios = require('axios')
  6. const config = require('../etc/config.json');
  7. const DailyRecordManager = require('./daily_records');
  8. const schedule = require('node-schedule');
  9. const manager = new DailyRecordManager();
  10. const rabbitMq = require('../src/mq/rabbit-mq');
  11. const helper = require('../src/helper');
  12. const PullDataService = require('../src/PullDataService');
  13. var _24HourRanges = []
  14. var jobs = []
  15. var cur_day_data = new Map()
  16. var cur_timeRange = null
  17. var pullDataService = new PullDataService(redis_help)
  18. var cur_page = 0
  19. const messageHandler = async (msg) => {
  20. let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
  21. PullDataConfig = JSON.parse(PullDataConfig)
  22. _24HourRanges = PullDataConfig
  23. CMD.startScheduler(taskCallback)
  24. };
  25. const taskCallback = async (context) => {
  26. var { timeRange, executionTime } = context;
  27. console.log('执行任务:', {
  28. 时间区间: timeRange.name,
  29. 开始时间: timeRange.start,
  30. 执行时间: executionTime
  31. });
  32. const today = helper.getLocalDate();
  33. cur_day_data.forEach((v,k)=>{
  34. if(k==today){
  35. }else{
  36. cur_day_data.set(k, {})
  37. }
  38. })
  39. if(!cur_day_data.has(today)){
  40. cur_day_data.set(today,{})
  41. }
  42. if(!cur_day_data.get(today)[timeRange.start]){
  43. cur_day_data.get(today)[timeRange.start] = {name:timeRange.name,interval_minute:timeRange.interval_minute,finish_count:0}
  44. }
  45. console.log("cur_day_data[today]:",cur_day_data.get(today),today)
  46. const existingRecord = await manager.getRecord(today);
  47. if (existingRecord) {
  48. // 如果存在今天的记录,则更新
  49. const result = await manager.updateRecord(today, cur_day_data.get(today));
  50. console.log("更新记录结果:", result);
  51. } else {
  52. // 如果不存在今天的记录,则创建新记录
  53. const result = await manager.createRecord(today, cur_day_data.get(today));
  54. console.log("创建新记录结果:", result);
  55. }
  56. cur_timeRange = timeRange
  57. cur_timeRange.finish_count = 0
  58. cur_page = 0
  59. await processTask();
  60. };
  61. // 启动消费者
  62. async function startConsumer() {
  63. try {
  64. await rabbitMq.consumerDirectMsg(messageHandler,"exchange_update_pull_config","updatePullConig");
  65. } catch (error) {
  66. console.error('启动消费者失败:', error);
  67. }
  68. }
  69. async function processDetailTask(task_item) {
  70. try{
  71. console.log(" start processDetailTask")
  72. let materialId_list = task_item.materialId_list
  73. let materialId_data_list = task_item.materialId_data_list
  74. let response = await pullDataService.get_detail(materialId_list)
  75. if(!response.success){
  76. console.log("get_detail:",response)
  77. throw {msg:response,timeRange:cur_timeRange,fun:"get_detail",materialId_list:materialId_list}
  78. }
  79. if(cur_timeRange!=null){
  80. cur_timeRange.finish_count+=response.data.list.length
  81. const today = helper.getLocalDate();
  82. cur_day_data.get(today)[cur_timeRange.start].finish_count = cur_timeRange.finish_count
  83. await manager.updateRecord(today,cur_day_data.get(today));
  84. }
  85. for (let index = 0; index < response.data.list.length; index++) {
  86. const element = response.data.list[index];
  87. const info = materialId_data_list[element.materialId]
  88. if(element.hasAnchorInfo){
  89. await origin_data_controllers.createOriginData(
  90. {
  91. video_id:element.awemeId,
  92. materialId:element.materialId,
  93. video_link:element.awemeUrl,
  94. title:CMD.subTitle(element.title),
  95. publish_time:new Date(element.publishTime),
  96. kepp_num:info.kepp_num,
  97. comment_num:info.comment_num,
  98. like_num:info.like_num,
  99. shared_num:info.shared_num,
  100. is_guajian:1,
  101. guajian_link:element.landingUrl,
  102. status:0,
  103. createTime_new:new Date(element.createTime),
  104. updateTime_new:new Date(element.updateTime)
  105. }
  106. )
  107. }
  108. }
  109. console.log(" finish processDetailTask")
  110. }catch (error) {
  111. console.error("processDetailTask:",error)
  112. }finally{
  113. // global.setTimeout(processDetailTask,500)
  114. }
  115. }
  116. async function process_material_list(response,page) {
  117. try{
  118. let FilterConfig = await redis_help.getKeyValue("FilterConfig")
  119. FilterConfig = JSON.parse(FilterConfig)
  120. let materialId_list = []
  121. let materialId_data_list = []
  122. if(!response.success){
  123. throw {msg:response,timeRange:cur_timeRange,fun:"get_novel_material_list"}
  124. }
  125. if(response.data == undefined||response.data == null){
  126. throw {msg:response,timeRange:cur_timeRange,fun:"get_novel_material_list"}
  127. }
  128. if(response.data.list == undefined||response.data.list == null){
  129. throw "没有数据了"
  130. }
  131. if(response.data.list.length<=0){
  132. throw "没有数据了"
  133. }
  134. if(cur_timeRange!=null){ //统计
  135. const today = helper.getLocalDate();
  136. // console.log("response.data:",response.data,today)
  137. if(cur_day_data.get(today)[cur_timeRange.start]){
  138. console.log("response.data.count:",response.data.count)
  139. console.log("(cur_day_data[today][timeRange.start]:",cur_day_data.get(today)[cur_timeRange.start])
  140. cur_timeRange.count = cur_timeRange.count + response.data.list.length
  141. cur_timeRange.cur_page = page
  142. cur_day_data.get(today)[cur_timeRange.start].start_time = cur_timeRange.start_time
  143. cur_day_data.get(today)[cur_timeRange.start].pull_day = cur_timeRange.pull_day
  144. cur_day_data.get(today)[cur_timeRange.start].pull_time = cur_timeRange.pull_time ||{}
  145. cur_day_data.get(today)[cur_timeRange.start].count = response.data.count
  146. cur_day_data.get(today)[cur_timeRange.start].cur_page = page
  147. }else{
  148. console.log("cur_day_data空")
  149. }
  150. manager.updateRecord(today,cur_day_data.get(today));
  151. }
  152. for (let index = 0; index < response.data.list.length; index++) {
  153. const origin_element = response.data.list[index];
  154. materialId_list.push(origin_element.materialId)
  155. materialId_data_list[origin_element.materialId] = {
  156. kepp_num:0,
  157. comment_num:0,
  158. like_num:0,
  159. shared_num:0
  160. }
  161. }
  162. let result = await origin_data_controllers.findNonExistentMaterialIds(materialId_list)
  163. return {materialId_list:result.data.nonExistentIds,materialId_data_list:materialId_data_list,count:response.data.list.length}
  164. }catch(e){
  165. console.log("materialId_list:",e)
  166. return null
  167. }
  168. }
  169. async function processTask(){
  170. try{
  171. let FilterConfig = await redis_help.getKeyValue("FilterConfig")
  172. FilterConfig = JSON.parse(FilterConfig)
  173. let response = await pullDataService.get_novel_material_list(cur_timeRange,cur_page,500)
  174. if(response.success){
  175. if(response.data.list.length<=0){
  176. cur_page = -1;
  177. throw response
  178. }
  179. }else{
  180. cur_page = -1;
  181. throw response
  182. }
  183. let detail_item = await process_material_list(response,cur_page,cur_timeRange)
  184. if(detail_item.materialId_list.length<=0){
  185. }else{
  186. await processDetailTask(detail_item)
  187. }
  188. console.log("processTask over!")
  189. }catch(e){
  190. console.error("PULL_DATA: error:",e)
  191. }finally{
  192. if(cur_page!=-1){
  193. cur_page++;
  194. global.setTimeout(processTask,1000)
  195. }
  196. }
  197. }
  198. CMD.subTitle = function(title){
  199. if(title.length>299){
  200. return title.substring(0, 299);
  201. }
  202. return title
  203. }
  204. CMD.isRight = function(FilterConfig,origin_element){
  205. if(origin_element.likeCount.count < FilterConfig.like_num){// console.log("点赞数正确")
  206. return false
  207. }
  208. if(origin_element.favoriteCount.count < FilterConfig.kepp_num){ // console.log("收藏数正确")
  209. return false
  210. }
  211. if(origin_element.shareCount.count < FilterConfig.shared_num){ // console.log("分享数正确")
  212. return false
  213. }
  214. if(origin_element.commentCount.count < FilterConfig.comment_num){ // console.log("评论")
  215. return false
  216. }
  217. return true
  218. }
  219. CMD.init = async function(){
  220. // await startConsumer();
  221. redis_help.connect(async ()=>{
  222. startConsumer()
  223. let PullDataConfig = await redis_help.getKeyValue("PullDataConfig")
  224. PullDataConfig = JSON.parse(PullDataConfig)
  225. // processTask()
  226. _24HourRanges = PullDataConfig
  227. // 定义任务回调函数
  228. let key = helper.getLocalDate();
  229. let record_res = await manager.getRecord(key);
  230. if(record_res!=null){
  231. if(!cur_day_data.has(key)){
  232. cur_day_data.set(key,record_res.content)
  233. }
  234. }
  235. await pullDataService.getToken()
  236. // 启动调度器
  237. CMD.startScheduler(taskCallback);
  238. })
  239. }
  240. // 生成24小时的时间区间
  241. CMD.generate24HourRanges = function() {
  242. // const ranges = [];
  243. // const today = new Date();
  244. // today.setMinutes(0);
  245. // today.setSeconds(0);
  246. // today.setMilliseconds(0);
  247. // for (let hour = 0; hour < 24; hour++) {
  248. // const timeString = `${hour.toString().padStart(2, '0')}:00`;
  249. // let periodName = '';
  250. // // 设置当天的小时
  251. // today.setHours(hour);
  252. // const timestamp = today.getTime();
  253. // // 根据时间划分时段
  254. // if (hour >= 0 && hour < 6) {
  255. // periodName = '凌晨区间';
  256. // } else if (hour >= 6 && hour < 9) {
  257. // periodName = '早晨区间';
  258. // } else if (hour >= 9 && hour < 12) {
  259. // periodName = '上午区间';
  260. // } else if (hour >= 12 && hour < 14) {
  261. // periodName = '中午区间';
  262. // } else if (hour >= 14 && hour < 18) {
  263. // periodName = '下午区间';
  264. // } else if (hour >= 18 && hour < 22) {
  265. // periodName = '晚上区间';
  266. // } else {
  267. // periodName = '深夜区间';
  268. // }
  269. // ranges.push({
  270. // name: periodName,
  271. // start: timeString,
  272. // timestamp: timestamp,
  273. // hour:hour,
  274. // timeFormat: new Date(timestamp).toLocaleString() // 可读的时间格式
  275. // });
  276. // }
  277. let ranges = [
  278. {
  279. name: '凌晨区间',
  280. start: '00:00',
  281. timestamp: 1733760000000,
  282. hour: 0,
  283. timeFormat: '12/10/2024, 12:00:00 AM'
  284. },
  285. {
  286. name: '凌晨区间',
  287. start: '01:00',
  288. timestamp: 1733763600000,
  289. hour: 1,
  290. timeFormat: '12/10/2024, 1:00:00 AM'
  291. },
  292. {
  293. name: '凌晨区间',
  294. start: '02:00',
  295. timestamp: 1733767200000,
  296. hour: 2,
  297. timeFormat: '12/10/2024, 2:00:00 AM'
  298. },
  299. {
  300. name: '凌晨区间',
  301. start: '03:00',
  302. timestamp: 1733770800000,
  303. hour: 3,
  304. timeFormat: '12/10/2024, 3:00:00 AM'
  305. },
  306. {
  307. name: '凌晨区间',
  308. start: '04:00',
  309. timestamp: 1733774400000,
  310. hour: 4,
  311. timeFormat: '12/10/2024, 4:00:00 AM'
  312. },
  313. {
  314. name: '凌晨区间',
  315. start: '05:00',
  316. timestamp: 1733778000000,
  317. hour: 5,
  318. timeFormat: '12/10/2024, 5:00:00 AM'
  319. },
  320. {
  321. name: '早晨区间',
  322. start: '06:00',
  323. timestamp: 1733781600000,
  324. hour: 6,
  325. timeFormat: '12/10/2024, 6:00:00 AM'
  326. },
  327. {
  328. name: '早晨区间',
  329. start: '07:00',
  330. timestamp: 1733785200000,
  331. hour: 7,
  332. timeFormat: '12/10/2024, 7:00:00 AM'
  333. },
  334. {
  335. name: '早晨区间',
  336. start: '08:00',
  337. timestamp: 1733788800000,
  338. hour: 8,
  339. timeFormat: '12/10/2024, 8:00:00 AM'
  340. },
  341. {
  342. name: '上午区间',
  343. start: '09:00',
  344. timestamp: 1733792400000,
  345. hour: 9,
  346. timeFormat: '12/10/2024, 9:00:00 AM'
  347. },
  348. {
  349. name: '上午区间',
  350. start: '10:00',
  351. timestamp: 1733796000000,
  352. hour: 10,
  353. timeFormat: '12/10/2024, 10:00:00 AM'
  354. },
  355. {
  356. name: '上午区间',
  357. start: '11:00',
  358. timestamp: 1733799600000,
  359. hour: 11,
  360. timeFormat: '12/10/2024, 11:00:00 AM'
  361. },
  362. {
  363. name: '中午区间',
  364. start: '12:00',
  365. timestamp: 1733803200000,
  366. hour: 12,
  367. timeFormat: '12/10/2024, 12:00:00 PM'
  368. },
  369. {
  370. name: '中午区间',
  371. start: '13:00',
  372. timestamp: 1733806800000,
  373. hour: 13,
  374. timeFormat: '12/10/2024, 1:00:00 PM'
  375. },
  376. {
  377. name: '下午区间',
  378. start: '14:00',
  379. timestamp: 1733810400000,
  380. hour: 14,
  381. timeFormat: '12/10/2024, 2:00:00 PM'
  382. },
  383. {
  384. name: '下午区间',
  385. start: '15:00',
  386. timestamp: 1733814000000,
  387. hour: 15,
  388. timeFormat: '12/10/2024, 3:00:00 PM'
  389. },
  390. {
  391. name: '下午区间',
  392. start: '16:00',
  393. timestamp: 1733817600000,
  394. hour: 16,
  395. timeFormat: '12/10/2024, 4:00:00 PM'
  396. },
  397. {
  398. name: '下午区间',
  399. start: '17:00',
  400. timestamp: 1733821200000,
  401. hour: 17,
  402. timeFormat: '12/10/2024, 5:00:00 PM'
  403. },
  404. {
  405. name: '晚上区间',
  406. start: '18:00',
  407. timestamp: 1733824800000,
  408. hour: 18,
  409. timeFormat: '12/10/2024, 6:00:00 PM'
  410. },
  411. {
  412. name: '晚上区间',
  413. start: '19:00',
  414. timestamp: 1733828400000,
  415. hour: 19,
  416. timeFormat: '12/10/2024, 7:00:00 PM'
  417. },
  418. {
  419. name: '晚上区间',
  420. start: '20:45',
  421. timestamp: 1733832000000,
  422. hour: 20,
  423. timeFormat: '12/10/2024, 8:00:00 PM'
  424. },
  425. {
  426. name: '晚上区间',
  427. start: '21:00',
  428. timestamp: 1733835600000,
  429. hour: 21,
  430. timeFormat: '12/10/2024, 9:00:00 PM'
  431. },
  432. {
  433. name: '深夜区间',
  434. start: '22:00',
  435. timestamp: 1733839200000,
  436. hour: 22,
  437. timeFormat: '12/10/2024, 10:00:00 PM'
  438. },
  439. {
  440. name: '深夜区间',
  441. start: '23:00',
  442. timestamp: 1733842800000,
  443. hour: 23,
  444. timeFormat: '12/10/2024, 11:00:00 PM'
  445. }
  446. ]
  447. return ranges;
  448. }
  449. CMD.stopScheduler = function() {
  450. jobs.forEach(({ job }) => job.cancel());
  451. jobs = [];
  452. console.log('调度器已停止,所有任务已清除');
  453. }
  454. CMD.startScheduler = function(taskCallback) {
  455. CMD.stopScheduler()
  456. // 为每个时间点创建定时任务
  457. _24HourRanges.forEach(timeRange => {
  458. // 解析小时
  459. const hour = parseInt(timeRange.start.split(':')[0]);
  460. const minute = parseInt(timeRange.start.split(':')[1]);
  461. // 创建定时任务 - 在每天的指定小时整点执行
  462. const job = schedule.scheduleJob(`${minute} ${hour} * * *`, async () => {
  463. try {
  464. console.log(`开始执行任务: ${timeRange.name} ${timeRange.start}`);
  465. await pullDataService.getToken()
  466. // 执行回调函数
  467. await taskCallback({
  468. timeRange,
  469. executionTime: new Date()
  470. });
  471. } catch (error) {
  472. // 记录失败
  473. console.error('任务执行错误:', error);
  474. }
  475. });
  476. jobs.push({
  477. job,
  478. timeRange
  479. });
  480. console.log(`已安排任务: ${timeRange.name} ${timeRange.start}`);
  481. });
  482. console.log(`调度器已启动,共设置 ${jobs.length} 个定时任务`);
  483. }
  484. if(!config.isDebug){
  485. CMD.init()
  486. }