build(deps): bump twisted from 18.9.0 to 19.7.0

.gitignore

@@ -0,0 +1,9 @@
+*.pyc
+*.log
+*.out
+biliob_spider.log
+debug.py
+mail.py
+get_data/color.py
+get_data/face.py
+.vscode/*

1

@@ -0,0 +1,4 @@
+nohup: ignoring input
+nohup: ignoring input
+nohup: ignoring input
+nohup: ignoring input

README.md

@@ -1,2 +1,3 @@
 # BiliOB
+
 BiliOB观测者是一个观测B站UP主及视频数据变化，并予以分析的Web应用程序。

biliob_analyzer/add_credit.py

@@ -0,0 +1,3 @@
+from db import db
+u = db['user']
+u.update_many({}, {'$inc': {'credit': 50}})

biliob_analyzer/add_focus.py

@@ -0,0 +1,11 @@
+from db import db
+from pymongo import MongoClient
+# 链接mongoDB
+
+coll = db['author']  # 获得collection的句柄
+docs = coll.find({'focus': {'$exists': False}}).batch_size(60)
+for each_doc in docs:
+    if 'mid' in each_doc:
+        each_doc['focus'] = True
+        coll.update_one({'mid': each_doc['mid']}, {'$set': each_doc})
+        print('已修复mid' + str(each_doc['mid']))

biliob_analyzer/add_focus_video.py

@@ -0,0 +1,11 @@
+from db import db
+from pymongo import MongoClient
+# 链接mongoDB
+
+coll = db['video']  # 获得collection的句柄
+docs = coll.find({'focus': {'$exists': False}}).batch_size(60)
+for each_doc in docs:
+    if 'aid' in each_doc:
+        each_doc['focus'] = True
+        coll.update_one({'aid': each_doc['aid']}, {'$set': each_doc})
+        print('已修复aid' + str(each_doc['aid']))

biliob_analyzer/add_keyword.py

@@ -0,0 +1,156 @@
+from pymongo import ReturnDocument
+import jieba
+from db import db
+from time import sleep
+# 载入字典
+from biliob_tracer.task import ProgressTask
+
+
+class KeywordAdder():
+
+    def __init__(self):
+        self.mongo_author = db['author']
+        self.mongo_video = db['video']
+        self.mongo_word = db['search_word']
+        jieba.load_userdict('./biliob_analyzer/dict.txt')
+
+    def get_video_kw_list(self, aid):
+        # 关键字从name和official中提取
+        video = self.mongo_video.find_one(
+            {'aid': aid}, {'_id': 0, 'title': 1, 'channel': 1, 'subChannel': 1, 'author': 1, 'tag': 1})
+        kw = []
+        for each_key in video:
+            if each_key != 'keyword' or each_key != 'tag':
+                kw.append(str(video[each_key]).lower())
+            elif each_key == 'tag':
+                kw += video['tag']
+            else:
+                kw += video['keyword']
+        seg_list = jieba.lcut_for_search(
+            ' '.join(kw), True)  # 搜索引擎模式
+
+        # 全名算作关键字
+        if 'author' in video and video['author'].lower() not in seg_list:
+            seg_list.append(video['author'].lower())
+
+        while ' ' in seg_list:
+            seg_list.remove(' ')
+        while '、' in seg_list:
+            seg_list.remove('、')
+        return list(set(seg_list))
+
+    def add_to_video(self, aid, seg_list):
+        sleep(0.01)
+        self.mongo_video.update_one({'aid': aid}, {'$set': {
+            'keyword': seg_list
+        }})
+
+    def add_video_kw(self, aid):
+        self.add_to_video(aid, self.get_video_kw_list(aid))
+        return True
+
+    def get_author_kw_list(self, mid):
+        # 关键字从name和official中提取
+        author = self.mongo_author.find_one(
+            {'mid': mid}, {'_id': 0, 'name': 1, 'official': 1, 'keyword': 1})
+        kw = []
+        for each_key in author:
+            if each_key != 'keyword':
+                kw.append(str(author[each_key]).lower())
+            else:
+                kw += author['keyword']
+        seg_list = jieba.lcut_for_search(
+            ' '.join(kw), True)  # 搜索引擎模式
+
+        # 全名算作关键字
+        if 'name' in author and author['name'].lower() not in seg_list:
+            seg_list.append(author['name'].lower())
+
+        while ' ' in seg_list:
+            seg_list.remove(' ')
+        while '、' in seg_list:
+            seg_list.remove('、')
+        return list(set(seg_list))
+
+    def add_author_kw(self, mid):
+        self.add_to_author(mid, self.get_author_kw_list(mid))
+        return True
+
+    def add_to_author(self, mid, seg_list):
+        sleep(0.01)
+        self.mongo_author.update_one(
+            {'mid': mid}, {'$set': {'keyword': seg_list}})
+
+    def add_all_author(self):
+        authors = self.mongo_author.find(
+            {
+                '$or': [
+                    {
+                        'keyword': []
+                    }, {
+                        'keyword': {
+                            '$exists': False
+                        }
+                    }
+                ]
+            }, {'_id': 0, 'mid': 1}).batch_size(200)
+        for each_author in authors:
+            mid = each_author['mid']
+            self.add_author_kw(mid)
+
+    def add_all_video(self):
+        videos = self.mongo_video.find({
+            '$or': [
+                {
+                    'keyword': []
+                }, {
+                    'keyword': {
+                        '$exists': False
+                    }
+                }
+            ]
+        }, {'_id': 0, 'aid': 1}).batch_size(200)
+        for each_video in videos:
+            aid = each_video['aid']
+            self.add_video_kw(aid)
+
+    def refresh_all_author(self):
+        authors = self.mongo_author.find(
+            {}, {'_id': 0, 'mid': 1}).batch_size(500)
+        for each_author in authors:
+            mid = each_author['mid']
+            print("[mid]"+str(mid))
+            self.add_author_kw(mid)
+
+    def refresh_all_video(self):
+        videos = self.mongo_video.find(
+            {}, {'_id': 0, 'aid': 1}).batch_size(500)
+        for each_video in videos:
+            aid = each_video['aid']
+            print("[aid]"+str(aid))
+            self.add_video_kw(aid)
+
+    def add_omitted(self):
+        total_value = self.mongo_word.count_documents({})
+        if self.mongo_word.count_documents({}) < 100:
+            return
+        t = ProgressTask("更新查询关键词字典", total_value=total_value,collection=db['tracer'])
+        d = open('./biliob_analyzer/dict.txt', 'r',
+                 encoding='utf8').read().split('\n')
+        for each in self.mongo_word.find():
+            if 'aid' in each and each['aid'] not in d:
+                d.append(each['aid'])
+            elif 'mid' in each and each['mid'] not in d:
+                d.append(each['mid'])
+            t.current_value += 1
+        pass
+        t.finished = True
+        o = open('./biliob_analyzer/dict.txt',
+                 'w', encoding='utf8', newline='')
+        for each in d:
+            o.write(each+'\n')
+        o.close()
+        self.mongo_word.delete_many({})
+        jieba.load_userdict('./biliob_analyzer/dict.txt')
+        self.refresh_all_author()
+        self.refresh_all_video()

biliob_analyzer/author_analyzer.py

@@ -0,0 +1,69 @@
+from db import db
+from pymongo import MongoClient
+from datetime import datetime
+from datetime import timedelta
+import logging
+
+logging.basicConfig(level=logging.INFO,
+                    format='[%(asctime)s] %(levelname)s @ %(name)s: %(message)s')
+logger = logging.getLogger(__name__)
+
+
+class AuthorAnalyzer(object):
+    def __init__(self):
+        self.db = db  # 获得数据库的句柄
+        self.coll = self.db['author']  # 获得collection的句柄
+
+    def author_filter(self):
+        pre_fans = -1
+        c_fans = -1
+        delta = timedelta(1)
+        pre_date = datetime
+        c_date = datetime
+        count_unfocus = 0
+        count_focus = 0
+        for each_doc in self.coll.find({'focus': True, 'cFans': {'lt': 50000}}):
+            flag_cool = 0
+            if 'data' in each_doc:
+                each_doc['data'].reverse()
+                for each_data in each_doc['data']:
+                    if pre_fans == -1:
+                        pre_fans = each_data['fans']
+                        pre_date = each_data['datetime']
+                        continue
+                    c_fans = each_data['fans']
+                    c_date = each_data['datetime']
+                    if pre_date + delta > c_date:
+                        continue
+                    if abs(c_fans-pre_fans) < 100:
+                        flag_cool += 1
+                    else:
+                        flag_cool = 0
+                    pre_fans = c_fans
+                    pre_date = c_date
+
+                    # 连续30日日均涨粉小于100且粉丝数小于100000则不追踪
+                    if flag_cool > 30 and each_data['fans'] < 100000:
+                        focus = False
+                        break
+                    elif flag_cool > 15 and each_data['fans'] < 5000:
+                        focus = False
+                        break
+                    elif flag_cool > 7 and each_data['fans'] < 1000:
+                        focus = False
+                        break
+                    else:
+                        focus = True
+
+                if focus:
+                    count_focus += 1
+                else:
+                    count_unfocus += 1
+                pre_fans = -1
+                c_fans = -1
+        logger.info("· 本轮筛选结果：")
+        logger.info("× 不再追踪总数："+str(count_unfocus))
+        logger.info("√ 持续追踪总数："+str(count_focus))
+
+    def fans_variation(self):
+        pass