add: 爬虫注册器

cwjokaka · Sep 9, 2019 · 1f39657 · 1f39657
1 parent c6f5640
commit 1f39657
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 6 deletions.
diff --git a/setting.py b/setting.py
@@ -1,12 +1,8 @@
-# from src.spider.spider_66_ip import Spider66Ip
-# from src.spider.spider_quan_wang_ip import SpiderQuanWangIp
-from src.database.memory_db import *
-
 DB = {
     'db_typy': 'memory'     # memory/redis
 }
 
-# SPIDER_LIST = [Spider66Ip, SpiderQuanWangIp]
+SPIDER_LIST = ['Spider66Ip', 'SpiderQuanWangIp', 'SpiderXiciIp']
 
 
 # 爬虫请求头

diff --git a/src/spider/spider_quan_wang_ip.py b/src/spider/spider_quan_wang_ip.py
@@ -8,7 +8,10 @@
 from src.spider.abs_spider import AbsSpider
 from bs4 import BeautifulSoup, Tag
 
+from src.spider.spider_register import spider_register
 
+
+@spider_register
 class SpiderQuanWangIp(AbsSpider):
     """
     全网IP代理爬虫 刷新速度:极快
@@ -84,4 +87,8 @@ def _judge_proxy_cover(self, cover_str: str):
         elif cover_str == '高匿':
             return ProxyCoverEnum.HIGH_COVER
         else:
-            return ProxyCoverEnum.UNKNOWN
+            return ProxyCoverEnum.UNKNOWN
+
+
+if __name__ == '__main__':
+    pass
diff --git a/src/spider/spider_register.py b/src/spider/spider_register.py
@@ -0,0 +1,7 @@
+spider_collection = {}
+
+
+def spider_register(cls):
+    spider_collection.update({cls.__name__: cls})
+    print(f'注册{cls.__name__}')
+    return cls