Revert "upd"

This reverts commit 532e147
cwjokaka · Sep 14, 2019 · 843d952 · 843d952
1 parent 532e147
commit 843d952
Show file tree

Hide file tree

Showing 4 changed files with 168 additions and 207 deletions.
diff --git a/main.py b/main.py
@@ -1,18 +1,15 @@
-import asyncio
 import typing
-from asyncio import AbstractEventLoop
 
 from src.database.memory_db import db_collection
 from src.entity.proxy_entity import ProxyEntity
 from src.spider.spiders import spider_collection
 from setting import SPIDER_LIST, DB_CONFIG
-from threading import Thread
 
 
-def crawl(event_loop: AbstractEventLoop):
+def crawl():
     proxies = []
     for spider_name in SPIDER_LIST:
-        proxies.extend(spider_collection[spider_name].crawl(event_loop))
+        proxies.extend(spider_collection[spider_name].crawl())
     return proxies
 
 
@@ -22,21 +19,12 @@ def save(proxies: typing.List[ProxyEntity]):
         db.set(f'{proxy.ip}:{proxy.port}', proxy)
 
 
-def start_event_loop(loop):
-    def init_loop(_loop):
-        asyncio.set_event_loop(_loop)
-        _loop.run_forever()
-    loop_thread = Thread(target=init_loop, args=(loop,))
-    loop_thread.setDaemon(True)
-    loop_thread.start()
 
 
-if __name__ == '__main__':
-    new_loop = asyncio.new_event_loop()
-    start_event_loop(new_loop)
 
+if __name__ == '__main__':
+    proxies = crawl()
     # 爬取
-    proxies = crawl(new_loop)
+    save(proxies)
     # 持久化
-    # save(proxies)
     print()
diff --git a/setting.py b/setting.py
@@ -7,8 +7,8 @@
 
 SPIDER_LIST = [
     'Spider66Ip',
-    # 'SpiderQuanWangIp',
-    # 'SpiderXiciIp'
+    'SpiderQuanWangIp',
+    'SpiderXiciIp'
 ]
 
 # 爬虫请求头

diff --git a/src/spider/abs_spider.py b/src/spider/abs_spider.py
@@ -1,7 +1,4 @@
-import asyncio
-from asyncio import AbstractEventLoop
-from collections import Coroutine
-from typing import List, Iterable
+from typing import List
 
 from src.entity.proxy_entity import ProxyEntity
 
@@ -11,17 +8,14 @@ class AbsSpider(object):
     def __init__(self, name='unknown') -> None:
         self._name = name
 
-    def crawl(self, event_loop: AbstractEventLoop):
+    def crawl(self):
         print(f'{self._name}开始爬取...')
-        # self.do_crawl()
-        # print(type(self.do_crawl()))
-        # print(isinstance(self.do_crawl(), Coroutine))
-        self.do_crawl(event_loop)
-        # print(f'{self._name}爬取完毕!共:{len(res)}个代理')
+        res = self.do_crawl()
+        print(f'{self._name}爬取完毕!共:{len(res)}个代理')
         # todo 持久化到数据库
 
-        # return ful.result()
+        return res
 
-    def do_crawl(self, event_loop) -> Iterable[ProxyEntity]:
-        raise NotImplementedError
+    def do_crawl(self) -> List[ProxyEntity]:
+        raise RuntimeError('do_crawl方法没有实现!')