first commit

cwjokaka · Sep 5, 2019 · 290e01b · 290e01b
1 parent 1c21566
commit 290e01b
Show file tree

Hide file tree

Showing 15 changed files with 115 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+.idea
+idea/
diff --git a/main.py b/main.py
@@ -0,0 +1,4 @@
+from src.spider.spider_66_ip import Spider66Ip
+
+if __name__ == '__main__':
+    Spider66Ip().crawl()
diff --git a/setting.py b/setting.py
@@ -0,0 +1,5 @@
+from src.spider import Spider66Ip
+
+DB_TYPE = 'memory'      # memory/redis
+
+SPIDER_LIST = [Spider66Ip]
diff --git a/src/__init__.py b/src/__init__.py
diff --git a/src/database/__init__.py b/src/database/__init__.py
diff --git a/src/database/abs_database.py b/src/database/abs_database.py
@@ -0,0 +1,10 @@
+class AbsDatabase(object):
+
+    def put(self, key, value):
+        raise RuntimeError('该put方法未实现!')
+
+    def get(self, key):
+        raise RuntimeError('该get方法未实现!')
+
+    def remove(self, key):
+        raise RuntimeError('该remove方法未实现!')
diff --git a/src/database/memory.py b/src/database/memory.py
@@ -0,0 +1,18 @@
+from src.database.abs_database import AbsDatabase
+
+
+class Memory(AbsDatabase):
+    """
+    数据库:基于内存实现
+    """
+    def __init__(self) -> None:
+        self._box = {}
+
+    def put(self, key, value):
+        self._box[key] = value
+
+    def get(self, key):
+        return self._box[key]
+
+    def remove(self, key):
+        return self._box.pop(key, None)
diff --git a/src/entity/__init__.py b/src/entity/__init__.py
diff --git a/src/entity/proxy_entity.py b/src/entity/proxy_entity.py
@@ -0,0 +1,38 @@
+class ProxyEntity(object):
+
+    def __init__(self, ip, port, source='', type='', check_count=0, region='', last_check_time=None):
+        self._ip = ip
+        self._port = port
+        self._source = source
+        self._type = type
+        self._check_count = check_count
+        self._region = region
+        self._last_check_time = last_check_time
+
+    @property
+    def ip(self):
+        return self._ip
+
+    @property
+    def port(self):
+        return self._port
+
+    @property
+    def source(self):
+        return self._source
+
+    @property
+    def type(self):
+        return self._type
+
+    @property
+    def check_count(self):
+        return self._check_count
+
+    @property
+    def region(self):
+        return self._region
+
+    @property
+    def last_check_time(self):
+        return self._last_check_time
diff --git a/src/exception/__init__.py b/src/exception/__init__.py
diff --git a/src/exception/ok_proxy_error.py b/src/exception/ok_proxy_error.py
@@ -0,0 +1,4 @@
+class OkProxyError(RuntimeError):
+
+    def __init__(self, msg) -> None:
+        self._msg = msg
diff --git a/src/spider/__init__.py b/src/spider/__init__.py
diff --git a/src/spider/abs_spider.py b/src/spider/abs_spider.py
@@ -0,0 +1,12 @@
+class AbsSpider(object):
+
+    def __init__(self, name='unknown') -> None:
+        self._name = name
+
+    def crawl(self):
+        print('开始爬取...')
+        self.do_crawl()
+        print('爬取完毕!')
+
+    def do_crawl(self):
+        raise RuntimeError('do_crawl方法没有实现!')
diff --git a/src/spider/spider_66_ip.py b/src/spider/spider_66_ip.py
@@ -0,0 +1,21 @@
+import requests
+
+from src.spider.abs_spider import AbsSpider
+from bs4 import BeautifulSoup
+
+class Spider66Ip(AbsSpider):
+    """
+    66IP代理爬虫
+    http://www.66ip.cn/
+    """
+    def __init__(self) -> None:
+        super().__init__('66IP代理爬虫')
+        self._base_url = 'http://www.66ip.cn'
+
+    def do_crawl(self):
+        for page in range(1, 2):
+            resp = requests.get(f'{self._base_url}/{page}.html')
+            resp.encoding = 'gb2312'
+            soup = BeautifulSoup(resp.text, 'lxml')
+            print(soup.find('table', attrs={'width': '100%', 'bordercolor': '#6699ff'}).find_all('tr'))
+            # print(resp.text)
diff --git a/test/__init__.py b/test/__init__.py
-Original file line number
+Diff line change
@@ Expand Up / @@ -102,3 +102,6 @@ venv.bak/ @@
     # mypy
     .mypy_cache/
+    .idea
+    idea/