Added pipelines

REMitchell · REMitchell · commit fb9cc07e3e30 · 2018-01-25T23:42:36.000-05:00
diff --git a/v2/Chapter05_Scrapy/wikiSpider/wikiSpider/articlePipelines.py b/v2/Chapter05_Scrapy/wikiSpider/wikiSpider/articlePipelines.py
@@ -0,0 +1,19 @@
+from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from wikiSpider.items import Article
+
+class ArticleSpider(CrawlSpider):
+    name = 'articlePipelines'
+    allowed_domains = ['wikipedia.org']
+    start_urls = ['https://en.wikipedia.org/wiki/Benevolent_dictator_for_life']
+    rules = [
+        Rule(LinkExtractor(allow='(/wiki/)((?!:).)*$'), callback='parse_items', follow=True),
+    ]
+
+    def parse_items(self, response):
+        article = Article()
+        article['url'] = response.url
+        article['title'] = response.css('h1::text').extract_first()
+        article['text'] = response.xpath('//div[@id="mw-content-text"]//text()').extract()
+        article['lastUpdated'] = response.css('li#footer-info-lastmod::text').extract_first()
+        return article
diff --git a/v2/Chapter05_Scrapy/wikiSpider/wikiSpider/pipelines.py b/v2/Chapter05_Scrapy/wikiSpider/wikiSpider/pipelines.py
@@ -1,11 +1,14 @@
-# -*- coding: utf-8 -*-
-
-# Define your item pipelines here
-#
-# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
-
+from datetime import datetime
+from wikiSpider.items import Article
+from string import whitespace
 
 class WikispiderPipeline(object):
-    def process_item(self, item, spider):
-        return item
+    def process_item(self, article, spider):
+        dateStr = article['lastUpdated']
+        # This page was last edited on 26 January 2018, at 03:56.
+        article['lastUpdated'] = article['lastUpdated'].replace('This page was last edited on', '')
+        article['lastUpdated'] = article['lastUpdated'].strip()
+        article['lastUpdated'] = datetime.strptime(article['lastUpdated'], '%d %B %Y, at %H:%M.')
+        article['text'] = [line for line in article['text'] if line not in whitespace]
+        article['text'] = ''.join(article['text'])
+        return article
