其实我只是想试试爬取图片而已,先看看网页，需要爬的地方有两个，一是封面图，二是下载地址，挺简单的
Item定义：
import scrapyclass TiantianmeijuItem(scrapy.Item): name = scrapy.Field() p_w_picpath_urls = scrapy.Field() p_w_picpaths = scrapy.Field() p_w_picpath_paths = scrapy.Field() episode = scrapy.Field() episode_url = scrapy.Field()
name是保存名字
p_w_picpath_urls和p_w_picpaths 是爬取图片的pipeline用的，一个是保存图片URL，一个是保存图片存放信息
p_w_picpath_paths其实没什么实际作用，只是记录下载成功的图片地址
epiosde和episode_url是保存集数和对应下载地址
Spider：
import scrapyfrom tiantianmeiju.items import TiantianmeijuItem import sysreload(sys) # Python2.5 初始化后会删除 sys.setdefaultencoding 这个方法，我们需要重新载入sys.setdefaultencoding('utf-8')class CacthUrlSpider(scrapy.Spider): name = 'meiju' allowed_domains = ['cn163.net'] start_urls = ["http://cn163.net/archives/{id}/".format(id=id) for id in ['16355', '13470', '18766', '18805']] def parse(self, response): item = TiantianmeijuItem() item['name'] = response.xpath('//*[@id="content"]/div[2]/div[2]/h2/text()').extract() item['p_w_picpath_urls'] = response.xpath('//*[@id="entry"]/div[2]/img/@src').extract() item['episode'] = response.xpath('//*[@id="entry"]/p[last()]/a/text()').extract() item['episode_url'] = response.xpath('//*[@id="entry"]/p[last()]/a/@href').extract() yield item 页面比较简单
Pipelines：这里写了两个管道，一个是把下载链接保存到文件，一个是下载图片
import jsonimport osfrom scrapy.pipelines.p_w_picpaths import ImagesPipelinefrom scrapy.exceptions import DropItemfrom scrapy.http import Requestfrom settings import IMAGES_STOREclass TiantianmeijuPipeline(object): def process_item(self, item, spider): return item class WriteToFilePipeline(object): def process_item(self, item, spider): item = dict(item) FolderName = item['name'][0].replace('/', '') downloadFile = 'download_urls.txt' with open(os.path.join(IMAGES_STORE, FolderName, downloadFile), 'w') as file: for name,url in zip(item['episode'], item['episode_url']): file.write('{name}: {url}\n'.format(name=name, url=url)) return item class MyImagesPipeline(ImagesPipeline): def get_media_requests(self, item, info): for p_w_picpath_url in item['p_w_picpath_urls']: yield Request(p_w_picpath_url, meta={'item': item}) def item_completed(self, results, item, info): p_w_picpath_paths = [x['path'] for ok,x in results if ok] if not p_w_picpath_paths: raise DropItem("Item contains no p_w_picpaths") item['p_w_picpath_paths'] = p_w_picpath_paths return item def file_path(self, request, response=None, info=None): item = request.meta['item'] FolderName = item['name'][0].replace('/', '') p_w_picpath_guid = request.url.split('/')[-1] filename = u'{}/{}'.format(FolderName, p_w_picpath_guid) return filename get_media_requests和item_completed，因为默认的图片储存路径是
/full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg，
我需要把full改成以美剧名字目录来保存，所以重写了file_path
settings打开pipelines相关配置：
ITEM_PIPELINES = { 'tiantianmeiju.pipelines.WriteToFilePipeline': 2, 'tiantianmeiju.pipelines.MyImagesPipeline': 1,}IMAGES_STORE = os.path.join(os.getcwd(), 'p_w_picpath') # 图片存储路径IMAGES_EXPIRES = 90IMAGES_MIN_HEIGHT = 110IMAGES_MIN_WIDTH = 110 爬下来之后就是这个效果了：