webscrapy/projects/electric/electric/spiders/newenergy.py

import time
import scrapy
from electric.items import ElectricItem

# 新能源网
class NgccSpider(scrapy.Spider):
    name = 'newenergy'
    allowed_domains = ['newenergy.giec.cas.cn']
    start_urls = [
        ('http://newenergy.giec.cas.cn/tyn/xydt/index{}.html', 31, '光伏'),
        ('http://newenergy.giec.cas.cn/tyn/jrjj/index{}.html', 31, '光伏'),
        ('http://newenergy.giec.cas.cn/tyn/gfdt/index{}.html', 31, '光伏'),
        ('http://newenergy.giec.cas.cn/tyn/grdt/index{}.html', 15, '光伏'),
        ('http://newenergy.giec.cas.cn/tyn/jcyy/index{}.html', 31, '光伏'),
        ('http://newenergy.giec.cas.cn/tyn/cpyjs/index{}.html', 31, '光伏'),
        ('http://newenergy.giec.cas.cn/fn/jrjj_15706/index{}.html', 31, '风能'),
        ('http://newenergy.giec.cas.cn/fn/fndt/index{}.html', 31, '风能'),
        ('http://newenergy.giec.cas.cn/fn/cydt/index{}.html', 31, '风能'),
        ('http://newenergy.giec.cas.cn/fn/fnzy/index{}.html', 2, '风能'),
        ('http://newenergy.giec.cas.cn/fn/cpyjs_15710/index{}.html', 23, '风能'),
        ('http://newenergy.giec.cas.cn/xsdt/index{}.html', 39, ''),
    ]

    custom_settings = {
        'DOWNLOAD_DELAY': 6,
        'DOWNLOAD_TIMEOUT': 1800,
    }

    def start_requests(self):
        for baseurl, maxpage, tag in self.start_urls:
            url = baseurl.format("")
            yield scrapy.Request(url=url, callback=self.parse_pages, errback=self.errback_httpbin, dont_filter=True, meta={"url": baseurl.format('_{}'), "maxpage": maxpage, "nextpage": 1, "tag":tag})

    def parse_pages(self, response):
        try:
            for item in response.xpath('//ul[@class="list_article"]/li'):
                url = item.css('a').attrib['href']
                yield response.follow(url=url, callback=self.parse_item, errback=self.errback_httpbin, meta={"url": url, "tag": response.meta['tag']})
        except:
            try:
                for item in response.xpath('//div/table[not(@class)]//tr'):
                    url = item.css('a').attrib['href']
                    yield response.follow(url=url, callback=self.parse_item, errback=self.errback_httpbin, meta={"url": url, "tag": response.meta['tag']})
            except:
                pass
        if response.meta['nextpage'] <= response.meta['maxpage']:
            nextpage = response.meta['url'].format(response.meta['nextpage'])
            yield response.follow(url=nextpage, callback=self.parse_pages, errback=self.errback_httpbin, meta={'url': response.meta['url'], "nextpage": response.meta['nextpage'] + 1, 'maxpage': response.meta['maxpage'], "tag": response.meta['tag']})

    def parse_item(self, response):
        url = response.url
        title = response.css('title::text').get()
        source = '新能源网'
        description = ''
        content = ''.join(response.xpath('//div[@class="TRS_Editor"]//text()').getall())
        date = time.time()
        column = response.meta['tag']
        yield ElectricItem(url=url, title=title, source=source,
                            description=description, content=content,
                            date=date, column=column)

    def errback_httpbin(self, failure):
        self.logger.error(repr(failure))