webscrapy/projects/electric/electric/spiders/zzsolar.py

import time
import scrapy
from electric.items import ElectricItem

## 郑州国际太阳能光伏展览会
class ZzsolarSpider(scrapy.Spider):
    name = 'zzsolar'
    allowed_domains = ['zzsolar.com.cn']
    start_urls = [
        ('https://zzsolar.com.cn/a/news/hyxw/list_13_{}.html', 122),
    ]

    custom_settings = {
        'DOWNLOAD_DELAY': 10,
    }

    def start_requests(self):
        for baseurl, maxpage in self.start_urls:
            url = baseurl.format(1)
            yield scrapy.Request(url=url, callback=self.parse_pages, errback=self.errback_httpbin, meta={"url": baseurl, "nextpage": 2, 'maxpage': maxpage}, dont_filter=True)

    def parse_pages(self, response):
        for item in response.xpath('//div[@class="cont thumblist1"]/ul/li'):
            url = item.css('a.fl').attrib['href']
            self.logger.info('parse item: {}'.format(url))
            yield response.follow(url=url, callback=self.parse_item, errback=self.errback_httpbin)
        if response.meta['nextpage'] <= response.meta['maxpage']:
            nextpage = response.meta['url'].format(response.meta['nextpage'])
            self.logger.info('next page: {}'.format(nextpage))
            yield response.follow(url=nextpage, callback=self.parse_pages, errback=self.errback_httpbin, meta={'url': response.meta['url'], "nextpage": response.meta['nextpage'] + 1, 'maxpage': response.meta['maxpage']})
    
    def parse_item(self, response):
        url = response.url
        title = response.css('h1::text').get()
        source = '郑州国际太阳能光伏展览会'
        description = ''
        content = response.xpath('//div[@class="showContxt"]//text()').getall()
        date = time.time()
        column = '光伏'
        yield ElectricItem(url=url, title=title, source=source,
                            description=description, content=content,
                            date=date, column=column)

    def errback_httpbin(self, failure):
        self.logger.error(repr(failure))