| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253 |
- import time
- import scrapy
- from electric.items import ElectricItem
- # 广西电力行业协会
- class GxepaSpider(scrapy.Spider):
- name = 'gxepa'
- allowed_domains = ['gxepa.org.cn']
- start_urls = [
- ('http://www.gxepa.org.cn/news_rdjj',102),
- ('http://www.gxepa.org.cn/news_xhyw',31),
- ('http://www.gxepa.org.cn/news_hyyw', 81),
- ]
- custom_settings = {
- 'DOWNLOAD_DELAY': 10,
- 'DOWNLOAD_TIMEOUT': 1800,
- }
- def start_requests(self):
- for url, maxpage in self.start_urls:
- yield scrapy.Request(url=url, callback=self.parse_pages, errback=self.errback_httpbin,
- meta={"url": url, "nextpage": 2, 'maxpage': maxpage},
- dont_filter=True)
- def parse_pages(self, response):
- for item in response.xpath('//div[@class="list-news"]/ul/li'):
- url = item.css('a').attrib['href']
- self.logger.info('parse item: {}'.format(url))
- yield response.follow(url=url, callback=self.parse_item, errback=self.errback_httpbin)
- if response.meta['nextpage'] <= response.meta['maxpage']:
- nextpage = response.meta['url'] + '?pageNo={}'.format(response.meta['nextpage'])
- self.logger.info('next page: {}'.format(nextpage))
- yield response.follow(url=nextpage, callback=self.parse_pages, errback=self.errback_httpbin,
- meta={'url': response.meta['url'], "nextpage": response.meta['nextpage'] + 1,
- 'maxpage': response.meta['maxpage']},
- dont_filter=True)
- def parse_item(self, response):
- url = response.url
- title = response.css('title::text').get()
- source = '广西电力行业协会'
- description = ''
- content = ''.join(''.join(response.xpath('//div[@id="ArtText"]//text()').getall()).split())
- date = time.time()
- column = ""
- self.logger.info('title: {}'.format(title))
- yield ElectricItem(url=url, title=title, source=source,
- description=description, content=content,
- date=date, column=column)
- def errback_httpbin(self, failure):
- self.logger.error(repr(failure))
|
