Scrapy python 규칙이 작동하지 않습니다.

Question

크레이그리스트의 첫 번째 페이지를 스크랩 할 수 있습니다. 그러나 Linkextractor는 다른 페이지에서 데이터를 가져 오지 않습니다. 규칙을 정의하는 데 문제가 있습니까?

import scrapy 
from craiglist.items import craiglistItem 
from scrapy.contrib.spiders import CrawlSpider, Rule 
from scrapy.linkextractors import LinkExtractor 


class ExampleSpider(CrawlSpider): 
    name = "craiglist" 
    allowed_domains = ["craiglist.org"] 
    start_urls = (
     'http://sfbay.craigslist.org/search/npo', 
    ) 
    rules = [ 
     Rule(LinkExtractor(restrict_xpaths='//a[@class="button next"]'),  callback='parse', follow= True) 
    ] 

    def parse(self, response): 
     titles = response.selector.xpath('//*[@id="sortable-results"]/ul/li/p') 
     items = [] 
     for title in titles: 
      item = craiglistItem() 
      item["title"] = title.select("a/text()").extract() 
      item["link"] = title.select("a/@href").extract() 
      items.append(item) 
     return items 

Answer 1

나는 코드를 수정했으며 현재 잘 작동한다. 아래는 작업 코드입니다.

import scrapy 
from craiglist.items import craiglistItem 
from scrapy.contrib.spiders import CrawlSpider, Rule 
from scrapy.linkextractors import LinkExtractor 
from scrapy.http import Request 


class ExampleSpider(CrawlSpider): 
    name = "craiglist" 
    allowed_domains = ["craigslist.org"] 
    start_urls = (
     'http://sfbay.craigslist.org/search/npo', 
    ) 
    rules = [ 
     Rule(LinkExtractor(restrict_xpaths='//a[@class="button next"]'), callback="parse_items", follow= True), 
    ] 

    def parse_start_url(self,response): 
     request=Request("http://sfbay.craigslist.org/search/npo", callback=self.parse_items) 
     return request 

    def parse_items(self, response): 
     titles = response.selector.xpath('//*[@id="sortable-results"]/ul/li/p') 
     items = [] 
     for title in titles: 
      item = craiglistItem() 
      item["title"] = title.select("a/text()").extract() 
      item["link"] = title.select("a/@href").extract() 
      #item["link"] = response.url 
      items.append(item) 
     return items