I wanted to scrape all items from each card and the firsr rule is working fine but the second rule meaning pagination rule is not working.
This is my code:
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class RealtorListSpider(CrawlSpider):
name = 'realtor_list'
allowed_domains = ['www.realtor.com']
start_urls = ['https://www.realtor.com/realestateagents/New-Orleans_LA/pg-1']
rules = (
Rule(LinkExtractor(restrict_xpaths='//*[#data-testid="component-agentCard"]'), callback='parse_item', follow=False),
Rule(LinkExtractor(restrict_xpaths='//a[#aria-label="Go to next page"]'), callback='parse_item', follow=True),
)
def parse_item(self, response):
yield{
'name': response.xpath('(//*[#class="jsx-3130164309 profile-Tiltle-main"]/text())[2]').get()
}
The problem is in your element selection in linkextractor, not in rule for pagination. Xpath expression doesn't contain link selection but selection is correct that's why I 've made the pagination in starting url and it's working fine.
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class RealtorListSpider(CrawlSpider):
name = 'realtor_list'
allowed_domains = ['www.realtor.com']
start_urls = ['https://www.realtor.com/realestateagents/New-Orleans_LA/pg-'+str(x) +'' for x in range(1,6)]
rules = (
Rule(LinkExtractor(restrict_xpaths='//*[#data-testid="component-agentCard"]'), callback='parse_item', follow=False),
)
def parse_item(self, response):
yield{
'name': response.xpath('(//*[#class="jsx-3130164309 profile-Tiltle-main"]/text())[2]').get()
}
Related
Im not sure why my script isnt scrapping any items, is the same script that im using for another website, maybe the classes im using are wrong.
`
import scrapy
import os
from scrapy.crawler import CrawlerProcess
from datetime import datetime
date = datetime.now().strftime("%d_%m_%Y")
class stiendaSpider(scrapy.Spider):
name = 'stienda'
start_urls = ['https://stienda.uy/tv']
def parse(self, response):
for products in response.css('.grp778'):
price = products.css('.precioSantander::text').get()
name = products.css('#catalogoProductos .tit::text').get()
if price and name:
yield {'name': name.strip(),
'price': price.strip()}
os.chdir('C:\\Users\\cabre\\Desktop\\scraping\\stienda\\data\\raw')
process = CrawlerProcess(
# settings={"FEEDS": {"items.csv": {"format": "csv"}}}
settings={"FEEDS": {f"stienda_{date}.csv": {"format": "csv"}}}
)
process.crawl(stiendaSpider)
process.start()
`
I tried several but I dont usnderstand why is not working..
I was able to get the name field, but the price attribute is rendered empty and filled in later from an ajax call. That is why it's not being extracted.
import scrapy
import os
from scrapy.crawler import CrawlerProcess
from datetime import datetime
date = datetime.now().strftime("%d_%m_%Y")
class stiendaSpider(scrapy.Spider):
name = 'stienda'
start_urls = ['https://stienda.uy/tv']
def parse(self, response):
for products in response.xpath('//div[#data-disp="1"]'):
name = products.css('.tit::text').get()
if name:
yield {'name': name.strip()}
You can see it if you look at the page source... all of the elements with the class 'precioSantander' are empty.
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scraper_api import ScraperAPIClient
class Spider(CrawlSpider):
allowed_domains = ['example.com']
client = ScraperAPIClient('xyz')
#Initilize method for taking argument from user
def __init__(self, category=None,location=None ,**kwargs):
#self.start_urls = [client.scrapyGet(url = 'http://example.com')]
super().__init__(**kwargs) # python3
rules = (
Rule(LinkExtractor(restrict_xpaths="//div[contains(#class,'on-click-container')]/a[contains(#href, '/biz/')]"), callback='parse_item', follow=True,process_request='set_proxy'),
#for next page
Rule(LinkExtractor(restrict_xpaths='//a[contains(#class,"next-link")]'),process_request='set_proxy')
)
def set_proxy(self,request):
pass
def parse_item(self, response):
# contains data
yield{
# BUSINESS INFORMATION
"Company Name": response.xpath('//title').extract_first(),
}
Here I don't understand how to write set_proxy function that send request to scraper API server. Please help about this.
I am very happy to having discovered the Scrapy Crawl Class with its Rule Objects. However when I am trying to extract urls which contain the word "login" with process_links it doesn't work. The solution I implemented comes from here: Example code for Scrapy process_links and process_request but it doesn't exclude the pages I want
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.loader import ItemLoader
from accenture.items import AccentureItem
class AccentureSpiderSpider(CrawlSpider):
name = 'accenture_spider'
start_urls = ['https://www.accenture.com/us-en/internet-of-things-index']
rules = (
Rule(LinkExtractor(restrict_xpaths='//a[contains(#href, "insight")]'), callback='parse_item',process_links='process_links', follow=True),
)
def process_links(self, links):
for link in links:
if 'login' in link.text:
continue # skip all links that have "login" in their text
yield link
def parse_item(self, response):
loader = ItemLoader(item=AccentureItem(), response=response)
url = response.url
loader.add_value('url', url)
yield loader.load_item()
My mistake was to use link.text
When using link.url it works fine :)
I created a crawl spider to download files. However the spider downloaded only the urls of the files and not the files themselves. I uploaded a question here Scrapy crawl spider does not download files? . While the the basic yield spider kindly suggested in the answers works perfectly, when I attempt to download files with items or item loaders the spider does not work! The original question does not include the items.py. So there it is:
ITEMS
import scrapy
from scrapy.item import Item, Field
class DepositsusaItem(Item):
# main fields
name = Field()
file_urls = Field()
files = Field()
# Housekeeping Fields
url = Field()
project = Field()
spider = Field()
server = Field()
date = Field()
pass
EDIT: added original code
EDIT: further corrections
SPIDER
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import datetime
import socket
from us_deposits.items import DepositsusaItem
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose
from urllib.parse import urljoin
class DepositsSpider(CrawlSpider):
name = 'deposits'
allowed_domains = ['doi.org']
start_urls = ['https://minerals.usgs.gov/science/mineral-deposit-database/#products', ]
rules = (
Rule(LinkExtractor(restrict_xpaths='//*[#id="products"][1]/p/a'),
callback='parse_x'),
)
def parse_x(self, response):
i = ItemLoader(item=DepositsusaItem(), response=response)
i.add_xpath('name', '//*[#class="container"][1]/header/h1/text()')
i.add_xpath('file_urls', '//span[starts-with(#data-url, "/catalog/file/get/")]/#data-url',
MapCompose(lambda i: urljoin(response.url, i))
)
i.add_value('url', response.url)
i.add_value('project', self.settings.get('BOT_NAME'))
i.add_value('spider', self.name)
i.add_value('server', socket.gethostname())
i.add_value('date', datetime.datetime.now())
return i.load_item()
SETTINGS
BOT_NAME = 'us_deposits'
SPIDER_MODULES = ['us_deposits.spiders']
NEWSPIDER_MODULE = 'us_deposits.spiders'
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {
'us_deposits.pipelines.UsDepositsPipeline': 1,
'us_deposits.pipelines.FilesPipeline': 2
}
FILES_STORE = 'C:/Users/User/Documents/Python WebCrawling Learning Projects'
PIPELINES
class UsDepositsPipeline(object):
def process_item(self, item, spider):
return item
class FilesPipeline(object):
def process_item(self, item, spider):
return item
It seems to me that using items and/or item loaders has nothing to do with your problem.
The only problems I see are in your settings file:
FilesPipeline is not activated (only us_deposits.pipelines.UsDepositsPipeline is)
FILES_STORE should be a string, not a set (an exception is raised when you activate the files pipeline)
ROBOTSTXT_OBEY = True will prevent the downloading of files
If I correct all of those issues, the file download works as expected.
I get "Twisted.internet.error.ReactorNotRestartable" when I run this spider. Actually, I get this error when I run any spider in the project directory. It was working fine when I last checked But since today it gives me this error. I am not familiar with twisted. Please help!
import scrapy
from quo.items import QuoItem
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from twisted.internet import reactor
class ISpider(CrawlSpider):
name='iShopE'
handle_httpstatus_list = [400]
def start_requests(self):
yield scrapy.Request('https://www.ishopping.pk/electronics/home-theatres.html')
def parse(self, response):
for href in response.xpath('//div[#class="category-products-"]').extract():
for product_page in response.xpath('//h2[#class="product-name"]/a/#href').extract():
url=response.urljoin(product_page)
yield scrapy.Request(url, callback=self.parse_productPage)
next_page=response.xpath('(//a[#class="next i-next"]/#href)[1]').extract()
if next_page:
yield scrapy.Request(response.urljoin(next_page[0]), callback=self.parse)
def parse_productPage(self,response):
url_ProductPage=response.url
item=QuoItem()
if url_ProductPage:
item['url_ProductPage']=url_ProductPage
for rev in response.xpath('//div[#class="product-essential"]'):
price=response.xpath('//div[#class="price-box"]/span[#class="regular-price"]/meta[#itemprop="price"]/#content').extract()
if price:
item['price']=price
newPrice=response.xpath('(//div[#class="price-box"]/p[#class="special-price"]/span[#class="price"]/text())[1]').extract()
if newPrice:
newPrice=" ".join(str(x) for x in newPrice)
newPrice=newPrice.strip('\n')
item['price'] =newPrice.replace(" ", "")
item['newPrice']=newPrice.replace(" ","")
oldPrice=response.xpath('(//div[#class="price-box"]/p[#class="old-price"]/span[#class="price"]/text())[1]').extract()
if oldPrice:
oldPrice=" ".join(str(x) for x in oldPrice)
oldPrice=oldPrice.strip('\n')
item['oldPrice'] =oldPrice.replace(" ", "")
Availability=response.xpath('//p[#class="availability in-stock"]/span[#class="value"]/text()').extract()
if Availability:
item['Availability']=Availability
Brand=response.xpath('(//div[#class="box-p-attr"]/span)[1]/text()').extract()
if Brand:
item['Brand']=Brand
deliveryTime=response.xpath('(//div[#class="box-p-attr"]/span)[2]/text()').extract()
if deliveryTime:
item['deliveryTime']=deliveryTime
Waranty=response.xpath('(//div[#class="box-p-attr"]/span)[3]/text()').extract()
if Waranty:
item['Waranty']=Waranty
title=response.xpath('//div[#class="product-name"]/h1[#itemprop="name"]/text()').extract()
if title:
item['Title']=title
yield item