How to Build A Web Crawler and Scraper for Craigslist Using Scrapy

    # You'll need the below modules to create your spider:
    from scrapy.spiders import CrawlSpider, Spider, Request, Rule
    from scrapy.linkextractors import LinkExtractor
    # Replace "SpiderNameItem" with the class name in your items.py.
    from rent_scraper.items import SpiderNameItem 


    class YourSpiderName(CrawlSpider):  # Make sure you're inheriting from the CrawlSpider class.
        
        name = 'spidername'  # This should already be filled with your spider's name.

        # Tells your spider to start crawling from this URL.
        start_urls = ['https://sfbay.craigslist.org/search/sfc/apa']

        rules = (
            # This rule extracts all links from the start URL page using the XPATH selector.
            # In this case it is only looking to extract links that are individual rental
            # posting links and not other links to ad sites or elsewhere on Craigslist.
            Rule(LinkExtractor(allow=(), restrict_xpaths=('//a[@class="result-title hdrlnk"]')),
            follow=True, callback='parse_item'),
            
            # This rule says to follow/do a GET request on the extracted link,
            # returning the response to the callback function,
            # which is the parse_tem() function below.
            Rule(LinkExtractor(allow=(), restrict_xpaths=('//a[contains(@class, "button next")]')),
            follow=True, callback='parse_item'),
        )

        def parse_item(self, response):
            """ Using XPATH, parses data from the HTML responses and returns a dictionary-like item. """

            # Specifies what an Item is; in this case it is referring to the items.py class name
            # and instantiating a dictionary for a single item (rental).
            item = SpiderNameItem()

            # In my item dictionary, insert the following values for each rental.
            # The below code uses XPATH selectors to select the specific data of interest
            # from the response of each rental link I crawl to.
            item['cl_id'] = response.xpath('//div/p[@class="postinginfo"]/text()').extract()
            item['price'] = response.xpath('//span/span[contains(@class, "price")]/text()').extract()
            item['attributes'] = response.xpath('//p[contains(@class, "attrgroup")]/span/b/text()').extract()
            item['neighborhood'] = response.xpath('//span/small/text()').extract()
            item['date'] = response.xpath('//time[contains(@class, "timeago")]/@datetime').extract_first()
            item['location'] = response.xpath('//a[contains(@href, "https://maps.google.com/")]/@href').extract()

            # Finally, once a single rental page and data has been scraped,
            # the crawler yields an iterator item that can later be called.
            yield item

    import scrapy


    class SpiderNameItem(scrapy.Item):

        # Dictionary keys created for scraping in craigslist.py.
        cl_id = scrapy.Field()
        price = scrapy.Field()
        attributes = scrapy.Field()
        neighborhood = scrapy.Field()
        date = scrapy.Field()
        location = scrapy.Field()

        # Additional dictionary keys created upon data cleanse in pipelines.py.
        # These were added fields because I created them later in the pipelines file,
        # and they were not collected in the initial scrape using my spider.
        # For example, the rental attributes (above) actually contain data on bedrooms,
        # bathrooms, and sqft, but it was easier for me to scrape all that data in one chunk
        # and separate the code used clean up that data into my pipelines.py file.
        bedrooms = scrapy.Field()
        bathrooms = scrapy.Field()
        sqft = scrapy.Field()
        latitude = scrapy.Field()
        longitude = scrapy.Field()
        latlng = scrapy.Field()

    from scrapy.exceptions import DropItem  # Module to handle (drop) bad items
    # The below are optional - import all the libraries you need here
    import re
    import urllib
    import geocoder

    # This should be customized to import from your model
    from model import connect_to_db_scrapy, Rental, UnitDetails


    class RentScraperPipeline(object):
        """ Scrubs scraped data of whitespaces and interesting characters
        and returns it as a data maintenance-friendly dictionary. """

        def process_item(self, item, spider):

            # Gets Craigslist posting ID
            if item['cl_id']:
                item['cl_id'] = re.split('\s', ''.join(item['cl_id']))[2]

            # Finds rental price; if none supplied or if it's listed at or below $50, drop item
            if item['price']:
                price = int(re.sub('[^\d.]+', '', ''.join(item['price'])))
                if price > 50:
                    item['price'] = price
                else:
                    raise DropItem('Missing price in %s' % item)
            else:
                raise DropItem('Missing price in %s' % item)

            # Finds bedrooms, bathrooms, and sqft, if provided
            if item['attributes']:
                attrs = item['attributes']

                for i, attribute in enumerate(attrs):
                    if "BR" in attrs[i]:
                        item['bedrooms'] = float(''.join(re.findall('\d+\.*\d*', attrs[i])))
                    if "Ba" in attrs[i]:
                        item['bathrooms'] = float(''.join(re.findall('\d+\.*\d*', attrs[i])))
                    if "BR" not in attrs[i] and "Ba" not in attrs[i]:
                        item['sqft'] = int(attrs[i])

            # Get neighborhood name, if provided
            if item['neighborhood']:
                item['neighborhood'] = re.sub('[^\s\w/\.]+', '', ''.join(item['neighborhood'])).rstrip().lstrip()

            # Get posting date in UTC isoformat time
            if item['date']:
                item['date'] = ''.join(item['date'])

            # Find Google maps web address; if none exists drop record
            if item['location']:
                location_url = urllib.unquote(''.join(item['location'])).decode('utf8')
                find_location = location_url.split('loc:')

                # If location on map is found convert to geolocation; if none exists drop record
                if len(find_location) > 1:
                    location = find_location[1]
                    geo_location = geocoder.google(location)

                    # If a geocoded location exists get latitude and longitude, otherwise drop record
                    if geo_location:
                        latitude = geo_location.lat
                        longitude = geo_location.lng
                        item['latitude'] = latitude
                        item['longitude'] = longitude
                        item['latlng'] = 'POINT(%s %s)' % (latitude, longitude)
                    else:
                        raise DropItem('Missing geolocation in %s' % item)

                else:
                    raise DropItem('Missing google maps location in %s' % item)

            else:
                raise DropItem('Missing google maps web address in %s' % item)

            return item


    class PostgresqlPipeline(object):
        """ Writes data to PostgreSQL database. """

        def __init__(self):
            """ Initializes database connection. """

            self.session = connect_to_db_scrapy()


        def process_item(self, item, spider):
            """ Method used to write data to database directly from the scraper pipeline. """

            cl_id = item.get('cl_id')
            price = item.get('price')
            date = item.get('date')
            neighborhood = item.get('neighborhood')
            bedrooms = item.get('bedrooms')
            bathrooms = item.get('bathrooms')
            sqft = item.get('sqft')
            latitude = item.get('latitude')
            longitude = item.get('longitude')
            latlng = item.get('latlng')

            try:
                # Create rental details for unit
                rental_details = UnitDetails(
                                    neighborhood=neighborhood,
                                    bedrooms=bedrooms,
                                    bathrooms=bathrooms,
                                    sqft=sqft,
                                    latitude=latitude,
                                    longitude=longitude,
                                    latlng=latlng
                                )

                # Add rental details to UnitDetails table
                self.session.add(rental_details)

                # Create rental unit
                rental = Rental(
                            cl_id=cl_id,
                            price=price,
                            date_posted=date,
                            unitdetails=rental_details
                        )

                # Add rental unit to Rental table
                self.session.add(rental)

                self.session.commit()

            except:
                # If the unit already exists in db or if data does not fit db model construct
                self.session.rollback()
                raise

            finally:
                # Close the database session at the end of all tries
                self.session.close()

    ITEM_PIPELINES = {
       'rent_scraper.pipelines.RentScraperPipeline': 100,
       # I commented this out, but this is an example of what you could do if you had a third pipeline.
       # 'rent_scraper.pipelines.JsonWriterPipeline': 200,
       'rent_scraper.pipelines.PostgresqlPipeline': 300
    }

    # Close Spider after number of items scraped
    CLOSESPIDER_ITEMCOUNT = 5

    # Close spider after number of crawled page responses have been requested
    CLOSESPIDER_PAGECOUNT = 10