For loop returning error when trying to loop through list with beauifulsoup

I am trying to create a loop that will loop though locations and extract out the necessary data and append it to the rest of the locations.

I feel that the code I have written is good but keep getting an error of:

AttributeError: 'NoneType' object has no attribute 'find_all'

but I know that shouldn't be the case.

Any help would be appreciated. Here is my code:

import urllib.request
from bs4 import BeautifulSoup as bs
import re
import pandas as pd
import requests


locations = ['las-vegas-nv','denver-co']
for location in locations:
    headers = {
        'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
    }



    url = 'https://www.apartments.com/{location}/1-bedrooms/'
    page = requests.get(url, headers=headers)
    soup = bs(page.text, 'lxml')
    table = soup.find("table", class_="rentTrendGrid")
    rows = []

    for tr in table.find_all('tr'):
        rows.append([td.text for td in tr.find_all(['th', 'td'])])

#header_row = rows[0]
    rows = list(zip(*rows[0:])) # tranpose the table
    df = pd.DataFrame(rows[1:], columns=rows[0])
    df['City'] = location
    dfs.append(df)
    df = pd.concat(dfs).reset_index(drop = True)
print(df)

1 answer

  • answered 2021-05-03 18:32 Ben Sorensen

    Andrej was right, super simple just had to put the 'f' in front.

    import urllib.request
    from bs4 import BeautifulSoup as bs
    import re
    import pandas as pd
    import requests
    
    
    locations = ['las-vegas-nv','denver-co']
    for location in locations:
        headers = {
            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
        }
    
    
    
        url = f'https://www.apartments.com/{location}/1-bedrooms/'
        page = requests.get(url, headers=headers)
        soup = bs(page.text, 'lxml')
        table = soup.find("table", class_="rentTrendGrid")
        rows = []
    
        for tr in table.find_all('tr'):
            rows.append([td.text for td in tr.find_all(['th', 'td'])])
    
    #header_row = rows[0]
        rows = list(zip(*rows[0:])) # tranpose the table
        df = pd.DataFrame(rows[1:], columns=rows[0])
        df['City'] = location
        dfs.append(df)
        df = pd.concat(dfs).reset_index(drop = True)
    print(df)