Python pyautogui loop with scrollIntoView becomes stale

I have a python script that's almost working but there is a problem that I cannot resolve. My loop is working but only for the first row of images. For some reason the webelement becomes stale after 3 images. I'm positive that there are more than 3 html elements that match my query though! I'm using this script to bring the elements into view driver.execute_script("arguments[0].scrollIntoView(true);", i). Any thoughts will be greatly appreciated.

import requests
from bs4 import BeautifulSoup
# from selenium.webdriver.chrome import options
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import time
from selenium.webdriver import ActionChains
from selenium.webdriver.common.action_chains import ActionChains
import pyautogui

# driver_path = '/usr/local/bin/chromedriver'
# driver = webdriver.Chrome(executable_path=driver_path)
options = Options()
driver = webdriver.Firefox(options=options)
driver.get("https://superrare.com/market?market-options=%257B%2522first%2522%3A30%2C%2522orderBy%2522%3A%2522RECENT_NFT_EVENT_BY_TOKEN_CONTRACT_ADDRESS_AND_TOKEN_ID__TIMESTAMP_DESC%2522%2C%2522fileTypes%2522%3A%255B%2522image%2Fjpeg%2522%2C%2522image%2Fpng%2522%255D%2C%2522listPrice%2522%3Afalse%2C%2522isGenesis%2522%3Afalse%2C%2522isSeries%2522%3Afalse%2C%2522neverReceivedOffer%2522%3Afalse%2C%2522reservePrice%2522%3Afalse%2C%2522liveAuctions%2522%3Afalse%2C%2522upcomingAuctions%2522%3Afalse%2C%2522hasSold%2522%3Afalse%2C%2522ownedByCreator%2522%3Afalse%2C%2522openOffers%2522%3Afalse%2C%2522artistsCollected%2522%3Afalse%2C%2522artistsYouFollow%2522%3Afalse%2C%2522artistsThatFollowYou%2522%3Afalse%2C%2522artistsFollowedByFollowed%2522%3Afalse%2C%2522lowerPriceRange%2522%3A0%2C%2522upperPriceRange%2522%3A100000%2C%2522numCreatorSales%2522%3Afalse%2C%2522lowerMintedRange%2522%3Anull%2C%2522upperMintedRange%2522%3Anull%2C%2522startCursor%2522%3A%2522WyJyZWNlbnRfbmZ0X2V2ZW50X2J5X3Rva2VuX2NvbnRyYWN0X2FkZHJlc3NfYW5kX3Rva2VuX2lkX190aW1lc3RhbXBfZGVzYyIsWyIyMDIyLTAxLTE5VDAxOjA3OjAxKzAwOjAwIiwiMHg0MWEzMjJiMjhkMGZmMzU0MDQwZTJjYmM2NzZmMDMyMGQ4Yzg4NTBkIiwxNDQxXV0%3D%2522%2C%2522endCursor%2522%3A%2522WyJyZWNlbnRfbmZ0X2V2ZW50X2J5X3Rva2VuX2NvbnRyYWN0X2FkZHJlc3NfYW5kX3Rva2VuX2lkX190aW1lc3RhbXBfZGVzYyIsWyIyMDIyLTAxLTE4VDEyOjEzOjMzKzAwOjAwIiwiMHhiOTMyYTcwYTU3NjczZDg5ZjRhY2ZmYmU4MzBlOGVkN2Y3NWZiOWUwIiwzMTY4OV1d%2522%2C%2522hasPreviousPage%2522%3Afalse%2C%2522hasNextPage%2522%3Atrue%257D")
actions = ActionChains(driver)

time.sleep(3)

ele=driver.find_element("xpath", '//div[@id="root"]')

total_height = ele.size["height"]
time.sleep(2)  

driver.set_window_size(1920, total_height)
time.sleep(2)



downloadimg = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//section[contains(@class,'md-media')]")))

for i in downloadimg:
    # bring the webelement into the viewport
    driver.execute_script("arguments[0].scrollIntoView(true);", i)
    downloadimg2 = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//img[contains(@class,'new-grid-img')]")))
    for img in downloadimg2:
        driver.execute_script("arguments[0].scrollIntoView(true);", img)
        time.sleep(2)
        # determine element's location; the +190 and -100 is to make the mouse show ON the element rather than at it's border
        location = img.location 
        size = img.size 
        x = location['x']+190
        y = location['y']-100   
        # move to the element 
        pyautogui.moveTo((int(x), int(y)), duration=2)
        time.sleep(2)
        # right click
        actions.context_click(img).perform()
        # move to open in new browser tab
        pyautogui.move(0, -25, duration=1)
        time.sleep(1)
        pyautogui.click()
        # move to the new browser tab and click 
        pyautogui.moveTo(480, 50, duration=2)
        pyautogui.click()
        # give time to the img to load
        time.sleep(4)
        # move to the image and click
        pyautogui.moveTo(780, 550, duration=2)
        pyautogui.click()
        # actions.context_click(i).perform()
        # right click and mobe to save img as and then click
        pyautogui.rightClick()
        pyautogui.move(20, 200, duration=2)
        pyautogui.click()
        time.sleep(2)
        # press enter in the dialog box to actually save the img to download folder
        pyautogui.press('enter')
        time.sleep(2)
        # close the tab and go back to the beginning of the loop
        pyautogui.hotkey('command', 'w')
   

1 answer

  • answered 2022-01-19 17:03 Prophet

    This specific web site is not initially loading all the images.
    So, when you are scrolling to the images in the second row only then this images are being loaded. This causes the web elements changes so that the initial list of pictures downloadimg becomes no more relevant, in Selenium terms Stale Elements.
    To make your code working you will need to get the downloadimg list again after each scrolling, something like this:

    downloadimg = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//img[contains(@class,'new-grid-img')]")))
    
    
    for i in downloadimg:
        # bring the webelement into the viewport
        driver.execute_script("arguments[0].scrollIntoView(true);", i)
        downloadimg = driver.find_element_by_css_selector('img.new-grid-img')
        time.sleep(2)
        # determine element's location; the +190 and -100 is to make the mouse show ON the element rather than at it's border
        location = i.location 
        size = i.size 
        x = location['x']+190
        y = location['y']-100   
        # move to the element 
        pyautogui.moveTo((int(x), int(y)), duration=2)
        time.sleep(2)
        # right click
        actions.context_click(i).perform()
        # move to open in new browser tab
        pyautogui.move(0, -25, duration=1)
        time.sleep(1)
        pyautogui.click()
        # move to the new browser tab and click 
        pyautogui.moveTo(480, 50, duration=2)
        pyautogui.click()
        # give time to the img to load
        time.sleep(4)
        # move to the image and click
        pyautogui.moveTo(780, 550, duration=2)
        pyautogui.click()
        # actions.context_click(i).perform()
        # right click and mobe to save img as and then click
        pyautogui.rightClick()
        pyautogui.move(20, 200, duration=2)
        pyautogui.click()
        time.sleep(2)
        # press enter in the dialog box to actually save the img to download folder
        pyautogui.press('enter')
        time.sleep(2)
        # close the tab and go back to the beginning of the loop
        pyautogui.hotkey('command', 'w')
    

    You should remove the redundant driver.quit() from your code as well

How many English words
do you know?
Test your English vocabulary size, and measure
how many words do you know
Online Test
Powered by Examplum