Python pyautogui loop with scrollIntoView becomes stale

2022-01-19 16:43 Remi Castonguay imported from Stackoverflow

I have a python script that's almost working but there is a problem that I cannot resolve. My loop is working but only for the first row of images. For some reason the webelement becomes stale after 3 images. I'm positive that there are more than 3 html elements that match my query though! I'm using this script to bring the elements into view driver.execute_script("arguments[0].scrollIntoView(true);", i). Any thoughts will be greatly appreciated.

import requests
from bs4 import BeautifulSoup
# from selenium.webdriver.chrome import options
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import time
from selenium.webdriver import ActionChains
from selenium.webdriver.common.action_chains import ActionChains
import pyautogui

# driver_path = '/usr/local/bin/chromedriver'
# driver = webdriver.Chrome(executable_path=driver_path)
options = Options()
driver = webdriver.Firefox(options=options)
driver.get("https://superrare.com/market?market-options=%257B%2522first%2522%3A30%2C%2522orderBy%2522%3A%2522RECENT_NFT_EVENT_BY_TOKEN_CONTRACT_ADDRESS_AND_TOKEN_ID__TIMESTAMP_DESC%2522%2C%2522fileTypes%2522%3A%255B%2522image%2Fjpeg%2522%2C%2522image%2Fpng%2522%255D%2C%2522listPrice%2522%3Afalse%2C%2522isGenesis%2522%3Afalse%2C%2522isSeries%2522%3Afalse%2C%2522neverReceivedOffer%2522%3Afalse%2C%2522reservePrice%2522%3Afalse%2C%2522liveAuctions%2522%3Afalse%2C%2522upcomingAuctions%2522%3Afalse%2C%2522hasSold%2522%3Afalse%2C%2522ownedByCreator%2522%3Afalse%2C%2522openOffers%2522%3Afalse%2C%2522artistsCollected%2522%3Afalse%2C%2522artistsYouFollow%2522%3Afalse%2C%2522artistsThatFollowYou%2522%3Afalse%2C%2522artistsFollowedByFollowed%2522%3Afalse%2C%2522lowerPriceRange%2522%3A0%2C%2522upperPriceRange%2522%3A100000%2C%2522numCreatorSales%2522%3Afalse%2C%2522lowerMintedRange%2522%3Anull%2C%2522upperMintedRange%2522%3Anull%2C%2522startCursor%2522%3A%2522WyJyZWNlbnRfbmZ0X2V2ZW50X2J5X3Rva2VuX2NvbnRyYWN0X2FkZHJlc3NfYW5kX3Rva2VuX2lkX190aW1lc3RhbXBfZGVzYyIsWyIyMDIyLTAxLTE5VDAxOjA3OjAxKzAwOjAwIiwiMHg0MWEzMjJiMjhkMGZmMzU0MDQwZTJjYmM2NzZmMDMyMGQ4Yzg4NTBkIiwxNDQxXV0%3D%2522%2C%2522endCursor%2522%3A%2522WyJyZWNlbnRfbmZ0X2V2ZW50X2J5X3Rva2VuX2NvbnRyYWN0X2FkZHJlc3NfYW5kX3Rva2VuX2lkX190aW1lc3RhbXBfZGVzYyIsWyIyMDIyLTAxLTE4VDEyOjEzOjMzKzAwOjAwIiwiMHhiOTMyYTcwYTU3NjczZDg5ZjRhY2ZmYmU4MzBlOGVkN2Y3NWZiOWUwIiwzMTY4OV1d%2522%2C%2522hasPreviousPage%2522%3Afalse%2C%2522hasNextPage%2522%3Atrue%257D")
actions = ActionChains(driver)

time.sleep(3)

ele=driver.find_element("xpath", '//div[@id="root"]')

total_height = ele.size["height"]
time.sleep(2)  

driver.set_window_size(1920, total_height)
time.sleep(2)



downloadimg = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//section[contains(@class,'md-media')]")))

for i in downloadimg:
    # bring the webelement into the viewport
    driver.execute_script("arguments[0].scrollIntoView(true);", i)
    downloadimg2 = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//img[contains(@class,'new-grid-img')]")))
    for img in downloadimg2:
        driver.execute_script("arguments[0].scrollIntoView(true);", img)
        time.sleep(2)
        # determine element's location; the +190 and -100 is to make the mouse show ON the element rather than at it's border
        location = img.location 
        size = img.size 
        x = location['x']+190
        y = location['y']-100   
        # move to the element 
        pyautogui.moveTo((int(x), int(y)), duration=2)
        time.sleep(2)
        # right click
        actions.context_click(img).perform()
        # move to open in new browser tab
        pyautogui.move(0, -25, duration=1)
        time.sleep(1)
        pyautogui.click()
        # move to the new browser tab and click 
        pyautogui.moveTo(480, 50, duration=2)
        pyautogui.click()
        # give time to the img to load
        time.sleep(4)
        # move to the image and click
        pyautogui.moveTo(780, 550, duration=2)
        pyautogui.click()
        # actions.context_click(i).perform()
        # right click and mobe to save img as and then click
        pyautogui.rightClick()
        pyautogui.move(20, 200, duration=2)
        pyautogui.click()
        time.sleep(2)
        # press enter in the dialog box to actually save the img to download folder
        pyautogui.press('enter')
        time.sleep(2)
        # close the tab and go back to the beginning of the loop
        pyautogui.hotkey('command', 'w')

1 answer

answered 2022-01-19 17:03 Prophet

This specific web site is not initially loading all the images.
So, when you are scrolling to the images in the second row only then this images are being loaded. This causes the web elements changes so that the initial list of pictures downloadimg becomes no more relevant, in Selenium terms Stale Elements.
To make your code working you will need to get the downloadimg list again after each scrolling, something like this:

downloadimg = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//img[contains(@class,'new-grid-img')]")))


for i in downloadimg:
    # bring the webelement into the viewport
    driver.execute_script("arguments[0].scrollIntoView(true);", i)
    downloadimg = driver.find_element_by_css_selector('img.new-grid-img')
    time.sleep(2)
    # determine element's location; the +190 and -100 is to make the mouse show ON the element rather than at it's border
    location = i.location 
    size = i.size 
    x = location['x']+190
    y = location['y']-100   
    # move to the element 
    pyautogui.moveTo((int(x), int(y)), duration=2)
    time.sleep(2)
    # right click
    actions.context_click(i).perform()
    # move to open in new browser tab
    pyautogui.move(0, -25, duration=1)
    time.sleep(1)
    pyautogui.click()
    # move to the new browser tab and click 
    pyautogui.moveTo(480, 50, duration=2)
    pyautogui.click()
    # give time to the img to load
    time.sleep(4)
    # move to the image and click
    pyautogui.moveTo(780, 550, duration=2)
    pyautogui.click()
    # actions.context_click(i).perform()
    # right click and mobe to save img as and then click
    pyautogui.rightClick()
    pyautogui.move(20, 200, duration=2)
    pyautogui.click()
    time.sleep(2)
    # press enter in the dialog box to actually save the img to download folder
    pyautogui.press('enter')
    time.sleep(2)
    # close the tab and go back to the beginning of the loop
    pyautogui.hotkey('command', 'w')

You should remove the redundant driver.quit() from your code as well

See also questions close to this topic

Python File Tagging System does not retrieve nested dictionaries in dictionary

I am building a file tagging system using Python. The idea is simple. Given a directory of files (and files within subdirectories), I want to filter them out using a filter input and tag those files with a word or a phrase.

If I got the following contents in my current directory:

data/
   budget.xls
   world_building_budget.txt
a.txt
b.exe
hello_world.dat
world_builder.spec

and I execute the following command in the shell:

py -3 tag_tool.py -filter=world -tag="World-Building Tool"

My output will be:

These files were tagged with "World-Building Tool":
data/ world_building_budget.txt
hello_world.dat
world_builder.spec

My current output isn't exactly like this but basically, I am converting all files and files within subdirectories into a single dictionary like this:

def fs_tree_to_dict(path_):
    file_token = ''
    for root, dirs, files in os.walk(path_):
        tree = {d: fs_tree_to_dict(os.path.join(root, d)) for d in dirs}
        tree.update({f: file_token for f in files})
        return tree

Right now, my dictionary looks like this: key:''.

In the following function, I am turning the empty values '' into empty lists (to hold my tags):

def empty_str_to_list(d):
    for k,v in d.items():
        if v == '':
            d[k] = []
        elif isinstance(v, dict):
            empty_str_to_list(v)

When I run my entire code, this is my output:

hello_world.dat ['World-Building Tool']
world_builder.spec ['World-Building Tool']

But it does not see data/world_building_budget.txt. This is the full dictionary:

{'data': {'world_building_budget.txt': []},  
'a.txt': [],
'hello_world.dat': [], 
'b.exe': [], 
'world_builder.spec': []}

This is my full code:

import os, argparse


def fs_tree_to_dict(path_):
    file_token = ''
    for root, dirs, files in os.walk(path_):
        tree = {d: fs_tree_to_dict(os.path.join(root, d)) for d in dirs}
        tree.update({f: file_token for f in files})
        return tree


def empty_str_to_list(d):
    for k, v in d.items():
        if v == '':
            d[k] = []
        elif isinstance(v, dict):
            empty_str_to_list(v)


parser = argparse.ArgumentParser(description="Just an example", formatter_class=argparse.ArgumentDefaultsHelpFormatter)

parser.add_argument("--filter", action="store", help="keyword to filter files")
parser.add_argument("--tag", action="store", help="a tag phrase to attach to a file")
parser.add_argument("--get_tagged", action="store", help="retrieve files matching an existing tag")

args = parser.parse_args()

filter = args.filter
tag = args.tag
get_tagged = args.get_tagged
current_dir = os.getcwd()

files_dict = fs_tree_to_dict(current_dir)
empty_str_to_list(files_dict)

for k, v in files_dict.items():
    if filter in k:
        if v == []:
            v.append(tag)
            print(k, v)

        elif isinstance(v, dict):
            empty_str_to_list(v)

    if get_tagged in v:
        print(k, v)

Actaully i am working on a project and in it, it is showing no module name pip_internal plz help me for the same. I am using pycharm(conda interpreter

 File "C:\Users\pjain\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "C:\Users\pjain\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "C:\Users\pjain\AppData\Local\Programs\Python\Python310\Scripts\pip.exe\__main__.py", line 4, in <module>
  File "C:\Users\pjain\AppData\Local\Programs\Python\Python310\lib\site-packages\pip\_internal\__init__.py", line 4, in <module>
    from pip_internal.utils import _log

I am using pycharm with conda interpreter.

Looping the function if the input is not string

I'm new to python (first of all) I have a homework to do a function about checking if an item exists in a dictionary or not.

inventory = {"apple" : 50, "orange" : 50, "pineapple" : 70, "strawberry" : 30}

def check_item():
    x = input("Enter the fruit's name: ")
    if not x.isalpha():
        print("Error! You need to type the name of the fruit")
    elif x in inventory:
        print("Fruit found:", x)
        print("Inventory available:", inventory[x],"KG")
    else:
        print("Fruit not found")
check_item()

I want the function to loop again only if the input written is not string. I've tried to type return Under print("Error! You need to type the name of the fruit") but didn't work. Help

setTimeout in nested for loops in JavaScript not working like what I want
I am writing a JavaScript application to simulate a document download process in web automation. The page to interact has a paginated list of people like shown in the screenshot below. Clicking on the person's name will download this person's document.

The automation process downloads each person's document, then proceeds to the next page and continues to download in the same way.

I would like (1) a 2 second delay between each download and (2) a 2 second delay after hitting a new page and before the first download.

After some research, I came up with the following. It is very close to what I want except that it executes the download immediately after landing on the next page. Any idea on how I can tweak it to get a 2+ seconds of delay between landing on a new paginated page and the first download?
```
function downloadDoc(x)
{
  console.log("Downloading doc ", x);
}

function goToPage(p)
{  
  
   for(let i=0; i<5; i++) 
   {
      setTimeout(downloadDoc, i*2000, i);
   }

      console.log("Go to page ", p);  
}


for(let i=0; i<5; i++) 
{
   setTimeout(downloadDoc, i*2000, i);
}

for (let p = 2; p< 5; p++) 
{    
  setTimeout(goToPage, p * 12000, p);
}
```
UPDATE: Solved by following the provided answer by @jonas-wilms at https://stackoverflow.com/a/44476626/97109
```
async function downloadDoc(x)
{
  await timer (1000);
  console.log("Downloading doc ", x);
}

async function goToPage(p) 
{
    await timer(2000);
    console.log("Go to page ", p);
}


async function task(i) { // 3
  await timer(1000);
  //console.log(` ${i} done!`);
    downloadDoc(i);
}

async function main() {
  for(let i = 0; i < 8; i++) {
       await goToPage(i);
      
    for(let j = 0; j < 5; j++) {
        await task(j);

        if (j === 4) { 
          await timer(2000);
        }
    }
  }
}
    
main();

function timer(ms) { return new Promise(res => setTimeout(res, ms)); }
```
Selenium project filling up my C disk after using pyinstaller to convert it to exe

I wrote a script to scrape some info from a certain website, and it then saves it to an online spreadsheet. Testing the script with VScode was going flawlessly then I used pyinstaller to make it .exe for a client, but after working with it for like 2 hours I found out that 30 GB of my C disk were taken

How to wait for translated text to appear in DeepL.com

After entering original text in DeepL, it takes 1-2 sec for translated text to appear.

I'm unable to get translated text.

I've tried to Synchronize using:

from selenium.webdriver.support import expected_conditions as EC

# ...
# ...
# ...

WebDriverWait(driver, 10).until_not(EC.text_to_be_present_in_element((By.XPATH, "//div[@id='target-dummydiv']"), "\r\n"))
translatedText = driver.find_element(By.XPATH, "//div[@id='target-dummydiv']").text

Checking for non-empty translated text:

WebDriverWait(driver, 10).until(self.IsTranslatedTextNotEmpty)
translatedText = driver.find_element(By.XPATH, "//div[@id='target-dummydiv']").text

def IsTranslatedTextNotEmpty(self, driver: WebDriver):
    translatedText = driver.find_element(By.XPATH, "//div[@id='target-dummydiv']").text
    return len(translatedText) > 0

doesn't work.

Note: Synchronization using time.sleep(5) works.

How to get the pixel location of an image from cv2 matchTemplate result

In python I am importing mss to capture my screen, numpy to process the screen grab, cv2 to matchTemplate an image from the screen grab, and pyagutogui to control my mouse.

Below is code that takes a screenshot and if there is a match to an image with >85% accuracy it draws a bounding box around it.

This works fine. However, instead of showing the screen grab, I want to just click on the location of the matchTemplate on my screen. And in the code below I believe I am telling my mouse to move to a location in my numpy array??

My Question: How do I convert image location in my cv2.matchTemplate to image location on my computer screen?

import cv2
import numpy as np
import pyautogui as pg
import mss

sct = mss.mss()
screen_grab_dimensions = {
    'left': 0,
    'top': 0,
    'width': 750,
    'height': 500
}
img = cv2.imread('img.png')
w = img.shape[1]
h = img.shape[0]

scr = np.array(sct.grab(screen_grab_dimensions))
# Cut off alpha
scr_remove = scr[:,:,:3]

result = cv2.matchTemplate(scr_remove, img, cv2.TM_CCOEFF_NORMED)
_, max_val, _, max_loc = cv2.minMaxLoc(result)
print(f"Max Val: {max_val} Max Loc: {max_loc}")
src = scr.copy()

if max_val > .85:
    cv2.rectangle(scr, max_loc, (max_loc[0] + w, max_loc[1] + h), (0, 255, 255), 2)
    print(f"Position: {pg.position(max_loc[0], max_loc[1])}")
    pg.moveTo(max_loc[1], max_loc[0])
    pg.click()
    # will delete below
    cv2.imshow('Screen Shot', scr)
    cv2.waitKey()
    cv2.destroyAllWindows()

The while loop inside the Python for loop is repeated only once

This is a part of the code that checks the notice that appears while working using pyautogui and resumes the work when it disappears.

Capture the guide and make it into an image, check it with pyautogui, and if it exists, move on to the next one.

    number = 0
    for mark in sub_list:
        pyautogui.moveTo(1105, 356)
        pyautogui.click()
        time.sleep(random.uniform(0.5, 1.2))
        pyautogui.moveTo(291, 529)
        pyautogui.click()
        time.sleep(random.uniform(0.5, 1.2))
        pyautogui.moveTo(190, 207)
        pyautogui.click()
        time.sleep(random.uniform(0.5, 1.2))
        pyautogui.moveTo(347, 529)
        pyautogui.click()
        pyautogui.write(sub_list[number])
        print(sub_list[number])
        time.sleep(random.uniform(0.5, 1.2))
        pyautogui.moveTo(702, 531)
        pyautogui.click()
    
        icon_to_click = "Recycle Bin"
    
        r = None
        while r is None:
            r = pyautogui.locateOnScreen('rb.png', grayscale=True)
        print(icon_to_click + ' now loaded')
    
        pyautogui.write(''.join(a_record))
        time.sleep(random.uniform(0.5, 1.2))
        pyautogui.moveTo(210, 616)
        pyautogui.click()
    
        tiu = str(sub_list[number]) + "." + str(main_domain)
        sub_domain.append(tiu)
        print(sub_domain[number])
        time.sleep(random.uniform(4, 5))
        number = number + 1

However, it works fine once, but from the second iteration it can't find the image and stops.

<pre>
    icon_to_click = "Recycle Bin"

    r = None
    while r is None:
        r = pyautogui.locateOnScreen('rb.png', grayscale=True)
    print(icon_to_click + ' now loaded')
</pre>

Should I change the settings in this part? Help

use pyautogui in a certain open program in windows
I would like to use image search in a certain open application, for example, I want it to search the image only in the windows "Calculator" application, how could I do that?

Today pyautogui searches the whole screen, is it possible to limit only one open application?
```
def main():
    try:
        while True:
           button7location = pyautogui.locateOnScreen('images/calc7Key.png', region=(0,0,1920, 1080), confidence=.5)
           print(button7location)

    except KeyboardInterrupt:
        print('\nDone.')

main()
```

How to fix StaleElementReferenceException on Python Selenium?

I am getting the following error while using Selenium in python:

Traceback (most recent call last):
  File "C:\Users\pedro\Desktop\PyCharm\tutorial\index.py", line 30, in <module>
    select_object_unidades.select_by_value(unidades)
  File "D:\Anaconda3\lib\site-packages\selenium\webdriver\support\select.py", line 79, in select_by_value
    opts = self._el.find_elements(By.CSS_SELECTOR, css)
  File "D:\Anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py", line 761, in find_elements
    return self._execute(Command.FIND_CHILD_ELEMENTS,
  File "D:\Anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py", line 710, in _execute
    return self._parent.execute(command, params)
  File "D:\Anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 425, in execute
    self.error_handler.check_response(response)
  File "D:\Anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 247, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=101.0.4951.41)
Stacktrace:
Backtrace:
    Ordinal0 [0x0107B8F3+2406643]
    Ordinal0 [0x0100AF31+1945393]
    Ordinal0 [0x00EFC748+837448]
    Ordinal0 [0x00EFF154+848212]
    Ordinal0 [0x00EFF012+847890]
    Ordinal0 [0x00EFF2A0+848544]
    Ordinal0 [0x00F28FF5+1019893]
    Ordinal0 [0x00F2957B+1021307]
    Ordinal0 [0x00F1FF41+982849]
    Ordinal0 [0x00F442C4+1131204]
    Ordinal0 [0x00F1FA64+981604]
    Ordinal0 [0x00F44494+1131668]
    Ordinal0 [0x00F54682+1197698]
    Ordinal0 [0x00F44096+1130646]
    Ordinal0 [0x00F1E636+976438]
    Ordinal0 [0x00F1F546+980294]
    GetHandleVerifier [0x012E9612+2498066]
    GetHandleVerifier [0x012DC920+2445600]
    GetHandleVerifier [0x01114F2A+579370]
    GetHandleVerifier [0x01113D36+574774]
    Ordinal0 [0x01011C0B+1973259]
    Ordinal0 [0x01016688+1992328]
    Ordinal0 [0x01016775+1992565]
    Ordinal0 [0x0101F8D1+2029777]
    BaseThreadInitThunk [0x76BEFA29+25]
    RtlGetAppContainerNamedObjectPath [0x77C47A7E+286]
    RtlGetAppContainerNamedObjectPath [0x77C47A4E+238]

The relevant part is:

select_element_nivel_ensino = driver.find_element(By.ID, "formTurma:inputNivel")
select_object_nivel_ensino = Select(select_element_nivel_ensino)
all_available_options_nivel_ensino = create_list_nivel_ensino()

select_element_unidades = driver.find_element(By.ID, "formTurma:inputDepto")
select_object_unidades = Select(select_element_unidades)
all_available_options_unidades = create_list_unidades()

for nivel_ensino in all_available_options_nivel_ensino:
    select_object_nivel_ensino.select_by_value(nivel_ensino)
    for unidades in all_available_options_unidades:
        select_object_unidades.select_by_value(unidades)
        driver.find_element(By.NAME, "formTurma:j_id_jsp_1370969402_11").click()

The error happens at the end of the second FOR, after the click, running the loop only once and givin the error showed up there.

What does this error mean and how I can fix this?

I have a stale element reference but no Idea how to get around it in my case

So I am having a problem. I am trying to automate part of our software testing with Python Selenium, however now I have fallen into a bit of a problem. I have the code below.

After the selection of a drop down (country_list_element = Select(driver.find_element(By.ID, 'countryLst'))) to South Africa, then there is an ajax overlay. After this, I need to select another drop down, (id_type_element = Select(driver.find_element(By.ID, 'c6'))) to 'Passport Number'. Now this exists in HTML, however when I run this, I get an error

Message: stale element reference: element is not attached to the page document

I have the XPATH of the ajax overlay and stored it into a variable "LOADING_ELEMENT_XPATH" and am trying to wait for it to go away, and then carry on.

What am I doing wrong here? I just need to be able to select the drop down value.

I attached the HTML at the bottom

import os
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

### Variables ###
PATH = 'C:\Selenium_Drivers\chromedriver.exe'
ENROLMENT_URL = **hidden for privacy**
SHORT_TIMEOUT  = 5   # give enough time for the loading element to appear
LONG_TIMEOUT = 30  # give enough time for loading to finish
LOADING_ELEMENT_XPATH = '/html/body/div[7]'

first_name = 'PyLenium'
last_name = 'Test1'
cell_number = '0790000000'
email_address = f'{first_name.casefold()}.{last_name.casefold()}@gmail.com'.replace(' ', '')
country_list = '1'
# id_number = 
passport_number = f'{first_name.casefold()}.{last_name.casefold()}!'.replace(' ', '')
password = 'Password1!'


driver = webdriver.Chrome(PATH)

driver.get(ENROLMENT_URL)                               # Navigate to the requested URL
driver.maximize_window()
first_name_element = driver.find_element(By.ID, 'c1')
first_name_element.send_keys(first_name)
last_name_element = driver.find_element(By.ID, 'c2')
last_name_element.send_keys(last_name)
cell_number_element = driver.find_element(By.ID, 'cell')
cell_number_element.send_keys(cell_number)
email_address_element = driver.find_element(By.ID, 'c3')
email_address_element.send_keys(email_address)
country_list_element = Select(driver.find_element(By.ID, 'countryLst'))
country_list_element.select_by_visible_text('South Africa')
id_type_element = Select(driver.find_element(By.ID, 'c6'))

WebDriverWait(driver, 30).until(EC.invisibility_of_element_located((By.XPATH,LOADING_ELEMENT_XPATH)))

id_type_element.select_by_visible_text('Passport Number')

time.sleep(10)

HTML of the page is below. The troublesome drop down menu is the one with 'id="c6"'. Note about this:

when one selects the drop down value of "South Africa" in the 'id="countryLst"' drop down, the 'id="c6"' drop down dynamically changes, however the HTML that I added is after South Africa is selected.
this drop down ('id="c6"') is the one that has a "stale element"
What I want to be doing, is select countryLst as South Africa, and then select c6 as Passport Number

<div class="col-md-4">
    <div class="input-group" style="margin-bottom: 1em;">
        <span class="input-group-addon "><i class="fa fa-globe" aria-hidden="true" style="font-size: 21px;"></i></span>
</div>
    <span id="countryLst_ctl">
        <select name="countryLst" id="countryLst" class="listbox fullWidth" size="1">
            <option value="0">Country of Origin</option>
            <option value="1">South Africa</option>
            <option value="2">United Arab Emirates</option>
        </select>
    </span>                        
</div>
<div class="col-md-4">
    <div class="input-group" style="margin-bottom: 1em;">
        <span class="input-group-addon">
            <img src="/assets/images/id-card.png" style="width: 18px;">
        </span>
        <span id="c6_ctl">
            <select name="c6" id="c6" class="listbox fullWidth" size="1">
                <option value="0">--Select ID Type--</option>
                <option value="1">RSA ID Number</option>
                <option value="2">Passport Number</option>
            </select>
        </span>                      
    </div>
</div>

How to handle StaleElementReferenceException (Selenium Python)?

I am writing a scraper for https://etherscan.io/ using selenium in python. I am getting StaleElementReferenceException in my code when it clicks on a button and not finding any good source to solve this error.

This is the code

url = "https://etherscan.io/login?"
driver.implicitly_wait(30)
driver.get(url)
username = driver.find_element_by_xpath('//*[@id="ContentPlaceHolder1_txtUserName"]')
username.send_keys('user')
password = driver.find_element_by_xpath('//*[@id="ContentPlaceHolder1_txtPassword"]')
password.send_keys('pass')
login = driver.find_element_by_xpath('//*[@id="ContentPlaceHolder1_btnLogin"]')
login.click()
more = driver.find_element_by_xpath('//*[@id="moreMegaMenu"]')
more.click()
word_cloud = driver.find_element_by_xpath('//*[@id="LI41"]/a').click()
soup = BeautifulSoup(driver.page_source, 'html.parser')
wait = WebDriverWait(driver, 10)
dropdown = driver.find_elements_by_css_selector('.dropdown-toggle')
for ele in dropdown:
    ele.click()
    time.sleep(1)
    account = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'.d-block')))
    driver.execute_script("arguments[0].click();", account)
    time.sleep(1)
    driver.execute_script("window.history.go(-1)")

Error Message:

StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=100.0.4896.75)

StaleElementReferenceException            Traceback (most recent call last)
<ipython-input-108-2804d338c51f> in <module>
      1 dropdown = driver.find_elements_by_css_selector('.dropdown-toggle')
      2 for ele in dropdown:
----> 3     ele.click() # this is the line where it is showing error
      4     time.sleep(1)
      5     account = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'.d-block')))

Related questions

Python pyautogui loop with scrollIntoView becomes stale

1 answer

See also questions close to this topic