Forums

Selenium Not working properly

Hello, I wrote this python scraping code to check if the website is open for aplications or not if so. it would give me a notification to my mobile:

import seleniumwire.undetected_chromedriver as uc

import os.path

import random

import time

import pandas as pd

from selenium import webdriver

from selenium.webdriver.chrome.service import Service

from selenium.common.exceptions import TimeoutException

from selenium.webdriver.common.by import By

from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.support.ui import Select, WebDriverWait

from selenium.webdriver.chrome.options import Options

from twocaptcha import TwoCaptcha

from pushbullet import Pushbullet


chrome_options = Options()

chrome_options.add_argument("--headless")

chrome_options.add_argument("--no-sandbox")

chrome_options.add_argument("--disable-dev-shm-usage")



df = pd.read_excel('visa.xlsx')


chrome_driver_path = r"C:\ProgramData\chocolatey\bin\chromedriver.exe"


def initialize_webdriver():

    # Chrome options setup
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--no-sandbox")

    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument('--window-size=1920,1080')

    chrome_options.add_argument("--ignore-certificate-errors")
    chrome_options.add_argument("--allow-running-insecure-content")

    service = Service(chrome_driver_path)


    web = webdriver.Chrome(service=service, options=chrome_options)
    # chrome_options.add_argument("user-agent=")
    # Proxy options setup
    seleniumwire_options = {
        'proxy': {
            'http': 'http://;kj;jkl:Iklj;kjiou_country-af@geo.iproyal.com:12321',
            'https': 'http://[oioljkljlj;:jkhjhk_country-af@geo.iproyal.com:12321',
            'no_proxy': 'localhost,555555555'
        }
    }


    # Navigate to the specified URL
    return web


# Use the function in your main script
if __name__ == '__main__':
    web = initialize_webdriver()
    time.sleep(5)
    web.get('website')
    print("not")
    time.sleep(5)
    print('with')

# Loop through each entry in the dataframe
for i in df.index:
    while True:  # Start of the retry loop
        try:
            entry = df.loc[i]
            # print(entry)

            wait = WebDriverWait(web, 10)

            # Select Visa Type
            VisaType = wait.until(EC.element_to_be_clickable((By.NAME, 'visa_type')))
            Select(VisaType).select_by_value(str(entry['Visa Type']))
            time.sleep(random.uniform(1, 3))

            # Select Nationality
            Nationality = wait.until(EC.element_to_be_clickable((By.NAME, 'nationality')))
            Select(Nationality).select_by_value(str(entry['Nationality']))
            time.sleep(random.uniform(1, 3))

            # Select Type of Passport
            Type_of_passport = wait.until(EC.element_to_be_clickable((By.NAME, 'passport_type')))
            Select(Type_of_passport).select_by_value(str(entry['Type of passport']))
            time.sleep(random.uniform(1, 3))

            # Select Place of Issue
            Place_of_Issue = wait.until(EC.element_to_be_clickable((By.NAME, 'issuer_agent')))
            Select(Place_of_Issue).select_by_value(str(entry['Place of Issue']))
            time.sleep(random.uniform(1, 3))
            time.sleep(random.uniform(1, 3))

            # Handle CAPTCHA
            captcha_img = web.find_element(By.CLASS_NAME, 'ecaptcha')
            captcha_img.screenshot('captcha.png')
            api_key = os.getenv('APIKEY_2CAPTCHA',lkjkljlkjlkjklj')
            solver = TwoCaptcha(api_key)

            try:
                result = solver.normal('captcha.png')
            except Exception as e:
                print(e)
                break  # Exit the loop if an error occurs with CAPTCHA solving

            code = result['code']
            print(code)

            # Wait for the CAPTCHA input field to be clickable
            captcha_input_field = wait.until(EC.element_to_be_clickable((By.ID, 'id_captcha_1')))
            captcha_input_field.send_keys(code)
            time.sleep(random.uniform(1, 3))

            # Wait for the submit button to be clickable
            submit_button = wait.until(EC.element_to_be_clickable((By.ID, 'first_step_submit_btn')))
            time.sleep(random.uniform(1, 3))

            # Scroll to the button and then click
            web.execute_script("arguments[0].scrollIntoView(true);", submit_button)
            submit_button.click()

            # Check for invalid CAPTCHA message
            try:
                wait.until(EC.visibility_of_element_located(
                    (By.CSS_SELECTOR, "span.help-block.m-b-none[style='color: #ff0000']")), 10)
                print("Invalid CAPTCHA. Refreshing and retrying...")
                web.refresh()
                continue  # Restart the current iteration
            except TimeoutException:
                # If no invalid CAPTCHA message is found, proceed to check for visa list full message
                pass

            # Check for "Visa application list full" message
            try:
                wait.until(
                    EC.visibility_of_element_located((By.CSS_SELECTOR, "div.alert.alert-danger-new.alert-dismissable")))
                print("Visa application list full. Waiting and retrying...")
                time.sleep(600)  # Wait for 10 minutes (600 seconds) before retrying
                web.refresh()
                continue  # Restart the current iteration
            except TimeoutException:
                # If no visa list full message is found, break the loop and proceed
                break

        except Exception as e:
            print(f"An error occurred: {e}")
            break  # Exit the loop if an unexpected error occurs
        time.sleep(120)

    # Scroll down a bit after filling the Occupation field
    web.execute_script("window.scrollBy(0, 300);")  # Adjust the scroll value as needed

    # Wait for a moment to let the page load
    time.sleep(3)  # Adjust the wait time as needed
    # Scroll down a bit after filling the Occupation field
    web.execute_script("window.scrollBy(0, 400);")  # Adjust the scroll value as needed

    # Wait for the first name input field to be visible and interactable
    first_name_field = WebDriverWait(web, 10).until(EC.element_to_be_clickable((By.ID, 'id_first_name')))
    # Send keys to the first name input field
    first_name_field.send_keys(str(entry['First name']))

    api_key = "o.gkjlkjlAxlkjlk"
    pb = Pushbullet(api_key)
    push = pb.push_note("alarm", "Site baz Shoda")
    Surname = web.find_element(By.ID, 'id_last_name')
    Surname.send_keys(str(entry['Surname']))
# Wait for user input before closing
input("Press Enter to exit...")

I run this code in pycharm and it runs perfectly without any problem. But when i try it in pythonanywhere, i get this error:

00:24 ~ $ python iran_visa.py
Traceback (most recent call last):
  File "/home/qasemshakeri/.local/lib/python3.9/site-packages/selenium/webdriver/common/service.py", line 71, in start
    self.process = subprocess.Popen(cmd, env=self.env,
  File "/usr/local/lib/python3.9/subprocess.py", line 951, in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,
  File "/usr/local/lib/python3.9/subprocess.py", line 1821, in _execute_child
    raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: 'C:\\ProgramData\\chocolatey\\bin\\chromedriver.exe'

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/qasemshakeri/iran_visa.py", line 69, in <module>
    web = initialize_webdriver()
  File "/home/qasemshakeri/iran_visa.py", line 48, in initialize_webdriver
    web = webdriver.Chrome(service=service, options=chrome_options)
  File "/home/qasemshakeri/.local/lib/python3.9/site-packages/selenium/webdriver/chrome/webdriver.py", line 70, in __init__
    super(WebDriver, self).__init__(DesiredCapabilities.CHROME['browserName'], "goog",
  File "/home/qasemshakeri/.local/lib/python3.9/site-packages/selenium/webdriver/chromium/webdriver.py", line 90, in __init__
    self.service.start()
  File "/home/qasemshakeri/.local/lib/python3.9/site-packages/selenium/webdriver/common/service.py", line 81, in start
    raise WebDriverException(
selenium.common.exceptions.WebDriverException: Message: 'C:\ProgramData\chocolatey\bin\chromedriver.exe' executable needs to be in PATH. 
Please see https://chromedriver.chromium.org/home

I used python 3.9 in my pycharm platform.

[edit by admin: formatting]

qasemshakeri | 8 posts | Jan. 18, 2024, 12:39 a.m. | permalink

I think that the problem is the line 

chrome_driver_path = r"C:\ProgramData\chocolatey\bin\chromedriver.exe"

...combined with this:

service = Service(chrome_driver_path)

...and the service=service in your construction of the webdriver.Chrome object. All of those are trying to make Selenium use a Chromedriver from the location where you have it installed on your own Windows machine, so they won't work (and are unnecessary) on PythonAnywhere.

Check out [this minimal Selenium example)[https://help.pythonanywhere.com/pages/selenium] -- it shows you the options you should use when running it on PythonAnywhere.

Staff giles | 12096 posts | PythonAnywhere staff | Jan. 18, 2024, 11:36 a.m. | permalink

Is pythonanywhere works also with selenium_wire, and undetected selenium driver?

Because i need to use proxy in my code, and run it with a proxy. I am not sure if i can do it with selenium, and selenium webdriver alone. or am i wrong ?

qasemshakeri | 8 posts | Jan. 18, 2024, 5:33 p.m. | permalink

I don't think it will work, but it may.

Staff glenn | 9718 posts | PythonAnywhere staff | Jan. 18, 2024, 5:44 p.m. | permalink