import sys import os import time import logging from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException, NoSuchElementException, WebDriverException from selenium.webdriver.common.action_chains import ActionChains # ----------------- CONFIG ------------------- SHOW_LOGS = False MAX_RETRIES = 2 WAIT_TIME = 20 LOGIN_WAIT_TIME = 10 LOG_FILE = '/home/auto11/public_html/scripts/pilkington_scraper.log' # ------------------------------------------- # Set up logging logging.basicConfig( level=logging.INFO, format='[%(asctime)s] %(levelname)s: %(message)s', handlers=[ logging.FileHandler(LOG_FILE), logging.StreamHandler() if SHOW_LOGS else logging.NullHandler() ] ) logger = logging.getLogger(__name__) # Setup Chrome options chrome_options = Options() chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--headless") # Comment out for debugging # Set a consistent viewport size chrome_options.add_argument("--window-size=1920,1080") # Initialize WebDriver driver = webdriver.Chrome(options=chrome_options) wait = WebDriverWait(driver, 15) # Retrieve part number from command-line arguments partNo = sys.argv[1] # Retrieve credentials from environment variables or use defaults USERNAME = "PPG6014" PASSWORD = "Order2107" TERMS_URL = "https://www.buypgwautoglass.com/registrationtandc.asp?PB=0&ShipToNo=&accept=Y" try: # Navigate to the login page driver.get(f"https://www.buypgwautoglass.com/rspAuthenticate.asp") logger.info("Website loaded.") # Login driver.find_element(By.NAME, "txtUsername").send_keys(USERNAME) driver.find_element(By.NAME, "txtPassword").send_keys(PASSWORD) driver.find_element(By.NAME, "button1").click() # Login button logger.info("Login submitted.") time.sleep(5) # wait for page load after redirec # print(driver.page_source) # Wait for the "I Agree" button to be clickable # WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.NAME, "I1"))) # logger.info("Agree button found and clickable.") driver.get("https://www.buypgwautoglass.com/Registration/registrationtandc.asp?REG=&PB=0&UserType=&ShipToNo=") driver.find_element(By.NAME, "I1").click() print(driver.page_source) # Click the "I Agree" button # driver.find_element(By.NAME, "I1").click() logger.info("Agree button clicked.") # Find by CSS selector # driver.find_element(By.CSS_SELECTOR, 'input[name="I1"]').click() # logger.info("Agree button clicked CSS.") # Or by XPath # driver.find_element(By.XPATH, '//input[@name="I1"]').click() # logger.info("Agree button clicked Xpath.") # Wait for the page to load after agreement # WebDriverWait(driver, 10).until(EC.url_contains("PartSearch")) # logger.info("Navigated to PartSearch page.") # Step 3: Navigate to part search URL # PART_SEARCH_URL = ( # f"https://www.buypgwautoglass.com/PartSearch/result.asp?" # f"REG=&PB=704&UserType=F&ShipToNo=4163&PartNo={partNo}&PartUID=410090&PartType=A" # ) # Wait for the form to be present # WebDriverWait(driver, 10).until( # EC.presence_of_element_located((By.ID, "formSearchAG")) # ) logger.info("Part search page showing.") # Wait for form to load # WebDriverWait(driver, 10).until( # EC.presence_of_element_located((By.ID, "formSearchAG")) # ) # # Set part number # part_input = WebDriverWait(driver, 10).until( # EC.visibility_of_element_located((By.ID, "PartNo")) # ) # part_number = "DW01494" # part_input.clear() # part_input.send_keys(part_number) # # Set hidden field: PartUID (can be blank or generated, depending on site logic) # driver.execute_script("document.getElementById('PartUID').value = '';") # # Set hdnVIN to 'N' (already present, but force-set in case of JS overrides) # driver.execute_script("document.getElementById('hdnVIN').value = 'N';") # # Set hdnCaptcha1 to empty string (or a value if CAPTCHA is expected) # driver.execute_script("document.getElementById('hdnCaptcha1').value = '';") # # Select Part Type = Auto Glass # part_type_radio = driver.find_element(By.ID, "PartTypeA") # if not part_type_radio.is_selected(): # part_type_radio.click() # # Submit the form # search_btn = driver.find_element(By.ID, "btnSearchAG") # search_btn.click() driver.get("https://www.buypgwautoglass.com/PartSearch/result.asp?REG=&PB=704&UserType=F&ShipToNo=4163&PartNo=DW01494&PartUID=410090&PartType=A") # Wait for result page # WebDriverWait(driver, 10).until( # EC.presence_of_element_located((By.TAG_NAME, "body")) # ) # print("Form submitted with all hidden fields set.") time.sleep(5) # Optional: print result page title print("Result Page Title:", driver.title) # driver.get(f"https://www.buypgwautoglass.com/PartSearch/result.asp?REG=&PB=704&UserType=F&ShipToNo=4163&PartNo={partNo}&PartUID=410090&PartType=A") # logger.info("Part page loaded.") # time.sleep(10) # wait for page load after redirect print(driver.page_source) # WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.ID, "form1"))) logger.info("Part page showing.") # Step 4: Parse prices rows = driver.find_elements( By.XPATH, "//tr[.//td[@class='partdesc' and contains(., 'Windshield')]]//td[@align='center']/font" ) prices = [] for row in rows: text = row.text.strip() if text.startswith('$'): try: prices.append(float(text.replace('$', '').replace(',', ''))) except ValueError: logger.warning(f"Skipping invalid price: {text}") continue if prices: logger.info(f"Prices: {prices}") logger.info(f"Lowest price: ${min(prices):.2f}") else: logger.warning("No prices found.") except TimeoutException as e: logger.error(f"Timeout error: {e}") except NoSuchElementException as e: logger.error(f"Element not found: {e}") except WebDriverException as e: logger.error(f"WebDriver error: {e}") except Exception as e: logger.error(f"An error occurred: {e}") finally: driver.quit() logger.info("WebDriver closed.")