Update Scraper.py

Scraper.py

@@ -7,6 +7,8 @@ from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
 
 def scrape_amazon(search_term, pincode, num_pages=5):
     options = Options()
@@ -14,8 +16,6 @@ def scrape_amazon(search_term, pincode, num_pages=5):
     options.add_argument('--disable-blink-features=AutomationControlled')
     options.add_argument('--disable-gpu')
     options.add_argument('--no-sandbox')
-    options.add_argument('--disable-dev-shm-usage')
-    options.add_argument('--window-size=1920,1080')
 
     driver = webdriver.Chrome(service=Service(), options=options)
 
@@ -27,6 +27,7 @@ def scrape_amazon(search_term, pincode, num_pages=5):
         driver.get(url)
 
         time.sleep(random.uniform(3, 5))
+
         driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
         time.sleep(random.uniform(2, 4))
 
@@ -54,7 +55,7 @@ def scrape_amazon(search_term, pincode, num_pages=5):
 
             try:
                 price_elem = product.find_element(By.XPATH, ".//span[@class='a-price-whole']")
-                selling_price = price_elem.text.replace(',', '').strip()
+                selling_price = (price_elem.text).replace(',', '').strip()
             except:
                 try:
                     price_elem = product.find_element(By.XPATH, ".//span[@class='a-offscreen']")
@@ -64,8 +65,7 @@ def scrape_amazon(search_term, pincode, num_pages=5):
 
             try:
                 mrp_elem = product.find_element(By.XPATH, ".//span[@class='a-price a-text-price']//span[@class='a-offscreen']")
-                raw_price = mrp_elem.get_attribute("textContent")
-                mrp = raw_price.replace('₹', '').replace(',', '').strip()
+                mrp = mrp_elem.get_attribute("textContent").replace('₹', '').replace(',', '').strip()
             except:
                 mrp = "No Price"
 
@@ -106,7 +106,7 @@ def scrape_amazon(search_term, pincode, num_pages=5):
                 reviews = "No Reviews"
 
             try:
-                product.find_element(By.XPATH, ".//span[contains(@class, 'a-color-secondary') and contains(text(), 'Sponsored')]")
+                ad_elem = product.find_element(By.XPATH, ".//span[contains(@class, 'a-color-secondary') and contains(text(), 'Sponsored')]")
                 ad_status = "Ad"
             except:
                 ad_status = "Not Ad"
@@ -136,13 +136,15 @@ def scrape_amazon(search_term, pincode, num_pages=5):
     driver.quit()
 
     df = pd.DataFrame(all_products)
+
     today_date = datetime.now().strftime("%Y-%m-%d")
-    filename_base = f"{search_term}_scrape_{today_date}"
-    excel_path = f"{filename_base}.xlsx"
-    df.to_excel(excel_path, index=False)
+    filename_base = f"{search_term}_scrape_{today_date}.xlsx"
+    df.to_excel(filename_base, index=False)
+
+    print(f"\nSaved: {filename_base}")
+    return filename_base
 
-    return excel_path
 
+#  Interface function for Gradio
 def scrape_amazon_interface(search_term, pincode, num_pages):
-    excel_path = scrape_amazon(search_term, pincode, num_pages)
-    return excel_path
+    return scrape_amazon(search_term, pincode, num_pages)