package parser; import java.time.Duration; import java.util.Collections; import org.openqa.selenium.By; import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.remote.RemoteWebDriver; import org.openqa.selenium.support.ui.WebDriverWait; public class ParserBase { protected boolean checkIfElementExists(WebDriver driver, String xpath) { return !driver.findElements(By.xpath(xpath)).isEmpty(); } protected boolean checkIfElementExists(WebElement element, String xpath) { return !element.findElements(By.xpath(xpath)).isEmpty(); } protected float formatFloat(String string) { float result = -1f; try { result = Float.parseFloat(string); } catch (NumberFormatException e) { // Empty by design } return result; } protected JavascriptExecutor getJsExecutor(ChromeDriver driver) { return driver; } protected ChromeDriver getSeleniumDriver() { ChromeOptions options = new ChromeOptions(); System.setProperty("webdriver.chrome.driver", System.getProperty("user.dir") + "/chromedriver.exe"); System.setProperty("webdriver.chrome.silentOutput", "true"); // Fixing 255 Error crashes options.addArguments("--no-sandbox"); options.addArguments("--disable-dev-shm-usage"); // Options to trick bot detection // Removing webdriver property options.addArguments("--disable-blink-features=AutomationControlled"); options.setExperimentalOption("excludeSwitches", Collections.singletonList("enable-automation")); options.setExperimentalOption("useAutomationExtension", null); // Changing the user agent / browser fingerprint options.addArguments("window-size=1920,1080"); options.addArguments( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"); // Other options.addArguments("disable-infobars"); ChromeDriver driver = new ChromeDriver(options); driver.executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"); return driver; } protected WebDriverWait getWaitDriver(ChromeDriver driver) { return new WebDriverWait(driver, Duration.ofSeconds(30)); } protected WebDriverWait getWaitDriver(RemoteWebDriver driver, int duration) { return new WebDriverWait(driver, Duration.ofSeconds(duration)); } protected void scrollElementIntoView(WebDriver driver, WebElement element) { ((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView();", element); } protected void scrollElementIntoViewCenter(ChromeDriver driver, WebElement element) { ((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView({ block: 'center' });", element); } protected void scrollToTopOfPage(ChromeDriver driver) { ((JavascriptExecutor) driver) .executeScript("document.body.scrollTop = document.documentElement.scrollTop = 0;"); } }