|
|
@@ -1,5 +1,17 @@
|
|
|
package parser;
|
|
|
|
|
|
+import com.google.common.base.Stopwatch;
|
|
|
+import com.google.common.base.Strings;
|
|
|
+import mysql.Mysql;
|
|
|
+import object.ResultDTO;
|
|
|
+import org.openqa.selenium.By;
|
|
|
+import org.openqa.selenium.StaleElementReferenceException;
|
|
|
+import org.openqa.selenium.WebElement;
|
|
|
+import org.openqa.selenium.chrome.ChromeOptions;
|
|
|
+import org.openqa.selenium.remote.RemoteWebDriver;
|
|
|
+import org.openqa.selenium.support.ui.ExpectedConditions;
|
|
|
+import org.openqa.selenium.support.ui.WebDriverWait;
|
|
|
+
|
|
|
import java.net.MalformedURLException;
|
|
|
import java.net.URL;
|
|
|
import java.time.Duration;
|
|
|
@@ -13,34 +25,17 @@ import java.util.concurrent.TimeUnit;
|
|
|
import java.util.logging.Level;
|
|
|
import java.util.logging.Logger;
|
|
|
|
|
|
-import org.openqa.selenium.By;
|
|
|
-import org.openqa.selenium.Platform;
|
|
|
-import org.openqa.selenium.StaleElementReferenceException;
|
|
|
-import org.openqa.selenium.WebElement;
|
|
|
-import org.openqa.selenium.chrome.ChromeDriver;
|
|
|
-import org.openqa.selenium.chrome.ChromeOptions;
|
|
|
-import org.openqa.selenium.remote.DesiredCapabilities;
|
|
|
-import org.openqa.selenium.remote.RemoteWebDriver;
|
|
|
-import org.openqa.selenium.support.ui.ExpectedConditions;
|
|
|
-import org.openqa.selenium.support.ui.WebDriverWait;
|
|
|
-
|
|
|
-import com.google.common.base.Stopwatch;
|
|
|
-import com.google.common.base.Strings;
|
|
|
-
|
|
|
-import mysql.Mysql;
|
|
|
-import object.ResultDTO;
|
|
|
-
|
|
|
public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
|
|
|
- private static final String DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV = "//div[contains(@class,'tabs')]/div[not(@class)]/div[1]/div";
|
|
|
+ private static final String DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV = "//div[contains(@class,'tabs')" +
|
|
|
+ "]/div[not(@class)]/div[1]/div";
|
|
|
private static final String SOCCER_RESULTS_TABLE_NAME = "SoccerResults";
|
|
|
private static final String DATE_PATTERN = "yyyyMMdd";
|
|
|
+ final int sportsId = Mysql.getInstance().getSportId("soccer");
|
|
|
private LocalDateTime baseDate;
|
|
|
private int sportId;
|
|
|
private int countryId;
|
|
|
private int leagueId;
|
|
|
-
|
|
|
- final int sportsId = Mysql.getInstance().getSportId("soccer");
|
|
|
private int currentLeagueId;
|
|
|
private int currentCountryId;
|
|
|
private List<ResultDTO> resultsToInsert = new ArrayList<>();
|
|
|
@@ -52,8 +47,8 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
System.out.println("Starting to get matches at date " + dateFormatted);
|
|
|
Stopwatch stopwatch = Stopwatch.createStarted();
|
|
|
getMatchesByDateSelenium(dateFormatted);
|
|
|
- System.out.println(
|
|
|
- "Done with date " + dateFormatted + " took " + stopwatch.elapsed(TimeUnit.SECONDS) + " seconds");
|
|
|
+ System.out.println("Done with date " + dateFormatted + " took " + stopwatch.elapsed(TimeUnit.SECONDS) + " " +
|
|
|
+ "seconds");
|
|
|
stopwatch.stop();
|
|
|
}
|
|
|
|
|
|
@@ -93,7 +88,6 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
try {
|
|
|
parseSoccerMatchesSelenium(soccerUrl);
|
|
|
} catch (MalformedURLException e) {
|
|
|
- // TODO Auto-generated catch block
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
@@ -117,34 +111,39 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
|
|
|
// Changing the user agent / browser fingerprint
|
|
|
options.addArguments("window-size=1920,1080");
|
|
|
- options.addArguments(
|
|
|
- "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36");
|
|
|
+ options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like " +
|
|
|
+ "Gecko) " + "Chrome/74.0.3729.169 Safari/537.36");
|
|
|
|
|
|
// Other
|
|
|
options.addArguments("disable-infobars");
|
|
|
|
|
|
+
|
|
|
+ System.out.println("Getting page");
|
|
|
+
|
|
|
RemoteWebDriver driver = new RemoteWebDriver(new URL("http://nordh.xyz:4444/wd/hub"), options);
|
|
|
+
|
|
|
sportId = Mysql.getInstance().getSportId("soccer");
|
|
|
driver.get(soccerUrl);
|
|
|
|
|
|
+
|
|
|
+ System.out.println("Waiting for elements");
|
|
|
WebDriverWait wait = getWaitDriver(driver, 90);
|
|
|
|
|
|
- wait.until(ExpectedConditions
|
|
|
- .numberOfElementsToBeMoreThan(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV), 1));
|
|
|
+ wait.until(ExpectedConditions.numberOfElementsToBeMoreThan(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV), 1));
|
|
|
|
|
|
- if (!checkIfElementExists(driver, "//p[text()='EU Odds']")) {
|
|
|
- WebElement oddsFormat = driver
|
|
|
- .findElement(By.xpath("//p[@class='self-center text-xs text-orange-main'][1]"));
|
|
|
+ System.out.println("Switching odds");
|
|
|
+ if (!checkIfElementExists(driver, "//p[text()='Decimal Odds']")) {
|
|
|
+ WebElement oddsFormat = driver.findElement(By.xpath("//p[@class='self-center text-xs " +
|
|
|
+ "text-orange-main'][1]"));
|
|
|
|
|
|
oddsFormat.click();
|
|
|
|
|
|
- driver.findElement(By.xpath("//a[text()='EU Odds']")).click();
|
|
|
+ driver.findElement(By.xpath("//a[text()='Decimal Odds']")).click();
|
|
|
wait.withTimeout(Duration.ofSeconds(30));
|
|
|
}
|
|
|
|
|
|
- List<WebElement> divs = driver
|
|
|
- .findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
|
|
|
-
|
|
|
+ List<WebElement> divs = driver.findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
|
|
|
+ System.out.println("Start Working on matches");
|
|
|
boolean staleElementRetried = false;
|
|
|
for (int i = 0; i < divs.size(); i++) {
|
|
|
try {
|
|
|
@@ -189,8 +188,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
resultsToInsert.clear();
|
|
|
}
|
|
|
}
|
|
|
- divs = driver
|
|
|
- .findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
|
|
|
+ divs = driver.findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
|
|
|
} catch (StaleElementReferenceException e) {
|
|
|
if (staleElementRetried) {
|
|
|
System.out.println("Stale element have already been retried continueing");
|
|
|
@@ -203,8 +201,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- String message = "INSERTING LAST " + resultsToInsert.size()
|
|
|
- + " results";
|
|
|
+ String message = "INSERTING LAST " + resultsToInsert.size() + " results";
|
|
|
Logger.getGlobal().log(Level.INFO, message);
|
|
|
Mysql.getInstance().addResults(resultsToInsert);
|
|
|
resultsToInsert.clear();
|
|
|
@@ -230,9 +227,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
time.add(0);
|
|
|
time.add(0);
|
|
|
}
|
|
|
- final LocalDateTime dt = baseDate.withHour(time.get(0))
|
|
|
- .withMinute(time.get(1)).withSecond(0)
|
|
|
- .withNano(0);
|
|
|
+ final LocalDateTime dt = baseDate.withHour(time.get(0)).withMinute(time.get(1)).withSecond(0).withNano(0);
|
|
|
|
|
|
result.setGameDate(dt);
|
|
|
}
|
|
|
@@ -246,6 +241,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
} catch (NumberFormatException e) {
|
|
|
// Skip this one
|
|
|
} catch (IndexOutOfBoundsException e) {
|
|
|
+ System.out.println("Index out of bounds for result: ");
|
|
|
System.out.println(result);
|
|
|
}
|
|
|
|
|
|
@@ -274,10 +270,8 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
|
|
|
result.setHomeTeam(homeTeamName);
|
|
|
result.setAwayTeam(awayTeamName);
|
|
|
|
|
|
- result.setHomeTeamId(
|
|
|
- Mysql.getInstance().getOrInsertTeam(homeTeamName, countryId, leagueId, sportId));
|
|
|
- result.setAwayTeamId(
|
|
|
- Mysql.getInstance().getOrInsertTeam(awayTeamName, countryId, leagueId, sportId));
|
|
|
+ result.setHomeTeamId(Mysql.getInstance().getOrInsertTeam(homeTeamName, countryId, leagueId, sportId));
|
|
|
+ result.setAwayTeamId(Mysql.getInstance().getOrInsertTeam(awayTeamName, countryId, leagueId, sportId));
|
|
|
}
|
|
|
|
|
|
}
|