New Url

Odds/src/main/Main.java

@@ -3,6 +3,7 @@ package main;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.time.LocalDateTime;
 
 import parser.OddsPortal;
 
@@ -47,15 +48,15 @@ public class Main {
 
     private static void updateFromOddsportal() {
         final OddsPortal op = new OddsPortal();
-        System.out.println("Getting Yesterdays matches");
-        op.getYesterdaysMatches();
-        System.out.println("Getting Todays Matches");
-        op.getTodaysMatches();
-        System.out.println("Getting tomorrows matches");
-        op.getTomorrowsMatches();
-
-        System.out.println("Getting next matches");
-        op.getNextDaysMatches();
+
+        System.out.println("Getting One days ago matches");
+        op.getMatches(LocalDateTime.now().plusDays(-1));
+        System.out.println("Getting Today days ago matches");
+        op.getMatches(LocalDateTime.now().plusDays(0));
+        System.out.println("Getting Tomorrow days matches");
+        op.getMatches(LocalDateTime.now().plusDays(1));
+        System.out.println("Getting Next days matches");
+        op.getMatches(LocalDateTime.now().plusDays(2));
 
     }
 

Odds/src/parser/OddsPortal.java

@@ -9,16 +9,19 @@ import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.openqa.selenium.By;
+import org.openqa.selenium.StaleElementReferenceException;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.chrome.ChromeOptions;
 import org.openqa.selenium.remote.RemoteWebDriver;
 import org.openqa.selenium.support.ui.ExpectedConditions;
 import org.openqa.selenium.support.ui.WebDriverWait;
 
+import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
 
 import mysql.Mysql;
@@ -40,108 +43,14 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
     private List<ResultDTO> resultsToInsert = new ArrayList<>();
     private String currentSeason;
 
-//    Beh�ver g�ras om till selenium
-//    public void getHistoricMatches(String sport, String country, String league, String year) {
-//        final String url = "https://www.oddsportal.com/";
-//        final String resultsPage = "/results";
-//        final WebClient webClient = new WebClient();
-//        webClient.getOptions().setUseInsecureSSL(true);
-//        webClient.getOptions().setCssEnabled(false);
-//        webClient.getOptions().setJavaScriptEnabled(true);
-//        webClient.getOptions().setThrowExceptionOnScriptError(false);
-//        Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
-//
-//        league = league.replace(" ", "-");
-//        league = league.replace("\\.", "");
-//        country = country.replace(" ", "-");
-//        league = league.replace("\\.", "");
-//        final Mysql mysql = Mysql.getInstance();
-//
-//        int currentParsePage = 1;
-//
-//        final String urlYearPart;
-//        if (year.equals(String.valueOf(LocalDate.now().getYear()))) {
-//            urlYearPart = "";
-//        } else {
-//            urlYearPart = "-" + year;
-//        }
-//
-//        try {
-//            sportId = mysql.getSportId(sport);
-//            countryId = mysql.getCountryId(country);
-//            leagueId = mysql.getLeagueId(sportId, countryId, league);
-//            String season = "";
-//
-//            final HtmlPage leaguePage = webClient
-//                    .getPage(url + "/" + sport + "/" + country + "/" + league + urlYearPart + resultsPage);
-//            final List<HtmlAnchor> yearFilter = leaguePage.getByXPath("//ul[contains(@class,'main-filter')]//a");
-//            for (final HtmlAnchor a : yearFilter) {
-//                final String active = ((HtmlSpan) a.getParentNode().getParentNode()).getAttribute(CLASS);
-//                if (active.contains("active") && !active.contains("inactive")) {
-//                    season = a.asNormalizedText();
-//                    year = season.replace('/', '-');
-//                }
-//            }
-//
-//            HtmlDivision tournamentTableDiv = leaguePage.getHtmlElementById("tournamentTable");
-//            HtmlTable tournamentTable = (HtmlTable) tournamentTableDiv.getFirstChild();
-//
-//            gameDate = LocalDateTime.now();
-//            final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH);
-//            parseTournamentTable(sportId, countryId, leagueId, season, tournamentTable, gameDate, dateFormatter);
-//            final HtmlDivision paginationLinksDiv = (HtmlDivision) tournamentTableDiv.getLastChild();
-//            final List<HtmlAnchor> pagiantionLinks = paginationLinksDiv
-//                    .getByXPath(".//a[contains(@href, 'page') and not(.//span[contains(@class, 'arrow')])]");
-//            for (final HtmlAnchor a : pagiantionLinks) {
-//                // When done with start page click pagiantion
-//                final int parsePage = Integer.parseInt(a.getTextContent());
-//                if (parsePage > currentParsePage) {
-//                    a.click();
-//                    webClient.waitForBackgroundJavaScript(1000);
-//
-//                    tournamentTableDiv = leaguePage.getHtmlElementById("tournamentTable");
-//                    tournamentTable = (HtmlTable) tournamentTableDiv.getFirstChild();
-//                    parseTournamentTable(sportId, countryId, leagueId, season, tournamentTable, gameDate,
-//                            dateFormatter);
-//                    currentParsePage = parsePage;
-//                }
-//                // process new tournament table content
-//            }
-//        } catch (FailingHttpStatusCodeException | IOException e) {
-//            e.printStackTrace();
-//        } catch (final ClassCastException cce) {
-//            Log.getLog().info("Class cast exception message: " + cce.getMessage() + " \ncause: " + cce.getCause());
-//        } finally {
-//            Mysql.getInstance().setParsingForLeague(leagueId, sportId, countryId, gameDate, currentParsePage, year);
-//        }
-//        webClient.close();
-//        Log.getLog().info("DONE with " + country + " (" + countryId + ") league " + league + "(" + leagueId + ")");
-//    }
-
-    public void getNextDaysMatches() {
-        baseDate = LocalDateTime.now().plusDays(2);
-        final String dateTomorrow = LocalDate.now().plusDays(2).format(DateTimeFormatter.ofPattern(DATE_PATTERN));
-        getMatchesByDateSelenium(dateTomorrow);
-    }
-
-    public void getTodaysMatches() {
-        baseDate = LocalDateTime.now();
-        final String date = LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN));
-        getMatchesByDateSelenium(date);
-    }
-
-    // https://stackoverflow.com/questions/14439991/skip-particular-javascript-execution-in-html-unit
-    // Skip url
-    public void getTomorrowsMatches() {
-        baseDate = LocalDateTime.now().plusDays(1);
-        final String dateTomorrow = LocalDate.now().plusDays(1).format(DateTimeFormatter.ofPattern(DATE_PATTERN));
-        getMatchesByDateSelenium(dateTomorrow);
-    }
-
-    public void getYesterdaysMatches() {
-        baseDate = LocalDateTime.now().plusDays(-1);
-        final String date = LocalDate.now().plusDays(-1).format(DateTimeFormatter.ofPattern(DATE_PATTERN));
-        getMatchesByDateSelenium(date);
+    public void getMatches(LocalDateTime date) {
+        baseDate = date;
+        final String dateFormatted = date.format(DateTimeFormatter.ofPattern(DATE_PATTERN));
+        System.out.println("Starting to get matches at date " + dateFormatted);
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        getMatchesByDateSelenium(dateFormatted);
+        System.out.println(
+                "Done with date " + dateFormatted + " took " + stopwatch.elapsed(TimeUnit.SECONDS) + " seconds");
     }
 
     private int getCountryId(List<WebElement> links) {
@@ -176,7 +85,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
     }
 
     private void getMatchesByDateSelenium(String date) {
-        final String soccerUrl = "https://oddsportal.com/matches/soccer/" + date;
+        final String soccerUrl = "https://oddsportal.com/matches/football/" + date;
         try {
             parseSoccerMatchesSelenium(soccerUrl);
         } catch (MalformedURLException e) {
@@ -186,7 +95,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
     }
 
     private void parseSoccerMatchesSelenium(final String soccerUrl) throws MalformedURLException {
-        // ChromeDriver driver = getSeleniumDriver();
+//        ChromeDriver driver = getSeleniumDriver();
 //        DesiredCapabilities capabilities = new DesiredCapabilities();
 //        capabilities.setCapability("browserName", "Chrome");
 //        capabilities.setPlatform(Platform.LINUX);
@@ -232,68 +141,83 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
         List<WebElement> divs = driver
                 .findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
 
+        boolean staleElementRetried = false;
         for (int i = 0; i < divs.size(); i++) {
-            WebElement element = divs.get(i);
-            scrollElementIntoView(driver, element);
-            ResultDTO result = new ResultDTO(SOCCER_RESULTS_TABLE_NAME);
+            try {
+                WebElement element = divs.get(i);
+                scrollElementIntoView(driver, element);
+                ResultDTO result = new ResultDTO(SOCCER_RESULTS_TABLE_NAME);
 
-            boolean somethingWrong = false;
-            List<WebElement> subDivs = element.findElements(By.xpath("./div"));
-            if (subDivs.size() == 3) {
-                WebElement competitionsDiv = subDivs.get(0);
-                List<WebElement> links = competitionsDiv.findElements(By.xpath(".//a"));
+                boolean somethingWrong = false;
+                List<WebElement> subDivs = element.findElements(By.xpath("./div"));
+                if (subDivs.size() == 3) {
+                    WebElement competitionsDiv = subDivs.get(0);
+                    List<WebElement> links = competitionsDiv.findElements(By.xpath(".//a"));
 
-                countryId = getCountryId(links);
-                leagueId = getLeagueId(links);
-                result.setLeagueId(leagueId);
-                result.setCountryId(countryId);
+                    countryId = getCountryId(links);
+                    leagueId = getLeagueId(links);
+                    result.setLeagueId(leagueId);
+                    result.setCountryId(countryId);
 
-                WebElement firstResultsDiv = subDivs.get(2);
+                    WebElement firstResultsDiv = subDivs.get(2);
 
-                setGameDate(result, firstResultsDiv);
+                    setGameDate(result, firstResultsDiv);
 
-                setTeamsInfo(result, firstResultsDiv);
+                    setTeamsInfo(result, firstResultsDiv);
 
-                setScoring(result, firstResultsDiv);
+                    setScoring(result, firstResultsDiv);
 
-                setOdds(result, firstResultsDiv);
+                    setOdds(result, firstResultsDiv);
 
-            } else if (subDivs.size() == 1) {
+                } else if (subDivs.size() == 1) {
 
-                result.setLeagueId(leagueId);
-                result.setCountryId(countryId);
-                WebElement firstResultsDiv = subDivs.get(0);
+                    result.setLeagueId(leagueId);
+                    result.setCountryId(countryId);
+                    WebElement firstResultsDiv = subDivs.get(0);
 
-                setGameDate(result, firstResultsDiv);
-                setTeamsInfo(result, firstResultsDiv);
+                    setGameDate(result, firstResultsDiv);
+                    setTeamsInfo(result, firstResultsDiv);
 
-                setScoring(result, firstResultsDiv);
+                    setScoring(result, firstResultsDiv);
 
-                setOdds(result, firstResultsDiv);
-            } else {
-                somethingWrong = true;
-                String message = "Subdiv size = " + subDivs.size();
-                Logger.getGlobal().log(Level.WARNING, message);
-            }
+                    setOdds(result, firstResultsDiv);
+                } else {
+                    somethingWrong = true;
+                    String message = "Subdiv size = " + subDivs.size();
+                    Logger.getGlobal().log(Level.WARNING, message);
+                }
 
-            if (!somethingWrong) {
-                final Mysql mysql = Mysql.getInstance();
+                if (!somethingWrong) {
+                    final Mysql mysql = Mysql.getInstance();
 
-                result.setCountryId(countryId);
-                result.setLeagueId(leagueId);
-                result.setSeason(getLastSeason(leagueId, countryId));
+                    result.setCountryId(countryId);
+                    result.setLeagueId(leagueId);
+                    result.setSeason(getLastSeason(leagueId, countryId));
 
-                resultsToInsert.add(result);
+                    resultsToInsert.add(result);
+//
+                    if (resultsToInsert.size() > 100) {
+                        Logger.getGlobal().log(Level.INFO, "INSERTING 100 results");
+//                    System.out.println("INSERTING RESULT " + result.getHomeTeam() + "-" + result.getAwayTeam()
+//                            + " league " + result.getLeagueId() + " score " + result.getHomeScore() + "-"
+//                            + result.getAwayScore());
+                        mysql.addResult(result);
+                        resultsToInsert.clear();
+                    }
 
-                if (resultsToInsert.size() > 100) {
-                    Logger.getGlobal().log(Level.INFO, "INSERTING 100 results");
-                    mysql.addResults(resultsToInsert);
-                    resultsToInsert.clear();
                 }
-
+                divs = driver
+                        .findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
+            } catch (StaleElementReferenceException e) {
+                if (staleElementRetried) {
+                    System.out.println("Stale element have already been retried continueing");
+                    staleElementRetried = false;
+                    i++;
+                }
+                System.out.println("Retrying stale element");
+                i--;
+                staleElementRetried = true;
             }
-            divs = driver
-                    .findElements(By.xpath(DIV_CONTAINS_CLASS_TABS_DIV_NOT_CLASS_DIV_1_DIV));
         }
 
         String message = "INSERTING LAST " + resultsToInsert.size()
@@ -305,108 +229,6 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
         driver.quit();
     }
 
-    /*
-     * Beh�ver g�ras om till Selenium
-     * private void parseTournamentTable(int sportId, int countryId, int leagueId,
-     * String season,
-     * HtmlTable tournamentTable, LocalDateTime gameDate,
-     * DateTimeFormatter dateFormatter) {
-     * for (final HtmlTableRow tr : tournamentTable.getRows()) {
-     * if (tr.getAttribute(CLASS).contains("deactivate")) {
-     * String homeTeam;
-     * String awayTeam;
-     * int homeScore = -1;
-     * int awayScore = -1;
-     * float odds1 = 0f;
-     * float oddsX = 0f;
-     * float odds2 = 0f;
-     * boolean overtime = false;
-     *
-     * final HtmlTableCell timeCell = tr.getCell(0);
-     * final HtmlTableCell participantsCell = tr.getCell(1);
-     *
-     * // Game Time
-     * final String[] timeValue = timeCell.asNormalizedText().split(":");
-     * if (gameDate != null) {
-     * gameDate = gameDate.withHour(Integer.valueOf(timeValue[0]));
-     * gameDate = gameDate.withMinute(Integer.valueOf(timeValue[1]));
-     * }
-     *
-     * // Teams
-     * final String[] participantsValue =
-     * participantsCell.asNormalizedText().split(" - ");
-     * homeTeam = participantsValue[0].trim();
-     * awayTeam = participantsValue[1].trim();
-     *
-     * final List<HtmlTableCell> cells = tr.getCells();
-     * for (final HtmlTableCell tc : cells) {
-     * // Score
-     * if (tc.getAttribute(CLASS).contains("table-score")) {
-     * final String[] scoreValue = tc.asNormalizedText().split(":");
-     * if (scoreValue[0].matches("\\D+")) {
-     * continue;
-     * }
-     * homeScore = Integer.valueOf(scoreValue[0]);
-     * if (scoreValue[1].matches("\\D+")) {
-     * overtime = true;
-     * }
-     * awayScore = Integer.valueOf(scoreValue[1].replaceAll("\\D+", ""));
-     * }
-     *
-     * if (tc.getAttribute(CLASS).contains("odds-nowrp")) {
-     * if (tc.asNormalizedText().matches("[+-][0-9][0-9][0-9]")) {
-     * if (odds1 == 0F) {
-     * odds1 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
-     * } else if (oddsX == 0F) {
-     * oddsX = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
-     * } else if (odds2 == 0F) {
-     * odds2 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
-     * }
-     * } else if (tc.asNormalizedText().matches("[0-9]+.[0-9]+")) {
-     * if (odds1 == 0F) {
-     * odds1 = Float.valueOf(tc.asNormalizedText());
-     * } else if (oddsX == 0F) {
-     * oddsX = Float.valueOf(tc.asNormalizedText());
-     * } else if (odds2 == 0F) {
-     * odds2 = Float.valueOf(tc.asNormalizedText());
-     * }
-     * }
-     * }
-     * }
-     *
-     * if (gameDate != null && homeTeam != null && awayTeam != null && odds1 != 0 &&
-     * oddsX != 0 && odds2 != 0
-     * && !Strings.isNullOrEmpty(season)) { // All set.. update sql result table
-     * Mysql.getInstance()
-     * .addResult(new ResultDTO(SOCCER_RESULTS_TABLE_NAME, gameDate, homeTeam,
-     * awayTeam, homeScore,
-     * awayScore, overtime, odds1,
-     * oddsX, odds2, countryId, season, leagueId, sportId));
-     * } else {
-     * Log.getLog()
-     * .info(String.
-     * format("Failed to insert result %s %s %s odds %s %s %s season %s", gameDate,
-     * homeTeam, awayTeam, odds1,
-     * oddsX, odds2, season));
-     * }
-     *
-     * } else if (tr.getAttribute(CLASS).contains("center nob-border")) { // Datum
-     * rader
-     * final List<HtmlSpan> dateSpan =
-     * tr.getByXPath(".//span[contains(@class, 'datet')]");
-     * final String dateString = dateSpan.get(0).asNormalizedText();
-     * if (dateString.toLowerCase().contains("yesterday")) {
-     * gameDate = LocalDateTime.now().minusDays(1);
-     * } else if (dateString.toLowerCase().contains("today")) {
-     * gameDate = LocalDateTime.now();
-     * } else {
-     * gameDate = LocalDate.parse(dateString, dateFormatter).atStartOfDay();
-     * }
-     * }
-     * }
-     * }
-     */
-
     private void setGameDate(ResultDTO result, WebElement firstResultsDiv) {
         final List<Integer> time = new ArrayList<>();
         if (checkIfElementExists(firstResultsDiv, "./div/a/div[1]/div/p")) {
@@ -438,9 +260,7 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
             result.setOddsX(Float.parseFloat(oddsDivs.get(1).getText()));
             result.setOdds2(Float.parseFloat(oddsDivs.get(2).getText()));
         } catch (NumberFormatException e) {
-            System.out.println(
-                    "failed to parse odds " + oddsDivs.get(0).getText() + ", " + oddsDivs.get(1).getText() + ", "
-                            + oddsDivs.get(2).getText());
+            // Skip this one
         } catch (IndexOutOfBoundsException e) {
             System.out.println(result);
         }
@@ -472,4 +292,5 @@ public class OddsPortal extends ParserBase implements ParserJoinedFunctions {
         result.setAwayTeamId(
                 Mysql.getInstance().getOrInsertTeam(awayTeamName, countryId, leagueId, sportId));
     }
+
 }