|
|
@@ -10,6 +10,8 @@ import java.util.Locale;
|
|
|
import java.util.logging.Level;
|
|
|
import java.util.logging.Logger;
|
|
|
|
|
|
+import org.eclipse.jetty.util.log.Log;
|
|
|
+
|
|
|
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
|
|
|
import com.gargoylesoftware.htmlunit.WebClient;
|
|
|
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
|
|
|
@@ -25,358 +27,316 @@ import mysql.Mysql;
|
|
|
|
|
|
public class OddsPortal implements ParserJoinedFunctions {
|
|
|
|
|
|
- private LocalDateTime baseDate;
|
|
|
- private int currentParsePage;
|
|
|
- private int sportId;
|
|
|
- private int countryId;
|
|
|
- private int leagueId;
|
|
|
- private LocalDateTime gameDate;
|
|
|
-
|
|
|
- public void getYesterdaysMatches() {
|
|
|
- baseDate = LocalDateTime.now().plusDays(-1);
|
|
|
- final String date = LocalDate.now().plusDays(-1).format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
|
|
- getMatchesByDate(date);
|
|
|
- }
|
|
|
-
|
|
|
- public void getTodaysMatches() {
|
|
|
- baseDate = LocalDateTime.now();
|
|
|
- final String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
|
|
- getMatchesByDate(date);
|
|
|
- }
|
|
|
-
|
|
|
- public void getTomorrowsMatches() {
|
|
|
- baseDate = LocalDateTime.now().plusDays(1);
|
|
|
- final String dateTomorrow = LocalDate.now().plusDays(1).format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
|
|
- getMatchesByDate(dateTomorrow);
|
|
|
- }
|
|
|
-
|
|
|
- public void getNextDaysMatches() {
|
|
|
- baseDate = LocalDateTime.now().plusDays(2);
|
|
|
- final String dateTomorrow = LocalDate.now().plusDays(2).format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
|
|
- getMatchesByDate(dateTomorrow);
|
|
|
- }
|
|
|
-
|
|
|
- // https://stackoverflow.com/questions/14439991/skip-particular-javascript-execution-in-html-unit
|
|
|
- // Skip url
|
|
|
- private void getMatchesByDate(String date) {
|
|
|
- final String soccerUrl = "https://oddsportal.com/matches/soccer/" + date;
|
|
|
- // final String hockeyUrl = "https://oddsportal.com/matches/hockey/" + date;
|
|
|
-
|
|
|
- final WebClient webClient = new WebClient();
|
|
|
- webClient.getOptions().setUseInsecureSSL(true);
|
|
|
- webClient.getOptions().setCssEnabled(false);
|
|
|
- webClient.getOptions().setJavaScriptEnabled(true);
|
|
|
- webClient.getOptions().setThrowExceptionOnScriptError(false);
|
|
|
- Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
|
|
|
-
|
|
|
- webClient.waitForBackgroundJavaScript(3000);
|
|
|
- parseSoccerMatches(soccerUrl, webClient, date);
|
|
|
-
|
|
|
- webClient.close();
|
|
|
- }
|
|
|
-
|
|
|
- private void parseSoccerMatches(final String soccerUrl, final WebClient webClient, String date) {
|
|
|
- try {
|
|
|
- System.out.println("Getting Webpage");
|
|
|
- final HtmlPage soccerMatches = webClient.getPage(soccerUrl);
|
|
|
- final HtmlTable matchesTable = soccerMatches.getFirstByXPath("//table[contains(@class, table-main)]");
|
|
|
- final List<HtmlTableRow> rows = matchesTable.getRows();
|
|
|
- String countryName = "";
|
|
|
- String leagueName = "";
|
|
|
- int i = 1;
|
|
|
- final int size = rows.size();
|
|
|
- for (final HtmlTableRow tr : rows) {
|
|
|
- System.out.println("Processing " + i++ + " of " + size);
|
|
|
- if (tr.getAttribute("class").equals("dark center")) {
|
|
|
- final List<HtmlAnchor> countryLeague = tr.getByXPath(".//a");
|
|
|
- countryName = countryLeague.get(0).asNormalizedText().toLowerCase().trim();
|
|
|
- leagueName = countryLeague.get(1).asNormalizedText().toLowerCase().trim();
|
|
|
- leagueName = leagueName.replaceAll(" ", "-");
|
|
|
- leagueName = leagueName.replaceAll("\\.", "");
|
|
|
- countryName = countryName.replaceAll(" ", "-");
|
|
|
- countryName = countryName.replaceAll("\\.", "");
|
|
|
- } else {
|
|
|
- final List<HtmlTableCell> cells = tr.getCells();
|
|
|
- final String[] time = cells.get(0).asNormalizedText().split(":");
|
|
|
- final String[] teams = cells.get(1).asNormalizedText().split(" - ");
|
|
|
- float odds1 = 0F;
|
|
|
- float oddsX = 0F;
|
|
|
- float odds2 = 0F;
|
|
|
- int homeScore = -1;
|
|
|
- int awayScore = -1;
|
|
|
- boolean overtime = false;
|
|
|
-
|
|
|
- boolean abandon = false;
|
|
|
-
|
|
|
- try {
|
|
|
- for (final HtmlTableCell tc : cells) {
|
|
|
- if (tc.getAttribute("class").contains("live-score")) {
|
|
|
- abandon = true;
|
|
|
- break;
|
|
|
- }
|
|
|
- // Score
|
|
|
- if (tc.getAttribute("class").contains("table-score")) {
|
|
|
- final String[] scoreValue = tc.asNormalizedText().split(":");
|
|
|
- homeScore = Integer.valueOf(scoreValue[0]);
|
|
|
- if (scoreValue[1].matches("\\D+")) {
|
|
|
- overtime = true;
|
|
|
- }
|
|
|
- awayScore = Integer.valueOf(scoreValue[1].replaceAll("\\D+", ""));
|
|
|
- }
|
|
|
- if (tc.getAttribute("class").contains("odds-nowrp")) {
|
|
|
- if (tc.asNormalizedText().matches("[+-][0-9][0-9][0-9]")) {
|
|
|
- if (odds1 == 0F) {
|
|
|
- odds1 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
- } else if (oddsX == 0F) {
|
|
|
- oddsX = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
- } else if (odds2 == 0F) {
|
|
|
- odds2 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
- }
|
|
|
- } else if (tc.asNormalizedText().matches("[0-9].[0-9]+")) {
|
|
|
- if (odds1 == 0F) {
|
|
|
- odds1 = Float.valueOf(tc.asNormalizedText());
|
|
|
- } else if (oddsX == 0F) {
|
|
|
- oddsX = Float.valueOf(tc.asNormalizedText());
|
|
|
- } else if (odds2 == 0F) {
|
|
|
- odds2 = Float.valueOf(tc.asNormalizedText());
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
- } catch (final NumberFormatException e) {
|
|
|
- System.out.println("Failed to get the match between " + teams[0].trim() + " and " + teams[1].trim()
|
|
|
- + " at " + baseDate.withHour(Integer.valueOf(time[0])).withMinute(Integer.valueOf(time[1]))
|
|
|
- + " odds1: " + odds1 + " oddsX: " + oddsX + " odds2: " + odds2 + " homeScore " + homeScore
|
|
|
- + " awayScore " + awayScore + " overtime: " + (overtime ? "true" : "false"));
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- if (abandon) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- final Mysql mysql = Mysql.getInstance();
|
|
|
- final int leagueId = mysql.addLeague(leagueName, countryName, "soccer");
|
|
|
- final int countryId = mysql.getCountryId(countryName);
|
|
|
- final int sportId = mysql.getSportId("soccer");
|
|
|
-
|
|
|
- // String season = mysql.getLastParsedYear(leagueName, countryId); // TODO This
|
|
|
- // don't work
|
|
|
- String season = String.valueOf(LocalDate.parse(date, DateTimeFormatter.ofPattern("yyyyMMdd")).getYear());
|
|
|
- if (Strings.isNullOrEmpty(season)) {
|
|
|
- season = String.valueOf(LocalDateTime.now().getYear());
|
|
|
- }
|
|
|
-
|
|
|
- final LocalDateTime dt = baseDate.withHour(Integer.valueOf(time[0])).withMinute(Integer.valueOf(time[1]))
|
|
|
- .withSecond(0).withNano(0);
|
|
|
- mysql.addResult("SoccerResults",
|
|
|
- dt,
|
|
|
- teams[0].trim(),
|
|
|
- teams[1].trim(),
|
|
|
- homeScore,
|
|
|
- awayScore,
|
|
|
- overtime,
|
|
|
- odds1,
|
|
|
- oddsX,
|
|
|
- odds2,
|
|
|
- countryId,
|
|
|
- season,
|
|
|
- leagueId,
|
|
|
- sportId);
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (FailingHttpStatusCodeException | IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- } catch (final SQLException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void getHistoricMatches(String sport, String country, String league, String year) {
|
|
|
- final String url = "https://www.oddsportal.com/";
|
|
|
- final String resultsPage = "/results";
|
|
|
- final WebClient webClient = new WebClient();
|
|
|
- webClient.getOptions().setUseInsecureSSL(true);
|
|
|
- webClient.getOptions().setCssEnabled(false);
|
|
|
- webClient.getOptions().setJavaScriptEnabled(true);
|
|
|
- webClient.getOptions().setThrowExceptionOnScriptError(false);
|
|
|
- Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
|
|
|
-
|
|
|
- league = league.replaceAll(" ", "-");
|
|
|
- league = league.replaceAll("\\.", "");
|
|
|
- country = country.replaceAll(" ", "-");
|
|
|
- league = league.replaceAll("\\.", "");
|
|
|
- final Mysql mysql = Mysql.getInstance();
|
|
|
-
|
|
|
- currentParsePage = 1;
|
|
|
-
|
|
|
- final String urlYearPart;
|
|
|
- if (year.equals(String.valueOf(LocalDate.now().getYear()))) {
|
|
|
- urlYearPart = "";
|
|
|
- } else {
|
|
|
- urlYearPart = "-" + year;
|
|
|
- }
|
|
|
-
|
|
|
- try {
|
|
|
- sportId = mysql.getSportId(sport);
|
|
|
- countryId = mysql.getCountryId(country);
|
|
|
- leagueId = mysql.getLeagueId(sportId, countryId, league);
|
|
|
- String season = "";
|
|
|
-
|
|
|
- final HtmlPage leaguePage
|
|
|
- = webClient.getPage(url + "/" + sport + "/" + country + "/" + league + urlYearPart + resultsPage);
|
|
|
- final List<HtmlAnchor> yearFilter = leaguePage.getByXPath("//ul[contains(@class,'main-filter')]//a");
|
|
|
- for (final HtmlAnchor a : yearFilter) {
|
|
|
- System.out.println("Year filter: " + a.getHrefAttribute());
|
|
|
- final String active = ((HtmlSpan) a.getParentNode().getParentNode()).getAttribute("class");
|
|
|
- if (active.contains("active") && !active.contains("inactive")) {
|
|
|
- season = a.asNormalizedText();
|
|
|
- year = season.replace('/', '-');
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- HtmlDivision tournamentTableDiv = leaguePage.getHtmlElementById("tournamentTable");
|
|
|
- HtmlTable tournamentTable = (HtmlTable) tournamentTableDiv.getFirstChild();
|
|
|
-
|
|
|
- gameDate = LocalDateTime.now();
|
|
|
- final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH);
|
|
|
- parseTournamentTable(sportId, countryId, leagueId, season, tournamentTable, gameDate, dateFormatter);
|
|
|
- final HtmlDivision paginationLinksDiv = (HtmlDivision) tournamentTableDiv.getLastChild();
|
|
|
- final List<HtmlAnchor> pagiantionLinks
|
|
|
- = paginationLinksDiv.getByXPath(".//a[contains(@href, 'page') and not(.//span[contains(@class, 'arrow')])]");
|
|
|
- for (final HtmlAnchor a : pagiantionLinks) {
|
|
|
- System.out.println("Continuing with Pagination: " + a.getHrefAttribute());
|
|
|
- // When done with start page click pagiantion
|
|
|
- final int parsePage = Integer.valueOf(a.getTextContent());
|
|
|
- if (parsePage > currentParsePage) {
|
|
|
- a.click();
|
|
|
- webClient.waitForBackgroundJavaScript(1000);
|
|
|
-
|
|
|
- tournamentTableDiv = leaguePage.getHtmlElementById("tournamentTable");
|
|
|
- tournamentTable = (HtmlTable) tournamentTableDiv.getFirstChild();
|
|
|
- parseTournamentTable(sportId, countryId, leagueId, season, tournamentTable, gameDate, dateFormatter);
|
|
|
- currentParsePage = parsePage;
|
|
|
- }
|
|
|
- // process new tournament table content
|
|
|
- }
|
|
|
- } catch (FailingHttpStatusCodeException | IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- } catch (final SQLException sqle) {
|
|
|
- sqle.printStackTrace();
|
|
|
- } catch (final ClassCastException cce) {
|
|
|
- System.out.println("No pagination table");
|
|
|
- // cce.printStackTrace();
|
|
|
- } finally {
|
|
|
- Mysql.getInstance().setParsingForLeague(leagueId, sportId, countryId, gameDate, currentParsePage, year);
|
|
|
- }
|
|
|
- webClient.close();
|
|
|
- System.out.println("DONE with " + country + " (" + countryId + ") league " + league + "(" + leagueId + ")");
|
|
|
- }
|
|
|
-
|
|
|
- private void parseTournamentTable(int sportId, int countryId, int leagueId, String season, HtmlTable tournamentTable,
|
|
|
- LocalDateTime gameDate, DateTimeFormatter dateFormatter) throws SQLException {
|
|
|
- for (final HtmlTableRow tr : tournamentTable.getRows()) {
|
|
|
- if (tr.getAttribute("class").contains("deactivate")) {
|
|
|
- String homeTeam;
|
|
|
- String awayTeam;
|
|
|
- int homeScore = -1;
|
|
|
- int awayScore = -1;
|
|
|
- float odds1 = 0f;
|
|
|
- float oddsX = 0f;
|
|
|
- float odds2 = 0f;
|
|
|
- boolean overtime = false;
|
|
|
-
|
|
|
- final HtmlTableCell timeCell = tr.getCell(0);
|
|
|
- final HtmlTableCell participantsCell = tr.getCell(1);
|
|
|
-
|
|
|
- // Game Time
|
|
|
- final String[] timeValue = timeCell.asNormalizedText().split(":");
|
|
|
- gameDate = gameDate.withHour(Integer.valueOf(timeValue[0]));
|
|
|
- gameDate = gameDate.withMinute(Integer.valueOf(timeValue[1]));
|
|
|
-
|
|
|
- // Teams
|
|
|
- final String[] participantsValue = participantsCell.asNormalizedText().split(" - ");
|
|
|
- homeTeam = participantsValue[0].trim();
|
|
|
- awayTeam = participantsValue[1].trim();
|
|
|
-
|
|
|
- final List<HtmlTableCell> cells = tr.getCells();
|
|
|
- for (final HtmlTableCell tc : cells) {
|
|
|
- // Score
|
|
|
- if (tc.getAttribute("class").contains("table-score")) {
|
|
|
- final String[] scoreValue = tc.asNormalizedText().split(":");
|
|
|
- if (scoreValue[0].matches("\\D+")) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- homeScore = Integer.valueOf(scoreValue[0]);
|
|
|
- if (scoreValue[1].matches("\\D+")) {
|
|
|
- overtime = true;
|
|
|
- }
|
|
|
- awayScore = Integer.valueOf(scoreValue[1].replaceAll("\\D+", ""));
|
|
|
- }
|
|
|
-
|
|
|
- if (tc.getAttribute("class").contains("odds-nowrp")) {
|
|
|
- if (tc.asNormalizedText().matches("[+-][0-9][0-9][0-9]")) {
|
|
|
- if (odds1 == 0F) {
|
|
|
- odds1 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
- } else if (oddsX == 0F) {
|
|
|
- oddsX = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
- } else if (odds2 == 0F) {
|
|
|
- odds2 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
- }
|
|
|
- } else if (tc.asNormalizedText().matches("[0-9].[0-9]+")) {
|
|
|
- if (odds1 == 0F) {
|
|
|
- odds1 = Float.valueOf(tc.asNormalizedText());
|
|
|
- } else if (oddsX == 0F) {
|
|
|
- oddsX = Float.valueOf(tc.asNormalizedText());
|
|
|
- } else if (odds2 == 0F) {
|
|
|
- odds2 = Float.valueOf(tc.asNormalizedText());
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if (gameDate != null && homeTeam != null && awayTeam != null && odds1 != 0 && oddsX != 0 && odds2 != 0
|
|
|
- && !Strings.isNullOrEmpty(season)) { // All set.. update sql result table
|
|
|
- System.out.println("Adding game between " + homeTeam + " and " + awayTeam + " with score " + homeScore + "-"
|
|
|
- + awayScore);
|
|
|
- Mysql.getInstance().addResult("SoccerResults",
|
|
|
- gameDate,
|
|
|
- homeTeam,
|
|
|
- awayTeam,
|
|
|
- homeScore,
|
|
|
- awayScore,
|
|
|
- overtime,
|
|
|
- odds1,
|
|
|
- oddsX,
|
|
|
- odds2,
|
|
|
- countryId,
|
|
|
- season,
|
|
|
- leagueId,
|
|
|
- sportId);
|
|
|
- } else {
|
|
|
- System.out.println(String.format(
|
|
|
- "Not adding, missing somethind.. gameDate: %s, homeTeam %s, awayTeam %s, odds1 %s, oddsX %s, odds2 %s, "
|
|
|
- + "season %s",
|
|
|
- gameDate,
|
|
|
- homeTeam,
|
|
|
- awayTeam,
|
|
|
- odds1,
|
|
|
- oddsX,
|
|
|
- odds2,
|
|
|
- season));
|
|
|
- }
|
|
|
-
|
|
|
- } else if (tr.getAttribute("class").contains("center nob-border")) { // Datum rader
|
|
|
- final List<HtmlSpan> dateSpan = tr.getByXPath(".//span[contains(@class, 'datet')]");
|
|
|
- final String dateString = dateSpan.get(0).asNormalizedText();
|
|
|
- if (dateString.toLowerCase().contains("yesterday")) {
|
|
|
- gameDate = LocalDateTime.now().minusDays(1);
|
|
|
- } else if (dateString.toLowerCase().contains("today")) {
|
|
|
- gameDate = LocalDateTime.now();
|
|
|
- } else {
|
|
|
- gameDate = LocalDate.parse(dateString, dateFormatter).atStartOfDay();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
+ private static final String CLASS = "class";
|
|
|
+ private static final String DATE_PATTERN = "yyyyMMdd";
|
|
|
+ private LocalDateTime baseDate;
|
|
|
+ private int sportId;
|
|
|
+ private int countryId;
|
|
|
+ private int leagueId;
|
|
|
+ private LocalDateTime gameDate;
|
|
|
+
|
|
|
+ public void getYesterdaysMatches() {
|
|
|
+ baseDate = LocalDateTime.now().plusDays(-1);
|
|
|
+ final String date = LocalDate.now().plusDays(-1).format(DateTimeFormatter.ofPattern(DATE_PATTERN));
|
|
|
+ getMatchesByDate(date);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void getTodaysMatches() {
|
|
|
+ baseDate = LocalDateTime.now();
|
|
|
+ final String date = LocalDate.now().format(DateTimeFormatter.ofPattern(DATE_PATTERN));
|
|
|
+ getMatchesByDate(date);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void getTomorrowsMatches() {
|
|
|
+ baseDate = LocalDateTime.now().plusDays(1);
|
|
|
+ final String dateTomorrow = LocalDate.now().plusDays(1).format(DateTimeFormatter.ofPattern(DATE_PATTERN));
|
|
|
+ getMatchesByDate(dateTomorrow);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void getNextDaysMatches() {
|
|
|
+ baseDate = LocalDateTime.now().plusDays(2);
|
|
|
+ final String dateTomorrow = LocalDate.now().plusDays(2).format(DateTimeFormatter.ofPattern(DATE_PATTERN));
|
|
|
+ getMatchesByDate(dateTomorrow);
|
|
|
+ }
|
|
|
+
|
|
|
+ // https://stackoverflow.com/questions/14439991/skip-particular-javascript-execution-in-html-unit
|
|
|
+ // Skip url
|
|
|
+ private void getMatchesByDate(String date) {
|
|
|
+ final String soccerUrl = "https://oddsportal.com/matches/soccer/" + date;
|
|
|
+ // final String hockeyUrl = "https://oddsportal.com/matches/hockey/" + date;
|
|
|
+
|
|
|
+ final WebClient webClient = new WebClient();
|
|
|
+ webClient.getOptions().setUseInsecureSSL(true);
|
|
|
+ webClient.getOptions().setCssEnabled(false);
|
|
|
+ webClient.getOptions().setJavaScriptEnabled(true);
|
|
|
+ webClient.getOptions().setThrowExceptionOnScriptError(false);
|
|
|
+ Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
|
|
|
+
|
|
|
+ webClient.waitForBackgroundJavaScript(3000);
|
|
|
+ parseSoccerMatches(soccerUrl, webClient, date);
|
|
|
+
|
|
|
+ webClient.close();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void parseSoccerMatches(final String soccerUrl, final WebClient webClient, String date) {
|
|
|
+ try {
|
|
|
+ final HtmlPage soccerMatches = webClient.getPage(soccerUrl);
|
|
|
+ final HtmlTable matchesTable = soccerMatches.getFirstByXPath("//table[contains(@class, table-main)]");
|
|
|
+ final List<HtmlTableRow> rows = matchesTable.getRows();
|
|
|
+ String countryName = "";
|
|
|
+ String leagueName = "";
|
|
|
+ int i = 1;
|
|
|
+ for (final HtmlTableRow tr : rows) {
|
|
|
+ if (tr.getAttribute(CLASS).equals("dark center")) {
|
|
|
+ final List<HtmlAnchor> countryLeague = tr.getByXPath(".//a");
|
|
|
+ countryName = countryLeague.get(0).asNormalizedText().toLowerCase().trim();
|
|
|
+ leagueName = countryLeague.get(1).asNormalizedText().toLowerCase().trim();
|
|
|
+ leagueName = leagueName.replace(" ", "-");
|
|
|
+ leagueName = leagueName.replace("\\.", "");
|
|
|
+ countryName = countryName.replace(" ", "-");
|
|
|
+ countryName = countryName.replace("\\.", "");
|
|
|
+ } else {
|
|
|
+ final List<HtmlTableCell> cells = tr.getCells();
|
|
|
+ final String[] time = cells.get(0).asNormalizedText().split(":");
|
|
|
+ final String[] teams = cells.get(1).asNormalizedText().split(" - ");
|
|
|
+ float odds1 = 0F;
|
|
|
+ float oddsX = 0F;
|
|
|
+ float odds2 = 0F;
|
|
|
+ int homeScore = -1;
|
|
|
+ int awayScore = -1;
|
|
|
+ boolean overtime = false;
|
|
|
+
|
|
|
+ boolean abandon = false;
|
|
|
+
|
|
|
+ try {
|
|
|
+ for (final HtmlTableCell tc : cells) {
|
|
|
+ if (tc.getAttribute(CLASS).contains("live-score")) {
|
|
|
+ abandon = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ // Score
|
|
|
+ if (tc.getAttribute(CLASS).contains("table-score")) {
|
|
|
+ final String[] scoreValue = tc.asNormalizedText().split(":");
|
|
|
+ homeScore = Integer.valueOf(scoreValue[0]);
|
|
|
+ if (scoreValue[1].matches("\\D+")) {
|
|
|
+ overtime = true;
|
|
|
+ }
|
|
|
+ awayScore = Integer.valueOf(scoreValue[1].replaceAll("\\D+", ""));
|
|
|
+ }
|
|
|
+ if (tc.getAttribute(CLASS).contains("odds-nowrp")) {
|
|
|
+ if (tc.asNormalizedText().matches("[+-][0-9][0-9][0-9]")) {
|
|
|
+ if (odds1 == 0F) {
|
|
|
+ odds1 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
+ } else if (oddsX == 0F) {
|
|
|
+ oddsX = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
+ } else if (odds2 == 0F) {
|
|
|
+ odds2 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
+ }
|
|
|
+ } else if (tc.asNormalizedText().matches("[0-9].[0-9]+")) {
|
|
|
+ if (odds1 == 0F) {
|
|
|
+ odds1 = Float.valueOf(tc.asNormalizedText());
|
|
|
+ } else if (oddsX == 0F) {
|
|
|
+ oddsX = Float.valueOf(tc.asNormalizedText());
|
|
|
+ } else if (odds2 == 0F) {
|
|
|
+ odds2 = Float.valueOf(tc.asNormalizedText());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ } catch (final NumberFormatException e) {
|
|
|
+ Log.getLog().info(
|
|
|
+ "Failed to get the match between %s and %s at %s odds1 %s oddsX %s odds2 %s homeScore %s awayScore %s overtime %s",
|
|
|
+ teams[0].trim(), teams[1].trim(),
|
|
|
+ baseDate.withHour(Integer.valueOf(time[0])).withMinute(Integer.valueOf(time[1])), odds1,
|
|
|
+ oddsX, odds2, homeScore, awayScore, (overtime ? "true" : "false"));
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (abandon) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ final Mysql mysql = Mysql.getInstance();
|
|
|
+ final int leaguesId = mysql.addLeague(leagueName, countryName, "soccer");
|
|
|
+ final int countrysId = mysql.getCountryId(countryName);
|
|
|
+ final int sportsId = mysql.getSportId("soccer");
|
|
|
+
|
|
|
+ // String season = mysql.getLastParsedYear(leagueName, countryId); // TODO This
|
|
|
+ // don't work
|
|
|
+ String season = String
|
|
|
+ .valueOf(LocalDate.parse(date, DateTimeFormatter.ofPattern(DATE_PATTERN)).getYear());
|
|
|
+ if (Strings.isNullOrEmpty(season)) {
|
|
|
+ season = String.valueOf(LocalDateTime.now().getYear());
|
|
|
+ }
|
|
|
+
|
|
|
+ final LocalDateTime dt = baseDate.withHour(Integer.valueOf(time[0]))
|
|
|
+ .withMinute(Integer.valueOf(time[1])).withSecond(0).withNano(0);
|
|
|
+ mysql.addResult("SoccerResults", dt, teams[0].trim(), teams[1].trim(), homeScore, awayScore,
|
|
|
+ overtime, odds1, oddsX, odds2, countrysId, season, leaguesId, sportsId);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (FailingHttpStatusCodeException | IOException | SQLException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public void getHistoricMatches(String sport, String country, String league, String year) {
|
|
|
+ final String url = "https://www.oddsportal.com/";
|
|
|
+ final String resultsPage = "/results";
|
|
|
+ final WebClient webClient = new WebClient();
|
|
|
+ webClient.getOptions().setUseInsecureSSL(true);
|
|
|
+ webClient.getOptions().setCssEnabled(false);
|
|
|
+ webClient.getOptions().setJavaScriptEnabled(true);
|
|
|
+ webClient.getOptions().setThrowExceptionOnScriptError(false);
|
|
|
+ Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
|
|
|
+
|
|
|
+ league = league.replace(" ", "-");
|
|
|
+ league = league.replace("\\.", "");
|
|
|
+ country = country.replace(" ", "-");
|
|
|
+ league = league.replace("\\.", "");
|
|
|
+ final Mysql mysql = Mysql.getInstance();
|
|
|
+
|
|
|
+ int currentParsePage = 1;
|
|
|
+
|
|
|
+ final String urlYearPart;
|
|
|
+ if (year.equals(String.valueOf(LocalDate.now().getYear()))) {
|
|
|
+ urlYearPart = "";
|
|
|
+ } else {
|
|
|
+ urlYearPart = "-" + year;
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ sportId = mysql.getSportId(sport);
|
|
|
+ countryId = mysql.getCountryId(country);
|
|
|
+ leagueId = mysql.getLeagueId(sportId, countryId, league);
|
|
|
+ String season = "";
|
|
|
+
|
|
|
+ final HtmlPage leaguePage = webClient
|
|
|
+ .getPage(url + "/" + sport + "/" + country + "/" + league + urlYearPart + resultsPage);
|
|
|
+ final List<HtmlAnchor> yearFilter = leaguePage.getByXPath("//ul[contains(@class,'main-filter')]//a");
|
|
|
+ for (final HtmlAnchor a : yearFilter) {
|
|
|
+ final String active = ((HtmlSpan) a.getParentNode().getParentNode()).getAttribute(CLASS);
|
|
|
+ if (active.contains("active") && !active.contains("inactive")) {
|
|
|
+ season = a.asNormalizedText();
|
|
|
+ year = season.replace('/', '-');
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ HtmlDivision tournamentTableDiv = leaguePage.getHtmlElementById("tournamentTable");
|
|
|
+ HtmlTable tournamentTable = (HtmlTable) tournamentTableDiv.getFirstChild();
|
|
|
+
|
|
|
+ gameDate = LocalDateTime.now();
|
|
|
+ final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH);
|
|
|
+ parseTournamentTable(sportId, countryId, leagueId, season, tournamentTable, gameDate, dateFormatter);
|
|
|
+ final HtmlDivision paginationLinksDiv = (HtmlDivision) tournamentTableDiv.getLastChild();
|
|
|
+ final List<HtmlAnchor> pagiantionLinks = paginationLinksDiv
|
|
|
+ .getByXPath(".//a[contains(@href, 'page') and not(.//span[contains(@class, 'arrow')])]");
|
|
|
+ for (final HtmlAnchor a : pagiantionLinks) {
|
|
|
+ // When done with start page click pagiantion
|
|
|
+ final int parsePage = Integer.parseInt(a.getTextContent());
|
|
|
+ if (parsePage > currentParsePage) {
|
|
|
+ a.click();
|
|
|
+ webClient.waitForBackgroundJavaScript(1000);
|
|
|
+
|
|
|
+ tournamentTableDiv = leaguePage.getHtmlElementById("tournamentTable");
|
|
|
+ tournamentTable = (HtmlTable) tournamentTableDiv.getFirstChild();
|
|
|
+ parseTournamentTable(sportId, countryId, leagueId, season, tournamentTable, gameDate,
|
|
|
+ dateFormatter);
|
|
|
+ currentParsePage = parsePage;
|
|
|
+ }
|
|
|
+ // process new tournament table content
|
|
|
+ }
|
|
|
+ } catch (FailingHttpStatusCodeException | IOException | SQLException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } catch (final ClassCastException cce) {
|
|
|
+ } finally {
|
|
|
+ Mysql.getInstance().setParsingForLeague(leagueId, sportId, countryId, gameDate, currentParsePage, year);
|
|
|
+ }
|
|
|
+ webClient.close();
|
|
|
+ Log.getLog().info("DONE with " + country + " (" + countryId + ") league " + league + "(" + leagueId + ")");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void parseTournamentTable(int sportId, int countryId, int leagueId, String season,
|
|
|
+ HtmlTable tournamentTable, LocalDateTime gameDate, DateTimeFormatter dateFormatter) throws SQLException {
|
|
|
+ for (final HtmlTableRow tr : tournamentTable.getRows()) {
|
|
|
+ if (tr.getAttribute(CLASS).contains("deactivate")) {
|
|
|
+ String homeTeam;
|
|
|
+ String awayTeam;
|
|
|
+ int homeScore = -1;
|
|
|
+ int awayScore = -1;
|
|
|
+ float odds1 = 0f;
|
|
|
+ float oddsX = 0f;
|
|
|
+ float odds2 = 0f;
|
|
|
+ boolean overtime = false;
|
|
|
+
|
|
|
+ final HtmlTableCell timeCell = tr.getCell(0);
|
|
|
+ final HtmlTableCell participantsCell = tr.getCell(1);
|
|
|
+
|
|
|
+ // Game Time
|
|
|
+ final String[] timeValue = timeCell.asNormalizedText().split(":");
|
|
|
+ if (gameDate != null) {
|
|
|
+ gameDate = gameDate.withHour(Integer.valueOf(timeValue[0]));
|
|
|
+ gameDate = gameDate.withMinute(Integer.valueOf(timeValue[1]));
|
|
|
+ }
|
|
|
+
|
|
|
+ // Teams
|
|
|
+ final String[] participantsValue = participantsCell.asNormalizedText().split(" - ");
|
|
|
+ homeTeam = participantsValue[0].trim();
|
|
|
+ awayTeam = participantsValue[1].trim();
|
|
|
+
|
|
|
+ final List<HtmlTableCell> cells = tr.getCells();
|
|
|
+ for (final HtmlTableCell tc : cells) {
|
|
|
+ // Score
|
|
|
+ if (tc.getAttribute(CLASS).contains("table-score")) {
|
|
|
+ final String[] scoreValue = tc.asNormalizedText().split(":");
|
|
|
+ if (scoreValue[0].matches("\\D+")) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ homeScore = Integer.valueOf(scoreValue[0]);
|
|
|
+ if (scoreValue[1].matches("\\D+")) {
|
|
|
+ overtime = true;
|
|
|
+ }
|
|
|
+ awayScore = Integer.valueOf(scoreValue[1].replaceAll("\\D+", ""));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (tc.getAttribute(CLASS).contains("odds-nowrp")) {
|
|
|
+ if (tc.asNormalizedText().matches("[+-][0-9][0-9][0-9]")) {
|
|
|
+ if (odds1 == 0F) {
|
|
|
+ odds1 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
+ } else if (oddsX == 0F) {
|
|
|
+ oddsX = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
+ } else if (odds2 == 0F) {
|
|
|
+ odds2 = convertAmericanOddsToDecimal(Integer.valueOf(tc.asNormalizedText()));
|
|
|
+ }
|
|
|
+ } else if (tc.asNormalizedText().matches("[0-9].[0-9]+")) {
|
|
|
+ if (odds1 == 0F) {
|
|
|
+ odds1 = Float.valueOf(tc.asNormalizedText());
|
|
|
+ } else if (oddsX == 0F) {
|
|
|
+ oddsX = Float.valueOf(tc.asNormalizedText());
|
|
|
+ } else if (odds2 == 0F) {
|
|
|
+ odds2 = Float.valueOf(tc.asNormalizedText());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (gameDate != null && homeTeam != null && awayTeam != null && odds1 != 0 && oddsX != 0 && odds2 != 0
|
|
|
+ && !Strings.isNullOrEmpty(season)) { // All set.. update sql result table
|
|
|
+ Mysql.getInstance().addResult("SoccerResults", gameDate, homeTeam, awayTeam, homeScore, awayScore,
|
|
|
+ overtime, odds1, oddsX, odds2, countryId, season, leagueId, sportId);
|
|
|
+ }
|
|
|
+
|
|
|
+ } else if (tr.getAttribute(CLASS).contains("center nob-border")) { // Datum rader
|
|
|
+ final List<HtmlSpan> dateSpan = tr.getByXPath(".//span[contains(@class, 'datet')]");
|
|
|
+ final String dateString = dateSpan.get(0).asNormalizedText();
|
|
|
+ if (dateString.toLowerCase().contains("yesterday")) {
|
|
|
+ gameDate = LocalDateTime.now().minusDays(1);
|
|
|
+ } else if (dateString.toLowerCase().contains("today")) {
|
|
|
+ gameDate = LocalDateTime.now();
|
|
|
+ } else {
|
|
|
+ gameDate = LocalDate.parse(dateString, dateFormatter).atStartOfDay();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|