|
@@ -2,30 +2,35 @@ package parser;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
import java.io.IOException;
|
|
|
import java.net.URL;
|
|
import java.net.URL;
|
|
|
-import java.nio.charset.Charset;
|
|
|
|
|
|
|
+import java.time.Duration;
|
|
|
import java.time.LocalDateTime;
|
|
import java.time.LocalDateTime;
|
|
|
|
|
+import java.util.ArrayList;
|
|
|
|
|
+import java.util.List;
|
|
|
import java.util.logging.Level;
|
|
import java.util.logging.Level;
|
|
|
import java.util.logging.Logger;
|
|
import java.util.logging.Logger;
|
|
|
|
|
|
|
|
-import javax.sql.rowset.spi.SyncResolver;
|
|
|
|
|
-
|
|
|
|
|
-import org.apache.commons.io.IOUtils;
|
|
|
|
|
|
|
+import org.openqa.selenium.By;
|
|
|
|
|
+import org.openqa.selenium.JavascriptExecutor;
|
|
|
|
|
+import org.openqa.selenium.WebElement;
|
|
|
|
|
+import org.openqa.selenium.chrome.ChromeDriver;
|
|
|
|
|
+import org.openqa.selenium.support.ui.ExpectedConditions;
|
|
|
|
|
+import org.openqa.selenium.support.ui.WebDriverWait;
|
|
|
|
|
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
-import com.gargoylesoftware.htmlunit.BrowserVersion;
|
|
|
|
|
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
|
|
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
|
|
|
-import com.gargoylesoftware.htmlunit.Page;
|
|
|
|
|
import com.gargoylesoftware.htmlunit.WebClient;
|
|
import com.gargoylesoftware.htmlunit.WebClient;
|
|
|
import com.gargoylesoftware.htmlunit.WebResponse;
|
|
import com.gargoylesoftware.htmlunit.WebResponse;
|
|
|
import com.gargoylesoftware.htmlunit.html.HtmlPage;
|
|
import com.gargoylesoftware.htmlunit.html.HtmlPage;
|
|
|
|
|
|
|
|
|
|
+import object.ResultDTO;
|
|
|
|
|
+
|
|
|
|
|
+public class ExpektParser extends ParserBase {
|
|
|
|
|
+
|
|
|
|
|
+ List<ResultDTO> result = new ArrayList<>();
|
|
|
|
|
|
|
|
-public class ExpektParser {
|
|
|
|
|
-
|
|
|
|
|
public ExpektParser() {
|
|
public ExpektParser() {
|
|
|
- getJSON();
|
|
|
|
|
- //getSoccerMatches();
|
|
|
|
|
|
|
+ // getSoccerMatches();
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
public static JsonNode get(URL url) throws Exception {
|
|
public static JsonNode get(URL url) throws Exception {
|
|
@@ -33,19 +38,115 @@ public class ExpektParser {
|
|
|
return mapper.readTree(url);
|
|
return mapper.readTree(url);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ public List<ResultDTO> seleniumParseSoccerMatches() {
|
|
|
|
|
+ List<ResultDTO> result = new ArrayList<>();
|
|
|
|
|
+ String url = "https://www.expekt.se/sv-se/sports#filter/football";
|
|
|
|
|
+
|
|
|
|
|
+ ParserBase pb = new ParserBase();
|
|
|
|
|
+ ChromeDriver driver = pb.getSeleniumDriver();
|
|
|
|
|
+ driver.get(url);
|
|
|
|
|
+
|
|
|
|
|
+ JavascriptExecutor js = driver;
|
|
|
|
|
+
|
|
|
|
|
+ WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(30));
|
|
|
|
|
+
|
|
|
|
|
+ wait.until(ExpectedConditions
|
|
|
|
|
+ .numberOfElementsToBeMoreThan(By.xpath("//div[contains(@class, 'KambiBC-betoffer-labels')]"), 0));
|
|
|
|
|
+
|
|
|
|
|
+ List<WebElement> headers = driver.findElements(By.xpath(
|
|
|
|
|
+ "//div[contains(@class,'CollapsibleContainer__CollapsibleWrapper') and not(contains(@class, 'expanded'))]"));
|
|
|
|
|
+
|
|
|
|
|
+ for (WebElement header : headers) {
|
|
|
|
|
+ WebElement title = header.findElement(By.xpath(".//div[contains(@class,'CollapsibleContainer__Title')]"));
|
|
|
|
|
+ scrollElementIntoViewCenter(driver, header);
|
|
|
|
|
+ if (title.getText().contains("Imorgon")) {
|
|
|
|
|
+ header.click();
|
|
|
|
|
+ scrollElementIntoViewCenter(driver, title);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ List<WebElement> expandableDivs = driver
|
|
|
|
|
+ .findElements(By.xpath("//div[contains(@class, 'KambiBC-betoffer-labels')]"));
|
|
|
|
|
+
|
|
|
|
|
+ scrollElementIntoViewCenter(driver, expandableDivs.get(0));
|
|
|
|
|
+ for (WebElement element : expandableDivs) {
|
|
|
|
|
+ if (!checkIfElementExists(element, ".//following-sibling::ul")) {
|
|
|
|
|
+ element.click();
|
|
|
|
|
+ scrollElementIntoView(driver, element);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ wait.until(ExpectedConditions.numberOfElementsToBeMoreThan(
|
|
|
|
|
+ By.xpath("//ul[contains(@class, 'KambiBC-list-view__column KambiBC-list-view__event-list')]"),
|
|
|
|
|
+ expandableDivs.size() - 3));
|
|
|
|
|
+
|
|
|
|
|
+ List<WebElement> leagueLists = driver
|
|
|
|
|
+ .findElements(
|
|
|
|
|
+ By.xpath("//ul[contains(@class, 'KambiBC-list-view__column KambiBC-list-view__event-list')]"));
|
|
|
|
|
+
|
|
|
|
|
+ List<WebElement> matches = driver.findElements(By.xpath("//div[@class='KambiBC-event-item__event-wrapper']"));
|
|
|
|
|
+
|
|
|
|
|
+ for (WebElement element : matches) {
|
|
|
|
|
+
|
|
|
|
|
+ List<WebElement> participants = element
|
|
|
|
|
+ .findElements(By.xpath(".//div[@class='KambiBC-event-participants__name']"));
|
|
|
|
|
+ if (participants.size() != 2) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ String homeTeamName = participants.get(0).getText();
|
|
|
|
|
+ String awayTeamName = participants.get(1).getText();
|
|
|
|
|
+
|
|
|
|
|
+ if (!checkIfElementExists(element,
|
|
|
|
|
+ ".//div[@class='KambiBC-list-view__column KambiBC-bet-offers-list__column KambiBC-bet-offers-list__column--num-3']")) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ WebElement oddsContainer = element.findElement(By.xpath(
|
|
|
|
|
+ ".//div[@class='KambiBC-list-view__column KambiBC-bet-offers-list__column KambiBC-bet-offers-list__column--num-3']"));
|
|
|
|
|
+
|
|
|
|
|
+ List<WebElement> odds = oddsContainer
|
|
|
|
|
+ .findElements(By.xpath(".//div[@class='OutcomeButton__Odds-sc-lxwzc0-6 iYOCD']"));
|
|
|
|
|
+
|
|
|
|
|
+ int od = 0;
|
|
|
|
|
+ float odds1 = -1f;
|
|
|
|
|
+ float oddsX = -1f;
|
|
|
|
|
+ float odds2 = -1f;
|
|
|
|
|
+ for (WebElement o : odds) {
|
|
|
|
|
+ if (od == 0) {
|
|
|
|
|
+ odds1 = Float.valueOf(o.getText());
|
|
|
|
|
+ } else if (od == 1) {
|
|
|
|
|
+ oddsX = Float.valueOf(o.getText());
|
|
|
|
|
+ } else if (od == 2) {
|
|
|
|
|
+ odds2 = Float.valueOf(o.getText());
|
|
|
|
|
+ }
|
|
|
|
|
+ od++;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ result.add(new ResultDTO(homeTeamName, awayTeamName, odds1, oddsX, odds2));
|
|
|
|
|
+ System.out.println(homeTeamName + "-" + awayTeamName + " (" + odds1 + "," + oddsX + "," + odds2 + ")");
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ driver.close();
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // EXPEKT
|
|
|
// https://eu-offering.kambicdn.org/offering/v2018/expektse/listView/football.json?lang=sv_SE&market=SE
|
|
// https://eu-offering.kambicdn.org/offering/v2018/expektse/listView/football.json?lang=sv_SE&market=SE
|
|
|
|
|
|
|
|
-private void getJSON() {
|
|
|
|
|
- try {
|
|
|
|
|
- URL url = new URL("https://eu-offering.kambicdn.org/offering/v2018/expektse/listView/football.json?lang=sv_SE&market=SE");
|
|
|
|
|
- System.out.println("Getting url");
|
|
|
|
|
- JsonNode node = get(url);
|
|
|
|
|
- System.out.println("Got URL");
|
|
|
|
|
- System.out.println(node.elements().next());
|
|
|
|
|
- } catch (Exception e) {
|
|
|
|
|
- e.printStackTrace();
|
|
|
|
|
|
|
+ // UNIBET
|
|
|
|
|
+ // https://eu-offering.kambicdn.org/offering/v2018/ubse/listView/football.json?lang=sv_SE&market=SE
|
|
|
|
|
+
|
|
|
|
|
+ public void getJSON() {
|
|
|
|
|
+ try {
|
|
|
|
|
+ URL url = new URL(
|
|
|
|
|
+ "https://eu-offering.kambicdn.org/offering/v2018/expektse/listView/football.json?lang=sv_SE&market=SE");
|
|
|
|
|
+ System.out.println("Getting url");
|
|
|
|
|
+ JsonNode node = get(url);
|
|
|
|
|
+ System.out.println("Got URL");
|
|
|
|
|
+ System.out.println(node.elements().next());
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ e.printStackTrace();
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
-}
|
|
|
|
|
|
|
|
|
|
private void getSoccerMatches() {
|
|
private void getSoccerMatches() {
|
|
|
String url = "https://www.expekt.se/sv-se/sports#filter/football";
|
|
String url = "https://www.expekt.se/sv-se/sports#filter/football";
|
|
@@ -57,31 +158,32 @@ private void getJSON() {
|
|
|
webClient.getOptions().setThrowExceptionOnScriptError(false);
|
|
webClient.getOptions().setThrowExceptionOnScriptError(false);
|
|
|
Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
|
|
Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
|
|
|
|
|
|
|
|
- //webClient.waitForBackgroundJavaScript(20000);
|
|
|
|
|
|
|
+ // webClient.waitForBackgroundJavaScript(20000);
|
|
|
try {
|
|
try {
|
|
|
System.out.println("Starting to Get page with wait 20 seconds " + LocalDateTime.now());
|
|
System.out.println("Starting to Get page with wait 20 seconds " + LocalDateTime.now());
|
|
|
|
|
|
|
|
webClient.waitForBackgroundJavaScript(3000);
|
|
webClient.waitForBackgroundJavaScript(3000);
|
|
|
- HtmlPage page = webClient.getPage(url);
|
|
|
|
|
-
|
|
|
|
|
|
|
+ HtmlPage page = webClient.getPage(url);
|
|
|
|
|
+
|
|
|
page = webClient.getPage(url);
|
|
page = webClient.getPage(url);
|
|
|
|
|
|
|
|
System.out.println(page.asNormalizedText());
|
|
System.out.println(page.asNormalizedText());
|
|
|
|
|
|
|
|
WebResponse webResponse = page.getWebResponse();
|
|
WebResponse webResponse = page.getWebResponse();
|
|
|
int i = 0;
|
|
int i = 0;
|
|
|
- // while (!webResponse.getContentAsString().contains("KambiBC-event-groups-list")){
|
|
|
|
|
- // webClient.waitForBackgroundJavaScript(500);
|
|
|
|
|
- // if (i++ < 50) {
|
|
|
|
|
- // System.out.println("Waiting");
|
|
|
|
|
-
|
|
|
|
|
- // } else {
|
|
|
|
|
- // break;
|
|
|
|
|
- // }
|
|
|
|
|
|
|
+ // while
|
|
|
|
|
+ // (!webResponse.getContentAsString().contains("KambiBC-event-groups-list")){
|
|
|
|
|
+ // webClient.waitForBackgroundJavaScript(500);
|
|
|
|
|
+ // if (i++ < 50) {
|
|
|
|
|
+ // System.out.println("Waiting");
|
|
|
|
|
+
|
|
|
|
|
+ // } else {
|
|
|
|
|
+ // break;
|
|
|
|
|
+ // }
|
|
|
// }
|
|
// }
|
|
|
System.out.println("Done waiting " + LocalDateTime.now());
|
|
System.out.println("Done waiting " + LocalDateTime.now());
|
|
|
|
|
|
|
|
- //System.out.println(webResponse.getContentAsString());
|
|
|
|
|
|
|
+ // System.out.println(webResponse.getContentAsString());
|
|
|
} catch (FailingHttpStatusCodeException | IOException e) {
|
|
} catch (FailingHttpStatusCodeException | IOException e) {
|
|
|
// TODO Auto-generated catch block
|
|
// TODO Auto-generated catch block
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|