From 8e6ae530f38fe78b3a55f5f2d69e8119a93b7bdf Mon Sep 17 00:00:00 2001 From: Andreas Schildbach Date: Sat, 27 Oct 2018 08:51:25 +0200 Subject: [PATCH] INVG, NS, NVV, OEBB, SNCB, VGS: Don't parse HTML any more when querying for nearby stations. --- .../pte/AbstractHafasLegacyProvider.java | 62 ------------------- .../src/de/schildbach/pte/InvgProvider.java | 19 +----- enabler/src/de/schildbach/pte/NsProvider.java | 30 ++------- .../src/de/schildbach/pte/NvvProvider.java | 25 +------- .../src/de/schildbach/pte/OebbProvider.java | 24 +------ .../src/de/schildbach/pte/SncbProvider.java | 24 +------ .../src/de/schildbach/pte/VgsProvider.java | 24 +------ 7 files changed, 9 insertions(+), 199 deletions(-) diff --git a/enabler/src/de/schildbach/pte/AbstractHafasLegacyProvider.java b/enabler/src/de/schildbach/pte/AbstractHafasLegacyProvider.java index 77baf960..3abec17c 100644 --- a/enabler/src/de/schildbach/pte/AbstractHafasLegacyProvider.java +++ b/enabler/src/de/schildbach/pte/AbstractHafasLegacyProvider.java @@ -2388,68 +2388,6 @@ public abstract class AbstractHafasLegacyProvider extends AbstractHafasProvider } } - protected AbstractHafasProvider setHtmlNearbyStationsPattern(final Pattern htmlNearbyStationsPattern) { - this.htmlNearbyStationsPattern = htmlNearbyStationsPattern; - return this; - } - - private Pattern htmlNearbyStationsPattern = Pattern.compile("(.*?)", - Pattern.DOTALL); - - private static final Pattern P_HTML_NEARBY_FINE_COORDS = Pattern - .compile("REQMapRoute0\\.Location0\\.X=(-?\\d+)&(?:amp;)?REQMapRoute0\\.Location0\\.Y=(-?\\d+)&"); - private static final Pattern P_HTML_NEARBY_STATIONS_FINE_LOCATION = Pattern - .compile("[\\?&;]input=(\\d+)&[^\"]*\">([^<]*)<"); - private static final Pattern P_HTML_NEARBY_STATIONS_MESSAGES = Pattern - .compile("(Ihre Eingabe kann nicht interpretiert werden)"); - - protected final NearbyLocationsResult htmlNearbyStations(final HttpUrl url) throws IOException { - final List stations = new ArrayList<>(); - - final CharSequence page = httpClient.get(url); - String oldZebra = null; - - final Matcher mCoarse = htmlNearbyStationsPattern.matcher(page); - - final Matcher mMessage = P_HTML_NEARBY_STATIONS_MESSAGES.matcher(page); - if (mMessage.find()) { - if (mMessage.group(1) != null) - return new NearbyLocationsResult(null, NearbyLocationsResult.Status.INVALID_ID); - } - - while (mCoarse.find()) { - final String zebra = mCoarse.group(1); - if (oldZebra != null && zebra.equals(oldZebra)) - throw new IllegalArgumentException("missed row? last:" + zebra); - else - oldZebra = zebra; - - final Matcher mFineLocation = P_HTML_NEARBY_STATIONS_FINE_LOCATION.matcher(mCoarse.group(2)); - - if (mFineLocation.find()) { - int parsedLon = 0; - int parsedLat = 0; - final String parsedId = mFineLocation.group(1); - final String parsedName = ParserUtils.resolveEntities(mFineLocation.group(2)); - - final Matcher mFineCoords = P_HTML_NEARBY_FINE_COORDS.matcher(mCoarse.group(2)); - - if (mFineCoords.find()) { - parsedLon = Integer.parseInt(mFineCoords.group(1)); - parsedLat = Integer.parseInt(mFineCoords.group(2)); - } - - final String[] placeAndName = splitStationName(parsedName); - stations.add(new Location(LocationType.STATION, parsedId, parsedLat, parsedLon, placeAndName[0], - placeAndName[1])); - } else { - throw new IllegalArgumentException("cannot parse '" + mCoarse.group(2) + "' on " + url); - } - } - - return new NearbyLocationsResult(null, stations); - } - private static final Pattern P_LINE_SBAHN = Pattern.compile("SN?\\d*"); private static final Pattern P_LINE_TRAM = Pattern.compile("STR\\w{0,5}"); private static final Pattern P_LINE_BUS = Pattern.compile("BUS\\w{0,5}"); diff --git a/enabler/src/de/schildbach/pte/InvgProvider.java b/enabler/src/de/schildbach/pte/InvgProvider.java index a455e552..a3de7b0a 100644 --- a/enabler/src/de/schildbach/pte/InvgProvider.java +++ b/enabler/src/de/schildbach/pte/InvgProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 the original author or authors. + * Copyright the original author or authors. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.Date; -import java.util.EnumSet; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; @@ -42,7 +41,6 @@ import de.schildbach.pte.dto.Departure; import de.schildbach.pte.dto.Line; import de.schildbach.pte.dto.Location; import de.schildbach.pte.dto.LocationType; -import de.schildbach.pte.dto.NearbyLocationsResult; import de.schildbach.pte.dto.Position; import de.schildbach.pte.dto.Product; import de.schildbach.pte.dto.QueryDeparturesResult; @@ -67,7 +65,6 @@ public class InvgProvider extends AbstractHafasLegacyProvider { public InvgProvider() { super(NetworkId.INVG, API_BASE, "dn", PRODUCTS_MAP); - setRequestUrlEncoding(Charsets.UTF_8); setStationBoardCanDoEquivs(false); setJsonNearbyLocationsEncoding(Charsets.UTF_8); @@ -116,20 +113,6 @@ public class InvgProvider extends AbstractHafasLegacyProvider { return super.splitStationName(address); } - @Override - public NearbyLocationsResult queryNearbyLocations(final EnumSet types, final Location location, - final int maxDistance, final int maxStations) throws IOException { - if (location.type == LocationType.STATION && location.hasId()) { - final HttpUrl.Builder url = stationBoardEndpoint.newBuilder().addPathSegment(apiLanguage); - url.addQueryParameter("near", "Anzeigen"); - url.addQueryParameter("distance", Integer.toString(maxDistance != 0 ? maxDistance / 1000 : 50)); - url.addQueryParameter("input", normalizeStationId(location.id)); - return htmlNearbyStations(url.build()); - } else { - throw new IllegalArgumentException("cannot handle: " + location); - } - } - private static final Pattern P_DEPARTURES_HEAD_COARSE = Pattern.compile(".*?" // + "(?:" // + "
.*?
.*?(
.*?
.*?
.*?
).*?
.*?" // diff --git a/enabler/src/de/schildbach/pte/NsProvider.java b/enabler/src/de/schildbach/pte/NsProvider.java index 267eee90..ad2a1162 100644 --- a/enabler/src/de/schildbach/pte/NsProvider.java +++ b/enabler/src/de/schildbach/pte/NsProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 the original author or authors. + * Copyright the original author or authors. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,14 +17,10 @@ package de.schildbach.pte; -import java.io.IOException; -import java.util.EnumSet; import java.util.Set; -import java.util.regex.Pattern; -import de.schildbach.pte.dto.Location; -import de.schildbach.pte.dto.LocationType; -import de.schildbach.pte.dto.NearbyLocationsResult; +import com.google.common.base.Charsets; + import de.schildbach.pte.dto.Product; import okhttp3.HttpUrl; @@ -37,31 +33,13 @@ public class NsProvider extends AbstractHafasLegacyProvider { private static final Product[] PRODUCTS_MAP = { Product.HIGH_SPEED_TRAIN, Product.HIGH_SPEED_TRAIN, Product.HIGH_SPEED_TRAIN, Product.REGIONAL_TRAIN, Product.SUBURBAN_TRAIN, Product.BUS, Product.FERRY, Product.SUBWAY, Product.TRAM, Product.ON_DEMAND }; - private static final Pattern HTML_NEARBY_STATIONS_PATTERN = Pattern - .compile("(.*?)", Pattern.DOTALL); public NsProvider() { super(NetworkId.NS, API_BASE, "nn", PRODUCTS_MAP); - - setHtmlNearbyStationsPattern(HTML_NEARBY_STATIONS_PATTERN); + setJsonNearbyLocationsEncoding(Charsets.UTF_8); setStationBoardHasLocation(true); } - @Override - public NearbyLocationsResult queryNearbyLocations(final EnumSet types, final Location location, - final int maxDistance, final int maxLocations) throws IOException { - if (location.type == LocationType.STATION && location.hasId()) { - final HttpUrl.Builder url = stationBoardEndpoint.newBuilder().addPathSegment(apiLanguage); - url.addQueryParameter("near", "Anzeigen"); - url.addQueryParameter("distance", Integer.toString(maxDistance != 0 ? maxDistance / 1000 : 50)); - url.addQueryParameter("input", normalizeStationId(location.id)); - url.addQueryParameter("L", "profi"); - return htmlNearbyStations(url.build()); - } else { - throw new IllegalArgumentException("cannot handle: " + location); - } - } - @Override public Set defaultProducts() { return Product.ALL; diff --git a/enabler/src/de/schildbach/pte/NvvProvider.java b/enabler/src/de/schildbach/pte/NvvProvider.java index 114be848..6376e868 100644 --- a/enabler/src/de/schildbach/pte/NvvProvider.java +++ b/enabler/src/de/schildbach/pte/NvvProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 the original author or authors. + * Copyright the original author or authors. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,15 +17,10 @@ package de.schildbach.pte; -import java.io.IOException; -import java.util.EnumSet; import java.util.regex.Matcher; import com.google.common.base.Charsets; -import de.schildbach.pte.dto.Location; -import de.schildbach.pte.dto.LocationType; -import de.schildbach.pte.dto.NearbyLocationsResult; import de.schildbach.pte.dto.Product; import de.schildbach.pte.util.StringReplaceReader; @@ -42,7 +37,6 @@ public class NvvProvider extends AbstractHafasLegacyProvider { public NvvProvider() { super(NetworkId.NVV, API_BASE, "dn", PRODUCTS_MAP); - setRequestUrlEncoding(Charsets.UTF_8); setJsonNearbyLocationsEncoding(Charsets.UTF_8); httpClient.setTrustAllCertificates(true); @@ -79,23 +73,6 @@ public class NvvProvider extends AbstractHafasLegacyProvider { return super.splitStationName(address); } - @Override - public NearbyLocationsResult queryNearbyLocations(final EnumSet types, final Location location, - final int maxDistance, final int maxLocations) throws IOException { - if (location.hasLocation()) { - return nearbyLocationsByCoordinate(types, location.lat, location.lon, maxDistance, maxLocations); - } else if (location.type == LocationType.STATION && location.hasId()) { - final HttpUrl.Builder url = stationBoardEndpoint.newBuilder().addPathSegment(apiLanguage); - url.addQueryParameter("near", "Anzeigen"); - url.addQueryParameter("distance", Integer.toString(maxDistance != 0 ? maxDistance / 1000 : 50)); - url.addQueryParameter("input", normalizeStationId(location.id)); - url.addQueryParameter("L", "vs_rmv"); - return htmlNearbyStations(url.build()); - } else { - throw new IllegalArgumentException("cannot handle: " + location); - } - } - @Override protected void addCustomReplaces(final StringReplaceReader reader) { reader.replace("
    ", " "); diff --git a/enabler/src/de/schildbach/pte/OebbProvider.java b/enabler/src/de/schildbach/pte/OebbProvider.java index bd8aa031..f7df74c3 100644 --- a/enabler/src/de/schildbach/pte/OebbProvider.java +++ b/enabler/src/de/schildbach/pte/OebbProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 the original author or authors. + * Copyright the original author or authors. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,14 +17,9 @@ package de.schildbach.pte; -import java.io.IOException; -import java.util.EnumSet; import java.util.Set; import java.util.regex.Matcher; -import de.schildbach.pte.dto.Location; -import de.schildbach.pte.dto.LocationType; -import de.schildbach.pte.dto.NearbyLocationsResult; import de.schildbach.pte.dto.Product; import okhttp3.HttpUrl; @@ -41,26 +36,9 @@ public class OebbProvider extends AbstractHafasLegacyProvider { public OebbProvider() { super(NetworkId.OEBB, API_BASE, "dn", PRODUCTS_MAP); - setDominantPlanStopTime(true); } - @Override - public NearbyLocationsResult queryNearbyLocations(final EnumSet types, final Location location, - final int maxDistance, final int maxLocations) throws IOException { - if (location.hasLocation()) { - return nearbyLocationsByCoordinate(types, location.lat, location.lon, maxDistance, maxLocations); - } else if (location.type == LocationType.STATION && location.hasId()) { - final HttpUrl.Builder url = stationBoardEndpoint.newBuilder().addPathSegment(apiLanguage); - url.addQueryParameter("near", "Suchen"); - url.addQueryParameter("distance", Integer.toString(maxDistance != 0 ? maxDistance / 1000 : 50)); - url.addQueryParameter("input", normalizeStationId(location.id)); - return htmlNearbyStations(url.build()); - } else { - throw new IllegalArgumentException("cannot handle: " + location); - } - } - @Override public Set defaultProducts() { return Product.ALL; diff --git a/enabler/src/de/schildbach/pte/SncbProvider.java b/enabler/src/de/schildbach/pte/SncbProvider.java index dc010f75..dbbb0e0f 100644 --- a/enabler/src/de/schildbach/pte/SncbProvider.java +++ b/enabler/src/de/schildbach/pte/SncbProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 the original author or authors. + * Copyright the original author or authors. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,17 +17,12 @@ package de.schildbach.pte; -import java.io.IOException; import java.util.Calendar; -import java.util.EnumSet; import java.util.Set; import java.util.regex.Matcher; import com.google.common.base.Charsets; -import de.schildbach.pte.dto.Location; -import de.schildbach.pte.dto.LocationType; -import de.schildbach.pte.dto.NearbyLocationsResult; import de.schildbach.pte.dto.Product; import de.schildbach.pte.util.ParserUtils; @@ -45,7 +40,6 @@ public class SncbProvider extends AbstractHafasLegacyProvider { public SncbProvider() { super(NetworkId.SNCB, API_BASE, "nn", PRODUCTS_MAP); - setRequestUrlEncoding(Charsets.UTF_8); setJsonNearbyLocationsEncoding(Charsets.UTF_8); setStationBoardHasLocation(true); @@ -71,22 +65,6 @@ public class SncbProvider extends AbstractHafasLegacyProvider { return super.splitStationName(address); } - @Override - public NearbyLocationsResult queryNearbyLocations(final EnumSet types, final Location location, - final int maxDistance, final int maxLocations) throws IOException { - if (location.hasLocation()) { - return nearbyLocationsByCoordinate(types, location.lat, location.lon, maxDistance, maxLocations); - } else if (location.type == LocationType.STATION && location.hasId()) { - final HttpUrl.Builder url = stationBoardEndpoint.newBuilder().addPathSegment(apiLanguage); - url.addQueryParameter("near", "Zoek"); - url.addQueryParameter("distance", Integer.toString(maxDistance != 0 ? maxDistance / 1000 : 50)); - url.addQueryParameter("input", normalizeStationId(location.id)); - return htmlNearbyStations(url.build()); - } else { - throw new IllegalArgumentException("cannot handle: " + location); - } - } - @Override public Set defaultProducts() { return Product.ALL; diff --git a/enabler/src/de/schildbach/pte/VgsProvider.java b/enabler/src/de/schildbach/pte/VgsProvider.java index 26de6c7c..4aafd668 100644 --- a/enabler/src/de/schildbach/pte/VgsProvider.java +++ b/enabler/src/de/schildbach/pte/VgsProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 the original author or authors. + * Copyright the original author or authors. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,13 +17,8 @@ package de.schildbach.pte; -import java.io.IOException; -import java.util.EnumSet; import java.util.regex.Matcher; -import de.schildbach.pte.dto.Location; -import de.schildbach.pte.dto.LocationType; -import de.schildbach.pte.dto.NearbyLocationsResult; import de.schildbach.pte.dto.Product; import okhttp3.HttpUrl; @@ -40,7 +35,6 @@ public class VgsProvider extends AbstractHafasLegacyProvider { public VgsProvider() { super(NetworkId.VGS, API_BASE, "dn", PRODUCTS_MAP); - setStationBoardHasStationTable(false); httpClient.setTrustAllCertificates(true); httpClient.setSslAcceptAllHostnames(true); @@ -72,20 +66,4 @@ public class VgsProvider extends AbstractHafasLegacyProvider { return super.splitStationName(address); } - - @Override - public NearbyLocationsResult queryNearbyLocations(final EnumSet types, final Location location, - final int maxDistance, final int maxLocations) throws IOException { - if (location.hasLocation()) { - return nearbyLocationsByCoordinate(types, location.lat, location.lon, maxDistance, maxLocations); - } else if (location.type == LocationType.STATION && location.hasId()) { - final HttpUrl.Builder url = stationBoardEndpoint.newBuilder().addPathSegment(apiLanguage); - url.addQueryParameter("near", "Anzeigen"); - url.addQueryParameter("distance", Integer.toString(maxDistance != 0 ? maxDistance / 1000 : 50)); - url.addQueryParameter("input", normalizeStationId(location.id)); - return htmlNearbyStations(url.build()); - } else { - throw new IllegalArgumentException("cannot handle: " + location); - } - } }