diff --git a/src/de/schildbach/pte/OebbProvider.java b/src/de/schildbach/pte/OebbProvider.java index bf72bc4a..aa1432cb 100644 --- a/src/de/schildbach/pte/OebbProvider.java +++ b/src/de/schildbach/pte/OebbProvider.java @@ -24,7 +24,7 @@ public class OebbProvider implements NetworkProvider public boolean hasCapabilities(final Capability... capabilities) { for (final Capability capability : capabilities) - if (capability == Capability.DEPARTURES || /* capability == Capability.CONNECTIONS || */capability == Capability.LOCATION_STATION_ID) + if (capability == Capability.DEPARTURES || capability == Capability.CONNECTIONS || capability == Capability.LOCATION_STATION_ID) return true; return false; @@ -108,35 +108,38 @@ public class OebbProvider implements NetworkProvider WALKSPEED_MAP.put(WalkSpeed.FAST, "85"); } - private String connectionsQueryUri(final LocationType fromType, final String from, final LocationType viaType, final String via, - final LocationType toType, final String to, final Date date, final boolean dep, final WalkSpeed walkSpeed) + private String connectionsQuery(final LocationType fromType, final String from, final LocationType viaType, final String via, + final LocationType toType, final String to, final Date date, final boolean dep, final WalkSpeed walkSpeed) throws IOException { final DateFormat DATE_FORMAT = new SimpleDateFormat("dd.MM.yy"); final DateFormat TIME_FORMAT = new SimpleDateFormat("HH:mm"); final StringBuilder uri = new StringBuilder(); - uri.append("http://fahrplan.oebb.at/bin/query.exe/dn?ld=web05&OK"); - uri.append("&REQ0HafasSearchForw=").append(dep ? "1" : "0"); - uri.append("&REQ0JourneyDate=").append(ParserUtils.urlEncode(DATE_FORMAT.format(date))); - uri.append("&REQ0JourneyStopsS0G=").append(ParserUtils.urlEncode(from)); + uri.append("queryPageDisplayed=yes"); + uri.append("&start.x=0"); + uri.append("&start.y=0"); + uri.append("&start=Suchen"); uri.append("&REQ0JourneyStopsS0A=").append(locationType(fromType)); + uri.append("&REQ0JourneyStopsS0G=").append(ParserUtils.urlEncode(from)); uri.append("&REQ0JourneyStopsS0ID="); // "tupel"? if (via != null) { - uri.append("&REQ0JourneyStops1.0G=").append(ParserUtils.urlEncode(via)); uri.append("&REQ0JourneyStops1.0A=").append(locationType(viaType)); + uri.append("&REQ0JourneyStops1.0G=").append(ParserUtils.urlEncode(via)); uri.append("&REQ0JourneyStops1.0ID="); } - uri.append("&REQ0JourneyStopsZ0G=").append(ParserUtils.urlEncode(to)); uri.append("&REQ0JourneyStopsZ0A=").append(locationType(toType)); + uri.append("&REQ0JourneyStopsZ0G=").append(ParserUtils.urlEncode(to)); uri.append("&REQ0JourneyStopsZ0ID="); + uri.append("&REQ0JourneyDate=").append(ParserUtils.urlEncode(DATE_FORMAT.format(date))); + uri.append("&wDayExt0=").append(ParserUtils.urlEncode("Mo|Di|Mi|Do|Fr|Sa|So")); uri.append("&REQ0JourneyTime=").append(ParserUtils.urlEncode(TIME_FORMAT.format(date))); + uri.append("&REQ0HafasSearchForw=").append(dep ? "1" : "0"); + uri.append("&existHafasDemo3=yes"); uri.append("&REQ0JourneyProduct_list=").append(ParserUtils.urlEncode("0:1111111111010000-000000")); uri.append("&REQ0JourneyDep_Foot_speed=").append(WALKSPEED_MAP.get(walkSpeed)); + uri.append("&existBikeEverywhere=yes"); uri.append("&existHafasAttrInc=yes"); - uri.append("&existHafasDemo3=yes"); - uri.append("&queryPageDisplayed=yes"); - uri.append("&start=Suchen"); return uri.toString(); } @@ -152,25 +155,38 @@ public class OebbProvider implements NetworkProvider throw new IllegalArgumentException(locationType.toString()); } + private static final String QUERY_CONNECTIONS_FORM_URL = "http://fahrplan.oebb.at/bin/query.exe/dn?"; + private static final Pattern P_QUERY_CONNECTIONS_FORM_ACTION = Pattern + .compile("
]*>(.*?)", Pattern.DOTALL); private static final Pattern P_ADDRESSES = Pattern.compile("]*>\\s*(.*?)\\s*", Pattern.DOTALL); - private static final Pattern P_CHECK_CONNECTIONS_ERROR = Pattern - .compile("(keine Verbindung gefunden werden)|(liegt nach dem Ende der Fahrplanperiode|liegt vor Beginn der Fahrplanperiode)"); public QueryConnectionsResult queryConnections(final LocationType fromType, final String from, final LocationType viaType, final String via, final LocationType toType, final String to, final Date date, final boolean dep, final WalkSpeed walkSpeed) throws IOException { - final String uri = connectionsQueryUri(fromType, from, viaType, via, toType, to, date, dep, walkSpeed); - final CharSequence page = ParserUtils.scrape(uri); + // get base url and cookies from form + final CharSequence form = ParserUtils.scrape(QUERY_CONNECTIONS_FORM_URL, false, null, null, true); + final Matcher m = P_QUERY_CONNECTIONS_FORM_ACTION.matcher(form); + if (!m.find()) + throw new IllegalStateException("cannot find form: '" + form + "' on " + QUERY_CONNECTIONS_FORM_URL); + final String baseUri = m.group(1); - final Matcher mError = P_CHECK_CONNECTIONS_ERROR.matcher(page); + // query + final String query = connectionsQuery(fromType, from, viaType, via, toType, to, date, dep, walkSpeed); + final CharSequence page = ParserUtils.scrape(baseUri, true, query, null, true); + + final Matcher mError = P_QUERY_CONNECTIONS_ERROR.matcher(page); if (mError.find()) { if (mError.group(1) != null) return QueryConnectionsResult.NO_CONNECTIONS; if (mError.group(2) != null) return QueryConnectionsResult.INVALID_DATE; + if (mError.group(3) != null) + return QueryConnectionsResult.SESSION_TIMEOUT; } List fromAddresses = null; @@ -205,19 +221,29 @@ public class OebbProvider implements NetworkProvider if (fromAddresses != null || viaAddresses != null || toAddresses != null) return new QueryConnectionsResult(QueryConnectionsResult.Status.AMBIGUOUS, fromAddresses, viaAddresses, toAddresses); else - return queryConnections(uri, page); + return queryConnections(baseUri, page); } public QueryConnectionsResult queryMoreConnections(final String uri) throws IOException { - final CharSequence page = ParserUtils.scrape(uri); + final CharSequence page = ParserUtils.scrape(uri, false, null, null, true); + + final Matcher mError = P_QUERY_CONNECTIONS_ERROR.matcher(page); + if (mError.find()) + { + if (mError.group(1) != null) + return QueryConnectionsResult.NO_CONNECTIONS; + if (mError.group(2) != null) + return QueryConnectionsResult.INVALID_DATE; + if (mError.group(3) != null) + return QueryConnectionsResult.SESSION_TIMEOUT; + } return queryConnections(uri, page); } - private static final Pattern P_CONNECTIONS_FORM_ACTION = Pattern.compile("" // - + "]*>" // action - , Pattern.DOTALL); + private static final Pattern P_CONNECTIONS_FORM_ACTION = Pattern + .compile("]*>.*?" // action + "\n(.*?)\n
.*?" // header @@ -272,8 +298,20 @@ public class OebbProvider implements NetworkProvider final Matcher mFormAction = P_CONNECTIONS_FORM_ACTION.matcher(firstPage); if (!mFormAction.find()) throw new IOException("cannot find form action in '" + firstPage + "' on " + firstUri); - final String uri = mFormAction.group(1) + "&guiVCtrl_connection_detailsOut_add_group_overviewOut=yes"; - final CharSequence page = ParserUtils.scrape(uri); + final String baseUri = mFormAction.group(1); + final String query = "sortConnections=minDeparture&guiVCtrl_connection_detailsOut_add_group_overviewOut=yes"; + final CharSequence page = ParserUtils.scrape(baseUri, true, query, null, true); + + final Matcher mError = P_QUERY_CONNECTIONS_ERROR.matcher(page); + if (mError.find()) + { + if (mError.group(1) != null) + return QueryConnectionsResult.NO_CONNECTIONS; + if (mError.group(2) != null) + return QueryConnectionsResult.INVALID_DATE; + if (mError.group(3) != null) + return QueryConnectionsResult.SESSION_TIMEOUT; + } // parse page final Matcher mPage = P_CONNECTIONS_PAGE.matcher(page); @@ -306,7 +344,7 @@ public class OebbProvider implements NetworkProvider final Date departureTime = ParserUtils.joinDateTime(departureDate, ParserUtils.parseTime(mConFine.group(4))); final Date arrivalTime = ParserUtils.joinDateTime(arrivalDate != null ? arrivalDate : departureDate, ParserUtils .parseTime(mConFine.group(5))); - final String link = uri + "#" + id; // TODO use print link? + final String link = firstUri + "#" + id; // TODO use print link? final Connection connection = new Connection(id, link, departureTime, arrivalTime, null, null, 0, from, 0, to, new ArrayList(1)); @@ -314,7 +352,7 @@ public class OebbProvider implements NetworkProvider } else { - throw new IllegalArgumentException("cannot parse '" + mConCoarse.group(1) + "' on " + uri); + throw new IllegalArgumentException("cannot parse '" + mConCoarse.group(1) + "' on " + baseUri + " (POST)"); } } @@ -390,16 +428,16 @@ public class OebbProvider implements NetworkProvider } else { - throw new IllegalArgumentException("cannot parse '" + set + "' on " + uri); + throw new IllegalArgumentException("cannot parse '" + set + "' on " + baseUri + " (POST)"); } } } - return new QueryConnectionsResult(uri, from, to, currentDate, linkEarlier, linkLater, connections); + return new QueryConnectionsResult(baseUri, from, to, currentDate, linkEarlier, linkLater, connections); } else { - throw new IllegalArgumentException("cannot parse '" + headSet + "' on " + uri); + throw new IllegalArgumentException("cannot parse '" + headSet + "' on " + baseUri + " (POST)"); } } else diff --git a/src/de/schildbach/pte/ParserUtils.java b/src/de/schildbach/pte/ParserUtils.java index dfe16994..ab0e672c 100644 --- a/src/de/schildbach/pte/ParserUtils.java +++ b/src/de/schildbach/pte/ParserUtils.java @@ -23,15 +23,17 @@ import java.io.OutputStreamWriter; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.io.Writer; +import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URL; -import java.net.URLConnection; import java.net.URLEncoder; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; +import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -40,26 +42,39 @@ import java.util.regex.Pattern; */ public final class ParserUtils { - private static final String SCRAPE_USER_AGENT = "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6 (.NET CLR 3.5.30729)"; + private static final String SCRAPE_USER_AGENT = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.10) Gecko/20100915 Ubuntu/10.04 (lucid) Firefox/3.6.10"; private static final int SCRAPE_INITIAL_CAPACITY = 4096; private static final int SCRAPE_CONNECT_TIMEOUT = 5000; private static final int SCRAPE_READ_TIMEOUT = 10000; + private static final String SCRAPE_DEFAULT_ENCODING = "ISO-8859-1"; + + private static String stateCookie; + + public static void resetState() + { + stateCookie = null; + } public static CharSequence scrape(final String url) throws IOException { - return scrape(url, null, null); + return scrape(url, false, null, null, false); } - public static CharSequence scrape(final String url, final String request, final String encoding) throws IOException + public static CharSequence scrape(final String url, final boolean isPost, final String request, String encoding, final boolean cookieHandling) + throws IOException { + if (encoding == null) + encoding = SCRAPE_DEFAULT_ENCODING; + + final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY); int tries = 3; while (true) { try { - final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY); - final URLConnection connection = new URL(url).openConnection(); + final HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setDoInput(true); connection.setDoOutput(request != null); connection.setConnectTimeout(SCRAPE_CONNECT_TIMEOUT); @@ -68,28 +83,50 @@ public final class ParserUtils // workaround to disable Vodafone compression connection.addRequestProperty("Cache-Control", "no-cache"); + if (cookieHandling && stateCookie != null) + { + connection.addRequestProperty("Cookie", stateCookie); + } + if (request != null) { - final Writer writer = new OutputStreamWriter(connection.getOutputStream(), encoding != null ? encoding : "ISO-8859-1"); + if (isPost) + { + connection.setRequestMethod("POST"); + connection.addRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + connection.addRequestProperty("Content-Length", Integer.toString(request.length())); + } + + final Writer writer = new OutputStreamWriter(connection.getOutputStream(), encoding); writer.write(request); writer.close(); } - final Reader pageReader = new InputStreamReader(connection.getInputStream(), encoding != null ? encoding : "ISO-8859-1"); - - final char[] buf = new char[SCRAPE_INITIAL_CAPACITY]; - while (true) - { - final int read = pageReader.read(buf); - if (read == -1) - break; - buffer.append(buf, 0, read); - } - + final Reader pageReader = new InputStreamReader(connection.getInputStream(), encoding); + copy(pageReader, buffer); pageReader.close(); if (buffer.length() > 0) + { + if (cookieHandling) + { + for (final Map.Entry> entry : connection.getHeaderFields().entrySet()) + { + if (entry.getKey().equalsIgnoreCase("set-cookie")) + { + for (final String value : entry.getValue()) + { + if (value.startsWith("NSC_")) + { + stateCookie = value.split(";", 2)[0]; + } + } + } + } + } + return buffer; + } else { if (tries-- > 0) @@ -108,6 +145,19 @@ public final class ParserUtils } } + private static long copy(final Reader reader, final StringBuilder builder) throws IOException + { + final char[] buffer = new char[SCRAPE_INITIAL_CAPACITY]; + long count = 0; + int n = 0; + while (-1 != (n = reader.read(buffer))) + { + builder.append(buffer, 0, n); + count += n; + } + return count; + } + private static final Pattern P_ENTITY = Pattern.compile("&(?:#(x[\\da-f]+|\\d+)|(amp|quot|apos));"); public static String resolveEntities(final CharSequence str) diff --git a/src/de/schildbach/pte/VbbProvider.java b/src/de/schildbach/pte/VbbProvider.java index 72f33781..b2c58f27 100644 --- a/src/de/schildbach/pte/VbbProvider.java +++ b/src/de/schildbach/pte/VbbProvider.java @@ -122,7 +122,7 @@ public final class VbbProvider implements NetworkProvider + DATE_FORMAT.format(now) + ""; final String uri = "http://www.vbb-fahrinfo.de/hafas/extxml/extxml.exe/dn"; - final CharSequence page = ParserUtils.scrape(uri, request, null); + final CharSequence page = ParserUtils.scrape(uri, false, request, null, false); final Matcher mError = P_STATION_LOCATION_ERROR.matcher(page); if (mError.find())