mirrors/public-transport-enabler
1
0
Fork 0
mirror of https://gitlab.com/oeffi/public-transport-enabler.git synced 2025-07-17 13:59:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Clean up scrape methods.

Browse source
This commit is contained in:
Andreas Schildbach 2014-12-19 16:48:29 +01:00
parent 05e3ac96cd
commit 55a30f56de
4 changed files with 35 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

24
enabler/src/de/schildbach/pte/AbstractEfaProvider.java
View file

@ -256,7 +256,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
// System.out.println(uri); // System.out.println(uri);
// System.out.println(parameters); // System.out.println(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), httpPost ? parameters.substring(1) : null, UTF_8, null); final CharSequence page = ParserUtils.scrape(uri.toString(), httpPost ? parameters.substring(1) : null, UTF_8);
final ResultHeader header = new ResultHeader(SERVER_PRODUCT); final ResultHeader header = new ResultHeader(SERVER_PRODUCT);
try try
@ -381,7 +381,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -460,7 +460,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -569,7 +569,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -641,7 +641,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -914,7 +914,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, "NSC_", 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, "NSC_");
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -1488,7 +1488,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -1704,7 +1704,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser(); final XmlPullParser pp = parserFactory.newPullParser();
@ -2173,7 +2173,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_", 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
return queryTrips(uri.toString(), is); return queryTrips(uri.toString(), is);
@ -2212,7 +2212,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_", 3); is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
return queryTripsMobile(uri.toString(), from, via, to, is); return queryTripsMobile(uri.toString(), from, via, to, is);
@ -2244,7 +2244,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_", 3); is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
return queryTrips(uri.toString(), is); return queryTrips(uri.toString(), is);
@ -2276,7 +2276,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try try
{ {
is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_", 3); is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
is.mark(512); is.mark(512);

6
enabler/src/de/schildbach/pte/AbstractHafasProvider.java
View file

@ -359,7 +359,7 @@ public abstract class AbstractHafasProvider extends AbstractNetworkProvider
protected final SuggestLocationsResult jsonGetStops(final String uri) throws IOException protected final SuggestLocationsResult jsonGetStops(final String uri) throws IOException
{ {
final CharSequence page = ParserUtils.scrape(uri, null, jsonGetStopsEncoding, null); final CharSequence page = ParserUtils.scrape(uri, null, jsonGetStopsEncoding);
// System.out.println(uri); // System.out.println(uri);
// System.out.println(page); // System.out.println(page);
@ -871,7 +871,7 @@ public abstract class AbstractHafasProvider extends AbstractNetworkProvider
try try
{ {
final String endpoint = extXmlEndpoint != null ? extXmlEndpoint : queryEndpoint; final String endpoint = extXmlEndpoint != null ? extXmlEndpoint : queryEndpoint;
final InputStream is = ParserUtils.scrapeInputStream(endpoint, request, null, null, null, 3); final InputStream is = ParserUtils.scrapeInputStream(endpoint, request, null, null, null);
firstChars = ParserUtils.peekFirstChars(is); firstChars = ParserUtils.peekFirstChars(is);
reader = new InputStreamReader(is, ISO_8859_1); reader = new InputStreamReader(is, ISO_8859_1);
@ -2377,7 +2377,7 @@ public abstract class AbstractHafasProvider extends AbstractNetworkProvider
protected final NearbyStationsResult jsonNearbyStations(final String uri) throws IOException protected final NearbyStationsResult jsonNearbyStations(final String uri) throws IOException
{ {
final CharSequence page = ParserUtils.scrape(uri, null, jsonNearbyStationsEncoding, null); final CharSequence page = ParserUtils.scrape(uri, null, jsonNearbyStationsEncoding);
try try
{ {

8
enabler/src/de/schildbach/pte/AbstractTsiProvider.java
View file

@ -205,7 +205,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(stopFinderEndpoint); final StringBuilder uri = new StringBuilder(stopFinderEndpoint);
uri.append(parameters); uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3); final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8);
try try
{ {
final List<SuggestedLocation> locations = new ArrayList<SuggestedLocation>(); final List<SuggestedLocation> locations = new ArrayList<SuggestedLocation>();
@ -295,7 +295,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(stopFinderEndpoint); final StringBuilder uri = new StringBuilder(stopFinderEndpoint);
uri.append(parameters); uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3); final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8);
try try
{ {
@ -339,7 +339,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(stopFinderEndpoint); final StringBuilder uri = new StringBuilder(stopFinderEndpoint);
uri.append(parameters); uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3); final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8);
try try
{ {
final JSONObject head = new JSONObject(page.toString()); final JSONObject head = new JSONObject(page.toString());
@ -753,7 +753,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(tripEndpoint); final StringBuilder uri = new StringBuilder(tripEndpoint);
uri.append(parameters); uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3); final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null);
try try
{ {
final JSONObject head = new JSONObject(page.toString()); final JSONObject head = new JSONObject(page.toString());

37
enabler/src/de/schildbach/pte/util/ParserUtils.java
View file

@ -68,7 +68,7 @@ public final class ParserUtils
public static final CharSequence scrape(final String url) throws IOException public static final CharSequence scrape(final String url) throws IOException
{ {
return scrape(url, null, null, null, null); return scrape(url, null);
} }
public static final CharSequence scrape(final String url, final String authorization) throws IOException public static final CharSequence scrape(final String url, final String authorization) throws IOException
@ -76,32 +76,25 @@ public final class ParserUtils
return scrape(url, null, null, null, authorization); return scrape(url, null, null, null, authorization);
} }
public static final CharSequence scrape(final String url, final String postRequest, Charset encoding, final String sessionCookieName) public static final CharSequence scrape(final String url, final String postRequest, final Charset encoding) throws IOException
throws IOException
{ {
return scrape(url, postRequest, encoding, sessionCookieName, null, 3); return scrape(url, postRequest, encoding, null);
} }
public static final CharSequence scrape(final String url, final String postRequest, Charset encoding, final String sessionCookieName, final String authorization) public static final CharSequence scrape(final String urlStr, final String postRequest, final Charset requestEncoding,
throws IOException final String sessionCookieName) throws IOException
{ {
return scrape(url, postRequest, encoding, sessionCookieName, authorization, 3); return scrape(urlStr, postRequest, requestEncoding, sessionCookieName, null);
} }
public static final CharSequence scrape(final String urlStr, final String postRequest, Charset requestEncoding, final String sessionCookieName, private static final CharSequence scrape(final String urlStr, final String postRequest, Charset requestEncoding, final String sessionCookieName,
int tries) throws IOException final String authorization) throws IOException
{
return scrape(urlStr, postRequest, requestEncoding, sessionCookieName, null, tries);
}
public static final CharSequence scrape(final String urlStr, final String postRequest, Charset requestEncoding, final String sessionCookieName,
final String authorization, int tries) throws IOException
{ {
if (requestEncoding == null) if (requestEncoding == null)
requestEncoding = SCRAPE_DEFAULT_ENCODING; requestEncoding = SCRAPE_DEFAULT_ENCODING;
final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY); final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY);
final InputStream is = scrapeInputStream(urlStr, postRequest, requestEncoding, null, sessionCookieName, authorization, tries); final InputStream is = scrapeInputStream(urlStr, postRequest, requestEncoding, null, sessionCookieName, authorization);
final Reader pageReader = new InputStreamReader(is, requestEncoding); final Reader pageReader = new InputStreamReader(is, requestEncoding);
copy(pageReader, buffer); copy(pageReader, buffer);
pageReader.close(); pageReader.close();
@ -123,21 +116,23 @@ public final class ParserUtils
public static final InputStream scrapeInputStream(final String url) throws IOException public static final InputStream scrapeInputStream(final String url) throws IOException
{ {
return scrapeInputStream(url, null, null, null, null, null, 3); return scrapeInputStream(url, null, null, null, null);
} }
public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, Charset requestEncoding, final String referer, public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, final Charset requestEncoding,
final String sessionCookieName, int tries) throws IOException final String referer, final String sessionCookieName) throws IOException
{ {
return scrapeInputStream(urlStr, postRequest, requestEncoding, referer, sessionCookieName, null, tries); return scrapeInputStream(urlStr, postRequest, requestEncoding, referer, sessionCookieName, null);
} }
public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, Charset requestEncoding, final String referer, public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, Charset requestEncoding, final String referer,
final String sessionCookieName, final String authorization, int tries) throws IOException final String sessionCookieName, final String authorization) throws IOException
{ {
if (requestEncoding == null) if (requestEncoding == null)
requestEncoding = SCRAPE_DEFAULT_ENCODING; requestEncoding = SCRAPE_DEFAULT_ENCODING;
int tries = 3;
while (true) while (true)
{ {
final URL url = new URL(urlStr); final URL url = new URL(urlStr);