mirrors/public-transport-enabler
1
0
Fork 0
mirror of https://gitlab.com/oeffi/public-transport-enabler.git synced 2025-07-16 09:29:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Clean up scrape methods.

Browse source
This commit is contained in:
Andreas Schildbach 2014-12-19 16:48:29 +01:00
parent 05e3ac96cd
commit 55a30f56de
4 changed files with 35 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

24
enabler/src/de/schildbach/pte/AbstractEfaProvider.java
View file

@ -256,7 +256,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
// System.out.println(uri);
// System.out.println(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), httpPost ? parameters.substring(1) : null, UTF_8, null);
final CharSequence page = ParserUtils.scrape(uri.toString(), httpPost ? parameters.substring(1) : null, UTF_8);
final ResultHeader header = new ResultHeader(SERVER_PRODUCT);
try
@ -381,7 +381,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -460,7 +460,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -569,7 +569,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -641,7 +641,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -914,7 +914,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, "NSC_", 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, "NSC_");
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -1488,7 +1488,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -1704,7 +1704,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null, 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpReferer, null);
firstChars = ParserUtils.peekFirstChars(is);
final XmlPullParser pp = parserFactory.newPullParser();
@ -2173,7 +2173,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_", 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is);
return queryTrips(uri.toString(), is);
@ -2212,7 +2212,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_", 3);
is = ParserUtils.scrapeInputStream(uri.toString(), httpPost ? parameters.substring(1) : null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is);
return queryTripsMobile(uri.toString(), from, via, to, is);
@ -2244,7 +2244,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_", 3);
is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is);
return queryTrips(uri.toString(), is);
@ -2276,7 +2276,7 @@ public abstract class AbstractEfaProvider extends AbstractNetworkProvider
try
{
is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_", 3);
is = ParserUtils.scrapeInputStream(uri.toString(), null, null, httpRefererTrip, "NSC_");
firstChars = ParserUtils.peekFirstChars(is);
is.mark(512);

6
enabler/src/de/schildbach/pte/AbstractHafasProvider.java
View file

@ -359,7 +359,7 @@ public abstract class AbstractHafasProvider extends AbstractNetworkProvider
protected final SuggestLocationsResult jsonGetStops(final String uri) throws IOException
{
final CharSequence page = ParserUtils.scrape(uri, null, jsonGetStopsEncoding, null);
final CharSequence page = ParserUtils.scrape(uri, null, jsonGetStopsEncoding);
// System.out.println(uri);
// System.out.println(page);
@ -871,7 +871,7 @@ public abstract class AbstractHafasProvider extends AbstractNetworkProvider
try
{
final String endpoint = extXmlEndpoint != null ? extXmlEndpoint : queryEndpoint;
final InputStream is = ParserUtils.scrapeInputStream(endpoint, request, null, null, null, 3);
final InputStream is = ParserUtils.scrapeInputStream(endpoint, request, null, null, null);
firstChars = ParserUtils.peekFirstChars(is);
reader = new InputStreamReader(is, ISO_8859_1);
@ -2377,7 +2377,7 @@ public abstract class AbstractHafasProvider extends AbstractNetworkProvider
protected final NearbyStationsResult jsonNearbyStations(final String uri) throws IOException
{
final CharSequence page = ParserUtils.scrape(uri, null, jsonNearbyStationsEncoding, null);
final CharSequence page = ParserUtils.scrape(uri, null, jsonNearbyStationsEncoding);
try
{

8
enabler/src/de/schildbach/pte/AbstractTsiProvider.java
View file

@ -205,7 +205,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(stopFinderEndpoint);
uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8);
try
{
final List<SuggestedLocation> locations = new ArrayList<SuggestedLocation>();
@ -295,7 +295,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(stopFinderEndpoint);
uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8);
try
{
@ -339,7 +339,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(stopFinderEndpoint);
uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8);
try
{
final JSONObject head = new JSONObject(page.toString());
@ -753,7 +753,7 @@ public abstract class AbstractTsiProvider extends AbstractNetworkProvider
final StringBuilder uri = new StringBuilder(tripEndpoint);
uri.append(parameters);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null, 3);
final CharSequence page = ParserUtils.scrape(uri.toString(), null, UTF_8, null);
try
{
final JSONObject head = new JSONObject(page.toString());

37
enabler/src/de/schildbach/pte/util/ParserUtils.java
View file

@ -68,7 +68,7 @@ public final class ParserUtils
public static final CharSequence scrape(final String url) throws IOException
{
return scrape(url, null, null, null, null);
return scrape(url, null);
}
public static final CharSequence scrape(final String url, final String authorization) throws IOException
@ -76,32 +76,25 @@ public final class ParserUtils
return scrape(url, null, null, null, authorization);
}
public static final CharSequence scrape(final String url, final String postRequest, Charset encoding, final String sessionCookieName)
throws IOException
public static final CharSequence scrape(final String url, final String postRequest, final Charset encoding) throws IOException
{
return scrape(url, postRequest, encoding, sessionCookieName, null, 3);
return scrape(url, postRequest, encoding, null);
}
public static final CharSequence scrape(final String url, final String postRequest, Charset encoding, final String sessionCookieName, final String authorization)
throws IOException
public static final CharSequence scrape(final String urlStr, final String postRequest, final Charset requestEncoding,
final String sessionCookieName) throws IOException
{
return scrape(url, postRequest, encoding, sessionCookieName, authorization, 3);
return scrape(urlStr, postRequest, requestEncoding, sessionCookieName, null);
}
public static final CharSequence scrape(final String urlStr, final String postRequest, Charset requestEncoding, final String sessionCookieName,
int tries) throws IOException
{
return scrape(urlStr, postRequest, requestEncoding, sessionCookieName, null, tries);
}
public static final CharSequence scrape(final String urlStr, final String postRequest, Charset requestEncoding, final String sessionCookieName,
final String authorization, int tries) throws IOException
private static final CharSequence scrape(final String urlStr, final String postRequest, Charset requestEncoding, final String sessionCookieName,
final String authorization) throws IOException
{
if (requestEncoding == null)
requestEncoding = SCRAPE_DEFAULT_ENCODING;
final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY);
final InputStream is = scrapeInputStream(urlStr, postRequest, requestEncoding, null, sessionCookieName, authorization, tries);
final InputStream is = scrapeInputStream(urlStr, postRequest, requestEncoding, null, sessionCookieName, authorization);
final Reader pageReader = new InputStreamReader(is, requestEncoding);
copy(pageReader, buffer);
pageReader.close();
@ -123,21 +116,23 @@ public final class ParserUtils
public static final InputStream scrapeInputStream(final String url) throws IOException
{
return scrapeInputStream(url, null, null, null, null, null, 3);
return scrapeInputStream(url, null, null, null, null);
}
public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, Charset requestEncoding, final String referer,
final String sessionCookieName, int tries) throws IOException
public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, final Charset requestEncoding,
final String referer, final String sessionCookieName) throws IOException
{
return scrapeInputStream(urlStr, postRequest, requestEncoding, referer, sessionCookieName, null, tries);
return scrapeInputStream(urlStr, postRequest, requestEncoding, referer, sessionCookieName, null);
}
public static final InputStream scrapeInputStream(final String urlStr, final String postRequest, Charset requestEncoding, final String referer,
final String sessionCookieName, final String authorization, int tries) throws IOException
final String sessionCookieName, final String authorization) throws IOException
{
if (requestEncoding == null)
requestEncoding = SCRAPE_DEFAULT_ENCODING;
int tries = 3;
while (true)
{
final URL url = new URL(urlStr);