mirrors/public-transport-enabler
1
0
Fork 0
mirror of https://gitlab.com/oeffi/public-transport-enabler.git synced 2025-07-18 00:09:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

retry when socket times out

Browse source 
git-svn-id: https://public-transport-enabler.googlecode.com/svn/trunk@79 0924bc21-9374-b0fa-ee44-9ff1593b38f0
This commit is contained in:
andreas.schildbach 2010-08-13 09:18:40 +00:00
parent edd72a64c4
commit a3be919104
1 changed files with 40 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

64
src/de/schildbach/pte/ParserUtils.java
View file

@ -23,6 +23,7 @@ import java.io.OutputStreamWriter;
import java.io.Reader; import java.io.Reader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.io.Writer; import java.io.Writer;
import java.net.SocketTimeoutException;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
import java.net.URLEncoder; import java.net.URLEncoder;
@ -51,34 +52,49 @@ public final class ParserUtils
public static CharSequence scrape(final String url, final String request) throws IOException public static CharSequence scrape(final String url, final String request) throws IOException
{ {
final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY); int tries = 3;
final URLConnection connection = new URL(url).openConnection();
connection.setDoInput(true);
connection.setDoOutput(request != null);
connection.setConnectTimeout(SCRAPE_CONNECT_TIMEOUT);
connection.setReadTimeout(SCRAPE_READ_TIMEOUT);
connection.addRequestProperty("User-Agent", SCRAPE_USER_AGENT);
if (request != null)
{
final Writer writer = new OutputStreamWriter(connection.getOutputStream(), "ISO-8859-1");
writer.write(request);
writer.close();
}
final Reader pageReader = new InputStreamReader(connection.getInputStream(), "ISO-8859-1");
final char[] buf = new char[SCRAPE_INITIAL_CAPACITY];
while (true) while (true)
{ {
final int read = pageReader.read(buf); try
if (read == -1) {
break; final StringBuilder buffer = new StringBuilder(SCRAPE_INITIAL_CAPACITY);
buffer.append(buf, 0, read); final URLConnection connection = new URL(url).openConnection();
} connection.setDoInput(true);
connection.setDoOutput(request != null);
connection.setConnectTimeout(SCRAPE_CONNECT_TIMEOUT);
connection.setReadTimeout(SCRAPE_READ_TIMEOUT);
connection.addRequestProperty("User-Agent", SCRAPE_USER_AGENT);
pageReader.close(); if (request != null)
return buffer; {
final Writer writer = new OutputStreamWriter(connection.getOutputStream(), "ISO-8859-1");
writer.write(request);
writer.close();
}
final Reader pageReader = new InputStreamReader(connection.getInputStream(), "ISO-8859-1");
final char[] buf = new char[SCRAPE_INITIAL_CAPACITY];
while (true)
{
final int read = pageReader.read(buf);
if (read == -1)
break;
buffer.append(buf, 0, read);
}
pageReader.close();
return buffer;
}
catch (final SocketTimeoutException x)
{
if (tries-- > 0)
System.out.println("socket timed out, retrying...");
else
throw x;
}
}
} }
private static final Pattern P_ENTITY = Pattern.compile("&(?:#(x[\\da-f]+|\\d+)|(amp|quot|apos));"); private static final Pattern P_ENTITY = Pattern.compile("&(?:#(x[\\da-f]+|\\d+)|(amp|quot|apos));");