public String crawlPageContentByPost(String url, String pram, String encode)
throws ClientProtocolException, IOException {
String content = "";
try {
content = doPostByGoagent(url, pram, encode);
if (content == null || content.equals("")) {
content = doPostByGoagent(url, pram, encode);
// System.out.println("启用公司代理");
// content = postByCompanyProxy(url, pram, encode);
// if (content == null || content.equals("")) {
// System.out.println("5秒后启用本机");
// Thread.sleep(5000);
// content = doPost(url, pram, encode);
// }
}
} catch (Exception e) {
try {
content = doPostByGoagent(url, pram, encode);
// System.out.println("goagent连接失败,启用公司代理");
// content = postByCompanyProxy(url, pram, encode);
// if (content == null || content.equals("")) {
// System.out.println("公司代理连接失败,启用本机");
// content = doPost(url, pram, encode);
// }
} catch (Exception e2) {
try {
content = doPostByGoagent(url, pram, encode);
// e2.printStackTrace();
// content = postByCompanyProxy(url, pram, encode);
// System.out.println("公司代理连接失败,启用本机");
// content = doPost(url, pram, encode);
} catch (Exception e3) {
e3.printStackTrace();
}
}
}
return content;
}
private String doPostByGoagent(String url, String parm, String encode)
throws ClientProtocolException, IOException {
String result = "";
HttpPost httpRequst = new HttpPost(url);// 创建HttpPost对象
HttpHost proxy = new HttpHost("127.0.0.1", 8087, null);
StringEntity entity = new StringEntity(parm);
entity.setContentType("application/x-www-form-urlencoded");
entity.setContentEncoding(encode);
httpRequst.setEntity(entity);
DefaultHttpClient httpClient = new DefaultHttpClient();
httpClient.getParams().setParameter(
CoreConnectionPNames.CONNECTION_TIMEOUT, 8000);// 连接时间20s
httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,
8000);// 数据传输时间60s
httpClient.getParams().setParameter(ConnRouteParams.DEFAULT_PROXY,
proxy);
HttpResponse httpResponse = httpClient.execute(httpRequst);
// System.out.println(httpResponse.getStatusLine().getStatusCode());
if (httpResponse.getStatusLine().getStatusCode() == 200) {
HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity.getContentEncoding() != null) {
if ("gzip".equalsIgnoreCase(httpEntity.getContentEncoding()
.getValue())) {
httpEntity = new GzipDecompressingEntity(httpEntity);
} else if ("deflate".equalsIgnoreCase(httpEntity
.getContentEncoding().getValue())) {
httpEntity = new DeflateDecompressingEntity(httpEntity);
}
}
result = enCodetoString(httpEntity, encode);// 取出应答字符串
}
return result;
} |
