@SuppressWarnings("unused")
private String postByCompanyProxy(String url, String parm, String encode)
throws ClientProtocolException, IOException {
int count = 5;
String result = "";
String urlString = url;
boolean okProxy = false;
boolean newProxy = false;
int oldProxyUsecount = 0;
for (int i = 0; i <= count; i++) {
try {
if (newProxy || oldProxyUsecount > 2 || ip.equals("")) {
okProxy = postByCompanyProxyBoolean(url, parm, encode);
}
if (okProxy) {
System.out.println("正在使用代理" + ip + ":" + port);
HttpPost httpRequst = new HttpPost(url);// 创建HttpPost对象
StringEntity entity = new StringEntity(parm);
entity.setContentType("application/x-www-form-urlencoded");
httpRequst.setEntity(entity);
httpRequst.getParams().setParameter(
CoreProtocolPNames.HTTP_CONTENT_CHARSET, encode);
HttpResponse httpResponse = httpPostClient
.execute(httpRequst);// 其中HttpGet是HttpUriRequst的子类
if (httpResponse.getStatusLine().getStatusCode() == 200) {
HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity.getContentEncoding() != null) {
if ("gzip".equalsIgnoreCase(httpEntity
.getContentEncoding().getValue())) {
httpEntity = new GzipDecompressingEntity(
httpEntity);
} else if ("deflate".equalsIgnoreCase(httpEntity
.getContentEncoding().getValue())) {
httpEntity = new DeflateDecompressingEntity(
httpEntity);
}
}
result = enCodetoString(httpEntity, encode);// 取出应答字符串
// System.out.println(result);
if (resultTest(result)) {
return result;
} else if (result.contains("function JumpSelf")
&& result.contains("WebShieldSessionVerify")) {
int indexs = result
.indexOf("&WebShieldSessionVerify");
int indexe = result.indexOf("\";}</script>");
String verify = result.substring(indexs, indexe);
urlString = urlString + verify;
newProxy = false;
} else if (result.contains("function JumpSelf")
&& !result.contains("WebShieldSessionVerify")) {
urlString = url;
newProxy = false;
}
} else if (httpResponse.getStatusLine().getStatusCode() == 302) {
System.out.println("重定向了");
Header header = httpResponse.getFirstHeader("location");
if (header != null) {
urlString = header.getValue();
System.out.println(urlString);
if (urlString.contains("tabid=26")) {
urlString = "http://www.landchina.com"
+ urlString;
result = getByHttpClient(urlString, encode,
httpPostClient);
if (resultTest(result)) {
System.out.println(i + "公司代理成功抓取" + url);
return result;
}
newProxy = false;
}
newProxy = false;
}
} else {
httpRequst.abort();
}
} else {
oldProxyUsecount++;
}
} catch (ClientProtocolException e) {
newProxy = true;
System.out.println(ip + "代理ip拒绝了");
} catch (IOException e) {
oldProxyUsecount++;
System.out.println(ip + "代理读取超时");
}
}
return "";
} |
