/**
* 新ip第一次访问时要先通过安全验证，这时只能得到首页的内容，所以在post前线验证一次 <功能详细描述> [参数说明]
*
* @return void [返回类型说明]
* @exception throws [违例类型] [违例说明]
* @see [类、类#方法、类#成员]
*/
private Boolean postByCompanyProxyBoolean(String url, String parm,
String encode) throws ClientProtocolException, IOException {
int count = 10;
String result = "";
String urlString = url;
String proxy = "";
HttpHost proxyHost = null;
boolean newProxy = false;
int oldProxyUsecount = 0;
for (int i = 0; i <= count; i++) {
try {
if (newProxy || oldProxyUsecount > 2 || ip.equals("")) {
oldProxyUsecount = 0;
String[] proxys = null;
try {
while (proxy.equals("") || !proxy.contains(":")) {
System.out.println("ip为空,正在提取");
proxy = doGet(ipUrl, "gbk");
}
proxys = proxy.replaceAll("\"|//|/|\r\n| | ", "")
.split(":");
} catch (Exception e) {
while (proxy.equals("") || !proxy.contains(":")) {
System.out.println("ip为空,正在提取");
proxy = doGet(ipUrl, "gbk");
}
proxys = proxy.replaceAll("\"|//|/|\r\n| ", "").split(
":");
}
ip = proxys[0];
port = Integer.parseInt(proxys[1]);
proxyHost = new HttpHost(ip, port, null);
}
System.out.println("正在使用代理" + ip + ":" + port);
HttpPost httpRequst = new HttpPost(url);// 创建HttpPost对象
StringEntity entity = new StringEntity(parm);
entity.setContentType("application/x-www-form-urlencoded");
httpRequst.setEntity(entity);
httpRequst.getParams().setParameter(
CoreProtocolPNames.HTTP_CONTENT_CHARSET, encode);
httpPostClient.getParams().setParameter(
CoreConnectionPNames.CONNECTION_TIMEOUT, 10000);// 连接时间20s
httpPostClient.getParams().setParameter(
CoreConnectionPNames.SO_TIMEOUT, 8000);// 数据传输时间60s
httpPostClient.getParams().setParameter(
ConnRouteParams.DEFAULT_PROXY, proxyHost);
HttpResponse httpResponse = httpPostClient.execute(httpRequst);// 其中HttpGet是HttpUriRequst的子类
if (httpResponse.getStatusLine().getStatusCode() == 200) {
HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity.getContentEncoding() != null) {
if ("gzip".equalsIgnoreCase(httpEntity
.getContentEncoding().getValue())) {
httpEntity = new GzipDecompressingEntity(httpEntity);
} else if ("deflate".equalsIgnoreCase(httpEntity
.getContentEncoding().getValue())) {
httpEntity = new DeflateDecompressingEntity(
httpEntity);
}
}
result = enCodetoString(httpEntity, encode);// 取出应答字符串
// System.out.println(result);
if (resultTest(result)) {
return true;
} else if (result.contains("function JumpSelf")
&& result.contains("WebShieldSessionVerify")) {
int indexs = result.indexOf("&WebShieldSessionVerify");
int indexe = result.indexOf("\";}</script>");
String verify = result.substring(indexs, indexe);
urlString = urlString + verify;
if (urlString.contains("tabid=26")
&& !urlString.contains("landchina")) {
urlString = "http://www.landchina.com" + urlString;
result = getByHttpClient(urlString, encode,
httpPostClient);
if (resultTest(result)) {
System.out.println(ip + "公司代理成功抓取" + url);
return true;
}
newProxy = false;
} else if (urlString.contains("tabid=26")
&& urlString.contains("landchina")) {
result = getByHttpClient(urlString, encode,
httpPostClient);
if (resultTest(result)) {
System.out.println(ip + "公司代理成功抓取" + url);
return true;
}
newProxy = false;
}
newProxy = false;
} else if (result.contains("function JumpSelf")
&& !result.contains("WebShieldSessionVerify")) {
urlString = url;
newProxy = false;
}
} else if (httpResponse.getStatusLine().getStatusCode() == 302) {
System.out.println("重定向了");
Header header = httpResponse.getFirstHeader("location");
if (header != null) {
urlString = header.getValue();
System.out.println(urlString);
if (urlString.contains("tabid=26")
&& !urlString.contains("landchina")) {
urlString = "http://www.landchina.com" + urlString;
result = getByHttpClient(urlString, encode,
httpPostClient);
if (resultTest(result)) {
System.out.println(ip + "公司代理成功抓取" + url);
return true;
}
newProxy = false;
} else if (urlString.contains("tabid=26")
&& urlString.contains("landchina")) {
result = getByHttpClient(urlString, encode,
httpPostClient);
if (resultTest(result)) {
System.out.println(ip + "公司代理成功抓取" + url);
return true;
}
newProxy = false;
}
newProxy = false;
}
} else {
httpRequst.abort();
}
} catch (ClientProtocolException e) {
newProxy = true;
System.out.println(ip + "代理ip拒绝了");
} catch (IOException e) {
oldProxyUsecount++;
System.out.println(ip + "代理读取超时");
}
}
return false;
} |
