电子技术论坛_中国专业的电子工程师学习交流社区-中电网技术论坛

标题: httpclient常用基本抓取类（6） [打印本页]

作者: look_w 时间: 2019-4-20 10:58 标题: httpclient常用基本抓取类（6）

private String getByHttpClient(String url, String encode,
         HttpClient httpClient) {
      int count = 2;
      String result = "";
      String urlString = url;
      for (int i = 0; i <= count; i++) {
         try {
            HttpGet httpRequst = new HttpGet(urlString);
            httpRequst.setHeader("Content-Type",
                     "application/x-www-form-urlencoded");
            HttpResponse httpResponse = httpClient.execute(httpRequst);// 其中HttpGet是HttpUriRequst的子类
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                  HttpEntity httpEntity = httpResponse.getEntity();
                  if (httpEntity.getContentEncoding() != null) {
                     if ("gzip".equalsIgnoreCase(httpEntity
                              .getContentEncoding().getValue())) {
                        httpEntity = new GzipDecompressingEntity(httpEntity);
                     } else if ("deflate".equalsIgnoreCase(httpEntity
                              .getContentEncoding().getValue())) {
                        httpEntity = new DeflateDecompressingEntity(
                                 httpEntity);
                     }
                  }
                  result = enCodetoString(httpEntity, encode);// 取出应答字符串
                  if (resultTest(result)) {
                     System.out.println(ip + "公司代理成功抓取" + url);
                     return result;
                  } else if (result.contains("function JumpSelf")
                        && result.contains("WebShieldSessionVerify")) {
                     int indexs = result.indexOf("&WebShieldSessionVerify");
                     int indexe = result.indexOf("\";}</script>");
                     String verify = result.substring(indexs, indexe);
                     urlString = urlString + verify;
                  } else if (result.contains("function JumpSelf")
                        && !result.contains("WebShieldSessionVerify")) {
                     urlString = url;
                  }
            } else
                  httpRequst.abort();
         } catch (ClientProtocolException e) {
            System.out.println(ip + "代理ip拒绝了");
         } catch (IOException e) {
            System.out.println(ip + "代理读取超时");
         }
      }
      return "";
}

欢迎光临电子技术论坛_中国专业的电子工程师学习交流社区-中电网技术论坛 (http://bbs.eccn.com/)