电子技术论坛_中国专业的电子工程师学习交流社区-中电网技术论坛

标题: httpclient常用基本抓取类（4） [打印本页]

作者: look_w 时间: 2019-4-20 10:55 标题: httpclient常用基本抓取类（4）

public String crawlPageContentByPost(String url, String pram, String encode)
         throws ClientProtocolException, IOException {
      String content = "";
      try {
         content = doPostByGoagent(url, pram, encode);
         if (content == null || content.equals("")) {
            content = doPostByGoagent(url, pram, encode);
            // System.out.println("启用公司代理");
            // content = postByCompanyProxy(url, pram, encode);
            // if (content == null || content.equals("")) {
            // System.out.println("5秒后启用本机");
            // Thread.sleep(5000);
            // content = doPost(url, pram, encode);
            // }
         }
      } catch (Exception e) {
         try {
            content = doPostByGoagent(url, pram, encode);
            // System.out.println("goagent连接失败,启用公司代理");
            // content = postByCompanyProxy(url, pram, encode);
            // if (content == null || content.equals("")) {
            // System.out.println("公司代理连接失败,启用本机");
            // content = doPost(url, pram, encode);
            // }
         } catch (Exception e2) {
            try {
                  content = doPostByGoagent(url, pram, encode);
                  // e2.printStackTrace();
                  // content = postByCompanyProxy(url, pram, encode);
                  // System.out.println("公司代理连接失败,启用本机");
                  // content = doPost(url, pram, encode);
            } catch (Exception e3) {
                  e3.printStackTrace();
            }

         }

      }

      return content;
}

private String doPostByGoagent(String url, String parm, String encode)
         throws ClientProtocolException, IOException {
      String result = "";
      HttpPost httpRequst = new HttpPost(url);// 创建HttpPost对象
      HttpHost proxy = new HttpHost("127.0.0.1", 8087, null);
      StringEntity entity = new StringEntity(parm);
      entity.setContentType("application/x-www-form-urlencoded");
      entity.setContentEncoding(encode);
      httpRequst.setEntity(entity);
      DefaultHttpClient httpClient = new DefaultHttpClient();
      httpClient.getParams().setParameter(
            CoreConnectionPNames.CONNECTION_TIMEOUT, 8000);// 连接时间20s
      httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,
            8000);// 数据传输时间60s
      httpClient.getParams().setParameter(ConnRouteParams.DEFAULT_PROXY,
            proxy);
      HttpResponse httpResponse = httpClient.execute(httpRequst);
      // System.out.println(httpResponse.getStatusLine().getStatusCode());
      if (httpResponse.getStatusLine().getStatusCode() == 200) {
         HttpEntity httpEntity = httpResponse.getEntity();
         if (httpEntity.getContentEncoding() != null) {
            if ("gzip".equalsIgnoreCase(httpEntity.getContentEncoding()
                     .getValue())) {
                  httpEntity = new GzipDecompressingEntity(httpEntity);
            } else if ("deflate".equalsIgnoreCase(httpEntity
                     .getContentEncoding().getValue())) {
                  httpEntity = new DeflateDecompressingEntity(httpEntity);
            }
         }
         result = enCodetoString(httpEntity, encode);// 取出应答字符串
      }
      return result;
}

欢迎光临电子技术论坛_中国专业的电子工程师学习交流社区-中电网技术论坛 (http://bbs.eccn.com/)