1.有的登录，会有跳转，httpcode为302 重定向，会返回Location节点，谷歌浏览器，可跟踪重定向地址
2.有重定向的地址需要 关闭 instanceFollowRedirects，设为false
/*
@author: qzf
@date: 2022/6/24 19:37
@describe: 可以带cookie请求，且获取cookie
@param:
@return:
*/
@Data
@Component
public class HttpUtilCookie {
private Map
private List
private String charset = “utf-8”;
private String postParam = “”;
private boolean instanceFollowRedirects = true;
private int connectTimeout = 10000;
private String httpMethod = “get”;
public HttpUtilCookie() {<br /> listResponseHeader = new ArrayList<>();<br /> listResponseHeader.add("Set-Cookie");<br /> listResponseHeader.add("Location");<br /> }<br /> /**<br /> * @author: qzf<br /> * @date: 2022/6/24 20:03<br /> * @describe: Get请求<br /> * @param:<br /> * @return:<br /> */<br /> public Map<String,Object> httpGet(String url){<br /> return this.httpRequest(url);<br /> }<br /> /**<br /> * @author: qzf<br /> * @date: 2022/6/24 20:04<br /> * @describe: Post请求<br /> * @param:<br /> * @return:<br /> */<br /> public Map<String,Object> httpPost(String url,Map<String, Object> param){<br /> StringBuffer buffer = new StringBuffer();<br /> if (param != null && !param.isEmpty()) {<br /> for (Map.Entry<String, Object> entry : param.entrySet()) {<br /> buffer.append(entry.getKey()).append("=")<br /> .append(URLEncoder.encode(entry.getValue()==null?"":entry.getValue().toString()))<br /> .append("&");<br /> }<br /> buffer.deleteCharAt(buffer.length() - 1);<br /> }<br /> this.postParam = buffer.toString();<br /> this.httpMethod = "post";<br /> return this.httpRequest(url);<br /> }<br /> /**<br /> * @author: qzf<br /> * @date: 2022/6/24 20:04<br /> * @describe: Post请求<br /> * @param:<br /> * @return:<br /> */<br /> public Map<String,Object> httpPostJson(String url,Map<String, Object> param){<br /> if (param != null && !param.isEmpty()) {<br /> this.postParam = JSONObject.toJSONString(param);<br /> }<br /> this.httpMethod = "post";<br /> return this.httpRequest(url);<br /> }<br /> /**<br /> * @author: qzf<br /> * @date: 2022/6/24 20:04<br /> * @describe: 公共请求<br /> * @param:<br /> * @return:<br /> */<br /> private Map<String,Object> httpRequest(String url) {<br /> Map<String,Object> resultMap = new HashMap<>();<br /> String result = "";<br /> BufferedReader in = null;<br /> PrintWriter out = null;<br /> try {URL realUrl = new URL(url);<br /> // 打开和URL之间的连接<br /> HttpURLConnection connection = (HttpURLConnection)realUrl.openConnection();<br /> ** if(!this.instanceFollowRedirects){**<br />** connection.setInstanceFollowRedirects(false);**<br />** }**<br />** if(this.connectTimeout != -1){**<br />** connection.setConnectTimeout(this.connectTimeout);**<br />** }**// 设置通用的请求属性<br /> connection.setRequestProperty("accept", "*/*");<br /> connection.setRequestProperty("connection", "Keep-Alive");<br /> connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");<br /> if(requestHeader != null) {<br /> for (Map.Entry item : requestHeader.entrySet()) {<br /> connection.setRequestProperty(item.getKey().toString(), item.getValue().toString());//设置header<br /> }<br /> }<br /> // 建立实际的连接<br /> if(httpMethod.equals("get")) {<br /> connection.connect();<br /> }else if(httpMethod.equals("post")) {<br /> // 发送POST请求必须设置如下两行<br /> connection.setDoOutput(true);<br /> connection.setDoInput(true);<br /> // 获取URLConnection对象对应的输出流<br /> out = new PrintWriter(connection.getOutputStream());<br /> // 发送请求参数<br /> out.print(this.postParam);<br /> // flush输出流的缓冲<br /> out.flush();<br /> }<br /> // 定义 BufferedReader输入流来读取URL的响应<br /> in = new BufferedReader(new InputStreamReader(connection.getInputStream(), charset));<br /> String line;<br /> while ((line = in.readLine()) != null) {<br /> result += line;<br /> }<br /> resultMap.put("result", StringUtils.trimToEmpty(result));<br /> if(this.listResponseHeader != null) {<br /> for (int i = 0; i < listResponseHeader.size(); i++) {<br /> resultMap.put(listResponseHeader.get(i), connection.getHeaderField(listResponseHeader.get(i)));<br /> }<br /> }} catch (Exception e) {<br /> System.out.println("发送请求出现异常！" + e);<br /> e.printStackTrace();<br /> return null;<br /> }<br /> // 使用finally块来关闭输入流<br /> finally {<br /> try {<br /> if (in != null) {<br /> in.close();<br /> }<br /> } catch (Exception e2) {<br /> e2.printStackTrace();<br /> }<br /> }return resultMap;<br /> }
}
//======================举例===================================
/*
@author: qzf
@date: 2022-06-15 13:58
@describe:爬取地下水数据
up：c2p6c2xhZG1pbi9TanpzbEAxMjM=
/
@Component
public class TaskDxs {
@Autowired
HttpUtilCookie httpUtilCookie;
@Autowired
private StStbprpBService stStbprpBService;
@Autowired
private HyDxsDatatimingService hyDxsDatatimingService;
private final String username = “sjzsladmin”;
private final String password = “c709ef0a96f1b8fbd9668f28319f949a”;
private final String loginUrl = “aHR0cDovLzIxOS4xNDguNTEuNjc6ODg2OS9jYXMtc2VydmVyLXdlYmFwcC9sb2dpbj9zZXJ2aWNlPWh0dHAlM2ElMmYlMmYyMTkuMTQ4LjUxLjY3JTNhODQwMCUyZkRYU1pMSVNULmFzcHg=”;
private final String searchDataUrl = “aHR0cDovLzIxOS4xNDguNTEuNjc6ODQwMC9CYXNlUmVmV2ViLmFzcHg/Y2xhc3NOYW1lPUlEdFN0YnBycEJGdW4mbWV0aG9kPUdldERYU1pMSVNUJnJhZD0=”;
private String lt = "";<br /> private String execution = "";<br /> private String JSESSIONID = "";<br /> private String CASTGC = "";<br /> private String LOCATION = "";<br /> private String ASPXAUTH = "";private final List<String> stcdList = Arrays.asList(new String[]{<br /> "30970166", //王家庄<br /> "30960641", //平安村<br /> "30970002", //新村<br /> "30970004", //董家庄<br /> "30970039", //候城<br /> "30970050", //西合流<br /> "30970046", //东两河<br /> "30970029", //海滩<br /> "30970033", //陈家庄<br /> "30960609", //旧城<br /> "30960613", //张家<br /> "30970017" //北白庄<br /> });<br /> /**<br /> * @author: qzf<br /> * @date: 2022/6/17 15:11<br /> * @describe: 获取登录需要的信息 lt<br /> * @param:<br /> * @return:<br /> */<br /> private void getLt(){<br /> String result = "";<br /> Map<String,Object> rst = httpUtilCookie.httpGet(loginUrl);<br /> if(rst != null){<br /> result = StringUtils.trim(rst.get("result").toString()).replaceAll("\\s","").replaceAll("\n","").replaceAll("\r","");<br /> String regFileName = "name=\"lt\"value=\"([^\"]+)\"";<br /> Matcher matcher = Pattern.compile(regFileName).matcher(result);<br /> if (matcher.find()) {<br /> lt = matcher.group(1);<br /> }<br /> regFileName = "name=\"execution\"value=\"([^\"]+)\"";<br /> matcher = Pattern.compile(regFileName).matcher(result);<br /> if (matcher.find()) {<br /> execution = matcher.group(1);<br /> }<br /> JSESSIONID = rst.get("Set-Cookie").toString();<br /> }<br /> }<br /> /**<br /> * @author: qzf<br /> * @date: 2022/6/17 15:29<br /> * @describe: 模拟登录<br /> * @param:<br /> * @return:<br /> */<br /> private void login(){<br /> Map<String,Object> params = new HashMap<>();<br /> params.put("username",username);<br /> params.put("password",password);<br /> params.put("lt",lt);<br /> params.put("execution",execution);<br /> params.put("_eventId","submit");<br /> params.put("geolocation","");<br /> params.put("btnsubmit","登录");<br /> httpUtilCookie.setInstanceFollowRedirects(false);<br /> Map<String,Object> header = new HashMap<>();<br /> header.put("Cookie",JSESSIONID);<br /> httpUtilCookie.setRequestHeader(header);<br /> Map<String, Object> rst = httpUtilCookie.httpPost(loginUrl,params);<br /> if(rst != null ){<br /> CASTGC = rst.get("Set-Cookie").toString();<br /> LOCATION = rst.get("Location").toString();<br /> }<br /> }private void getASPXAUTH(){<br /> httpUtilCookie.setInstanceFollowRedirects(false);<br /> Map<String,Object> header = new HashMap<>();<br /> header.put("Cookie",JSESSIONID+"; "+CASTGC);<br /> Map<String, Object> rst = httpUtilCookie.httpGet(LOCATION);<br /> if(rst != null ){<br /> ASPXAUTH = rst.get("Set-Cookie").toString();<br /> }<br /> }<br /> private int getSearchData(String date,String stcd){<br /> String result = "";<br /> Map<String,Object> header = new HashMap<>();<br /> //header.put("Cookie",JSESSIONID+"; "+ ASPXAUTH);<br /> header.put("Cookie", ASPXAUTH);<br /> Map<String,Object> params = new HashMap<>();<br /> params.put("txtDateB",date + " 08");<br /> params.put("txtDateE",date + " 08");<br /> params.put("radioValue","all");<br /> params.put("txtWorld", stcd);<br /> params.put("txtWorldname","");<br /> params.put("Cityid","1301");<br /> params.put("Countyid","");<br /> params.put("dCode","130100");<br /> params.put("multiSel","");<br /> params.put("checkTM","");<br /> params.put("isRelated","0");<br /> params.put("pagestart","0");<br /> params.put("pagelistcount","50");<br /> params.put("ispage","true");<br /> httpUtilCookie.setRequestHeader(header);<br /> Map<String, Object> rst = httpUtilCookie.httpPost(searchDataUrl+Math.random(), params);<br /> if(rst == null){<br /> return -1;<br /> }<br /> result = rst.get("result").toString();<br /> if(StringUtil.isEmpty(result)) {<br /> return -2;<br /> }<br /> if(result.contains("<html>")){<br /> return -3;<br /> }<br /> JSONArray data = null;<br /> try {<br /> data = JSONArray.parseArray(result);<br /> }catch (Exception e){<br /> e.printStackTrace();<br /> return -4;<br /> }<br /> if (data == null) {<br /> System.err.println("data null 返回结果错误");<br /> return -10;<br /> }<br /> if(data.size()==0){<br /> System.err.println("data size = 0 结果数为0");<br /> return -20;<br /> }<br /> JSONObject jsonObject = data.getJSONObject(0);<br /> StStbprpB stStbprpB = stStbprpBService.getByPrimaryKey(jsonObject.getString("STCD"));<br /> if(stStbprpB != null) {<br /> try {<br /> StStbprpB stStbprpBTmp = new StStbprpB();<br /> stStbprpBTmp.setStcd(stStbprpB.getStcd());<br /> stStbprpBTmp.setDtmel(jsonObject.getBigDecimal("GROUNDHEIGHT"));//地面高程<br /> stStbprpBService.update(stStbprpBTmp);HyDxsDatatiming hyDxsDatatiming = new HyDxsDatatiming();<br /> hyDxsDatatiming.setSt(jsonObject.getString("STCD"));<br /> hyDxsDatatiming.setTt(jsonObject.getString("DATATM").replaceAll("/","-"));//时间<br /> hyDxsDatatiming.setJkgc(jsonObject.getBigDecimal("GROUNDHEIGHT"));//地面高程<br /> hyDxsDatatiming.setH(jsonObject.getBigDecimal("GWBD"));//地下水埋深<br /> hyDxsDatatiming.sethJs(jsonObject.getBigDecimal("GWBD"));//地下水埋深<br /> hyDxsDatatiming.setMpZ(jsonObject.getBigDecimal("GWTMP"));//地下水位<br /> hyDxsDatatiming.setMpZJs(jsonObject.getBigDecimal("GWTMP"));//地下水位<br /> hyDxsDatatiming.setC(jsonObject.getBigDecimal("GWTMP"));//地下水温<br /> hyDxsDatatiming.setVt(jsonObject.getBigDecimal("V"));//电压<br /> hyDxsDatatimingService.save(hyDxsDatatiming);<br /> }<br /> catch (Exception e){<br /> e.printStackTrace();<br /> System.err.println("插入出错");<br /> }<br /> }return 0;}<br /> public void getData(String date) throws InterruptedException {<br /> this.getLt();<br /> this.login();<br /> this.getASPXAUTH();<br /> int rst = 0;<br /> for (String stcd : stcdList) {<br /> rst = getSearchData(date, stcd);<br /> if(rst != 0) {<br /> System.err.println("TaskDxs:错误代码:"+rst);<br /> }<br /> Thread.sleep(200);<br /> }}<br /> public static void main(String[] args){<br /> TaskDxs taskDxs = new TaskDxs();<br /> taskDxs.getLt();<br /> taskDxs.login();<br /> taskDxs.getASPXAUTH();<br /> taskDxs.getSearchData("2022-06-24 08","30970166");<br /> }<br />}
若有收获，就点个赞吧
0 人点赞
