学校招生数

src/test/java/spider/CollegeData.java

@@ -48,8 +48,8 @@ public class CollegeData {
     }
 //181
     public void collegeList() throws SQLException {
-//        int count = 0, total = 5000, page = 131;
-        int count = 0, total = 5000, page = 114;
+//        int count = 0, total = 5000, page = 79;
+        int count = 0, total = 5000, page = 1;
         JSONObject res=new JSONObject();
         do {
             try {
@@ -85,15 +85,25 @@ public class CollegeData {
                         ApiUtil.getInstance().login();
                         processColledgeCount=0;
                     }
-                    /**
-                     * 院校录取数据
-                     */
+
                     List<Integer> liberalScienceList= Arrays.asList(0,1);
                     List<Integer> typeList= Arrays.asList(2,3);
                     for(Integer liberalScience:liberalScienceList){
+                        /**
+                         * 院校录取数据
+                         */
                         for(Integer type:typeList){
-                            byHistoryNew(collegeId,liberalScience,type);
+//                            byHistoryNew(collegeId,liberalScience,type);
                         }
+
+                        /**
+                         * 专业录取数据
+                         */
+//                        byCollege(collegeId,liberalScience,2);
+                        /**
+                         * 招生计划数据
+                         */
+                        byCollege(collegeId,liberalScience,3);
                     }
                     processColledgeCount++;
 
@@ -144,7 +154,7 @@ public class CollegeData {
      * type 类型，普通省份不传或传1；312院校专业组省份(江苏、湖南、湖北、广东、福建):2021年以后数据传2，2020年以前数据传3 query false integer
      */
     private void byHistoryNew(Integer collegeId,Integer liberalScience,Integer type) throws IOException {
-        ThreadUtil.safeSleep(2 * 1000);
+//        ThreadUtil.safeSleep(2 * 1000);
 
         StringBuilder sb = new StringBuilder(host);
         sb.append("/query/collegeScore/byHistoryNew");
@@ -158,13 +168,19 @@ public class CollegeData {
             JSONObject dataObject = res.getJSONObject("data");
             JSONArray tdDatas = dataObject.getJSONArray("tdData");
             JSONArray thDatasArray = dataObject.getJSONArray("thData");
-//            Map<String,String> thDatasMap = new HashMap<>();
-//            thDatasArray.forEach(m->{
-//                JSONObject thDatasJSONObject = (JSONObject) m;
-//                if(thDatasJSONObject.getInteger("encryption")==1){
-//                    thDatasMap.put(thDatasJSONObject.getString("prop"),thDatasJSONObject.getString("prop"));
-//                }
-//            });
+            JSONObject thData1=new JSONObject(){{//增加最低/最高分数据解析
+                put("label","最低投档分");
+                put("prop","LowestScore");
+                put("encryption",1);
+            }};
+            thDatasArray.add(thData1);
+            JSONObject thData2=new JSONObject(){{//增加最低/最高分数据解析
+                put("label","最高投档分");
+                put("prop","MajorScoreCount");
+                put("encryption",1);
+            }};
+            thDatasArray.add(thData2);
+
             //font
             JSONObject font = res.getJSONObject("font");
             Integer fontId = font.getInteger("id");
@@ -179,17 +195,7 @@ public class CollegeData {
                 row.put("font_value",fontValue);
                 row.put("update_time",new Date());
                 //解密错位数据
-                thDatasArray.forEach(m->{
-                    JSONObject thDatasJSONObject = (JSONObject) m;
-                    if(thDatasJSONObject.getInteger("encryption")==1){
-                        String prop = thDatasJSONObject.getString("prop");
-                        String originalValue = row.getString(prop);
-                        if(StringUtils.isNotBlank(originalValue)){
-                            String realValue=getRealData(fontId,originalValue);
-                            row.put(prop,realValue);
-                        }
-                    }
-                });
+                replaceEncryptDate(fontId,row,thDatasArray);
 
                 Entity subjectTable4 = Entity.create("sy_colledge_enroll_data");
                 subjectTable4.putAll(row);
@@ -216,6 +222,142 @@ public class CollegeData {
         }
     }
 
+    /**
+     * 将数据替换为真实值
+     * @param fontId
+     * @param row
+     * @param thDatasArray
+     * @return
+     */
+    private JSONObject replaceEncryptDate(Integer fontId,JSONObject row,JSONArray thDatasArray){
+        //解密错位数据
+        thDatasArray.forEach(m->{
+            JSONObject thDatasJSONObject = (JSONObject) m;
+            if(thDatasJSONObject.getInteger("encryption")==1){
+                String prop = thDatasJSONObject.getString("prop");
+                String originalValue = row.getString(prop);
+                if(StringUtils.isNotBlank(originalValue)){
+                    String realValue=getRealData(fontId,originalValue);
+                    row.put(prop,realValue);
+                }
+            }
+        });
+
+        return row;
+    }
+
+    /**
+     * 专业录取数据
+     * @param collegeId
+     * @param liberalScience
+     * @throws IOException
+     */
+    private void byCollege(Integer collegeId,Integer liberalScience,Integer queryType) throws IOException {
+//        ThreadUtil.safeSleep(2 * 1000);
+
+        StringBuilder sb = new StringBuilder(host);
+        sb.append("/years");
+        sb.append("?queryType=").append(queryType);
+        sb.append("&provinceId=").append("18");
+        JSONObject res = ApiUtil.getInstance().httpSyncGet(sb.toString());
+//        System.out.println("data is "+res);
+        if(res.getInteger("error")==0){
+            //data
+            JSONArray dataYearArray = res.getJSONArray("data");
+            for (int j = 0; j < dataYearArray.size(); j++) {
+                JSONObject dataJSONObject = dataYearArray.getJSONObject(j);
+//                int yearLast= Calendar.getInstance().getWeekYear()-1;//补充上一个年份
+//                dataJSONObject.put("id",yearLast);
+//                dataJSONObject.put("year",yearLast);
+                Integer year = dataJSONObject.getInteger("year");
+
+                sb = new StringBuilder(host);
+                sb.append("/query/majorScore/byCollege");
+                sb.append("?provinceId=").append("18");
+                sb.append("&collegeId=").append(collegeId);
+                sb.append("&liberalScience=").append(liberalScience);
+                sb.append("&year=").append(year);
+                sb.append("&pageIndex=").append(1);
+                sb.append("&pageSize=").append(100);
+                res = ApiUtil.getInstance().httpSyncGet(sb.toString());
+//        System.out.println("data is "+res);
+                JSONObject rows = res.getJSONObject("data");
+                if(res.getInteger("error")==0){
+                    JSONArray tdDatas = rows.getJSONArray("tdData");
+                    JSONArray thDatasArray = rows.getJSONArray("thData");
+                    if(queryType==2){
+                        JSONObject thData1=new JSONObject(){{//增加最低/最高分数据解析
+                            put("label","平均投档分");
+                            put("prop","AvgScore");
+                            put("encryption",1);
+                        }};
+                        thDatasArray.add(thData1);
+                        JSONObject thData2=new JSONObject(){{//增加最低/最高分数据解析
+                            put("label","最高投档分");
+                            put("prop","MajorScoreCount");
+                            put("encryption",1);
+                        }};
+                        thDatasArray.add(thData2);
+                    }
+
+                    //font
+                    JSONObject font = res.getJSONObject("font");
+                    Integer fontId = font.getInteger("id");
+                    String fontValue = fontMap.get(fontId);
+
+                    Set<Entity> subjectList=new HashSet();
+                    for (int i = 0; i < tdDatas.size(); i++) {
+                        //sy_colledge_enroll_data
+                        JSONObject row = tdDatas.getJSONObject(i);
+//                        System.out.println("colledgeCode："+row.getString("CollegeCode"));
+                        if(queryType==3){
+                            row.remove("AvgScore");
+                            row.remove("HighestScore");
+                            row.remove("LiberalScience");
+                            row.remove("LowestScore");
+                            row.remove("LowestScoreRank");
+                            row.remove("LowestScore");
+                            row.remove("LowestScore");
+                        }
+                        row.put("font_id",fontId);
+                        row.put("font_value",fontValue);
+                        row.put("update_time",new Date());
+                        //解密错位数据
+                        replaceEncryptDate(fontId,row,thDatasArray);
+
+                        String tableName = StringUtils.EMPTY;
+                        if(queryType==2){
+                            tableName = "sy_colledge_major_enroll_data";
+                        }else if(queryType==3){
+                            tableName = "sy_colledge_recruit_data";
+                        }
+                        Entity subjectTable4 = Entity.create(tableName);
+                        subjectTable4.putAll(row);
+                        subjectList.add(subjectTable4);
+                        if(!isBatchInsert){
+                            try {
+                                DbUtil.use().insertOrUpdate(subjectTable4, "ID");
+                            } catch (SQLException e) {
+                                e.printStackTrace();
+                            }
+                        }
+                    }
+                    if(isBatchInsert){
+                        try {
+                            if(!CollectionUtils.isEmpty(subjectList)){
+                                DbUtil.use().insert(subjectList);
+                            }
+                        } catch (SQLException e) {
+                            e.printStackTrace();
+                        }
+                    }
+                }else {
+                    log.error("byCollege res is {}, request is {}  ",res,sb.toString());
+                }
+            }
+        }
+    }
+
     /**
      * 学校详情
      * @param id