collegeData

src/test/java/spider/CollegeData.java

@@ -32,10 +32,11 @@ public class CollegeData {
     private List<String> errors = new ArrayList<>();
     protected final Logger log = LoggerFactory.getLogger(this.getClass());
     Boolean isBatchInsert = false;
+    Map<Integer,String> liberalScienceMap = new HashMap<Integer, String>();
 
     @Test
     public void test() throws Exception {
-        System.out.println("010102".compareTo("030101"));
+        System.out.println(Calendar.getInstance().getWeekYear());
         System.out.println("030102".compareTo("030101"));
         System.out.println("030101".compareTo("030101"));
 //        getMajorData();
@@ -56,7 +57,7 @@ public class CollegeData {
             JSONObject jsonObject = (JSONObject)ll;
             provinceMap.put(jsonObject.getString("provinceName"),JSONObject.parseObject(JSONObject.toJSONString(jsonObject),ProvinceData.class));
         });
-        String processProvinceName= "江西";
+        String processProvinceName= "湖北";
         User currentUser = new User();
         switch (processProvinceName){
             case "湖南":
@@ -113,21 +114,25 @@ public class CollegeData {
                 break;
             case "河南":
                 /**
-                 * 河南	18574734705	75231013
+                 * 河南	18873220395	75231013
                  */
-                currentUser = new User().setCode("13437248803").setPassword("41186990").setIsHn(false).setIsNewGaokao(true).setProvinceName(processProvinceName);
+                currentUser = new User().setCode("18873220395").setPassword("75231013").setIsHn(false).setIsNewGaokao(false).setProvinceName(processProvinceName);
 
                 break;
             case "广西":
                 /** 河南 湖北 广东 广西
-                 * 广西	18873220395	50277878
+                 * 广西	18574734705	50277878
                  */
-                currentUser = new User().setCode("13437248803").setPassword("41186990").setIsHn(false).setIsNewGaokao(true).setProvinceName(processProvinceName);
+                currentUser = new User().setCode("18574734705").setPassword("50277878").setIsHn(false).setIsNewGaokao(false).setProvinceName(processProvinceName);
 
                 break;
         }
 
         ApiUtil.getInstance().login(currentUser);
+
+        //处理liberalScienceMap
+        getLiberalScience(provinceMap.get(processProvinceName));
+
         collegeList(provinceMap.get(processProvinceName));
 
 //        new YouZy().getUniversityRanking();
@@ -137,7 +142,7 @@ public class CollegeData {
 //        int count = 0, total = 5000, page = 79;
         Integer ProvinceId=provinceData.getId();
         String Location=provinceData.getProvinceName();//湖北的1-5页之前需要修正ProvinceId,表示学校所在省份
-        int count = 0, total = 5000, page = 1;
+        int count = 0, total = 5000, page = 89;
         JSONObject res=new JSONObject();
         do {
             try {
@@ -165,15 +170,15 @@ public class CollegeData {
                      * 选科数据 xuanke(collegeId);
                      */
                     List<Integer> colledgeIds = Arrays.asList(
-                            1,3,654,871,1526,718,652,1855);
+                            228,229,230,231,232,233,234,235,236,237,238);
                     if(colledgeIds.contains(collegeId)){
                         continue;
                     }
                     if("湖南".equalsIgnoreCase(provinceData.getProvinceName())){
                         if(processColledgeCount>5){
                             //湖南 超过5个学校换个账号
-                        ApiUtil.getInstance().login();
-                        processColledgeCount=0;
+                            ApiUtil.getInstance().login();
+                            processColledgeCount=0;
                         }
                     }
 
@@ -245,6 +250,30 @@ public class CollegeData {
         return result.toString();
     }
 
+    private void getLiberalScience(ProvinceData provinceData) throws IOException {
+//        ThreadUtil.safeSleep(2 * 1000);
+
+        StringBuilder sb = new StringBuilder(host);
+        sb.append("/liberalSciencesNew");
+        sb.append("?queryType=2")
+                .append("&provinceId=").append(provinceData.getId());
+        sb.append("&year=").append(2021);
+        JSONObject res = ApiUtil.getInstance().httpSyncGet(sb.toString());
+        if(res.getInteger("error")==0){
+            //data
+            JSONArray dataArray = res.getJSONArray("data");
+            dataArray.forEach(ll->{
+                JSONObject oo= (JSONObject) ll;
+                liberalScienceMap.put(oo.getInteger("liberalScience"),oo.getString("name"));
+            });
+        }else {
+            log.error("getLiberalScience is error {}",res);
+            return;
+        }
+        log.info("liberalScienceMap is {}",liberalScienceMap);
+    }
+
+
     /*
      * 院校录取数据
      * Request URL: https://apiv4.diyigaokao.com/query/collegeScore/byHistoryNew?liberalScience=1&collegeId=1&type=2
@@ -304,6 +333,7 @@ public class CollegeData {
                 row.put("update_time",new Date());
                 //解密错位数据
                 replaceEncryptDate(fontId,row,thDatasArray);
+                row.put("LiberalScienceType",liberalScienceMap.get(row.getInteger("LiberalScience")));
 
                 Entity subjectTable4 = Entity.create("sy_colledge_enroll_data");
                 subjectTable4.putAll(row);
@@ -384,6 +414,9 @@ public class CollegeData {
             for(int year:yearList){
                 sb = new StringBuilder(host);
                 if(queryType==2){
+                    if(year==Calendar.getInstance().getWeekYear()){
+                        continue;//当前年份无专业录取数据
+                    }
                     sb.append("/query/majorScore/byCollege");
                 }else if(queryType==3){
                     sb.append("/query/majorPlan/byCollege");
@@ -402,8 +435,8 @@ public class CollegeData {
                 if(res.getInteger("error")==0){
                     JSONArray tdDatas = rows.getJSONArray("tdData");
                     if(tdDatas.size()==0){
-                        log.error(collegeName+" "+year+"年无数据:"+JSONObject.toJSONString(provinceData));
-                        return;
+                        log.error(collegeName+" "+year+"年无"+(queryType==2?"专业录取":"招生计划")+"数据:"+JSONObject.toJSONString(provinceData));
+                        continue;
                     }
                     JSONArray thDatasArray = rows.getJSONArray("thData");
                     if(queryType==2){
@@ -441,6 +474,7 @@ public class CollegeData {
 //                            row.remove("LowestScore");
                         }
 //                        row.put("ProvinceId",ProvinceId);//湖南
+
                         row.put("Location",Location);//湖南
                         row.put("font_id",fontId);
                         row.put("font_value",fontValue);
@@ -450,9 +484,11 @@ public class CollegeData {
 
                         String tableName = StringUtils.EMPTY;
                         if(queryType==2){
+                            row.put("LiberalScienceType",liberalScienceMap.get(row.getInteger("LiberalScience")));
                             tableName = "sy_colledge_enroll_major_data";
                         }else if(queryType==3){
 //                            tableName = "sy_colledge_recruit_data_"+year+"_"+provinceData.getProvinceSpell();
+                            row.put("LiberalScienceType",liberalScienceMap.get(row.getInteger("Liberalscience")));
                             tableName = "sy_colledge_recruit_data";
                         }
                         Entity subjectTable4 = Entity.create(tableName);