Merge branch 'master' of http://121.4.203.192:9000/root/spider

* 'master' of http://121.4.203.192:9000/root/spider:
  spider
  一分一段全部数据
  一分一段数据
  代码重构
  修复问题

# Conflicts:
#	src/main/java/com/mingxue/spider/utils/ApiUtil.java

pom.xml

@@ -51,6 +51,16 @@
       <artifactId>fastjson</artifactId>
       <version>1.2.79</version>
     </dependency>
+    <dependency>
+      <groupId>org.jsoup</groupId>
+      <artifactId>jsoup</artifactId>
+      <version>1.15.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.seleniumhq.selenium</groupId>
+      <artifactId>selenium-java</artifactId>
+      <version>3.141.59</version>
+    </dependency>
     <dependency>
       <groupId>mysql</groupId>
       <artifactId>mysql-connector-java</artifactId>
@@ -63,7 +73,6 @@
     <dependency>
       <groupId>org.projectlombok</groupId>
       <artifactId>lombok</artifactId>
-      <version>1.18.12</version>
       <scope>test</scope>
     </dependency>
   </dependencies>

src/main/java/com/mingxue/spider/fliphtml5/QuickUpload.java

@@ -0,0 +1,96 @@
+package com.mingxue.spider.fliphtml5;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+import cn.hutool.core.lang.Dict;
+import cn.hutool.core.lang.UUID;
+import okhttp3.FormBody;
+import okhttp3.MediaType;
+import okhttp3.MultipartBody;
+import okhttp3.MultipartBody.Builder;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+public class QuickUpload {
+    private static String host = "https://fliphtml5.com";
+    private OkHttpClient.Builder builder;
+    private Request.Builder requestBuilder;
+    private OkHttpClient okHttpClient;
+    private User user = new User();
+
+    class User {
+        String mail = "duzejun111@163.com";
+        String pass = "duzejun";
+        String name;
+        String cId;
+        String uId;
+
+    }
+
+    private QuickUpload() {
+        builder = new OkHttpClient.Builder();
+        okHttpClient = builder.build();
+        requestBuilder = new Request.Builder();// 省的每次都new request操作,直接builder出来,随后需要什么往里加,build出来即可
+    }
+
+    public void login() {
+        try {
+            httpGetHtml("/login.php");
+            // umail=duzejun111%40163.com&upass=duzejun&isMobile=0
+            Dict data = Dict.create().set("umail", user.mail).set("upass", user.pass).set("isMobile", 0);
+            String res = httpPost("/userLoad/handle-user-log-in.php", data);
+            // {"status":"1","cId":"3173336","uId":"2953511","uLink":"hsucl","uName":"duzejun111","PrimaryAccountId":"0","uType":"0","mess":""}
+            JSONObject json = JSON.parseObject(res);
+            user.cId = json.getString("cId");
+            user.uId = json.getString("uId");
+            user.name = json.getString("uName");
+        } catch (Exception e) {
+            // TODO: handle exception
+        }
+    }
+
+    public void upload(String path) {
+        try {
+            String url = "https://s3.amazonaws.com/online.fliphtml5.com";
+            Builder builder = new MultipartBody.Builder();
+            File file = new File(path);
+            builder.addFormDataPart("file", file.getName(), MultipartBody.create(MediaType.parse("multipart/form-data"), file));
+            httpPost(url, builder.build());
+        } catch (Exception e) {
+            // TODO: handle exception
+        }
+    }
+
+    public String httpGetHtml(String url) throws IOException {
+        Request request = requestBuilder.url(host + url).build();
+        Response res = okHttpClient.newCall(request).execute();
+        return res.body().string();
+    }
+
+    public String httpPost(String url, Dict data) throws IOException {
+        FormBody.Builder builder = new FormBody.Builder();
+        for (String key : data.keySet()) {
+            builder.add(key, data.getStr(key));
+        }
+        return httpPost(host + url, builder.build());
+    }
+
+    public String httpPost(String url, RequestBody body) throws IOException {
+        Request request = requestBuilder.url(url).method("POST", body).build();
+        Response res = okHttpClient.newCall(request).execute();
+        return res.body().string();
+    }
+
+    public static void main(String[] args) {
+        QuickUpload qu = new QuickUpload();
+        qu.login();
+        qu.upload("e:/temp/Java编码规范.pdf");
+
+    }
+}

src/main/java/com/mingxue/spider/fliphtml5/QuickUpload2.java

@@ -0,0 +1,84 @@
+package com.mingxue.spider.fliphtml5;
+
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+
+public class QuickUpload2 {
+    private static String host = "https://fliphtml5.com";
+    private User user = new User();
+    ChromeDriver driver;
+
+    class User {
+        String mail = "duzejun111@163.com";
+        String pass = "duzejun";
+        String name;
+        String cId;
+        String uId;
+    }
+
+    private QuickUpload2() {
+        // 配置浏览器驱动地址
+        System.setProperty("webdriver.chrome.driver", "e:/temp/chromedriver.exe");
+        ChromeOptions option = new ChromeOptions();
+        option.addArguments("no-sandbox");// 禁用沙盒
+        // 通过ChromeOptions的setExperimentalOption方法，传下面两个参数来禁止掉谷歌受自动化控制的信息栏
+        option.setExperimentalOption("useAutomationExtension", false);
+        option.setExperimentalOption("excludeSwitches", Collections.singletonList("enable-automation"));
+        driver = new ChromeDriver(option);
+
+    }
+
+    public void login() {
+        try {
+            driver.get(host + "/login.php");
+            driver.findElement(By.id("loginEmail")).sendKeys(user.mail);
+            TimeUnit.SECONDS.sleep(1);
+            driver.findElement(By.id("loginPassword")).sendKeys(user.pass);
+            TimeUnit.SECONDS.sleep(1);
+            WebElement btn = driver.findElement(By.id("loginButton"));
+            btn.click();
+            TimeUnit.SECONDS.sleep(2);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void upload(String path) {
+        try {
+            TimeUnit.SECONDS.sleep(4);
+            driver.get(host + "/quick-upload/");
+            TimeUnit.SECONDS.sleep(2);
+            WebElement frm = driver.findElement(By.className("upload-pdf-form"));
+            driver.executeScript("arguments[0].style=\"display: block !important;\"", frm  );
+            TimeUnit.SECONDS.sleep(2);
+            WebElement inputFile = driver.findElement(By.id("upload-pdf-input"));
+            inputFile.sendKeys(path);
+            TimeUnit.SECONDS.sleep(2);
+            driver.executeScript("$(arguments[0]).fileupload()",inputFile);
+            TimeUnit.SECONDS.sleep(1000);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void close() {
+        driver.quit();
+        driver = null;
+
+    }
+
+    public static void main(String[] args) {
+        QuickUpload2 qu = new QuickUpload2();
+        qu.login();
+        qu.upload("e:/temp/Java编码规范.pdf");
+        qu.close();
+
+    }
+
+}

src/main/java/com/mingxue/spider/jlzy/Abstractservice.java

@@ -0,0 +1,58 @@
+package com.mingxue.spider.jlzy;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.mingxue.spider.helper.StringUtils;
+
+@SuppressWarnings("serial")
+public class Abstractservice {
+    protected Logger log = LoggerFactory.getLogger(getClass());
+    protected static String host = "https://apiv4.diyigaokao.com";
+    // 字体编码
+    static Map<Integer, String> fontMap = new HashMap<Integer, String>() {
+        {
+            put(1, "0923546781");
+            put(2, "0329547681");
+            put(3, "0359247681");
+            put(4, "0351426789");
+        }
+    };
+
+    /**
+     * 数据解密
+     * 
+     * @param row
+     * @param thData
+     */
+    protected void decrypt(JSONObject row, Integer fontId, JSONArray thData) {
+        // 解密错位数据
+        for (int i = 0; i < thData.size(); i++) {
+            JSONObject th = thData.getJSONObject(i);
+            if (th.getInteger("encryption") == 1) {
+                String prop = th.getString("prop");
+                String originalValue = row.getString(prop);
+                if (StringUtils.isNotBlank(originalValue)) {
+                    String realValue = getRealData(fontId, originalValue);
+                    row.put(prop, realValue);
+                }
+            }
+        }
+    }
+
+    protected String getRealData(Integer fontId, String data) {
+        if (StringUtils.isBlank(data)) {
+            return data;
+        }
+        StringBuilder result = new StringBuilder();
+        for (char index : data.toCharArray()) {
+            result.append(fontMap.get(fontId).toCharArray()[Integer.parseInt(String.valueOf(index))]);
+        }
+        return result.toString();
+    }
+}

src/main/java/com/mingxue/spider/jlzy/CollegeService.java

@@ -0,0 +1,112 @@
+package com.mingxue.spider.jlzy;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.mingxue.spider.utils.ApiUtil;
+
+import cn.hutool.core.collection.CollectionUtil;
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.thread.ThreadUtil;
+import cn.hutool.db.DbUtil;
+import cn.hutool.db.Entity;
+
+public class CollegeService extends Abstractservice {
+
+    /**
+     * 获取学校信息
+     */
+    public void list() {
+
+    }
+
+    /**
+     * 院校录取数据 2021
+     */
+
+    /**
+     * 专业录取数据 2021
+     */
+
+    /**
+     * 招生计划数据 2022
+     * 
+     * @param collegeId
+     * @param Location
+     * @param ProvinceId
+     * @throws SQLException
+     */
+    public void plan() throws SQLException {
+        List<Entity> list = DbUtil.use().find(Arrays.asList("id", "collegeCodeGb", "provinceId"), Entity.create());
+        ApiUtil.getInstance().login();
+        for (Entity row : list) {
+            Integer collegeId = row.getInt("id");
+            plan(collegeId);
+        }
+
+    }
+
+    private void plan(Integer collegeId) {
+        long start = System.currentTimeMillis();
+        ThreadUtil.safeSleep(2 * 1000);
+        Integer year = 2022;
+        final String tableName = "sy_colledge_recruit_data_" + year;
+        try {
+            // 科类： 1理科(物理)，0文科(历史)
+            List<Integer> liberalScienceList = Arrays.asList(0, 1);
+            List<Entity> entityList = new ArrayList<>();
+            for (Integer liberalScience : liberalScienceList) {
+                try {
+                    StringBuilder sb = new StringBuilder(host);
+                    sb.append("/query/majorPlan/byCollege");
+                    sb.append("?collegeId=").append(collegeId);
+                    sb.append("&liberalScience=").append(liberalScience);
+                    sb.append("&year=").append(year);
+                    sb.append("&pageIndex=").append(1);
+                    sb.append("&pageSize=").append(100);
+                    JSONObject res = ApiUtil.getInstance().httpSyncGet(sb.toString());
+                    if (res.getInteger("error") == 0) {
+                        JSONObject data = res.getJSONObject("data");
+                        JSONArray tdData = data.getJSONArray("tdData");
+                        JSONArray thData = data.getJSONArray("thData");
+                        // font
+                        Integer fontId = res.getJSONObject("font").getInteger("id");
+                        String fontValue = fontMap.get(fontId);
+                        for (int i = 0; i < tdData.size(); i++) {
+                            JSONObject row = tdData.getJSONObject(i);
+                            decrypt(row, fontId, thData);
+                            row.put("Location", "湖南");// 湖南
+                            row.put("font_id", fontId);
+                            row.put("font_value", fontValue);
+                            row.put("update_time", new Date());
+                            Entity entity = Entity.create(tableName);
+                            entity.putAll(row);
+                            entityList.add(entity);
+                        }
+                    }
+                } catch (Exception e) {
+                    log.error(e.getMessage(), e);
+                }
+                if (CollectionUtil.isNotEmpty(entityList)) {
+                    DbUtil.use().del(Entity.create(tableName).set("CollegeID", collegeId).set("Year", year));
+                    DbUtil.use().insert(entityList);
+                }
+            }
+        } catch (SQLException e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            log.info("获取{},{}年招生计划数用时:{}", collegeId, year, DateUtil.formatBetween(System.currentTimeMillis() - start));
+        }
+    }
+
+    public static void main(String[] args) {
+        ApiUtil.getInstance().login();
+        new CollegeService().plan(1526);
+    }
+
+}

src/main/java/com/mingxue/spider/jlzy/DataTransfer.java

@@ -0,0 +1,7 @@
+package com.mingxue.spider.jlzy;
+
+public class DataTransfer {
+ 
+    
+    
+}

src/main/java/com/mingxue/spider/jlzy/UserService.java

@@ -0,0 +1,6 @@
+package com.mingxue.spider.jlzy;
+
+public class UserService extends Abstractservice {
+
+  
+}

src/main/java/com/mingxue/spider/utils/ApiUtil.java

@@ -121,7 +121,7 @@ public class ApiUtil {
 
                 return true;
             } else {
-                log.error("{}",res);
+                log.error("账号{} {}",user.getCode(),res);
                 return false;
             }
         } catch (Exception e) {

src/main/java/com/mingxue/spider/utils/HttpUtils.java

@@ -42,38 +42,44 @@ import com.alibaba.fastjson.JSON;
 public class HttpUtils {
     private static final Logger log = LoggerFactory.getLogger(HttpUtils.class);
 
-    public static String postBody(String postUrl, String body,String uSign) {
+    public static String postBody(String postUrl, String body, String uSign, String accessToken) {
         try {
             CloseableHttpClient httpClient = HttpClients.createDefault();
             HttpPost httpPost = new HttpPost(postUrl);
             httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON));
-
-            if(StringUtils.isNotEmpty(uSign)){
+            if (StringUtils.isNotEmpty(uSign)) {
                 httpPost.setHeader("u-sign", uSign);
             }
+            if (StringUtils.isNotBlank(accessToken)) {
+                httpPost.setHeader("u-token", accessToken);
+            }
             httpPost.setHeader("Content-Type", "application/json");
             CloseableHttpResponse response = httpClient.execute(httpPost);
             String result = "";
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode == 200 && response != null) {
                 HttpEntity httpEntity = response.getEntity();
-
                 // 如果返回的内容不为空
                 if (httpEntity != null) {
                     result = EntityUtils.toString(httpEntity);
                 }
+            }else {
+                
             }
-
             return result;
         } catch (Exception e) {
-            log.error("postUrl is {}",postUrl);
+            log.error("postUrl is {}", postUrl);
             log.error(e.getMessage(), e);
         }
-
         return "";
     }
+
+    public static String postBody(String postUrl, String body, String uSign) {
+        return postBody(postUrl, body, uSign, null);
+    }
+
     public static String postBody(String postUrl, String body) {
-        return postBody(postUrl,body,null);
+        return postBody(postUrl, body, null, null);
     }
 
     /**

src/main/java/com/mingxue/spider/utils/UsignUtils.java

@@ -1,16 +1,16 @@
 package com.mingxue.spider.utils;
 
-import com.alibaba.fastjson.JSONObject;
-import com.mingxue.spider.helper.StringUtils;
-import jdk.nashorn.api.scripting.NashornScriptEngine;
-import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
-
 import java.io.FileReader;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.alibaba.fastjson.JSONObject;
+
+import jdk.nashorn.api.scripting.NashornScriptEngine;
+import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
+
 public class UsignUtils {
-    public static String getUsign(String jsParam,String jsonParam){
+    public static String getUsign(String jsParam, String jsonParam) {
         String[] options = new String[] {"--language=es6"};
         NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
 
@@ -18,10 +18,10 @@ public class UsignUtils {
             NashornScriptEngine engine = getEngine();
             JSONObject jsonObject = JSONObject.parseObject(jsonParam);
 
-            String aaaa= "u_sign('"+jsParam+"',"+jsonObject+")";
+            String aaaa = "u_sign('" + jsParam + "'," + jsonObject + ")";
 
-            Object o = engine.eval("u_sign('"+jsParam+"',"+jsonParam+")");
-//            System.out.println("value : "+o);
+            Object o = engine.eval("u_sign('" + jsParam + "'," + jsonParam + ")");
+            // System.out.println("value : "+o);
             return o.toString();
         } catch (Exception e) {
             e.printStackTrace();
@@ -29,15 +29,15 @@ public class UsignUtils {
         }
     }
 
-    public static String getUsign(String jsParam){
+    public static String getUsign(String jsParam) {
         String[] options = new String[] {"--language=es6"};
         NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
 
         try {
             NashornScriptEngine engine = getEngine();
 
-            Object o = engine.eval("u_sign('"+jsParam+"')");
-//            System.out.println("value : "+o);
+            Object o = engine.eval("u_sign('" + jsParam + "')");
+            // System.out.println("value : "+o);
             return o.toString();
         } catch (Exception e) {
             e.printStackTrace();
@@ -45,23 +45,23 @@ public class UsignUtils {
         }
     }
 
-    public static NashornScriptEngine getEngine(){
+    public static NashornScriptEngine getEngine() {
         /* 1.首先引入js引擎 */
-//		ScriptEngineManager manager = new ScriptEngineManager();
-        //引擎名称传入JavaScript、js、javascript、nashorn、Nashorn 均可等价
-//		ScriptEngine engine = manager.getEngineByName("nashorn");
+        // ScriptEngineManager manager = new ScriptEngineManager();
+        // 引擎名称传入JavaScript、js、javascript、nashorn、Nashorn 均可等价
+        // ScriptEngine engine = manager.getEngineByName("nashorn");
 
         String[] options = new String[] {"--language=es6"};
         NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
-        NashornScriptEngine engine = (NashornScriptEngine) factory.getScriptEngine(options);
+        NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine(options);
 
         try {
-			/* 2.然后读取js文件 */
+            /* 2.然后读取js文件 */
             String path = Thread.currentThread().getContextClassLoader().getResource("").getPath(); // 获取targe路径
 
-//            String requestPath = "/youzy.dms.basiclib.api.career.job.hits.inc";
-//            String jobCode="02030502";
-//            params =requestPath+"?jobCode="+jobCode;
+            // String requestPath = "/youzy.dms.basiclib.api.career.job.hits.inc";
+            // String jobCode="02030502";
+            // params =requestPath+"?jobCode="+jobCode;
 
             engine.eval(new FileReader(path + "js/usign.js"));
 
@@ -72,46 +72,49 @@ public class UsignUtils {
         }
     }
 
-    public static String getRequestPathAndParam(String requestPath,Map<String,Object> paramMap){
-        StringBuilder parameter= new StringBuilder();
-        int i=0;
-        for(Map.Entry<String,Object> entry:paramMap.entrySet()){
-            if(i==0){
-                parameter.append("?");
-            }else {
-                parameter.append("&");
+    public static String getRequestPathAndParam(String api, Map<String, Object> params) {
+        if (null != params) {
+            StringBuilder parameter = new StringBuilder();
+            int i = 0;
+            for (Map.Entry<String, Object> entry : params.entrySet()) {
+                if (i == 0) {
+                    parameter.append("?");
+                } else {
+                    parameter.append("&");
+                }
+                parameter.append(entry.getKey()).append("=").append(entry.getValue());
+                i++;
             }
-            parameter.append(entry.getKey()).append("=").append(entry.getValue());
-            i++;
+            return api + parameter.toString();
+        } else {
+            return api;
         }
-        String jsParam = requestPath+parameter.toString();
-        return jsParam;
     }
 
-
     public static void main(String[] args) {
         String requestPath = "/youzy.dms.basiclib.api.career.job.hits.inc";
         String jobCode = "02030502";
-        Map<String,Object> paramMap =new HashMap<>();
-        paramMap.put("jobCode",jobCode);
-        System.out.println(getUsign(getRequestPathAndParam(requestPath,paramMap)));
+        Map<String, Object> paramMap = new HashMap<>();
+        paramMap.put("jobCode", jobCode);
+        System.out.println(getUsign(getRequestPathAndParam(requestPath, paramMap)));
 
         requestPath = "/youzy.dms.basiclib.api.major.tree.query";
         String eduLevel = "ben";
-        paramMap =new HashMap<>();
-        paramMap.put("eduLevel",eduLevel);
-        System.out.println(getUsign(getRequestPathAndParam(requestPath,paramMap)));
+        paramMap = new HashMap<>();
+        paramMap.put("eduLevel", eduLevel);
+        System.out.println(getUsign(getRequestPathAndParam(requestPath, paramMap)));
 
         requestPath = "/youzy.dms.basiclib.api.major.tree.query";
-        String jsonParam="{\"count\":5,\"rankType\":1,\"eduLevel\":\"ben\"}";
-        paramMap =new HashMap<>();
-        System.out.println(getUsign(getRequestPathAndParam(requestPath,paramMap),jsonParam));
+        String jsonParam = "{\"count\":5,\"rankType\":1,\"eduLevel\":\"ben\"}";
+        paramMap = new HashMap<>();
+        System.out.println(getUsign(getRequestPathAndParam(requestPath, paramMap), jsonParam));
 
-        //u-sign: f6a8052b16fbe01064e4c4e86441a642
+        // u-sign: f6a8052b16fbe01064e4c4e86441a642
         requestPath = "/youzy.dms.basiclib.api.college.query";
-        jsonParam="{\"keyword\":\"\",\"provinceNames\":[],\"natureTypes\":[],\"eduLevel\":\"\",\"categories\":[],\"features\":[],\"pageIndex\":%s,\"pageSize\":20,\"sort\":11}";
-        jsonParam = String.format(jsonParam,2);
-        paramMap =new HashMap<>();
-        System.out.println(getUsign(getRequestPathAndParam(requestPath,paramMap),jsonParam));
+        jsonParam =
+            "{\"keyword\":\"\",\"provinceNames\":[],\"natureTypes\":[],\"eduLevel\":\"\",\"categories\":[],\"features\":[],\"pageIndex\":%s,\"pageSize\":20,\"sort\":11}";
+        jsonParam = String.format(jsonParam, 2);
+        paramMap = new HashMap<>();
+        System.out.println(getUsign(getRequestPathAndParam(requestPath, paramMap), jsonParam));
     }
 }

src/main/java/com/mingxue/spider/utils/WoffUtil.java

@@ -0,0 +1,5 @@
+package com.mingxue.spider.utils;
+
+public class WoffUtil {
+
+}

src/main/java/com/mingxue/spider/youzy/AbstractService.java

@@ -0,0 +1,73 @@
+package com.mingxue.spider.youzy;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.mingxue.spider.helper.StringUtils;
+import com.mingxue.spider.utils.HttpUtils;
+import com.mingxue.spider.utils.UsignUtils;
+
+import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.lang.Dict;
+import cn.hutool.core.util.StrUtil;
+
+public class AbstractService {
+    protected Logger logger=LoggerFactory.getLogger(getClass());
+    private static String host = "https://uwf7de983aad7a717eb.youzy.cn";
+    private static String accessToken;
+
+    protected String getAccessToken() {
+        return accessToken;
+    }
+
+    protected void setAccessToken(String accessToken) {
+        AbstractService.accessToken = accessToken;
+    }
+
+    protected Object postBody(String api) {
+        return postBody(api, null);
+    }
+
+    protected Map<String, String> buildProvinceMap() {
+        List<String> lines = new ArrayList<>();
+        IoUtil.readUtf8Lines(getClass().getResourceAsStream("/yzy_province.txt"), lines);
+        Map<String, String> map = new LinkedHashMap<>(lines.size());
+        for (int i = 1; i < lines.size(); i++) {
+            String line = lines.get(i);
+            if (StringUtils.isNotBlank(line)) {
+                List<String> sub = StrUtil.splitTrim(line, ",");
+                map.put(sub.get(2), sub.get(1));
+            }
+        }
+        return map;
+    }
+
+    protected Object postBody(String api, Dict body) {
+        String requestPath = UsignUtils.getRequestPathAndParam(api, body);
+        String jsonBody = JSON.toJSONString(body);
+        String usign = StringUtils.EMPTY;
+        if (StringUtils.isBlank(jsonBody)) {
+            usign = UsignUtils.getUsign(requestPath);
+        } else {
+            usign = UsignUtils.getUsign(requestPath, jsonBody);
+        }
+        String url = host + requestPath;
+        String res = HttpUtils.postBody(url, jsonBody, usign, accessToken);
+        JSONObject json = JSONObject.parseObject(res);
+        if (json.getBooleanValue("isSuccess")) {
+            return json.get("result");
+        }
+        return json;
+    }
+
+    protected String decode(String encStr) {
+        return UsignUtils.getUsign(encStr);
+    }
+}

src/main/java/com/mingxue/spider/youzy/PclService.java

@@ -0,0 +1,72 @@
+package com.mingxue.spider.youzy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.lang.Dict;
+import cn.hutool.db.DbUtil;
+import cn.hutool.db.Entity;
+
+/**
+ * 批次线
+ * 
+ * @author Duzj
+ *
+ */
+public class PclService extends AbstractService {
+    private String api = "/youzy.dms.datalib.api.enrolldata.pcl.query";
+    private String tableName = "yzy_pcl";
+
+    public void getAll() {
+        long start = System.currentTimeMillis();
+        Map<String, String> pros = buildProvinceMap();
+        for (Entry<String, String> item : pros.entrySet()) {
+            getByProvince(item.getKey(), item.getValue(), 0);
+        }
+        logger.info("获取全国批次线数据用时:{}", DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+
+    private void getByProvince(String code, String name, Integer yyyy) {
+        long start = System.currentTimeMillis();
+        Dict body = Dict.create().set("provinceCode", code).set("year", yyyy);
+        Object res = postBody(api, body);
+        JSONArray arr = (JSONArray)res;
+        for (int i = 0; i < arr.size(); i++) {
+            JSONObject item = arr.getJSONObject(i);
+            Integer year = item.getInteger("year");
+            JSONArray sub = item.getJSONArray("batches");
+            List<Entity> records = new ArrayList<>();;
+            for (int j = 0; j < sub.size(); j++) {
+                JSONObject row = sub.getJSONObject(j);
+                Entity entity = Entity.create(tableName);
+                entity.set("province_code", code).set("province_name", name);
+                entity.set("year", year).set("batch", row.getString("batch"));
+                entity.set("course", row.getString("course"));
+                entity.set("score", row.getDouble("score"));
+                entity.set("pressure_score", row.getDouble("pressureScore"));
+                entity.set("pressure_range", row.getString("pressureRange"));
+                records.add(entity);
+            }
+            try {
+                DbUtil.use().del(Entity.create(tableName).set("province_code", code).set("year", year));
+                DbUtil.use().insert(records);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+        logger.info("获取[{}]省市批次线数据用时:{}", name, DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+
+    public static void main(String[] args) {
+        // new PclService().getByProvince("51", "四川", 0);
+        long start = System.currentTimeMillis();
+        new PclService().getAll();
+        System.out.println(DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+}

src/main/java/com/mingxue/spider/youzy/ScoreService.java

@@ -0,0 +1,54 @@
+package com.mingxue.spider.youzy;
+
+import java.util.List;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+
+import cn.hutool.core.lang.Dict;
+
+/**
+ * 分数相当服务
+ * 
+ * @author Duzj
+ *
+ */
+public class ScoreService extends AbstractService {
+
+    /**
+     * @param provinceCode
+     * @return
+     */
+    public JSONObject planConfig(Integer provinceCode) {
+        String api = "/youzy.dms.datalib.api.enrolldata.plan.config.get?provinceCode=" + provinceCode;
+        JSONObject res = (JSONObject)postBody(api);
+        JSONArray arr = res.getJSONArray("years");
+        return null != arr && arr.size() > 0 ? arr.getJSONObject(0) : null;
+    }
+
+    public void plan(Integer provinceCode, String collegeCode, Integer year) {
+        JSONObject config = planConfig(provinceCode);
+        if (null == config || !year.equals(config.getInteger("year"))) {
+            return;
+        }
+        String api = "/youzy.dms.datalib.api.enrolldata.plan.encrypted.v2.query";
+        List<String> courses = config.getJSONArray("course").toJavaList(String.class);
+        List<String> batchs = config.getJSONArray("batch").toJavaList(String.class);
+        for (String course : courses) {
+            for (String batch : batchs) {
+                Dict body = Dict.create().set("collegeCode", collegeCode).set("year", year);
+                body.set("provinceCode", provinceCode).set("course", course).set("batch", batch);
+                JSONObject res = (JSONObject)postBody(api, body);
+                JSONArray rows = res.getJSONArray("uCodes");
+                if (null == rows || rows.size() == 0) {
+                    continue;
+                }
+                rows = JSON.parseArray(decode(rows.toJSONString()));
+                System.out.println(rows);
+            }
+        }
+
+    }
+
+}

src/main/java/com/mingxue/spider/youzy/UserService.java

@@ -0,0 +1,27 @@
+package com.mingxue.spider.youzy;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.alibaba.fastjson.JSONObject;
+
+import cn.hutool.core.lang.Dict;
+
+public class UserService extends AbstractService {
+
+    public void validate(String mobile, String password) {
+        if (StringUtils.isNotBlank(getAccessToken())) {
+            return;
+        }
+        String api = "/tocusers/youzy.toc.user.login.validate";
+        Dict body = Dict.create();
+        body.set("username", mobile).set("password", password);
+        JSONObject res = (JSONObject)postBody(api, body);
+        if (res.containsKey("validateUserOutput")) {
+            JSONObject user = res.getJSONObject("validateUserOutput");
+            JSONObject token = user.getJSONObject("userToken");
+            setAccessToken(token.getString("accessToken"));
+        } else {
+            System.out.println(res);
+        }
+    }
+}

src/main/java/com/mingxue/spider/youzy/YfydService.java

@@ -0,0 +1,113 @@
+package com.mingxue.spider.youzy;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.mingxue.spider.helper.StringUtils;
+
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.lang.Dict;
+import cn.hutool.db.DbUtil;
+import cn.hutool.db.Entity;
+
+/**
+ * 一分一段/位次查询
+ * 
+ * @author Duzj
+ *
+ */
+public class YfydService extends AbstractService {
+    private String api = "/youzy.dms.datalib.api.enrolldata.yfyd.query";
+    private String tableName = "yzy_yfyd";
+
+    public void getAll() {
+        long start = System.currentTimeMillis();
+        Map<String, String> pros = buildProvinceMap();
+        for (Entry<String, String> item : pros.entrySet()) {
+            getByProvince(item.getKey(), item.getValue());
+        }
+        logger.info("获取全国一分一段数据用时:{}", DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+
+    private void getByProvince(String code, String name) {
+        Map<Integer, JSONArray> map = getYearAndCourses(code);
+        for (Entry<Integer, JSONArray> entry : map.entrySet()) {
+            getByProvince(code, name, entry.getKey(), entry.getValue());
+        }
+    }
+
+    @SuppressWarnings("unused")
+    private void getByProvince(String code, String name, Integer year) {
+        getByProvince(code, name, year, getCourses(code, year));
+    }
+
+    private void getByProvince(String code, String name, Integer year, JSONArray courses) {
+        if (null == courses) {
+            return;
+        }
+
+        long start = System.currentTimeMillis();
+        List<Entity> records = new ArrayList<>();
+        for (int x = 0; x < courses.size(); x++) {
+            JSONObject obj = courses.getJSONObject(x);
+            String course = obj.getString("course");
+            if (StringUtils.isBlank(course)) {
+                continue;
+            }
+            List<String> types = obj.getJSONArray("bzTypes").toJavaList(String.class);
+            for (String type : types) {
+                // {"provinceCode":43,"score":"150","year":2022,"course":"历史","bzType":"本专"}
+                Dict body = Dict.create().set("provinceCode", code).set("year", year);
+                body.set("score", 0).set("course", course).set("bzType", type);
+                JSONObject res = (JSONObject)postBody(api, body);
+                JSONArray arr = res.getJSONArray("yfyds");
+                for (int i = 0; i < arr.size(); i++) {
+                    JSONObject item = arr.getJSONObject(i);
+                    Entity entity = Entity.create(tableName);
+                    entity.set("province_code", code).set("province_name", name);
+                    entity.set("year", year).set("course", course).set("bz_type", type);
+                    entity.set("min_score", item.getDouble("minScore"));
+                    entity.set("max_score", item.getDouble("maxScore"));
+                    entity.set("same_count", item.getInteger("sameCount"));
+                    entity.set("lowest_rank", item.getDouble("lowestRank"));
+                    entity.set("highest_rank", item.getDouble("highestRank"));
+                    records.add(entity);
+                }
+            }
+        }
+        try {
+            DbUtil.use().del(Entity.create(tableName).set("province_code", code).set("year", year));
+            DbUtil.use().insert(records);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        logger.info("获取[{}]省市[{}]年,一分一段数据用时:{}", name, year, DateUtil.formatBetween(System.currentTimeMillis() - start));
+
+    }
+
+    private JSONArray getCourses(String code, Integer year) {
+        return getYearAndCourses(code).get(year);
+    }
+
+    private Map<Integer, JSONArray> getYearAndCourses(String code) {
+        JSONObject res = (JSONObject)postBody("/youzy.dms.datalib.api.enrolldata.yfyd.config.get?provinceCode=" + code);
+        Map<Integer, JSONArray> map = new LinkedHashMap<>();
+        JSONArray arr = res.getJSONArray("configs");
+        for (int i = 0; i < arr.size(); i++) {
+            JSONObject item = arr.getJSONObject(i);
+            map.put(item.getInteger("year"), item.getJSONArray("courses"));
+        }
+        return map;
+    }
+
+    public static void main(String[] args) {
+        long start = System.currentTimeMillis();
+        new YfydService().getAll();
+        System.out.println(DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+}

src/main/java/com/mingxue/spider/youzy/YouZyServer.java

@@ -0,0 +1,13 @@
+package com.mingxue.spider.youzy;
+
+public class YouZyServer {
+
+    private static UserService userService = new UserService();
+
+    private static ScoreService scoreService = new ScoreService();
+
+    public static void main(String[] args) {
+        userService.validate("17307497333", "Xj123456");
+        scoreService.plan(43, "10013", 2021);
+    }
+}

src/main/resources/users.txt

@@ -1,13 +1,68 @@
-002180054390,10253537
+002180054473,36680365
+002180054472,18508278
+002180054471,24223525
+002180054470,14971017
+002180054469,60248551
+002180054468,84778723
+002180054466,81725092
+002180054465,17131135
+002180054464,86870442
+002180054458,11003994
+002180054457,92451161
+002180054456,27080352
+002180054455,88826291
+002180054436,82398377
+002180054435,28311630
+002180054434,29161329
+002180054430,57417569
+002180054428,27718255
+002180054427,61356610
+002180054426,74567923
+002180054425,84239091
+002180054424,45882217
+002180054423,31227224
+002180054422,1046269
+002180054421,17295510
+002180054417,33145594
+002180054415,56862009
+002180054414,10601003
+002180054412,43162060
+00218005441141604594
+002180054410,43787056
+002180054406,29717188
+002180054405,20317936
+002180054404,58424981
+002180054403,24902275
+002180054402,26151624
+002180054401,30656209
+002180054399,39515161
+002180054398,29340318
+002180054396,89362505
+002180054394,51313992
+002180054393,55512404
+002180054391,67577625
+002180054389,47306621
 002180054388,59296188
-002180054182,86934322
+002180054387,16944190
+002180054386,88928895
+002180054385,65072001
+002280024693,53643407
+002280024691,56921921
+002280024687,14281218
+002280024679,89432519
+002280024675,71891136
+002280024674,12729092
+002280024670,31082789
+002280024669,78132826
+002280024667,44017234
+002280024663,52185095
 002180054185,55900344
 002180054347,85968924
 002180054186,17464580
 002180054188,86684308
 002180054189,91506868
-002180054192,11088908
-002180054190,94862151
+002180054388,59296188
+002180054182,86934322
 002180054169,19027633
 002180054174,25057765
 002180054175,35491736
@@ -33,4 +88,64 @@
 002180054382,82653711
 002180054392,59831255
 002180054391,67577625
-
+002180054477,56734083
+002180054476,20211784
+002180054474,14426607
+002180054473,36680365
+002180054472,18508278
+002180054471,24223525
+002180054470,14971017
+002180054469,60248551
+002180054468,84778723
+002180054466,81725092
+002180054465,17131135
+002180054464,86870442
+002180054458,11003994
+002180054457,92451161
+002180054456,27080352
+002180054455,88826291
+002180054436,82398377
+002180054435,28311630
+002180054434,29161329
+002180054430,57417569
+002180054428,27718255
+002180054427,61356610
+002180054426,74567923
+002180054425,84239091
+002180054424,45882217
+002180054423,31227224
+002180054422,1046269
+002180054421,17295510
+002180054417,33145594
+002180054415,56862009
+002180054414,10601003
+002180054412,43162060
+00218005441141604594
+002180054410,43787056
+002180054406,29717188
+002180054405,20317936
+002180054404,58424981
+002180054403,24902275
+002180054402,26151624
+002180054401,30656209
+002180054399,39515161
+002180054398,29340318
+002180054396,89362505
+002180054394,51313992
+002180054393,55512404
+002180054391,67577625
+002180054389,47306621
+002180054388,59296188
+002180054387,16944190
+002180054386,88928895
+002180054385,65072001
+002280024693,53643407
+002280024691,56921921
+002280024687,14281218
+002280024679,89432519
+002280024675,71891136
+002280024674,12729092
+002280024670,31082789
+002280024669,78132826
+002280024667,44017234
+002280024663,52185095

src/main/resources/yzy_province.txt

@@ -0,0 +1,31 @@
+id,name,code
+844,安徽,34 
+834,北京,11 
+854,重庆,50 
+845,福建,35 
+851,广东,44 
+852,广西,45 
+860,甘肃,62 
+856,贵州,52 
+1128,河北,13
+848,河南,41 
+853,海南,46 
+849,湖北,42 
+850,湖南,43 
+841,黑龙江,23
+840,吉林,22 
+1,江苏,32
+846,江西,36 
+839,辽宁,21 
+838,内蒙古,15
+862,宁夏,64 
+861,青海,63 
+842,上海,31 
+855,四川,51 
+847,山东,37 
+837,山西,14 
+859,陕西,61 
+835,天津,12 
+1120,新疆,65
+857,云南,53 
+843,浙江,33

src/test/java/spider/CollegeData.java

@@ -35,6 +35,9 @@ public class CollegeData {
 
     @Test
     public void test() throws Exception {
+        System.out.println("010102".compareTo("030101"));
+        System.out.println("030102".compareTo("030101"));
+        System.out.println("030101".compareTo("030101"));
 //        getMajorData();
 
         /**
@@ -53,7 +56,7 @@ public class CollegeData {
             JSONObject jsonObject = (JSONObject)ll;
             provinceMap.put(jsonObject.getString("provinceName"),JSONObject.parseObject(JSONObject.toJSONString(jsonObject),ProvinceData.class));
         });
-        String processProvinceName= "山西";
+        String processProvinceName= "江西";
         User currentUser = new User();
         switch (processProvinceName){
             case "湖南":
@@ -107,6 +110,20 @@ public class CollegeData {
                  */
                 currentUser = new User().setCode("13437248803").setPassword("41186990").setIsHn(false).setIsNewGaokao(true).setProvinceName(processProvinceName);
 
+                break;
+            case "河南":
+                /**
+                 * 河南	18574734705	75231013
+                 */
+                currentUser = new User().setCode("13437248803").setPassword("41186990").setIsHn(false).setIsNewGaokao(true).setProvinceName(processProvinceName);
+
+                break;
+            case "广西":
+                /** 河南 湖北 广东 广西
+                 * 广西	18873220395	50277878
+                 */
+                currentUser = new User().setCode("13437248803").setPassword("41186990").setIsHn(false).setIsNewGaokao(true).setProvinceName(processProvinceName);
+
                 break;
         }
 
@@ -148,14 +165,16 @@ public class CollegeData {
                      * 选科数据 xuanke(collegeId);
                      */
                     List<Integer> colledgeIds = Arrays.asList(
-                            2478,2479,2480,2462,2463,2464,2465,2791,2792,2793,2794,2796);
+                            1,3,654,871,1526,718,652,1855);
                     if(colledgeIds.contains(collegeId)){
-//                        continue;
+                        continue;
                     }
-                    if(processColledgeCount>5){
-                        //超过5个学校换个账号
-//                        ApiUtil.getInstance().login();
-//                        processColledgeCount=0;
+                    if("湖南".equalsIgnoreCase(provinceData.getProvinceName())){
+                        if(processColledgeCount>5){
+                            //湖南 超过5个学校换个账号
+                        ApiUtil.getInstance().login();
+                        processColledgeCount=0;
+                        }
                     }
 
                     List<Integer> liberalScienceList= Arrays.asList(0,1);
@@ -180,11 +199,12 @@ public class CollegeData {
                         /**
                          * 专业录取数据
                          */
-//                        byCollege(collegeId,liberalScience,2,Location,ProvinceId);
+                        ThreadUtil.safeSleep(2 * 1000);
+                        byCollege(collegeId,collegeName,liberalScience,2,Location,ProvinceId,provinceData);
                         /**
                          * 招生计划数据
                          */
-                        byCollege(collegeId,collegeName,liberalScience,3,Location,ProvinceId,provinceData);
+//                        byCollege(collegeId,collegeName,liberalScience,3,Location,ProvinceId,provinceData);
                     }
                     processColledgeCount++;
 
@@ -360,7 +380,7 @@ public class CollegeData {
 //                dataJSONObject.put("year",yearLast);
 //                Integer year = dataJSONObject.getInteger("year");
 //            List<Integer> yearList = Arrays.asList(2022,2021,2020,2019);
-            List<Integer> yearList = Arrays.asList(2022);
+            List<Integer> yearList = Arrays.asList(2021);
             for(int year:yearList){
                 sb = new StringBuilder(host);
                 if(queryType==2){
@@ -377,6 +397,7 @@ public class CollegeData {
                 sb.append("&pageSize=").append(100);
                 res = ApiUtil.getInstance().httpSyncGet(sb.toString());
 //        System.out.println("data is "+res);
+                ThreadUtil.safeSleep(1 * 1000);
                 JSONObject rows = res.getJSONObject("data");
                 if(res.getInteger("error")==0){
                     JSONArray tdDatas = rows.getJSONArray("tdData");
@@ -429,17 +450,18 @@ public class CollegeData {
 
                         String tableName = StringUtils.EMPTY;
                         if(queryType==2){
-                            tableName = "sy_colledge_major_enroll_data";
+                            tableName = "sy_colledge_enroll_major_data";
                         }else if(queryType==3){
-                            tableName = "sy_colledge_recruit_data_2022_"+provinceData.getProvinceSpell();
+//                            tableName = "sy_colledge_recruit_data_"+year+"_"+provinceData.getProvinceSpell();
+                            tableName = "sy_colledge_recruit_data";
                         }
                         Entity subjectTable4 = Entity.create(tableName);
                         subjectTable4.putAll(row);
                         subjectList.add(subjectTable4);
                         if(!isBatchInsert){
                             try {
-//                                DbUtil.use().insertOrUpdate(subjectTable4, "ID");
-                                DbUtil.use().insert(subjectTable4);
+                                DbUtil.use().insertOrUpdate(subjectTable4, "ID");
+//                                DbUtil.use().insert(subjectTable4);
                             } catch (SQLException e) {
                                 e.printStackTrace();
                             }

src/test/java/spider/YouZy.java

@@ -12,6 +12,7 @@ import com.mingxue.spider.utils.UsignUtils;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
 
 import java.sql.SQLException;
 import java.util.*;
@@ -26,11 +27,12 @@ public class YouZy {
     private static Logger log = LoggerFactory.getLogger(YouZy.class);
 
     private static final String host = "https://uwf7de983aad7a717eb.youzy.cn";
+    Boolean isBatchInsert = true;
 
     @Test
     public void test() throws Exception {
         //高校
-        getUniversityList();
+//        getUniversityList();
 //        getMajorData();
 //        processVocationalHotAndLack();
         /**
@@ -40,11 +42,13 @@ public class YouZy {
 
 
         /**
-         * 职业库
+         * 专业库
          * eduLevel:"ben","zhuan"
          */
 
-//        List<String> majorList = getMajor(3, "ben,zhuan");
+        Integer majorLevel = 3;
+        List<String> majorList = getMajor(majorLevel, "ben,zhuan");
+        log.error("{}级专业编码为 {}",majorLevel,JSONObject.toJSONString(majorList));
 //        System.out.println(JSONObject.toJSONString(majorList));
 
 //        for (String majorCode : majorList) {
@@ -56,12 +60,22 @@ public class YouZy {
 //            }
 //            getCareerProspects(majorCode);
 //        }
+
+        //处理专业开设院校
+        for (String majorCode : majorList) {
+            if(Integer.parseInt(majorCode)<80202){
+                continue;
+            }
+            processUniversityByCode(majorCode,2022);
+        }
+
+
     }
 
 
     public void getUniversityList() {
 //        31 43 52  67
-        int count = 0, total = 5000, pageIndex = 96, pageSize=20;
+        int count = 0, total = 5000, pageIndex = 115, pageSize=20;
         do {
             //学校列表
             String requestPath = "/youzy.dms.basiclib.api.college.query";
@@ -95,7 +109,7 @@ public class YouZy {
                         /**
                          * 招生简单
                          */
-//                        enrollDetail(universityCode);
+                        enrollDetail(universityCode);
 
                         /**
                          * 特色专业
@@ -126,12 +140,12 @@ public class YouZy {
                         /**
                          * 获取学校图片
                          */
-                        getImgs(universityCode);
+//                        getImgs(universityCode);
 
                         /**
                          * VR
                          */
-                        getVR(universityCode);
+//                        getVR(universityCode);
                         /**
                          * 视频：需要将aliId存储到我们自己的vod中
                          */
@@ -149,6 +163,81 @@ public class YouZy {
         }while (count <= total);
     }
 
+    //处理专业开设院校
+    private void processUniversityByCode(String code,Integer year){
+
+        int count = 0, total = 5000, pageIndex = 1, pageSize=20;
+        if(Integer.parseInt(code)==80202){
+            pageIndex= 10;
+        }
+        do {
+            //学校列表
+            String requestPath = "/youzy.dms.basiclib.api.eduranking.mr.search";
+            String jsonParam="{\"year\":\"2022\",\"majorCode\":\"%s\",\"keyword\":\"\",\"collegeProvinceCodes\":[],\"features\":[],\"collegeCategories\":[],\"natureTypes\":[],\"pageIndex\":%s,\"pageSize\":20}";
+            jsonParam = String.format(jsonParam,code,pageIndex);
+            Map paramMap =new HashMap<>();
+            String res = getRes(requestPath,paramMap,jsonParam);
+            if (StringUtils.isEmpty(res)) {
+                log.error("res is empty");
+                return;
+            } else {
+                JSONObject resultJsonobject = JSONObject.parseObject(res);
+                if (resultJsonobject.getBoolean("isSuccess")) {
+                    JSONObject result = resultJsonobject.getJSONObject("result");
+                    total= result.getInteger("totalCount");
+                    long start = System.currentTimeMillis();
+                    JSONArray items = result.getJSONArray("items");
+                    Set<Entity> subjectList=new HashSet();
+
+                    for (int i=0;i<items.size();i++) {
+                        JSONObject rowUniversity = (JSONObject)items.get(i);
+                        JSONObject college = rowUniversity.getJSONObject("college");
+                        String universityCode = college.getString("code");
+                        String universityName = college.getString("cnName");
+                        rowUniversity.remove("college");
+                        rowUniversity.remove("hits");
+                        rowUniversity.remove("isUp");
+                        rowUniversity.put("majorCode",code);
+                        rowUniversity.put("year",year);
+                        rowUniversity.put("universityCode",universityCode);
+                        rowUniversity.put("universityName",universityName);
+
+                        Date now =new Date();
+                        rowUniversity.put("updateTime",now);
+
+                        //存储学校
+                        Entity universityTable = Entity.create("sy_major_university");
+                        universityTable.putAll(rowUniversity);
+                        subjectList.add(universityTable);
+                        if(!isBatchInsert){
+                            try {
+//                            DbUtil.use().insertOrUpdate(universityTable, "id");
+                                DbUtil.use().insert(universityTable);
+                            } catch (SQLException e) {
+                                e.printStackTrace();
+                            }
+                        }
+                    }
+                    if(isBatchInsert){
+                        try {
+                            if(!CollectionUtils.isEmpty(subjectList)){
+                                DbUtil.use().insert(subjectList);
+                            }
+                        } catch (SQLException e) {
+                            e.printStackTrace();
+                        }
+                    }
+                    log.error("完成专业编码 {}, 耗时{}",code, DateUtil.formatBetween(System.currentTimeMillis() - start));
+                }
+            }
+
+            log.error("专业编码 {}, 已完成page={}, 开始第{}页" ,code,pageIndex,(pageIndex+1));
+            count = pageIndex * pageSize;
+            pageIndex += 1;
+        }while (count <= total);
+
+    }
+
     private void getVideo(String universityCode,String universityName){
         String requestPath = "/op/Classrooms/Packs/Query";
         String jsonParam = "";
@@ -440,7 +529,7 @@ public class YouZy {
     }
 
     /**
-     * 招生简单
+     * 招生简章
      * @param collegeCode
      */
     private void enrollDetail(String collegeCode){
@@ -463,7 +552,7 @@ public class YouZy {
                     JSONObject row = (JSONObject) items.get(i);
                     Integer year = row.getInteger("year");
                     //只取2021、2022的数据
-                    if(year<2021) continue;
+                    if(year<=2021) continue;
                     String id = row.getString("id");
 
                     //处理detail
@@ -493,7 +582,7 @@ public class YouZy {
 
                             Date now =new Date();
                             row.put("createTime",createdAt);
-                            row.put("updateTime",now);
+                            row.put("update_time",now);
                             //存储学校
                             Entity universityTable = Entity.create("sy_university_enroll_brochure");
                             universityTable.putAll(row);