一分一段全部数据

src/main/java/com/mingxue/spider/youzy/YfydService.java

@@ -8,6 +8,7 @@ import java.util.Map.Entry;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.mingxue.spider.helper.StringUtils;
 
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.lang.Dict;
@@ -28,21 +29,36 @@ public class YfydService extends AbstractService {
         long start = System.currentTimeMillis();
         Map<String, String> pros = buildProvinceMap();
         for (Entry<String, String> item : pros.entrySet()) {
-            getByProvince(item.getKey(), item.getValue(), 2022);
+            getByProvince(item.getKey(), item.getValue());
         }
         logger.info("获取全国一分一段数据用时:{}", DateUtil.formatBetween(System.currentTimeMillis() - start));
     }
 
+    private void getByProvince(String code, String name) {
+        Map<Integer, JSONArray> map = getYearAndCourses(code);
+        for (Entry<Integer, JSONArray> entry : map.entrySet()) {
+            getByProvince(code, name, entry.getKey(), entry.getValue());
+        }
+    }
+
+    @SuppressWarnings("unused")
     private void getByProvince(String code, String name, Integer year) {
-        long start = System.currentTimeMillis();
-        List<Entity> records = new ArrayList<>();
-        JSONArray courses = getCourses(code, year);
+        getByProvince(code, name, year, getCourses(code, year));
+    }
+
+    private void getByProvince(String code, String name, Integer year, JSONArray courses) {
         if (null == courses) {
             return;
         }
+
+        long start = System.currentTimeMillis();
+        List<Entity> records = new ArrayList<>();
         for (int x = 0; x < courses.size(); x++) {
             JSONObject obj = courses.getJSONObject(x);
             String course = obj.getString("course");
+            if (StringUtils.isBlank(course)) {
+                continue;
+            }
             List<String> types = obj.getJSONArray("bzTypes").toJavaList(String.class);
             for (String type : types) {
                 // {"provinceCode":43,"score":"150","year":2022,"course":"历史","bzType":"本专"}
@@ -70,11 +86,15 @@ public class YfydService extends AbstractService {
         } catch (Exception e) {
             e.printStackTrace();
         }
-        logger.info("获取[{}]省市一分一段数据用时:{}", name, DateUtil.formatBetween(System.currentTimeMillis() - start));
+        logger.info("获取[{}]省市[{}]年,一分一段数据用时:{}", name, year, DateUtil.formatBetween(System.currentTimeMillis() - start));
 
     }
 
     private JSONArray getCourses(String code, Integer year) {
+        return getYearAndCourses(code).get(year);
+    }
+
+    private Map<Integer, JSONArray> getYearAndCourses(String code) {
         JSONObject res = (JSONObject)postBody("/youzy.dms.datalib.api.enrolldata.yfyd.config.get?provinceCode=" + code);
         Map<Integer, JSONArray> map = new LinkedHashMap<>();
         JSONArray arr = res.getJSONArray("configs");
@@ -82,7 +102,7 @@ public class YfydService extends AbstractService {
             JSONObject item = arr.getJSONObject(i);
             map.put(item.getInteger("year"), item.getJSONArray("courses"));
         }
-        return map.get(year);
+        return map;
     }
 
     public static void main(String[] args) {