一分一段数据

src/main/java/com/mingxue/spider/youzy/YfydService.java

@@ -0,0 +1,93 @@
+package com.mingxue.spider.youzy;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.lang.Dict;
+import cn.hutool.db.DbUtil;
+import cn.hutool.db.Entity;
+
+/**
+ * 一分一段/位次查询
+ * 
+ * @author Duzj
+ *
+ */
+public class YfydService extends AbstractService {
+    private String api = "/youzy.dms.datalib.api.enrolldata.yfyd.query";
+    private String tableName = "yzy_yfyd";
+
+    public void getAll() {
+        long start = System.currentTimeMillis();
+        Map<String, String> pros = buildProvinceMap();
+        for (Entry<String, String> item : pros.entrySet()) {
+            getByProvince(item.getKey(), item.getValue(), 2022);
+        }
+        logger.info("获取全国一分一段数据用时:{}", DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+
+    private void getByProvince(String code, String name, Integer year) {
+        long start = System.currentTimeMillis();
+        List<Entity> records = new ArrayList<>();
+        JSONArray courses = getCourses(code, year);
+        if (null == courses) {
+            return;
+        }
+        for (int x = 0; x < courses.size(); x++) {
+            JSONObject obj = courses.getJSONObject(x);
+            String course = obj.getString("course");
+            List<String> types = obj.getJSONArray("bzTypes").toJavaList(String.class);
+            for (String type : types) {
+                // {"provinceCode":43,"score":"150","year":2022,"course":"历史","bzType":"本专"}
+                Dict body = Dict.create().set("provinceCode", code).set("year", year);
+                body.set("score", 0).set("course", course).set("bzType", type);
+                JSONObject res = (JSONObject)postBody(api, body);
+                JSONArray arr = res.getJSONArray("yfyds");
+                for (int i = 0; i < arr.size(); i++) {
+                    JSONObject item = arr.getJSONObject(i);
+                    Entity entity = Entity.create(tableName);
+                    entity.set("province_code", code).set("province_name", name);
+                    entity.set("year", year).set("course", course).set("bz_type", type);
+                    entity.set("min_score", item.getDouble("minScore"));
+                    entity.set("max_score", item.getDouble("maxScore"));
+                    entity.set("same_count", item.getInteger("sameCount"));
+                    entity.set("lowest_rank", item.getDouble("lowestRank"));
+                    entity.set("highest_rank", item.getDouble("highestRank"));
+                    records.add(entity);
+                }
+            }
+        }
+        try {
+            DbUtil.use().del(Entity.create(tableName).set("province_code", code).set("year", year));
+            DbUtil.use().insert(records);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        logger.info("获取[{}]省市一分一段数据用时:{}", name, DateUtil.formatBetween(System.currentTimeMillis() - start));
+
+    }
+
+    private JSONArray getCourses(String code, Integer year) {
+        JSONObject res = (JSONObject)postBody("/youzy.dms.datalib.api.enrolldata.yfyd.config.get?provinceCode=" + code);
+        Map<Integer, JSONArray> map = new LinkedHashMap<>();
+        JSONArray arr = res.getJSONArray("configs");
+        for (int i = 0; i < arr.size(); i++) {
+            JSONObject item = arr.getJSONObject(i);
+            map.put(item.getInteger("year"), item.getJSONArray("courses"));
+        }
+        return map.get(year);
+    }
+
+    public static void main(String[] args) {
+        long start = System.currentTimeMillis();
+        new YfydService().getAll();
+        System.out.println(DateUtil.formatBetween(System.currentTimeMillis() - start));
+    }
+}