spider/src/test/java/spider/CollegeData.java

package spider;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;

import com.mingxue.spider.helper.StringUtils;
import org.junit.Test;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mingxue.spider.utils.ApiUtil;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.CharsetUtil;
import cn.hutool.db.DbUtil;
import cn.hutool.db.Entity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;

public class CollegeData {
    private static String host = "https://apiv4.diyigaokao.com";

    private List<String> errors = new ArrayList<>();
    protected final Logger log = LoggerFactory.getLogger(this.getClass());
    Boolean isBatchInsert = false;

    @Test
    public void test() throws Exception {
        // 湖南
        ApiUtil.getInstance().login();
        collegeList();
//        generateNineDate();
//         广东账号：18774924158，密码：123456
//         ApiUtil.getInstance().login("18774924158", "123456",false);
        // 湖北账号：18674898114， 密码：123456
        // ApiUtil.getInstance().login("18674898114", "123456",false);
        // 江西账号：13203226079，密码：123456
//        ApiUtil.getInstance().login("13203226079", "123456",false);
//        collegeList();
//         getCategoryMajor();
    }
//181
    public void collegeList() throws SQLException {
//        int count = 0, total = 5000, page = 131;
        int count = 0, total = 5000, page = 14;
        JSONObject res=new JSONObject();
        do {
            try {
                StringBuilder sb = new StringBuilder(host);
                sb.append("/college/list/byMultiple");
                sb.append("?provinceIds=&yxjbz=&levels=&collegeType=&hotTags=");
                sb.append("&pageIndex=").append(page).append("&pageSize=15");
                ThreadUtil.safeSleep(2 * 1000);

                res = ApiUtil.getInstance().httpSyncGet(sb.toString());
                total = res.getInteger("total");
                JSONArray data = res.getJSONArray("data");
                int processColledgeCount=0;
                for (int i = 0; i < data.size(); i++) {
                    long start = System.currentTimeMillis();
                    JSONObject row = data.getJSONObject(i);
                    final Integer collegeId = row.getInteger("id");
                    Entity entity = DbUtil.use().get("college", "id", collegeId);
                    if (null == entity) {
//                        detail(collegeId, 0);
                        ThreadUtil.safeSleep(1 * 1000);
                    }
                    /**
                     * 选科数据 xuanke(collegeId);
                     */
//                    List<Integer> colledgeIds = Arrays.asList(
//                            1211,1212,1213,1214,1215,1216,1217);
//                    if(colledgeIds.contains(collegeId)){
//                        continue;
//                    }
                    if(processColledgeCount>5){
                        //超过5个学校换个账号
                        ApiUtil.getInstance().login();
                        processColledgeCount=0;
                    }
                    /**
                     * 院校录取数据
                     */
                    List<Integer> liberalScienceList= Arrays.asList(1,2);
                    List<Integer> typeList= Arrays.asList(2,3);
                    for(Integer liberalScience:liberalScienceList){
                        for(Integer type:typeList){
                            byHistoryNew(collegeId,liberalScience,type);
                        }
                    }
                    processColledgeCount++;

                    ThreadUtil.safeSleep(1 * 1000);
                    log.error("学校 {}, {}, 耗时{}",collegeId,row.getString("collegeName"),DateUtil.formatBetween(System.currentTimeMillis() - start));
                }
                log.error("已完成page={}, 开始第{}页" ,page,(page+1));
                count = page * 15;
                page += 1;
            } catch (IOException e) {
                e.printStackTrace();
                log.error("error res is {}",res);
                return;
            }
        } while (count <= total);
        File dir = new File("E:/projects/wangmin/evaluation/code/spider/");
        FileUtil.writeLines(errors, new File(dir, ApiUtil.getInstance().getUser().getCode() + ".txt"), CharsetUtil.CHARSET_UTF_8);
        System.out.println("执行完成!");
    }

    /*
     * 院校录取数据
     * Request URL: https://apiv4.diyigaokao.com/query/collegeScore/byHistoryNew?liberalScience=1&collegeId=1&type=2
     * collegeId 院校ID true（必传） integer
     * liberalScience 科类： 1理科(物理)，0文科(历史), 2文理不分;  true integer
     * accessToken accessToken令牌 header false string
     * phase 批次 false integer
     * type 类型，普通省份不传或传1；312院校专业组省份(江苏、湖南、湖北、广东、福建):2021年以后数据传2，2020年以前数据传3 query false integer
     */
    private void byHistoryNew(Integer collegeId,Integer liberalScience,Integer type) throws IOException {
        ThreadUtil.safeSleep(2 * 1000);

        StringBuilder sb = new StringBuilder(host);
        sb.append("/query/collegeScore/byHistoryNew");
        sb.append("?liberalScience=").append(liberalScience);
        sb.append("&collegeId=").append(collegeId);
        sb.append("&type=").append(type);
        JSONObject res = ApiUtil.getInstance().httpSyncGet(sb.toString());
//        System.out.println("data is "+res);
        if(res.getInteger("error")==0){
            //data
            JSONObject dataObject = res.getJSONObject("data");
            JSONArray tdDatas = dataObject.getJSONArray("tdData");
            Set<Entity> subjectList=new HashSet();
            for (int i = 0; i < tdDatas.size(); i++) {
                //sy_colledge_enroll_data
                JSONObject row = tdDatas.getJSONObject(i);
                row.put("type",type);
                Entity subjectTable4 = Entity.create("sy_colledge_enroll_data");
                subjectTable4.putAll(row);
                subjectList.add(subjectTable4);
                if(!isBatchInsert){
                    try {
                        DbUtil.use().insertOrUpdate(subjectTable4, "ID");
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }
            }
            if(isBatchInsert){
                try {
                    if(!CollectionUtils.isEmpty(subjectList)){
                        DbUtil.use().insert(subjectList);
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }else {
            log.error("byHistoryNew res is {}, request is {}  ",res,sb.toString());
        }
    }

    /**
     * 学校详情
     * @param id
     * @param retry
     */
    private void detail(Integer id, Integer retry) {
        int maxRetry = 2;
        JSONObject res = null;
        try {
            String url = host + "/college/detail/" + id;
            res = ApiUtil.getInstance().httpSyncGet(url);
            if (Integer.valueOf(-1).equals(res.getInteger("error"))) {
                System.out.println(url);
                if (retry >= maxRetry) {
                    System.out.println("重试" + maxRetry + "次还不够:" + id);
                    errors.add(url + "\t" + res.getString("data"));
                    return;
                }
                ThreadUtil.safeSleep(1 * 1000);
                detail(id, retry + 1);
            } else {
                JSONObject data = res.getJSONObject("data");
                data.remove("rankMap");
                data.remove("collegeDetailHotMajor");
                data.remove("yuanxiList");
                Entity record = Entity.create("college");
                record.putAll(data);
                DbUtil.use().insertOrUpdate(record, "id");
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
            throw e;
        }
    }

    private void xuanke(Integer collegeId) {
        List<Integer> years = new ArrayList<Integer>(2);
        Integer provinceId = ApiUtil.getInstance().getUser().getProvinceId();
        try {
            JSONObject res = ApiUtil.getInstance().httpSyncGet(host + "/years?provinceId=" + provinceId + "&queryType=4");
            JSONArray rows = res.getJSONArray("data");
            for (int i = 0; i < rows.size(); i++) {
                years.add(rows.getJSONObject(i).getInteger("year"));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

        List<Entity> records = new ArrayList<>(12);
        for (Integer year : years) {
            int count = 0, total = 100, page = 1;
            do {
                try {
                    StringBuilder sb = new StringBuilder();
                    sb.append(host).append("/xuanke/list/byCollegeId");
                    sb.append("?provinceId=").append(provinceId);
                    sb.append("&collegeId=").append(collegeId);
                    sb.append("&year=").append(year);
                    sb.append("&page=").append(page).append("&pageSize=30");
                    JSONObject res = ApiUtil.getInstance().httpSyncGet(sb.toString());
                    JSONArray rows = res.getJSONArray("data");
                    for (int i = 0; i < rows.size(); i++) {
                        JSONObject row = rows.getJSONObject(i);
                        row.remove("Row");
                        Entity record = Entity.create("major");
                        record.putAll(row);
                        record.set("_provinceId", provinceId).set("_collegeId", collegeId).set("_year", year);
                        records.add(record);
                    }
                    total = res.getInteger("total");
                    count = page * 30;
                    page += 1;
                } catch (Exception e) {
                    e.printStackTrace();
                }

            } while (count <= total);
        }
        try {
            DbUtil.use().del(Entity.create("major").set("CollegeID", collegeId));
            DbUtil.use().insert(records);
            // DbUtil.use()
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 获取专业列表
     */
    public void getCategoryMajor() {
        Set<Entity> MajorSubjectList = new HashSet<>();// 一级专业
        Set<Entity> MajorCategoryList = new HashSet<Entity>();// 二级专业
        Set<Entity> MajorList = new HashSet<Entity>();// 三级专业
        try {
            JSONObject res = ApiUtil.getInstance().httpSyncGet(host + "/major/allCategoryMajor");
            JSONObject data = res.getJSONObject("data");
            // JSONArray bk = data.getJSONArray("BK");
            JSONArray zk = data.getJSONArray("ZK");
            // getData(bk,"本科",MajorSubjectList,MajorCategoryList,MajorList);
            MajorSubjectList.clear();// 一级专业
            MajorCategoryList.clear();// 二级专业
            MajorList.clear();// 三级专业
            getData(zk, "专科", MajorSubjectList, MajorCategoryList, MajorList);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private void getData(JSONArray bk, String Type, Set<Entity> MajorSubjectList, Set<Entity> MajorCategoryList, Set<Entity> MajorList) {
        for (int i = 0; i < bk.size(); i++) {
            JSONObject Subject = bk.getJSONObject(i);
            if ("专科".equalsIgnoreCase(Type)) {
                // 专科的ID在本科的基础上增加12
                Subject.put("ID", (Subject.getInteger("ID") + 12));
            }
            Subject.put("Type", Type);
            JSONArray CategoryArray = Subject.getJSONArray("Category");
            Subject.remove("Category");
            Entity subjectTable = Entity.create("sy_major_subject");
            subjectTable.putAll(Subject);

            MajorSubjectList.add(subjectTable);
            System.out.println(Type + "：MajorSubjectList：" + JSONObject.toJSONString(MajorSubjectList));

            for (int j = 0; j < CategoryArray.size(); j++) {
                JSONObject Category = CategoryArray.getJSONObject(j);
                Category.put("MajorSubjectCode", Subject.getString("SubjectCode"));// 上级ID填充下级
                Category.remove("MajorSubjectID");// ID转换为code作为关联

                Category.put("Type", Type);
                if ("专科".equalsIgnoreCase(Type)) {
                    Category.remove("ID");
                }

                // System.out.println("before Category："+Category);
                JSONArray MajorArray = Category.getJSONArray("Major");

                Category.remove("Major");
                // System.out.println("after Category："+Category);

                Entity categoryTable = Entity.create("sy_major_category");
                categoryTable.putAll(Category);
                MajorCategoryList.add(categoryTable);
                for (int k = 0; k < MajorArray.size(); k++) {
                    JSONObject Major = MajorArray.getJSONObject(k);
                    Major.put("MajorCategoryCode", Category.getString("MajorCategoryCode"));// 上级ID填充下级
                    Major.remove("MajorCategoryID");// ID转换为code关联
                    Major.put("Type", Type);
                    if ("专科".equalsIgnoreCase(Type)) {
                        Major.remove("ID");
                    }

                    Entity majorTable = Entity.create("sy_major_major");
                    majorTable.putAll(Major);
                    MajorList.add(majorTable);
                }
            }

            try {
                // DbUtil.use().del(Entity.create("sy_major_subject").set("Type", Type));
                DbUtil.use().insert(MajorSubjectList);
                MajorSubjectList.clear();

                // DbUtil.use().del(Entity.create("sy_major_category").set("Type", Type));
                DbUtil.use().insert(MajorCategoryList);
                MajorCategoryList.clear();

                // DbUtil.use().del(Entity.create("sy_major_major").set("Type", Type));
                DbUtil.use().insert(MajorList);
                MajorList.clear();

            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

    /**
     * 随机生成9科数据
     */
    private void generateNineDate(){
        int count = 6000;
//        JSONArray majorArray = new JSONArray();
        Set<Entity> MajorList = new HashSet<Entity>();
        for (int i =0 ;i<count;i++){
            Map major=new TreeMap();
            major.put("chinese",(int)(100-Math.random()*50));
            major.put("math",(int)(100-Math.random()*50));
            major.put("english",(int)(100-Math.random()*50));
            major.put("physics",(int)(100-Math.random()*50));
            major.put("chemistry",(int)(100-Math.random()*50));
            major.put("biology",(int)(100-Math.random()*50));
            major.put("history",(int)(100-Math.random()*50));
            major.put("politics",(int)(100-Math.random()*50));
            major.put("geography",(int)(100-Math.random()*50));

            Entity majorTable = Entity.create("major_nine");
            majorTable.putAll(major);
            MajorList.add(majorTable);

//            System.out.println(major);
        }


        try {
            DbUtil.use().insert(MajorList);
        } catch (SQLException e) {
            e.printStackTrace();
        }

        System.out.println("finished");

    }
}