职业库

src/test/java/spider/YouZy.java

@@ -28,18 +28,19 @@ public class YouZy {
     @Test
     public void test() throws Exception {
 //        processVocationalHotAndLack();
-//        getVocationalList();
-//        getCareerProspects("010101","c29e43498afdfe626ea17818b86fec4b");
+        /**
+         * 职业库三级树、职业概况、就业岗位
+         */
+        getVocationalList();
 
 
-        //eduLevel:"ben","zhuan"
-//        List<String> majorList = getMajor(3, "ben,zhuan");
-//        System.out.println(JSONObject.toJSONString(majorList));
-        List<String> majorList = getMajor(3, "ben,zhuan");
-        System.out.println(majorList.size());
+        /**
+         * 职业库
+         * eduLevel:"ben","zhuan"
+         */
 
-//        majorList = getMajor(3, "zhuan");
-//        System.out.println(majorList.size());
+        List<String> majorList = getMajor(3, "ben,zhuan");
+//        System.out.println(JSONObject.toJSONString(majorList));
 
         for (String majorCode : majorList) {
 //            if(majorCode.compareTo("430701")<=-1){
@@ -48,7 +49,6 @@ public class YouZy {
 //            if(!"081003".equalsIgnoreCase(majorCode)){
 //                continue;
 //            }
-
             getCareerProspects(majorCode);
         }
     }
@@ -159,13 +159,10 @@ public class YouZy {
             }
 
         }
-        if(true){
-            return;
-        }
+
         if (StringUtils.isEmpty(jobDirection)) {
             log.error("jobDirection is empty,code is {},major is {},resultJsonobject is {}",code,major
                     ,JSONObject.parseObject(res));
-
 //            return;
         }
 
@@ -230,18 +227,21 @@ public class YouZy {
         //插入表sy_vocational_post
         Entity subjectTable4 = Entity.create("sy_major_career_prospects");
         subjectTable4.putAll(insertObj4);
-//                dbInsertList.add(subjectTable4);
+                dbInsertList.add(subjectTable4);
 
         try {
             DbUtil.use().insertOrUpdate(subjectTable4, "code");
             log.info("finished code {}",code);
-//                    DbUtil.use().insert(dbInsertList);
+                    DbUtil.use().insert(dbInsertList);
         } catch (SQLException e) {
             e.printStackTrace();
         }
     }
 
-
+    /**
+     * 热门职业排名、未来10年紧缺职业TOP20
+     * @throws SQLException
+     */
     public void processVocationalHotAndLack() throws SQLException {
         List<Entity> dbInsertList = new LinkedList<>();
         JSONArray hitsArray = getVocationalHits();
@@ -317,12 +317,16 @@ public class YouZy {
         return resultJsonArray;
     }
 
+    /**
+     * 获取职业库
+     * @throws Exception
+     */
     public void getVocationalList() throws Exception {
-        String url = host + "/youzy.dms.basiclib.api.career.jobs.tree.get";
-        String uSign = "c08ac01cdae3d836849cc3ee3721332b";
-        String res = HttpUtils.postBody(url, "", uSign);
-        if (StringUtils.isEmpty(res)) {
-            log.error("res is empty");
+        String path = "/youzy.dms.basiclib.api.career.jobs.tree.get";
+        Map<String, Object> map = new HashMap<>();
+        String res = getRes(path, map);
+        if(StringUtils.isEmpty(res)){
+            log.error("getVocationalList's is empty");
             return;
         }
         JSONObject resultJsonobject = JSONObject.parseObject(res);
@@ -358,8 +362,8 @@ public class YouZy {
 //                    e.printStackTrace();
 //                }
 
-                dbInsertList.add(subjectTable);
-                dataSet.add(insertObj1);
+//                dbInsertList.add(subjectTable);
+//                dataSet.add(insertObj1);
 
                 JSONArray childrenArray1 = jsonobejct0.getJSONArray("children");
 //                log.info("childrenArray1 size is {}",childrenArray1.size());
@@ -380,9 +384,9 @@ public class YouZy {
                     insertObj2.put("ancestors", parentCode1 + "," + parentCode2);
                     dataSet.add(insertObj2);
 
-                    Entity subjectTable2 = Entity.create("sy_vocational");
-                    subjectTable2.putAll(insertObj2);
-                    dbInsertList.add(subjectTable2);
+//                    Entity subjectTable2 = Entity.create("sy_vocational");
+//                    subjectTable2.putAll(insertObj2);
+//                    dbInsertList.add(subjectTable2);
 
 //                    try {
 //                        DbUtil.use().insertOrUpdate(subjectTable2,"code");
@@ -409,10 +413,10 @@ public class YouZy {
                         insertObj3.put("ancestors", parentCode1 + "," + parentCode2 + "," + parentCode3);
 
 
-                        Entity subjectTable3 = Entity.create("sy_vocational");
-                        subjectTable3.putAll(insertObj3);
-                        dbInsertList.add(subjectTable3);
-                        dataSet.add(insertObj3);
+//                        Entity subjectTable3 = Entity.create("sy_vocational");
+//                        subjectTable3.putAll(insertObj3);
+//                        dbInsertList.add(subjectTable3);
+//                        dataSet.add(insertObj3);
 
 //                        try {
 //                            DbUtil.use().insertOrUpdate(subjectTable3,"code");
@@ -441,17 +445,22 @@ public class YouZy {
         log.info(JSONObject.toJSONString(res));
     }
 
+    /**
+     * 职业概况与就业岗位
+     * @param vocationalCode
+     * @throws Exception
+     */
     private void processVocationalOverviewAndPost(String vocationalCode) throws Exception {
         //1-1概述
-        String url = host + "/youzy.dms.basiclib.api.career.job.bycode.get?code=" + vocationalCode;
-        String body = "code=" + vocationalCode;
-        String uSign = "28b6defebdf24aab9d6b6f0b68d58c4f";
-        String res = HttpUtils.postBody(url, "", uSign);
+        String path = "/youzy.dms.basiclib.api.career.job.bycode.get";
+        Map<String, Object> map = new HashMap<>();
+        map.put("code", vocationalCode);
+
+        String res = getRes(path, map);
         if (StringUtils.isEmpty(res)) {
-            log.error("1-1概述：res is empty，the code is {}", vocationalCode);
+            log.error("/youzy.dms.basiclib.api.career.job.bycode.get is empty");
             return;
         }
-
         JSONObject resObject = JSONObject.parseObject(res);
 
         JSONObject insertObj3 = new JSONObject();
@@ -495,15 +504,16 @@ public class YouZy {
 
 
         //1-2、概述-相关专业
-        url = host + "/youzy.dms.basiclib.api.jobmajors.relevant.query?jobCode=" + vocationalCode;
-        ;
-//        body = "jobCode="+vocationalCode;
-        uSign = "6907cd5128cd30d61afb3774e90771a8";
-        res = HttpUtils.postBody(url, "", uSign);
+        path = "/youzy.dms.basiclib.api.jobmajors.relevant.query";
+        map = new HashMap<>();
+        map.put("jobCode", vocationalCode);
+
+        res = getRes(path, map);
         if (StringUtils.isEmpty(res)) {
-            log.error("概述-相关专业：res is emptythe code is {}", vocationalCode);
+            log.error("/youzy.dms.basiclib.api.jobmajors.relevant.query is empty");
             return;
         }
+
         JSONObject postMajorsJsonobject = JSONObject.parseObject(res);
 
         if (postMajorsJsonobject.getBoolean("isSuccess")) {
@@ -512,19 +522,19 @@ public class YouZy {
         }
 
         //插入表 sy_vocational_overview
-//        Entity subjectTable3 = Entity.create("sy_vocational_overview");
-//        subjectTable3.putAll(insertObj3);
-//        DbUtil.use().insertOrUpdate(subjectTable3,"code");
+        Entity subjectTable3 = Entity.create("sy_vocational_overview");
+        subjectTable3.putAll(insertObj3);
+        DbUtil.use().insertOrUpdate(subjectTable3,"code");
 
 
         //2、就业岗位
-        url = host + "/youzy.dms.basiclib.api.career.post.tabs.get?code=" + vocationalCode;
-        ;
-//        body = "jobCode="+vocationalCode;
-        uSign = "28b6defebdf24aab9d6b6f0b68d58c4f";
-        res = HttpUtils.postBody(url, "", uSign);
+        path = "/youzy.dms.basiclib.api.career.post.tabs.get";
+        map = new HashMap<>();
+        map.put("code", vocationalCode);
+
+        res = getRes(path, map);
         if (StringUtils.isEmpty(res)) {
-            log.error("2-1、就业岗位：res is emptythe code is {}", vocationalCode);
+            log.error("/youzy.dms.basiclib.api.career.post.tabs.get is empty");
             return;
         }
         JSONObject postJsonobject = JSONObject.parseObject(res);
@@ -564,15 +574,19 @@ public class YouZy {
 
     }
 
+    /**
+     * 岗位详情：薪资情况、就业形势、招聘需求量、收入排行-按行业、收入排行-按地区
+     * @param postName
+     * @throws Exception
+     */
     private void processPostDetails(String postName) throws Exception {
         //2-2、岗位详情
-        String url = host + "/youzy.dms.basiclib.api.career.post.byname.get?name=" + postName;
-        ;
-//        body = "jobCode="+vocationalCode;
-        String uSign = "e4b60f1264f61582957a67c1f23cede3";
-        String res = HttpUtils.postBody(url, "", uSign);
+        String path = "/youzy.dms.basiclib.api.career.post.byname.get";
+        Map<String, Object> map = new HashMap<>();
+        map.put("name", postName);
+
+        String res = getRes(path, map);
         if (StringUtils.isEmpty(res)) {
-            log.error("2-2、岗位详情：res is emptythe name is {}", postName);
             return;
         }
 
@@ -621,7 +635,6 @@ public class YouZy {
 
     /**
      * 获取请求的result
-     *
      * @param path /youzy.dms.basiclib.api.major.tree.query
      * @param map  参数
      * @return
@@ -640,6 +653,12 @@ public class YouZy {
         return resultString;
     }
 
+    /**
+     * 获取请求结果
+     * @param path
+     * @param map
+     * @return
+     */
     public String getRes(String path, Map<String, Object> map) {
         String requestPath = UsignUtils.getRequestPathAndParam(path, map);
         String usign = UsignUtils.getUsign(requestPath);