Browse Source

vacation crawler

albertshaw 8 years ago
parent
commit
c2e4ed508d

+ 5 - 3
pom.xml

@@ -327,9 +327,11 @@
             <artifactId>aliyun-java-sdk-sms</artifactId>
             <version>3.0.0-rc1</version>
         </dependency>
-        
-
- 
+        <dependency>
+			<groupId>org.apache.httpcomponents</groupId>
+			<artifactId>httpasyncclient</artifactId>
+			<version>4.1.3</version>
+		</dependency>
         
 	</dependencies>
 	<build>

+ 66 - 0
src/main/java/com/goafanti/admin/controller/AdminTriggerController.java

@@ -0,0 +1,66 @@
+package com.goafanti.admin.controller;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.goafanti.common.bo.Result;
+import com.goafanti.common.controller.BaseApiController;
+import com.goafanti.common.utils.FileUtils;
+import com.goafanti.common.utils.LoggerUtils;
+import com.goafanti.crawler.RequestUtils;
+import com.goafanti.crawler.callback.TransitFutureCallback;
+
+@RestController
+@RequestMapping(value = "/api/admin/trigger")
+public class AdminTriggerController extends BaseApiController {
+
+	@Value(value = "${upload.path}")
+	private String uploadPath = null;
+
+	@RequestMapping(value = "/vacationcrawler", method = RequestMethod.GET)
+	public Result vacationCrawlerStart(String id) {
+		Result res = new Result();
+		try {
+			TransitFutureCallback cb = new TransitFutureCallback("<a.*?href=\"(.*?)\".*?节假日安排.*?</a>");
+			Map<String, String> json = new HashMap<>();
+			cb.setResult(json);
+			RequestUtils.startRequest(
+					"http://sousuo.gov.cn/list.htm?n=40&p=0&t=paper&sort=pubtime&subchildtype=gc189&location=%E7%BB%BC%E5%90%88%E6%94%BF%E5%8A%A1%E5%85%B6%E4%BB%96&timetype=timeqb",
+					cb);
+			for (String j : json.keySet()) {
+				saveToFile("/json/vacations/" + j + ".json", json.get(j));
+			}
+		} catch (Exception e) {
+			res.getError().add(buildError("", e.getMessage()));
+		}
+		return res;
+	}
+
+	private void saveToFile(String filePath, String s) {
+		File file = new File(uploadPath + filePath);
+		file.getParentFile().mkdirs();
+		FileWriter fw = null;
+		try {
+			fw = new FileWriter(file);
+			fw.write(s);
+		} catch (IOException e) {
+			LoggerUtils.error(FileUtils.class, e.getMessage(), e);
+		} finally {
+			try {
+				if (fw != null) {
+					fw.close();
+				}
+			} catch (IOException e) {
+				fw = null;
+			}
+		}
+	}
+}

+ 5 - 5
src/main/java/com/goafanti/common/utils/FileUtils.java

@@ -31,7 +31,7 @@ public class FileUtils {
 	 * @param resultMap
 	 * @throws IOException
 	 */
-	public void out(HttpServletResponse response, String jsonStr) {
+	public static void out(HttpServletResponse response, String jsonStr) {
 		PrintWriter out = null;
 		try {
 			response.setHeader("Content-Type", "application/json");
@@ -39,7 +39,7 @@ public class FileUtils {
 			out = response.getWriter();
 			out.println(jsonStr);
 		} catch (Exception e) {
-			LoggerUtils.fmtError(getClass(), e, "输出数据失败");
+			LoggerUtils.fmtError(FileUtils.class, e, "输出数据失败");
 		} finally {
 			if (null != out) {
 				out.flush();
@@ -55,7 +55,7 @@ public class FileUtils {
 	 * @param fileName
 	 * @param workbook
 	 */
-	public void downloadExcel(HttpServletResponse response, String fileName, HSSFWorkbook workbook) {
+	public static void downloadExcel(HttpServletResponse response, String fileName, HSSFWorkbook workbook) {
 		String fn = null;
 		try {
 			fn = URLEncoder.encode(fileName, "UTF-8");
@@ -69,7 +69,7 @@ public class FileUtils {
 			out = response.getOutputStream();
 			workbook.write(out);
 		} catch (IOException e) {
-			LoggerUtils.fmtError(getClass(), e, "输出Excel失败");
+			LoggerUtils.fmtError(FileUtils.class, e, "输出Excel失败");
 		} finally {
 			if (null != out) {
 				try {
@@ -236,7 +236,7 @@ public class FileUtils {
 	private static String subStr(String s) {
 		return "企业" + s.substring(s.lastIndexOf(UNDERLINE) + 1, s.lastIndexOf("."));
 	}
-	
+
 	/**
 	 * 
 	 * @param response

+ 58 - 0
src/main/java/com/goafanti/crawler/RequestUtils.java

@@ -0,0 +1,58 @@
+package com.goafanti.crawler;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
+import org.apache.http.impl.nio.client.HttpAsyncClients;
+import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
+import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
+import org.apache.http.nio.reactor.ConnectingIOReactor;
+
+import com.goafanti.crawler.callback.CrawlerFutureCallback;
+
+public class RequestUtils {
+
+	public static void startRequest(String url, CrawlerFutureCallback callback) throws Exception {
+		CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault();
+		httpclient.start();
+		CountDownLatch latch = new CountDownLatch(1);
+		callback.setCountDownLatch(latch);
+		httpclient.execute(buildGetRequest(url), callback);
+		latch.await();
+		httpclient.close();
+	}
+
+	public static void startRequest(Map<String, CrawlerFutureCallback> requests) throws Exception {
+		if (requests.isEmpty()) {
+			return;
+		}
+		ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor();
+		PoolingNHttpClientConnectionManager cm = new PoolingNHttpClientConnectionManager(ioReactor);
+		cm.setMaxTotal(10);
+		CloseableHttpAsyncClient httpclient = HttpAsyncClients.custom().setConnectionManager(cm).build();
+		httpclient.start();
+		CountDownLatch latch = new CountDownLatch(requests.size());
+		CrawlerFutureCallback cb = null;
+		for (String key : requests.keySet()) {
+			cb = requests.get(key);
+			cb.setCountDownLatch(latch);
+			httpclient.execute(buildGetRequest(key), cb);
+		}
+		latch.await();
+		httpclient.close();
+	}
+
+	private static HttpGet buildGetRequest(String url) {
+		HttpGet request = new HttpGet(url);
+
+		request.setConfig(RequestConfig.custom().setConnectTimeout(10000).setConnectionRequestTimeout(10000)
+				.setSocketTimeout(1000).build());
+
+		request.setHeader("User-Agent",
+				"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
+		return request;
+	}
+}

+ 64 - 0
src/main/java/com/goafanti/crawler/callback/CrawlerFutureCallback.java

@@ -0,0 +1,64 @@
+package com.goafanti.crawler.callback;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.concurrent.FutureCallback;
+import org.apache.http.util.EntityUtils;
+
+import com.goafanti.common.utils.LoggerUtils;
+
+public abstract class CrawlerFutureCallback implements FutureCallback<HttpResponse> {
+
+	private CountDownLatch		countDownLatch	= null;
+
+	private Map<String, String>	result			= null;
+
+	@Override
+	public void completed(HttpResponse response) {
+		HttpEntity entity = response.getEntity();
+		if (entity != null) {
+			try {
+				handleWebpage(EntityUtils.toString(entity, "UTF-8"));
+			} catch (Exception e) {
+				LoggerUtils.error(getClass(), e.getMessage(), e);
+			}
+		}
+		countDown();
+	}
+
+	@Override
+	public void failed(Exception e) {
+		countDown();
+	}
+
+	@Override
+	public void cancelled() {
+		countDown();
+	}
+
+	protected void handleWebpage(String pageContent) {
+
+	}
+
+	private void countDown() {
+		if (countDownLatch != null) {
+			countDownLatch.countDown();
+		}
+	}
+
+	public void setCountDownLatch(CountDownLatch countDownLatch) {
+		this.countDownLatch = countDownLatch;
+	}
+
+	public Map<String, String> getResult() {
+		return result;
+	}
+
+	public void setResult(Map<String, String> result) {
+		this.result = result;
+	}
+
+}

+ 38 - 0
src/main/java/com/goafanti/crawler/callback/TransitFutureCallback.java

@@ -0,0 +1,38 @@
+package com.goafanti.crawler.callback;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.goafanti.crawler.RequestUtils;
+
+public class TransitFutureCallback extends CrawlerFutureCallback {
+
+	private String regex = null;
+
+	@SuppressWarnings("unused")
+	private TransitFutureCallback() {
+
+	}
+
+	public TransitFutureCallback(String regex) {
+		this.regex = regex;
+	}
+
+	@Override
+	protected void handleWebpage(String pageContent) {
+		Pattern pattern = Pattern.compile(regex);
+		Matcher m = pattern.matcher(pageContent);
+		Map<String, CrawlerFutureCallback> cbs = new HashMap<>();
+		while (m.find()) {
+			cbs.put(m.group(1), new VacationFutureCallback(getResult()));
+		}
+		try {
+			RequestUtils.startRequest(cbs);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+}

+ 152 - 0
src/main/java/com/goafanti/crawler/callback/VacationFutureCallback.java

@@ -0,0 +1,152 @@
+package com.goafanti.crawler.callback;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.alibaba.fastjson.JSON;
+
+public class VacationFutureCallback extends CrawlerFutureCallback {
+
+	private String		year			= null;
+
+	private Pattern		dateTime		= null;
+
+	private Pattern		dateTimePeriod	= null;
+
+	private Pattern		dateTimePeriod2	= null;
+
+	private Set<String>	vacations		= null;
+
+	private Set<String>	workDays		= null;
+
+	public VacationFutureCallback(Map<String, String> result) {
+		setResult(result);
+	}
+
+	@Override
+	protected void handleWebpage(String pageContent) {
+		Pattern pattern = Pattern.compile("<td class=\"b12c\">([\\s\\S]*?)</td>", Pattern.MULTILINE);
+		Matcher m = pattern.matcher(pageContent);
+		if (m.find()) {
+			handleNotice(m.group(1));
+		}
+	}
+
+	private void handleNotice(String notice) {
+		Matcher yearMatcher = Pattern.compile("关于(\\d+)年").matcher(notice);
+		if (yearMatcher.find()) {
+			year = yearMatcher.group(1);
+		} else {
+			return;
+		}
+		List<String> parts = new ArrayList<>();
+		Matcher linesMatcher = Pattern.compile("<p.*?>([\\s\\S]+?)</p>", Pattern.MULTILINE).matcher(notice);
+		while (linesMatcher.find()) {
+			parts.add(linesMatcher.group(1));
+		}
+		handleParts(parts);
+	}
+
+	private void handleParts(List<String> parts) {
+		dateTime = Pattern.compile("(\\d+)月(\\d+)日");
+		dateTimePeriod = Pattern.compile("(\\d+)月(\\d+)日至(\\d+)日");
+		dateTimePeriod2 = Pattern.compile("(\\d+)月(\\d+)日至(\\d+)月(\\d+)日");
+		vacations = new HashSet<>();
+		workDays = new HashSet<>();
+		for (String part : parts) {
+			for (String s : part.split("<.*?>")) {
+				if (s.matches("\\d+月\\d+日.*")) {
+					handleLine(s);
+				}
+			}
+		}
+		Calendar c = Calendar.getInstance();
+		int y = Integer.parseInt(year);
+		c.set(y, 0, 1);
+		while (c.get(Calendar.YEAR) == y) {
+			int dow = c.get(Calendar.DAY_OF_WEEK);
+			if (dow == 7 || dow == 1) {
+				String ckey = getDateStr(c);
+				if (!workDays.contains(ckey)) {
+					vacations.add(ckey);
+				}
+			}
+			c.add(Calendar.DATE, 1);
+		}
+		if (this.getResult() != null) {
+			this.getResult().put(year, JSON.toJSONString(vacations));
+		}
+	}
+
+	private void handleLine(String line) {
+		for (String s : line.split("。")) {
+			boolean isVacation = s.indexOf("上班") == -1;
+			if (findNoPeriod2(s, isVacation) && findNoPeriod(s, isVacation)) {
+				Matcher dtm = dateTime.matcher(s);
+				while (dtm.find()) {
+					if (isVacation) {
+						vacations.add(getDateStr(Integer.parseInt(dtm.group(1)), Integer.parseInt(dtm.group(2))));
+					} else {
+						workDays.add(getDateStr(Integer.parseInt(dtm.group(1)), Integer.parseInt(dtm.group(2))));
+					}
+				}
+			}
+		}
+	}
+
+	private boolean findNoPeriod2(String s, boolean isVacation) {
+		Matcher dtp2m = dateTimePeriod2.matcher(s);
+		if (dtp2m.find()) {
+			addPeriodToSet(dtp2m.group(1), dtp2m.group(2), dtp2m.group(3), dtp2m.group(4),
+					isVacation ? vacations : workDays);
+			return false;
+		}
+		return true;
+	}
+
+	private boolean findNoPeriod(String s, boolean isVacation) {
+		Matcher dtp2m = dateTimePeriod.matcher(s);
+		if (dtp2m.find()) {
+			addPeriodToSet(dtp2m.group(1), dtp2m.group(2), dtp2m.group(1), dtp2m.group(3),
+					isVacation ? vacations : workDays);
+			return false;
+		}
+		return true;
+	}
+
+	private void addPeriodToSet(String stM, String stD, String edM, String edD, Set<String> set) {
+		Calendar st = Calendar.getInstance();
+		st.set(Integer.parseInt(year), Integer.parseInt(stM) - 1, Integer.parseInt(stD));
+		Calendar ed = (Calendar) st.clone();
+		ed.set(Integer.parseInt(year), Integer.parseInt(edM) - 1, Integer.parseInt(edD));
+		while (ed.after(st)) {
+			set.add(getDateStr(st));
+			st.add(Calendar.DATE, 1);
+		}
+		set.add(getDateStr(ed));
+	}
+
+	private String getDateStr(Calendar c) {
+		return getDateStr(c.get(Calendar.MONTH) + 1, c.get(Calendar.DATE));
+	}
+
+	private String getDateStr(int m, int d) {
+		StringBuilder sb = new StringBuilder(year);
+		if (m < 10) {
+			sb.append(0);
+		}
+		sb.append(m);
+		if (d < 10) {
+			sb.append(0);
+		}
+		sb.append(d);
+		return sb.toString();
+	}
+
+}

+ 13 - 18
src/main/java/com/goafanti/patent/controller/PatentApiController.java

@@ -76,7 +76,6 @@ public class PatentApiController extends BaseApiController {
 	@Resource
 	private UserService						userService;
 
-
 	@Value(value = "${upload.private.path}")
 	private String							uploadPrivatePath	= null;
 
@@ -88,7 +87,8 @@ public class PatentApiController extends BaseApiController {
 		Result res = new Result();
 		res = checkCertify(res, TokenManager.getUserId());
 		if (res.getError().isEmpty()) {
-			res.setData(patentInfoService.savePatentInfo(patentInfo, TokenManager.getUserId(), TokenManager.getToken().getAid())); 
+			res.setData(patentInfoService.savePatentInfo(patentInfo, TokenManager.getUserId(),
+					TokenManager.getToken().getAid()));
 		}
 		return res;
 	}
@@ -194,7 +194,7 @@ public class PatentApiController extends BaseApiController {
 		res.setData(map);
 		return res;
 	}
-	
+
 	/**
 	 * 获取管理员
 	 */
@@ -226,7 +226,8 @@ public class PatentApiController extends BaseApiController {
 
 	/**
 	 * 管理端专利详情修改保存
-	 * @throws ParseException 
+	 * 
+	 * @throws ParseException
 	 */
 	@RequestMapping(value = "/managePatentInfo", method = RequestMethod.POST)
 	public Result managePatentInfo(String patentNumber, String patentName, String patentCatagory, String patentField,
@@ -246,7 +247,7 @@ public class PatentApiController extends BaseApiController {
 		patentInfo.setPatentCertificateUrl(patentCertificateUrl);
 		Date recordTime = null;
 		if (!StringUtils.isBlank(recordTimeFormattedDate)) {
-			 recordTime = DateUtils.parseDate(recordTimeFormattedDate, "yyyy-MM-dd");
+			recordTime = DateUtils.parseDate(recordTimeFormattedDate, "yyyy-MM-dd");
 		}
 		patentInfoService.updatePatentInfo(patentInfo, patentLog, recordTime);
 
@@ -315,7 +316,7 @@ public class PatentApiController extends BaseApiController {
 	@RequestMapping(value = "/replyConfirm", method = RequestMethod.POST)
 	public Result replyConfirm(String pid, Integer patentState) {
 		Result res = new Result();
-		if (StringUtils.isBlank(patentInfoService.updateNoticeOfCorrection(res, pid, patentState))){
+		if (StringUtils.isBlank(patentInfoService.updateNoticeOfCorrection(res, pid, patentState))) {
 			res.getError().add(buildError("", "确认失败!"));
 		}
 		return res;
@@ -430,8 +431,7 @@ public class PatentApiController extends BaseApiController {
 		if (res.getError().isEmpty()) {
 			exportComosites(response, composites);
 		} else {
-			FileUtils downloadUtils = new FileUtils();
-			downloadUtils.out(response, res.toString());
+			FileUtils.out(response, res.toString());
 		}
 		return res;
 	}
@@ -448,8 +448,7 @@ public class PatentApiController extends BaseApiController {
 		if (res.getError().isEmpty()) {
 			exportPendings(response, pendings);
 		} else {
-			FileUtils downloadUtils = new FileUtils();
-			downloadUtils.out(response, res.toString());
+			FileUtils.out(response, res.toString());
 		}
 		return res;
 	}
@@ -470,8 +469,7 @@ public class PatentApiController extends BaseApiController {
 		if (res.getError().isEmpty()) {
 			exportFees(response, fees);
 		} else {
-			FileUtils downloadUtils = new FileUtils();
-			downloadUtils.out(response, res.toString());
+			FileUtils.out(response, res.toString());
 		}
 
 		return res;
@@ -541,8 +539,7 @@ public class PatentApiController extends BaseApiController {
 					: calDeadline(obj.getPatentApplicationDateFormattedDate()));
 
 		}
-		FileUtils downloadUtils = new FileUtils();
-		downloadUtils.downloadExcel(response, sheetName + Calendar.getInstance().getTimeInMillis() + ".xls", workbook);
+		FileUtils.downloadExcel(response, sheetName + Calendar.getInstance().getTimeInMillis() + ".xls", workbook);
 	}
 
 	// 待缴年登印费专利管理报表
@@ -597,8 +594,7 @@ public class PatentApiController extends BaseApiController {
 					: calDeadline(obj.getAuthorizedDateFormattedDate()));
 
 		}
-		FileUtils downloadUtils = new FileUtils();
-		downloadUtils.downloadExcel(response, sheetName + Calendar.getInstance().getTimeInMillis() + ".xls", workbook);
+		FileUtils.downloadExcel(response, sheetName + Calendar.getInstance().getTimeInMillis() + ".xls", workbook);
 	}
 
 	private void exportComosites(HttpServletResponse response, List<PatentCompositeBo> composites) {
@@ -628,8 +624,7 @@ public class PatentApiController extends BaseApiController {
 			row.createCell(10).setCellValue(obj.getPatentApplicationFormattedDate());
 			row.createCell(11).setCellValue(obj.getAuthor());
 		}
-		FileUtils downloadUtils = new FileUtils();
-		downloadUtils.downloadExcel(response, sheetName + Calendar.getInstance().getTimeInMillis() + ".xls", workbook);
+		FileUtils.downloadExcel(response, sheetName + Calendar.getInstance().getTimeInMillis() + ".xls", workbook);
 
 	}
 

+ 1 - 0
src/main/resources/spring/spring-mvc.xml

@@ -105,4 +105,5 @@
 		<property name="appName" value="${app.name}" />
 	</bean>
 	<bean id="systemWebSocketHandler" class="com.goafanti.core.websocket.SystemWebSocketHandler" scope="singleton"></bean>
+	
 </beans>