基于Java和XxlCrawler获取各城市月度天气情况实践分享

http://www.tianqihoubao.com/lishi/changsha/month/202404.html

@PageSelect(cssQuery = "#content >table >tbody >tr")
@AllArgsConstructor
@NoArgsConstructor
public static class PageVo {
		
	@PageFieldSelect(cssQuery=" >td:eq(0)")
	@Excel(name = "日期")
	private String day;
		
	@PageFieldSelect(cssQuery = ">td:eq(1)")
	@Excel(name = "天气状况")
	private String weatherInfo;
		
	@PageFieldSelect(cssQuery = ">td:eq(2)")
	@Excel(name = "最低气温/最高气温")
	private String weatherTemp;
		
	@PageFieldSelect(cssQuery = ">td:eq(3)")
	@Excel(name = "风力风向(夜间/白天)")
	private String weatherWind;
}

private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36";
 
private List<PageVo> weatherList = new ArrayList<PageVo>();
	
private static final String COMMON_URL  = "http://www.tianqihoubao.com/lishi/changsha/month/";

@Test
public void fetchWeaterInfo() {
	int year = 2023;
    int month = 9; // 月份从 0 开始，3 表示4月份
    String monthStr = "";
    monthStr +=  month < 10 ? "0" + month : month;
    String targetUrl = COMMON_URL + year + monthStr + ".html";
    System.out.println(targetUrl);
	// 构造爬虫
	XxlCrawler crawler = new XxlCrawler.Builder()
		.setUrls(targetUrl)
		.setAllowSpread(false)// 不允许扩散爬取
		.setThreadCount(3)
		.setPauseMillis(3000)
		.setUserAgent(USER_AGENT)//设置user_agent
		.setIfPost(false)
		.setFailRetryCount(3)// 重试三次
		.setTimeoutMillis(1000 * 12)//超时时间，有些网站加载慢，一定要加这个时间
		.setPageParser(new PageParser<PageVo>() {
					
			@Override
			public void parse(Document html, Element pageVoElement, PageVo pageVo) {
				//System.out.println(pageVoElement);
				weatherList.add(pageVo);
			}
	}).build();
    crawler.start(true);// 启动
	// excel工具包
	weatherList.remove(0);//把第一行表头移除掉
	ExcelUtil<PageVo> util = new ExcelUtil<PageVo>(PageVo.class);
	util.exportExcel(weatherList, year + "年" + month + "月天气情况表");
	System.out.println("finished...");
}

weatherList.remove(0);//把第一行表头移除掉

http://www.tianqihoubao.com/lishi/changsha/month/202309.html
09:42:29.027 [main] INFO com.xuxueli.crawler.rundata.strategy.LocalRunData - >>>>>>>>>>> xxl-crawler addUrl success, link: http://www.tianqihoubao.com/lishi/changsha/month/202309.html
09:42:29.035 [main] INFO com.xuxueli.crawler.XxlCrawler - >>>>>>>>>>> xxl crawler start ...
09:42:29.038 [pool-1-thread-1] INFO com.xuxueli.crawler.thread.CrawlerThread - >>>>>>>>>>> xxl crawler, process link : http://www.tianqihoubao.com/lishi/changsha/month/202309.html
09:42:33.634 [pool-1-thread-1] INFO com.xuxueli.crawler.XxlCrawler - >>>>>>>>>>> xxl crawler is finished.
09:42:33.635 [pool-1-thread-2] INFO com.xuxueli.crawler.thread.CrawlerThread - >>>>>>>>>>> xxl crawler thread LocalRunData.getUrl interrupted.
09:42:33.635 [pool-1-thread-3] INFO com.xuxueli.crawler.thread.CrawlerThread - >>>>>>>>>>> xxl crawler thread LocalRunData.getUrl interrupted.
09:42:33.635 [pool-1-thread-1] INFO com.xuxueli.crawler.XxlCrawler - >>>>>>>>>>> xxl crawler stop.
09:42:33.636 [pool-1-thread-1] INFO com.xuxueli.crawler.thread.CrawlerThread - >>>>>>>>>>> xxl crawler thread LocalRunData.getUrl interrupted.
finished...

private volatile int timeoutMillis = XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT;     // 超时时间，毫秒
 
 // timeout default, ms
public static final int TIMEOUT_MILLIS_DEFAULT = 5*1000;

http://www.tianqihoubao.com/lishi/changsha/month/202404.html
11:03:27.685 [main] INFO com.xuxueli.crawler.rundata.strategy.LocalRunData - >>>>>>>>>>> xxl-crawler addUrl success, link: http://www.tianqihoubao.com/lishi/changsha/month/202404.html
11:03:27.693 [main] INFO com.xuxueli.crawler.XxlCrawler - >>>>>>>>>>> xxl crawler start ...
11:03:27.695 [pool-1-thread-2] INFO com.xuxueli.crawler.thread.CrawlerThread - >>>>>>>>>>> xxl crawler, process link : http://www.tianqihoubao.com/lishi/changsha/month/202404.html
11:03:32.697 [main] INFO com.xuxueli.crawler.XxlCrawler - >>>>>>>>>>> xxl crawler still running ...
11:03:33.456 [pool-1-thread-2] ERROR com.xuxueli.crawler.util.JsoupUtil - Read timeout
java.net.SocketTimeoutException: Read timeout
	at org.jsoup.internal.ConstrainableInputStream.read(ConstrainableInputStream.java:58)
	at sun.nio.cs.StreamDecoder.readBytes(Unknown Source)
	at sun.nio.cs.StreamDecoder.implRead(Unknown Source)
	at sun.nio.cs.StreamDecoder.read(Unknown Source)
	at java.io.InputStreamReader.read(Unknown Source)
	at java.io.BufferedReader.fill(Unknown Source)
	at java.io.BufferedReader.read1(Unknown Source)
	at java.io.BufferedReader.read(Unknown Source)
	at org.jsoup.parser.CharacterReader.bufferUp(CharacterReader.java:87)
	at org.jsoup.parser.CharacterReader.current(CharacterReader.java:246)
	at org.jsoup.parser.TokeniserState$1.read(TokeniserState.java:12)
	at org.jsoup.parser.Tokeniser.read(Tokeniser.java:62)
	at org.jsoup.parser.TreeBuilder.runParser(TreeBuilder.java:86)
	at org.jsoup.parser.TreeBuilder.parse(TreeBuilder.java:61)
	at org.jsoup.parser.Parser.parseInput(Parser.java:51)
	at org.jsoup.helper.DataUtil.parseInputStream(DataUtil.java:218)
	at org.jsoup.helper.HttpConnection$Response.parse(HttpConnection.java:962)
	at org.jsoup.helper.HttpConnection.get(HttpConnection.java:355)
	at com.xuxueli.crawler.util.JsoupUtil.load(JsoupUtil.java:77)
	at com.xuxueli.crawler.loader.strategy.JsoupPageLoader.load(JsoupPageLoader.java:17)
	at com.xuxueli.crawler.thread.CrawlerThread.processPage(CrawlerThread.java:167)
	at com.xuxueli.crawler.thread.CrawlerThread.run(CrawlerThread.java:84)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.lang.Thread.run(Unknown Source)