基于WebCollector的全站克隆工具

<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" ????????xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ????????xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> ???<modelVersion>4.0.0</modelVersion> ???<groupId>wyf</groupId> ???<artifactId>CloneSite</artifactId> ???<version>1.0-SNAPSHOT</version> ???<dependencies> ???????<!-- https://mvnrepository.com/artifact/cn.edu.hfut.dmic.webcollector/WebCollector --> ???????<dependency> ???????????<groupId>cn.edu.hfut.dmic.webcollector</groupId> ???????????<artifactId>WebCollector</artifactId> ???????????<version>2.71</version> ???????</dependency> ???????<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup --> ???????<dependency> ???????????<groupId>org.jsoup</groupId> ???????????<artifactId>jsoup</artifactId> ???????????<version>1.11.2</version> ???????</dependency> ???</dependencies> ???<build> ???????<plugins> ???????????<plugin> ???????????????<groupId>org.apache.maven.plugins</groupId> ???????????????<artifactId>maven-compiler-plugin</artifactId> ???????????????<configuration> ???????????????????<source>1.8</source> ???????????????????<target>1.8</target> ???????????????????<encoding>utf-8</encoding> ???????????????</configuration> ???????????</plugin> ???????</plugins> ???</build></project>

import cn.edu.hfut.dmic.webcollector.conf.Configuration;import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;import cn.edu.hfut.dmic.webcollector.model.Page;import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import java.io.BufferedWriter;import java.io.IOException;import java.io.OutputStream;import java.nio.charset.Charset;import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.Paths;public class Main extends BreadthCrawler {static String seed = "http://www.xqbase.com/computer.htm";//html网页前缀static String prefix = "http://www.xqbase.com/computer";static Path targetFolder = Paths.get("haha").toAbsolutePath();int maxRedirect = 0;public Main(String crawlPath, boolean autoParse) { ???super(crawlPath, autoParse); ???this.getConf().setMaxRedirect(maxRedirect); ???this.addSeed(seed);}boolean isInvalidPathChar(char var0) { ???return var0 < ' ' || "<>:\"|?*".indexOf(var0) != -1;}boolean isInvalidPath(String path) { ???for (int i = 0; i < path.length(); i++) if (isInvalidPathChar(path.charAt(i))) return true; ???return false;}/** * 将URL转化为本地的path，用于将网页内容保存到本地 * * @param url：绝对路径url * @param type:文件类型，用于决定保存成的后缀名称 */Path url2path(String url, String type) { ???int beg = url.indexOf(":") + 3; ???String path = url.substring(beg); ???//如果文件名包含不合法字符，那么使用hashcode ???if (isInvalidPath(path)) { ???????path = path.hashCode() + ""; ???} ???if (type != null && !path.endsWith("." + type)) { ???????path += '.' + type; ???} ???return targetFolder.resolve(path);}/** * now表示当前html网页url，resource表示资源文件url，返回二者的相对位置 * resourceType表示是否强制resourceURL发生变化 */String path2relative(String htmlUrl, String resourceUrl, String resourceType) { ???return url2path(htmlUrl, "html").getParent().relativize(url2path(resourceUrl, resourceType)).toString().replace('\\', '/');}/** * 递归创建目录，用于创建文件 */void mkdir(Path p) { ???p = p.toAbsolutePath(); ???if (Files.exists(p)) return; ???if (Files.notExists(p.getParent())) mkdir(p.getParent()); ???try { ???????Files.createDirectory(p); ???} catch (IOException e) { ???????e.printStackTrace(); ???}}/** * 保存文本文件 */void writeFile(Path path, String content, Charset encoding) { ???mkdir(path.getParent()); ???try (BufferedWriter cout = Files.newBufferedWriter(path, encoding)) { ???????cout.write(content); ???} catch (Exception e) { ???????e.printStackTrace(); ???}}/** * 保存二进制文件 */void writeFile(Path path, byte[] data) { ???mkdir(path.getParent()); ???try (OutputStream cout = Files.newOutputStream(path)) { ???????cout.write(data); ???} catch (Exception e) { ???????e.printStackTrace(); ???}}void src(Page page, CrawlDatums crawlDatums, Document doc) { ???String src[] = new String[]{"script", "svg", "img"}; ???for (int ind = 0; ind < src.length; ind++) { ???????String j = src[ind]; ???????for (Element i : doc.select(j)) { ???????????if (i.hasAttr("src") == false) continue; ???????????String s = i.absUrl("src"); ???????????if (s.trim().length() == 0) continue; ???????????i.attr("src", path2relative(page.url(), s, null)); ???????????CrawlDatum next = new CrawlDatum(s, "binary"); ???????????crawlDatums.add(next); ???????} ???}}void hrefOfResource(Page page, CrawlDatums crawlDatums, Document doc) { ???String href[] = new String[]{"link"}; ???for (int ind = 0; ind < href.length; ind++) { ???????String j = href[ind]; ???????for (Element i : doc.select(j)) { ???????????if (i.hasAttr("href") == false) continue; ???????????String s = i.absUrl("href"); ???????????if (s.trim().length() == 0) continue; ???????????i.attr("href", path2relative(page.url(), s, null)); ???????????CrawlDatum next = new CrawlDatum(s, "binary"); ???????????crawlDatums.add(next); ???????} ???}}void hrefOfHtml(Page page, CrawlDatums crawlDatums, Document doc) { ???for (Element i : doc.select("a")) { ???????if (i.hasAttr("href")) { ???????????String s = i.absUrl("href"); ???????????if (s.trim().length() == 0) continue; ???????????i.attr("href", path2relative(page.url(), s, "html")); ???????????if (s.startsWith(prefix)) { ???????????????crawlDatums.add(s); ???????????} ???????} ???}}@Overridepublic void visit(Page page, CrawlDatums crawlDatums) { ???if (page.matchType("binary")) { ???????writeFile(url2path(page.url(), null), page.content()); ???} else { ???????Document doc = page.doc(); ???????src(page, crawlDatums, doc); ???????hrefOfResource(page, crawlDatums, doc); ???????hrefOfHtml(page, crawlDatums, doc); ???????writeFile(url2path(page.url(), "html"), doc.html(), doc.charset()); ???}}public static void main(String[] args) throws Exception { ???//autoparse表示是否让引擎来控制url的解析 ???Main blog = new Main("webcollector", false); ???Configuration conf = blog.getConf(); ???conf.setConnectTimeout(3000); ???blog.start(Integer.MAX_VALUE);}}