Nutch教程——URLNormalizer源码详解

URL正规化（URLNormalize）对大多数网络爬虫来说是一个非常重要的流程,大部分爬虫的去重机制都是基于URL的去重,在实际中,不同的URL有可能对应同一个网页,例如下面的几个URL：

这些URL其实都指向http://nutcher.org/,在文件路径中,../表示上级目录, ./表示当前目录,所以/abc/../和/./都等价于/.如果爬虫没有URL正规化机制, 爬虫会认为http://nutcher.org/、http://nutcher.org/abc/../、http://nutcher.org/./是三个不同的页面,而三个URL实际都指向http://nutcher.org/,因此会将http://nutcher.org/爬取三次.

可以看出,URL正规化的主要目的就是防止网页的重复采集.这里我们介绍Nutch的URL正规化组件之一, urlnormalizer-basic,该组件在nutch中对应的类是BasicURLNormalizer.java，这里我们用源码注释的方式描述urlnormalizer-basic的工作机制：

  1. package org.apache.nutch.net.urlnormalizer.basic;
  3. import java.net.URL;
  4. import java.net.MalformedURLException;
  6. // Slf4j Logging imports
  7. import org.slf4j.Logger;
  8. import org.slf4j.LoggerFactory;
  10. // Nutch imports
  11. import org.apache.nutch.net.URLNormalizer;
  13. import org.apache.hadoop.conf.Configuration;
  14. import org.apache.hadoop.conf.Configured;
  15. import org.apache.oro.text.regex.*;
  17. /**
  18.  * Converts URLs to a normal form:
  19.  * <ul>
  20.  * <li>remove dot segments in path: <code>/./</code> or <code>/../</code></li>
  21.  * <li>remove default ports, e.g. 80 for protocol <code>http://</code></li>
  22.  * </ul>
  23.  * 将URL转换为正规的形式
  24.  * 移除URL中类似/./和/../的部分
  25.  * 移除默认端口，例如对于http协议，移除URL中指定的80端口
  26.  */
  27. public class BasicURLNormalizer extends Configured implements URLNormalizer {
  28.     public static final Logger LOG = LoggerFactory.getLogger(BasicURLNormalizer.class);
  30.     /*这里使用Perl规范的正则，借助了org.apache.oro.text.regex.Perl5Compiler类*/
  31.     private Perl5Compiler compiler = new Perl5Compiler();
  32.     private ThreadLocal<Perl5Matcher> matchers = new ThreadLocal<Perl5Matcher>() {
  33.         protected Perl5Matcher initialValue() {
  34.           return new Perl5Matcher();
  35.         }
  36.       };
  37.     private final Rule relativePathRule;
  38.     private final Rule leadingRelativePathRule;
  39.     private final Rule currentPathRule;
  40.     private final Rule adjacentSlashRule;
  42.     private final static java.util.regex.Pattern hasNormalizablePattern = java.util.regex.Pattern.compile("/\\.?\\.?/");
  44.     private Configuration conf;
  46.     public BasicURLNormalizer() {
  47.       try {
  48.         // this pattern tries to find spots like "/xx/../" in the url, which
  49.         // could be replaced by "/" xx consists of chars, different then "/"
  50.         // (slash) and needs to have at least one char different from "."
  51.         // 这里希望从URL中找到匹配/xx/../的部分,替换为/
  52.         // xx中的每个字符不能是'/'，且至少出现一个不为'.'的字符
  53.         relativePathRule = new Rule();
  54.         relativePathRule.pattern = (Perl5Pattern)
  55.           compiler.compile("(/[^/]*[^/.]{1}[^/]*/\\.\\./)",
  56.                            Perl5Compiler.READ_ONLY_MASK);
  57.         relativePathRule.substitution = new Perl5Substitution("/");
  60.         // this pattern tries to find spots like leading "/../" in the url,
  61.         // which could be replaced by "/"
  62.         // 如果URL中的路径，以/../起始,将其替换为/
  63.         leadingRelativePathRule = new Rule();
  64.         leadingRelativePathRule.pattern = (Perl5Pattern)
  65.           compiler.compile("^(/\\.\\./)+", Perl5Compiler.READ_ONLY_MASK);
  66.         leadingRelativePathRule.substitution = new Perl5Substitution("/");
  69.         // this pattern tries to find spots like "/./" in the url,
  70.         // which could be replaced by "/"
  71.         // 这里希望从URL中找到匹配/./的部分,替换为/
  72.         currentPathRule = new Rule();
  73.         currentPathRule.pattern = (Perl5Pattern)
  74.           compiler.compile("(/\\./)", Perl5Compiler.READ_ONLY_MASK);
  75.         currentPathRule.substitution = new Perl5Substitution("/");
  78.         // this pattern tries to find spots like "xx//yy" in the url,
  79.         // which could be replaced by a "/"
  80.         // 这里希望从URL中找到匹配//的部分,替换为/
  81.         adjacentSlashRule = new Rule();
  82.         adjacentSlashRule.pattern = (Perl5Pattern)
  83.           compiler.compile("/{2,}", Perl5Compiler.READ_ONLY_MASK);
  84.         adjacentSlashRule.substitution = new Perl5Substitution("/");
  86.       } catch (MalformedPatternException e) {
  87.         throw new RuntimeException(e);
  88.       }
  89.     }
  91.     public String normalize(String urlString, String scope)
  92.             throws MalformedURLException {
  93.         if ("".equals(urlString))                     // permit empty
  94.             return urlString;
  96.         urlString = urlString.trim();                 // remove extra spaces
  98.         URL url = new URL(urlString);
  100.         String protocol = url.getProtocol();
  101.         String host = url.getHost();
  102.         int port = url.getPort();
  103.         String file = url.getFile();
  105.         boolean changed = false;
  107.         if (!urlString.startsWith(protocol))        // protocol was lowercased
  108.             changed = true;
  110.         if ("http".equals(protocol) || "https".equals(protocol) || "ftp".equals(protocol)) {
  112.             if (host != null) {
  113.                 String newHost = host.toLowerCase();    // lowercase host
  114.                 if (!host.equals(newHost)) {
  115.                     host = newHost;
  116.                     changed = true;
  117.                 }
  118.             }
  120.             if (port == url.getDefaultPort()) {       // uses default port 如果URL中出现端口信息，检查是否为URL对应协议默认端口
  121.                 port = -1;                              // so don't specify it
  122.                 changed = true;
  123.             }
  125.             if (file == null || "".equals(file)) {    // add a slash
  126.                 file = "/";
  127.                 changed = true;
  128.             }
  130.             if (url.getRef() != null) {                 // remove the ref
  131.                 changed = true;
  132.             }
  134.             // check for unnecessary use of "/../"
  135.             String file2 = substituteUnnecessaryRelativePaths(file);
  137.             if (!file.equals(file2)) {
  138.                 changed = true;
  139.                 file = file2;
  140.             }
  142.         }
  144.         if (changed)
  145.             urlString = new URL(protocol, host, port, file).toString();
  147.         return urlString;
  148.     }
  150.     private String substituteUnnecessaryRelativePaths(String file) {
  152.         if (!hasNormalizablePattern.matcher(file).find())
  153.             return file;
  155.         String fileWorkCopy = file;
  156.         int oldLen = file.length();
  157.         int newLen = oldLen - 1;
  159.         // All substitutions will be done step by step, to ensure that certain
  160.         // constellations will be normalized, too
  161.         //
  162.         // URL正规化会一步一步进行，下面给出一个例子
  163.         //
  164.         // For example: "/aa/bb/../../cc/../foo.html will be normalized in the
  165.         // following manner:
  166.         //   "/aa/bb/../../cc/../foo.html"
  167.         //   "/aa/../cc/../foo.html"
  168.         //   "/cc/../foo.html"
  169.         //   "/foo.html"
  170.         //
  171.         // The normalization also takes care of leading "/../", which will be
  172.         // replaced by "/", because this is a rather a sign of bad webserver
  173.         // configuration than of a wanted link.  For example, urls like
  174.         // "http://www.foo.com/../" should return a http 404 error instead of
  175.         // redirecting to "http://www.foo.com".
  176.         //
  177.         Perl5Matcher matcher = (Perl5Matcher)matchers.get();
  179.         while (oldLen != newLen) {
  180.             // substitue first occurence of "/xx/../" by "/"
  181.             oldLen = fileWorkCopy.length();
  182.             fileWorkCopy = Util.substitute
  183.               (matcher, relativePathRule.pattern,
  184.                relativePathRule.substitution, fileWorkCopy, 1);
  186.             // remove leading "/../"
  187.             fileWorkCopy = Util.substitute
  188.               (matcher, leadingRelativePathRule.pattern,
  189.                leadingRelativePathRule.substitution, fileWorkCopy, 1);
  191.             // remove unnecessary "/./"
  192.             fileWorkCopy = Util.substitute
  193.             (matcher, currentPathRule.pattern,
  194.                     currentPathRule.substitution, fileWorkCopy, 1);
  197.             // collapse adjacent slashes with "/"
  198.             fileWorkCopy = Util.substitute
  199.             (matcher, adjacentSlashRule.pattern,
  200.               adjacentSlashRule.substitution, fileWorkCopy, 1);
  202.             newLen = fileWorkCopy.length();
  203.         }
  205.         return fileWorkCopy;
  206.     }
  209.     /**
  210.      * Class which holds a compiled pattern and its corresponding substition
  211.      * string.
  212.      */
  213.     private static class Rule {
  214.         public Perl5Pattern pattern;
  215.         public Perl5Substitution substitution;
  216.     }
  219.   public void setConf(Configuration conf) {
  220.     this.conf = conf;
  221.   }
  223.   public Configuration getConf() {
  224.     return this.conf;
  225.   }
  227. }