NUTCH-669 - Consolidate code for Fetcher and Fetcher2

git-svn-id: https://svn.apache.org/repos/asf/lucene/nutch/trunk@749289 13f79535-47bb-0310-9956-ffa450edef68

Author: siren

CHANGES.txt

@@ -372,6 +372,8 @@ Unreleased changes (1.0-dev)
 
 139. NUTCH-700 - Neko1.9.11 goes into a loop (Julien Nioche, siren)
 
+140. NUTCH-669 - Consolidate code for Fetcher and Fetcher2 (siren)
+
 
 Release 0.9 - 2007-04-02
 

bin/nutch

@@ -41,7 +41,6 @@ if [ $# = 0 ]; then
   echo "  generate          generate new segments to fetch from crawl db"
   echo "  freegen           generate new segments to fetch from text files"
   echo "  fetch             fetch a segment's pages"
-  echo "  fetch2            fetch a segment's pages using Fetcher2 implementation"
   echo "  parse             parse a segment's pages"
   echo "  readseg           read / dump segment data"
   echo "  mergesegs         merge several segments, with optional filtering and slicing"

src/java/org/apache/nutch/crawl/Crawl.java

@@ -24,7 +24,6 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import org.apache.nutch.fetcher.Fetcher;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.conf.*;
 import org.apache.hadoop.mapred.*;
@@ -36,6 +35,8 @@
 import org.apache.nutch.util.NutchConfiguration;
 import org.apache.nutch.util.NutchJob;
 
+import org.apache.nutch.fetcher.Fetcher;
+
 public class Crawl {
   public static final Log LOG = LogFactory.getLog(Crawl.class);
 
@@ -118,7 +119,7 @@ public static void main(String args[]) throws Exception {
         LOG.info("Stopping at depth=" + i + " - no more URLs to fetch.");
         break;
       }
-      fetcher.fetch(segment, threads);  // fetch it
+      fetcher.fetch(segment, threads, org.apache.nutch.fetcher.Fetcher.isParsing(conf));  // fetch it
       if (!Fetcher.isParsing(job)) {
         parseSegment.parse(segment);    // parse it, if needed
       }