[prev in list] [next in list] [prev in thread] [next in thread] 

List:       nutch-cvs
Subject:    [Nutch-cvs] svn commit: r240286 - in
From:       cutting () apache ! org
Date:       2005-08-26 16:51:27
Message-ID: 20050826165128.46554.qmail () minotaur ! apache ! org
[Download RAW message or body]

Author: cutting
Date: Fri Aug 26 09:51:25 2005
New Revision: 240286

URL: http://svn.apache.org/viewcvs?rev=240286&view=rev
Log:
Add reporter to RecordWriter.close().

Modified:
    lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/FetcherOutputFormat.java
  lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/Indexer.java
    lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/ParseOutputFormat.java


Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/FetcherOutputFormat.java
                
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/FetcherOutputFormat.java?rev=240286&r1=240285&r2=240286&view=diff
 ==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/FetcherOutputFormat.java \
                (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/FetcherOutputFormat.java \
Fri Aug 26 09:51:25 2005 @@ -29,6 +29,7 @@
 import org.apache.nutch.mapred.OutputFormat;
 import org.apache.nutch.mapred.RecordWriter;
 import org.apache.nutch.mapred.JobConf;
+import org.apache.nutch.mapred.Reporter;
 
 import org.apache.nutch.protocol.Content;
 
@@ -79,13 +80,13 @@
 
         }
 
-        public void close() throws IOException {
+        public void close(Reporter reporter) throws IOException {
           fetchOut.close();
           if (contentOut != null) {
             contentOut.close();
           }
           if (parseOut != null) {
-            parseOut.close();
+            parseOut.close(reporter);
           }
         }
 

Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/Indexer.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/Indexer.java?rev=240286&r1=240285&r2=240286&view=diff
 ==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/Indexer.java \
                (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/Indexer.java Fri Aug \
26 09:51:25 2005 @@ -89,18 +89,37 @@
       writer.setSimilarity(new NutchSimilarity());
 
       return new RecordWriter() {
+          boolean closed;
 
           public void write(WritableComparable key, Writable value)
             throws IOException {                  // unwrap & index doc
             writer.addDocument((Document)((ObjectWritable)value).get());
           }
           
-          public void close() throws IOException {
-            LOG.info("Optimizing index.");        // optimize & close index
-            writer.optimize();
-            writer.close();
-            fs.completeLocalOutput(perm, temp);   // copy to ndfs
-            fs.createNewFile(new File(perm, IndexSegment.DONE_NAME));
+          public void close(final Reporter reporter) throws IOException {
+            // spawn a thread to give progress heartbeats
+            Thread prog = new Thread() {
+                public void run() {
+                  while (!closed) {
+                    try {
+                      reporter.setStatus("closing");
+                      Thread.sleep(1000);
+                    } catch (InterruptedException e) { continue; }
+                      catch (Throwable e) { return; }
+                  }
+                }
+              };
+
+            try {
+              prog.start();
+              LOG.info("Optimizing index.");        // optimize & close index
+              writer.optimize();
+              writer.close();
+              fs.completeLocalOutput(perm, temp);   // copy to ndfs
+              fs.createNewFile(new File(perm, IndexSegment.DONE_NAME));
+            } finally {
+              closed = true;
+            }
           }
         };
     }

Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/ParseOutputFormat.java
                
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/ParseOutputFormat.java?rev=240286&r1=240285&r2=240286&view=diff
 ==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/ParseOutputFormat.java \
                (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/crawl/ParseOutputFormat.java \
Fri Aug 26 09:51:25 2005 @@ -79,7 +79,7 @@
           }
         }
         
-        public void close() throws IOException {
+        public void close(Reporter reporter) throws IOException {
           textOut.close();
           dataOut.close();
           crawlOut.close();




-------------------------------------------------------
SF.Net email is Sponsored by the Better Software Conference & EXPO
September 19-22, 2005 * San Francisco, CA * Development Lifecycle Practices
Agile & Plan-Driven Development * Managing Projects & Teams * Testing & QA
Security * Process Improvement & Measurement * http://www.sqe.com/bsce5sf
_______________________________________________
Nutch-cvs mailing list
Nutch-cvs@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nutch-cvs


[prev in list] [next in list] [prev in thread] [next in thread]