[prev in list] [next in list] [prev in thread] [next in thread] 

List:       flume-commits
Subject:    git commit: FLUME-1872: SpoolingDirectorySource doesn't delete tracker file when deletePolicy is imm
From:       brock () apache ! org
Date:       2013-01-25 21:51:21
Message-ID: 20130125215121.CFC38825638 () tyr ! zones ! apache ! org
[Download RAW message or body]

Updated Branches:
  refs/heads/trunk f32e8cd4c -> ea5ac469e


FLUME-1872: SpoolingDirectorySource doesn't delete tracker file when deletePolicy is \
immediate

(Mike Percy via Brock Noland)


Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/ea5ac469
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/ea5ac469
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/ea5ac469

Branch: refs/heads/trunk
Commit: ea5ac469ee31dafe0fb4d300afe9fc6774df278e
Parents: f32e8cd
Author: Brock Noland <brock@apache.org>
Authored: Fri Jan 25 15:50:07 2013 -0600
Committer: Brock Noland <brock@apache.org>
Committed: Fri Jan 25 15:50:54 2013 -0600

----------------------------------------------------------------------
 .../avro/ReliableSpoolingFileEventReader.java      |    2 +
 .../avro/TestReliableSpoolingFileEventReader.java  |   31 +++++++++++++--
 2 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flume/blob/ea5ac469/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
                
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java \
b/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
 index 28df24c..37f0ee1 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
                
+++ b/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
 @@ -371,6 +371,8 @@ public class ReliableSpoolingFileEventReader implements \
ReliableEventReader {  if (!fileToDelete.delete()) {
       throw new IOException("Unable to delete spool file: " + fileToDelete);
     }
+    // now we no longer need the meta file
+    deleteMetaFile();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/flume/blob/ea5ac469/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
                
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java \
b/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
 index 31ecf8e..9d708c1 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
                
+++ b/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
 @@ -93,11 +93,30 @@ public class TestReliableSpoolingFileEventReader {
 
   }
 
-  // FIXME: implement ignore pattern test
-  @Ignore
   @Test
-  public void testIgnorePattern() {
-    ReliableSpoolingFileEventReader parser;
+  public void testIgnorePattern() throws IOException {
+    ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder()
+        .spoolDirectory(WORK_DIR)
+        .ignorePattern("^file2$")
+        .deletePolicy(DeletePolicy.IMMEDIATE.toString())
+        .build();
+
+    List<File> before = listFiles(WORK_DIR);
+    Assert.assertEquals("Expected 5, not: " + before, 5, before.size());
+
+    List<Event> events;
+    do {
+      events = reader.readEvents(10);
+      reader.commit();
+    } while (!events.isEmpty());
+
+    List<File> after = listFiles(WORK_DIR);
+    Assert.assertEquals("Expected 1, not: " + after, 1, after.size());
+    Assert.assertEquals("file2", after.get(0).getName());
+    List<File> trackerFiles = listFiles(new File(WORK_DIR,
+        SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR));
+    Assert.assertEquals("Expected 0, not: " + trackerFiles, 0,
+        trackerFiles.size());
   }
 
   @Test
@@ -163,6 +182,10 @@ public class TestReliableSpoolingFileEventReader {
 
     List<File> after = listFiles(WORK_DIR);
     Assert.assertEquals("Expected 0, not: " + after, 0, after.size());
+    List<File> trackerFiles = listFiles(new File(WORK_DIR,
+        SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR));
+    Assert.assertEquals("Expected 0, not: " + trackerFiles, 0,
+        trackerFiles.size());
   }
 
   private static List<File> listFiles(File dir) {


[prev in list] [next in list] [prev in thread] [next in thread]