[prev in list] [next in list] [prev in thread] [next in thread] 

List:       hadoop-commits
Subject:    [hadoop] branch trunk updated: HDFS-15500. In-order deletion of snapshots: Diff lists must be update
From:       shashikant () apache ! org
Date:       2020-08-27 9:25:23
Message-ID: 159852032296.29891.3216677470861931802 () gitbox ! apache ! org
[Download RAW message or body]

This is an automated email from the ASF dual-hosted git repository.

shashikant pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 41182a9  HDFS-15500. In-order deletion of snapshots: Diff lists must be \
update only in the last snapshot. (#2233) 41182a9 is described below

commit 41182a9b6d81d0c8a4dc0a9cf89ea0ade815afd3
Author: Tsz-Wo Nicholas Sze <szetszwo@apache.org>
AuthorDate: Thu Aug 27 02:24:52 2020 -0700

    HDFS-15500. In-order deletion of snapshots: Diff lists must be update only in the \
                last snapshot. (#2233)
---
 .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java |  6 ++++++
 .../hdfs/server/namenode/snapshot/AbstractINodeDiffList.java |  6 +++++-
 .../hdfs/server/namenode/snapshot/DiffListByArrayList.java   |  2 ++
 .../namenode/snapshot/DirectoryWithSnapshotFeature.java      |  2 ++
 .../hdfs/server/namenode/snapshot/SnapshotManager.java       | 12 ++++++++++++
 5 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 index 15bf6b1..badf237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 @@ -1569,6 +1569,12 @@ public class FSNamesystem implements Namesystem, \
FSNamesystemMBean,  // null in some unit tests
       haContext.checkOperation(op);
     }
+
+    boolean assertsEnabled = false;
+    assert assertsEnabled = true; // Intentional side effect!!!
+    if (assertsEnabled && op == OperationCategory.WRITE) {
+      getSnapshotManager().initThreadLocals();
+    }
   }
   
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
 index 776adf1..16e3b75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
 @@ -76,7 +76,11 @@ abstract class AbstractINodeDiffList<N extends INode,
     if (diffs == null) {
       return;
     }
-    int snapshotIndex = diffs.binarySearch(snapshot);
+    final int snapshotIndex = diffs.binarySearch(snapshot);
+    // DeletionOrdered: only can remove the element at index 0 and no prior
+    //                  check snapshotIndex <= 0 since the diff may not exist
+    assert !SnapshotManager.isDeletionOrdered()
+        || (snapshotIndex <= 0 && prior == Snapshot.NO_SNAPSHOT_ID);
 
     D removed;
     if (snapshotIndex == 0) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
 index 95c23df..7fa3f05 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
 @@ -57,6 +57,8 @@ public class DiffListByArrayList<T extends Comparable<Integer>>
 
   @Override
   public T remove(int i) {
+    // DeletionOrdered: only can remove the element at index 0
+    assert !SnapshotManager.isDeletionOrdered() || i == 0;
     return list.remove(i);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
 index b8f7b65..c3a9aa1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
 @@ -175,6 +175,8 @@ public class DirectoryWithSnapshotFeature implements \
INode.Feature {  final INode.ReclaimContext reclaimContext,
         final INodeDirectory currentDir,
         final DirectoryDiff posterior) {
+      // DeletionOrdered: must not combine posterior
+      assert !SnapshotManager.isDeletionOrdered();
       diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
         /** Collect blocks for deleted files. */
         @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
 index b5b0971..789fa3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
 @@ -95,6 +95,18 @@ public class SnapshotManager implements SnapshotStatsMXBean {
   static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT
       = 5 * 60_000L; //5 minutes
 
+  private static final ThreadLocal<Boolean> DELETION_ORDERED
+      = new ThreadLocal<>();
+
+  static boolean isDeletionOrdered() {
+    final Boolean b = DELETION_ORDERED.get();
+    return b != null? b: false;
+  }
+
+  public void initThreadLocals() {
+    DELETION_ORDERED.set(isSnapshotDeletionOrdered());
+  }
+
   private final FSDirectory fsdir;
   private boolean captureOpenFiles;
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[prev in list] [next in list] [prev in thread] [next in thread]