[prev in list] [next in list] [prev in thread] [next in thread] 

List:       hadoop-commits
Subject:    [hadoop] branch branch-3.3 updated: HDFS-16732. [SBN READ] Avoid get location from observer when the
From:       xkrogen () apache ! org
Date:       2022-08-25 17:54:31
Message-ID: 166145007111.3617471.8239011148068173456 () gitbox2-he-fi ! apache ! org
[Download RAW message or body]

This is an automated email from the ASF dual-hosted git repository.

xkrogen pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
     new 3edddaf9fcf HDFS-16732. [SBN READ] Avoid get location from observer when the \
block report is delayed (#4756) 3edddaf9fcf is described below

commit 3edddaf9fcf97367a09f0c2fa9578e3548ea2559
Author: zhengchenyu <zhengchenyu16@gmail.com>
AuthorDate: Fri Aug 26 01:37:25 2022 +0800

    HDFS-16732. [SBN READ] Avoid get location from observer when the block report is \
delayed (#4756)  
    Signed-off-by: Erik Krogen <xkrogen@apache.org>
    
    (cherry picked from commit 231a4468cdb83f9c2ff8897e70fe7c3d23b58cf4)
---
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  | 36 +++++++++++++++++-----
 .../hdfs/server/namenode/ha/TestObserverNode.java  |  6 ++++
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 index 0df3f0499cb..cb0cf39760e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 @@ -103,6 +103,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.commons.text.CaseUtils;
 import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
 import static org.apache.hadoop.ha.HAServiceProtocol.HAServiceState.ACTIVE;
@@ -2138,14 +2139,8 @@ public class FSNamesystem implements Namesystem, \
FSNamesystemMBean,  }
             }
           }
-        } else if (haEnabled && haContext != null &&
-            haContext.getState().getServiceState() == OBSERVER) {
-          for (LocatedBlock b : res.blocks.getLocatedBlocks()) {
-            if (b.getLocations() == null || b.getLocations().length == 0) {
-              throw new ObserverRetryOnActiveException("Zero blocklocations "
-                  + "for " + srcArg);
-            }
-          }
+        } else if (isObserver()) {
+          checkBlockLocationsWhenObserver(res.blocks, srcArg);
         }
       } finally {
         readUnlock(operationName);
@@ -3390,6 +3385,10 @@ public class FSNamesystem implements Namesystem, \
FSNamesystemMBean,  logAuditEvent(false, operationName, src);
       throw e;
     }
+    if (needLocation && isObserver() && stat instanceof HdfsLocatedFileStatus) {
+      LocatedBlocks lbs = ((HdfsLocatedFileStatus) stat).getLocatedBlocks();
+      checkBlockLocationsWhenObserver(lbs, src);
+    }
     logAuditEvent(true, operationName, src);
     return stat;
   }
@@ -4088,6 +4087,14 @@ public class FSNamesystem implements Namesystem, \
FSNamesystemMBean,  logAuditEvent(false, operationName, src);
       throw e;
     }
+    if (needLocation && isObserver()) {
+      for (HdfsFileStatus fs : dl.getPartialListing()) {
+        if (fs instanceof HdfsLocatedFileStatus) {
+          LocatedBlocks lbs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks();
+          checkBlockLocationsWhenObserver(lbs, fs.toString());
+        }
+      }
+    }
     logAuditEvent(true, operationName, src);
     return dl;
   }
@@ -8733,4 +8740,17 @@ public class FSNamesystem implements Namesystem, \
                FSNamesystemMBean,
       throw new UnsupportedActionException(operationName + " not supported.");
     }
   }
+
+  private boolean isObserver() {
+    return haEnabled && haContext != null && haContext.getState().getServiceState() \
== OBSERVER; +  }
+
+  private void checkBlockLocationsWhenObserver(LocatedBlocks blocks, String src)
+      throws ObserverRetryOnActiveException {
+    for (LocatedBlock b : blocks.getLocatedBlocks()) {
+      if (b.getLocations() == null || b.getLocations().length == 0) {
+        throw new ObserverRetryOnActiveException("Zero blocklocations for " + src);
+      }
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java \
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
 index 29cae6f13ad..a9101171945 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
                
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
 @@ -369,6 +369,12 @@ public class TestObserverNode {
     dfs.open(testPath);
     assertSentTo(0);
 
+    dfs.getClient().listPaths("/", new byte[0], true);
+    assertSentTo(0);
+
+    dfs.getClient().getLocatedFileInfo(testPath.toString(), false);
+    assertSentTo(0);
+
     Mockito.reset(bmSpy);
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[prev in list] [next in list] [prev in thread] [next in thread]