[prev in list] [next in list] [prev in thread] [next in thread]
List: hadoop-commits
Subject: svn commit: r1497964 - in /hadoop/common/branches/branch-1.2: ./ src/core/org/apache/hadoop/fs/ src/
From: acmurthy () apache ! org
Date: 2013-06-29 0:57:09
Message-ID: 20130629005710.330862388847 () eris ! apache ! org
[Download RAW message or body]
Author: acmurthy
Date: Sat Jun 29 00:57:09 2013
New Revision: 1497964
URL: http://svn.apache.org/r1497964
Log:
Merge -r 1497961:1497963 from branch-1 to branch-1.2 to fix MAPREDUCE-5351. Fixed a \
memory leak in JobTracker due to stable FS objects in FSCache. Contributed by Sandy \
Ryza.
Added:
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
- copied unchanged from r1497963, \
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
Modified:
hadoop/common/branches/branch-1.2/CHANGES.txt
hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Modified: hadoop/common/branches/branch-1.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/CHANGES.txt?rev=1497964&r1=1497963&r2=1497964&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.2/CHANGES.txt Sat Jun 29 00:57:09 2013
@@ -40,6 +40,9 @@ Release 1.2.1 - Unreleased
HADOOP-9665. Fixed BlockDecompressorStream#decompress to return -1 rather
than throw EOF at end of file. (Zhijie Shen via acmurthy)
+ MAPREDUCE-5351. Fixed a memory leak in JobTracker due to stable FS objects in
+ FSCache. (Sandy Ryza via acmurthy)
+
Release 1.2.0 - 2013.05.05
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/fs/FileSystem.java?rev=1497964&r1=1497963&r2=1497964&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/fs/FileSystem.java \
(original)
+++ hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/fs/FileSystem.java \
Sat Jun 29 00:57:09 2013 @@ -36,6 +36,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.*;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -146,6 +147,14 @@ public abstract class FileSystem extends
public static void setDefaultUri(Configuration conf, String uri) {
setDefaultUri(conf, URI.create(fixName(uri)));
}
+
+ /** Get the number of entries in the filesystem cache
+ * @return the number of entries in the filesystem cache
+ */
+ @Private
+ public static int getCacheSize() {
+ return CACHE.map.size();
+ }
/** Called after a new FileSystem instance is constructed.
* @param name a uri whose authority section names the host, port, etc.
Modified: hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java?rev=1497964&r1=1497963&r2=1497964&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java \
(original)
+++ hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java \
Sat Jun 29 00:57:09 2013 @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal;
import org.apache.hadoop.security.UserGroupInformation;
@@ -105,8 +106,15 @@ public class CleanupQueue {
(ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs(
new PrivilegedExceptionAction<Object>() {
public Object run() throws IOException {
- p.getFileSystem(conf).delete(p, true);
- return null;
+ FileSystem fs = p.getFileSystem(conf);
+ try {
+ fs.delete(p, true);
+ return null;
+ } finally {
+ // So that we don't leave an entry in the FileSystem cache for
+ // every job.
+ fs.close();
+ }
}
});
Modified: hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1497964&r1=1497963&r2=1497964&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java \
(original)
+++ hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java \
Sat Jun 29 00:57:09 2013 @@ -3307,6 +3307,7 @@ public class JobInProgress {
* removing all delegation token etc.
*/
void cleanupJob() {
+ FileSystem tempDirFs = null;
synchronized (this) {
try {
// Definitely remove the local-disk copy of the job file
@@ -3324,6 +3325,7 @@ public class JobInProgress {
if (jobTempDir != null && conf.getKeepTaskFilesPattern() == null &&
!conf.getKeepFailedTaskFiles()) {
Path jobTempDirPath = new Path(jobTempDir);
+ tempDirFs = jobTempDirPath.getFileSystem(conf);
CleanupQueue.getInstance().addToQueue(
new PathDeletionContext(jobTempDirPath, conf, userUGI, jobId));
}
@@ -3341,12 +3343,15 @@ public class JobInProgress {
this.runningReduces = null;
}
- //close the user's FS
- try {
- fs.close();
- } catch (IOException ie) {
- LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) +
- " while closing FileSystem for " + userUGI);
+ // Close the user's FS. Or don't, in the common case of FS being the same
+ // FS as the temp directory FS, as it will be closed by the CleanupQueue.
+ if (tempDirFs != fs) {
+ try {
+ fs.close();
+ } catch (IOException ie) {
+ LOG.warn("Ignoring exception " + StringUtils.stringifyException(ie) +
+ " while closing FileSystem for " + userUGI);
+ }
}
}
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic