[prev in list] [next in list] [prev in thread] [next in thread]
List: hadoop-commits
Subject: hadoop git commit: YARN-7677. Docker image cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan
From: jlowe () apache ! org
Date: 2018-01-31 17:46:27
Message-ID: 6b705be921da4335be20b1d2ef2d856d () git ! apache ! org
[Download RAW message or body]
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 961dcb693 -> 874bdbc40
YARN-7677. Docker image cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan
(cherry picked from commit 12eaae383ad06de8f9959241b2451dec82cf9ceb)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src \
/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/874bdbc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/874bdbc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/874bdbc4
Branch: refs/heads/branch-3.0
Commit: 874bdbc4013b72e40b64fb874e83d3d14045a93d
Parents: 961dcb6
Author: Jason Lowe <jlowe@apache.org>
Authored: Wed Jan 31 11:37:54 2018 -0600
Committer: Jason Lowe <jlowe@apache.org>
Committed: Wed Jan 31 11:45:57 2018 -0600
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 31 ++++++++------------
.../nodemanager/LinuxContainerExecutor.java | 8 -----
.../launcher/ContainerLaunch.java | 15 ++++++++--
.../runtime/DefaultLinuxContainerRuntime.java | 6 ----
.../DelegatingLinuxContainerRuntime.java | 11 -------
.../runtime/DockerLinuxContainerRuntime.java | 7 -----
.../runtime/ContainerRuntime.java | 11 -------
.../launcher/TestContainerLaunch.java | 10 +++++--
8 files changed, 33 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 0b4dbc9..f5d454e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -334,7 +334,6 @@ public abstract class ContainerExecutor implements Configurable \
{ public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
Map<Path, List<String>> resources, List<String> command, Path logDir,
String user, String outFilename) throws IOException {
- updateEnvForWhitelistVars(environment);
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
@@ -352,6 +351,19 @@ public abstract class ContainerExecutor implements Configurable \
{ for (Map.Entry<String, String> env : environment.entrySet()) {
sb.env(env.getKey(), env.getValue());
}
+ // Whitelist environment variables are treated specially.
+ // Only add them if they are not already defined in the environment.
+ // Add them using special syntax to prevent them from eclipsing
+ // variables that may be set explicitly in the container image (e.g,
+ // in a docker image)
+ for(String var : whitelistVars) {
+ if (!environment.containsKey(var)) {
+ String val = getNMEnvVar(var);
+ if (val != null) {
+ sb.whitelistedEnv(var, val);
+ }
+ }
+ }
}
if (resources != null) {
@@ -651,23 +663,6 @@ public abstract class ContainerExecutor implements Configurable \
{ }
}
- /**
- * Propagate variables from the nodemanager's environment into the
- * container's environment if unspecified by the container.
- * @param env the environment to update
- * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
- */
- protected void updateEnvForWhitelistVars(Map<String, String> env) {
- for(String var : whitelistVars) {
- if (!env.containsKey(var)) {
- String val = getNMEnvVar(var);
- if (val != null) {
- env.put(var, val);
- }
- }
- }
- }
-
@VisibleForTesting
protected String getNMEnvVar(String varname) {
return System.getenv(varname);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java \
index 97f1a11..349dd97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java \
@@ -63,7 +63,6 @@ import java.net.InetSocketAddress; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -470,13 +469,6 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
- protected void updateEnvForWhitelistVars(Map<String, String> env) {
- if (linuxContainerRuntime.useWhitelistEnv(env)) {
- super.updateEnvForWhitelistVars(env);
- }
- }
-
- @Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
Container container = ctx.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index c8ca2c8..e9d136e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1077,6 +1077,9 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void env(String key, String value) throws IOException;
+ public abstract void whitelistedEnv(String key, String value)
+ throws IOException;
+
public abstract void echo(String echoStr) throws IOException;
public final void symlink(Path src, Path dst) throws IOException {
@@ -1197,6 +1200,11 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
+ public void whitelistedEnv(String key, String value) throws IOException {
+ line("export ", key, "=${", key, ":-", "\"", value, "\"}");
+ }
+
+ @Override
public void echo(final String echoStr) throws IOException {
line("echo \"" + echoStr + "\"");
}
@@ -1287,6 +1295,11 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
+ public void whitelistedEnv(String key, String value) throws IOException {
+ env(key, value);
+ }
+
+ @Override
public void echo(final String echoStr) throws IOException {
lineWithLenCheck("@echo \"", echoStr, "\"");
}
@@ -1385,8 +1398,6 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put(Environment.PWD.name(), pwd.toString());
- putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name());
-
if (!Shell.WINDOWS) {
environment.put("JVM_PID", "$$");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoo \
p/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index e9c58b8..6c3ae85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -36,7 +36,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
-import java.util.Map;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -73,11 +72,6 @@ public class DefaultLinuxContainerRuntime implements \
LinuxContainerRuntime { }
@Override
- public boolean useWhitelistEnv(Map<String, String> env) {
- return true;
- }
-
- @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
//nothing to do here at the moment.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoo \
p/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 517a4e2..9fe4927 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -93,17 +93,6 @@ public class DelegatingLinuxContainerRuntime implements \
LinuxContainerRuntime { }
}
- @Override
- public boolean useWhitelistEnv(Map<String, String> env) {
- try {
- LinuxContainerRuntime runtime = pickContainerRuntime(env);
- return runtime.useWhitelistEnv(env);
- } catch (ContainerExecutionException e) {
- LOG.debug("Unable to determine runtime");
- return false;
- }
- }
-
@VisibleForTesting
LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment) throws ContainerExecutionException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoo \
p/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 5e3e15c..30a1e05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -308,13 +308,6 @@ public class DockerLinuxContainerRuntime implements \
LinuxContainerRuntime { }
@Override
- public boolean useWhitelistEnv(Map<String, String> env) {
- // Avoid propagating nodemanager environment variables into the container
- // so those variables can be picked up from the Docker image instead.
- return false;
- }
-
- @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index aa294fc..7caa0ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -24,8 +24,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import java.util.Map;
-
/**
* An abstraction for various container runtime implementations. Examples
* include Process Tree, Docker, Appc runtimes etc. These implementations
@@ -85,13 +83,4 @@ public interface ContainerRuntime {
* and hostname
*/
String[] getIpAndHost(Container container) throws ContainerExecutionException;
-
- /**
- * Whether to propagate the whitelist of environment variables from the
- * nodemanager environment into the container environment.
- * @param env the container's environment variables
- * @return true if whitelist variables should be propagated, false otherwise
- * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
- */
- boolean useWhitelistEnv(Map<String, String> env);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874bdbc4/hadoop-yarn-project/hadoop \
-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nod \
emanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java \
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/sr \
c/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 5923f8e..5e57bb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage \
r/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -337,7 +337,8 @@ public class TestContainerLaunch extends \
BaseContainerManagerTest { \
Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); // Available in env \
and in whitelist Assert.assertTrue(shellContent.contains(
- "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\""));
+ "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}"
+ ));
fos.flush();
fos.close();
}
@@ -382,9 +383,12 @@ public class TestContainerLaunch extends \
BaseContainerManagerTest { // Whitelisted variable overridden by container
Assert.assertTrue(shellContent.contains(
"export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
- // Verify no whitelisted variables inherited from NM env
+ // Available in env but not in whitelist
Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
- Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME"));
+ // Available in env and in whitelist
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}"
+ ));
fos.flush();
fos.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic