diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 41e01b17febf..1b6965018a88 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -302,6 +302,20 @@ protected void openInternal(String[] additionalFilesNotFromConf, } } + /** + * Builds the map of common local resources (app jar + localized resources) for the Tez session. + * Extracted for testability to ensure resources are properly registered. + */ + @VisibleForTesting + Map buildCommonLocalResources() { + final Map commonLocalResources = new HashMap<>(); + commonLocalResources.put(DagUtils.getBaseName(appJarLr), appJarLr); + for (LocalResource lr : this.resources.localizedResources) { + commonLocalResources.put(DagUtils.getBaseName(lr), lr); + } + return commonLocalResources; + } + /** * Opens a Tez session without performing a complete rollback/cleanup on failure. * @@ -318,14 +332,9 @@ protected void openInternal(String[] additionalFilesNotFromConf, */ @VisibleForTesting void openInternalUnsafe(boolean isAsync, LogHelper console) throws TezException, IOException { - final Map commonLocalResources = new HashMap<>(); + final Map commonLocalResources = buildCommonLocalResources(); final boolean llapMode = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE)); - commonLocalResources.put(DagUtils.getBaseName(appJarLr), appJarLr); - for (LocalResource lr : this.resources.localizedResources) { - commonLocalResources.put(DagUtils.getBaseName(lr), lr); - } - if (llapMode) { // localize llap client jars addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java index 62e0d0cd27ca..bafd0e547dc1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java @@ -21,6 +21,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.hive.conf.HiveConf; @@ -36,6 +37,13 @@ public class TestTezSessionState { private static final Logger LOG = LoggerFactory.getLogger(TestTezSessionState.class.getName()); + private static SessionState createSessionState() { + HiveConf hiveConf = new HiveConfForTest(TestTezSessionState.class); + hiveConf.set("hive.security.authorization.manager", + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory"); + return SessionState.start(hiveConf); + } + @Test public void testSymlinkedLocalFilesAreLocalizedOnce() throws Exception { Path jarPath = Files.createTempFile("jar", ""); @@ -60,14 +68,12 @@ public void testSymlinkedLocalFilesAreLocalizedOnce() throws Exception { @Test public void testScratchDirDeletedInTheEventOfExceptionWhileOpeningSession() throws Exception { - HiveConf hiveConf = new HiveConfForTest(getClass()); - hiveConf.set("hive.security.authorization.manager", - "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory"); - SessionState.start(hiveConf); + SessionState ss = createSessionState(); + HiveConf hiveConf = ss.getConf(); final AtomicReference scratchDirPath = new AtomicReference<>(); - TezSessionState sessionState = new TezSessionState(SessionState.get().getSessionId(), hiveConf) { + TezSessionState sessionState = new TezSessionState(ss.getSessionId(), hiveConf) { @Override void openInternalUnsafe(boolean isAsync, SessionState.LogHelper console) throws TezException, IOException { super.openInternalUnsafe(isAsync, console); @@ -90,4 +96,40 @@ void openInternalUnsafe(boolean isAsync, SessionState.LogHelper console) throws Assert.assertFalse("Scratch dir is not supposed to exist after cleanup: " + scratchDirPath.get(), Files.exists(Paths.get(scratchDirPath.get()))); } + + /** + * Tests whether commonLocalResources is populated with app jar and localized resources when opening + * a Tez session. + */ + @Test + public void testCommonLocalResourcesPopulatedOnSessionOpen() throws Exception { + Path jarPath = Files.createTempFile("test-jar", ".jar"); + Files.write(jarPath, "testCommonLocalResourcesPopulated".getBytes(), StandardOpenOption.APPEND); + + SessionState ss = createSessionState(); + HiveConf hiveConf = ss.getConf(); + + TezSessionState.HiveResources resources = + new TezSessionState.HiveResources(new org.apache.hadoop.fs.Path("/tmp")); + + TezSessionState tempSession = new TezSessionState(ss.getSessionId(), hiveConf); + + LocalResource localizedLr = tempSession.createJarLocalResource(jarPath.toUri().toString()); + resources.localizedResources.add(localizedLr); + + final TezSessionState sessionStateForTest = new TezSessionState(ss.getSessionId(), hiveConf) { + @Override + void openInternalUnsafe(boolean isAsync, SessionState.LogHelper console) { + Map commonLocalResources = buildCommonLocalResources(); + Assert.assertEquals("commonLocalResources must contain exactly 2 jars (hive-exec app jar + localized test jar)", + 2, commonLocalResources.size()); + Assert.assertTrue("commonLocalResources must contain the hive-exec app jar", + commonLocalResources.keySet().stream().anyMatch(k -> k.contains("hive-exec"))); + Assert.assertTrue("commonLocalResources must contain the added localized test jar", + commonLocalResources.containsKey(DagUtils.getBaseName(localizedLr))); + } + }; + + sessionStateForTest.open(resources); + } } \ No newline at end of file