java.util.concurrent.ExecutionException: java.io.IOException: java.lang.reflect.UndeclaredThrowableException

Cask Community Issue Tracker | Chris Gianelloni | 9 months ago
  1. 0

    After enabling HDFS Data at Rest Encryption using Cloudera Manager and a Java KMS, Mapreduce programs appear to fail. The "cdap" user is the KMS admin, as are users in the "sysadmin" group. {code} 2016-03-21 23:48:54,979 - INFO [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.b.MapReduceRuntimeService@289] - Submitting MapReduce Job: job=PurchaseHistoryBuilder,=namespaceId=default, applicationId=PurchaseHistory, program=PurchaseHistoryBuilder, runid=5424fb96-efbf-11e5-b900-42010af00026 2016-03-21 23:48:54,996 - INFO [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.b.i.LocalClientProtocolProvider@42] - Using framework: yarn 2016-03-21 23:48:55,659 - WARN [MapReduceRunner-PurchaseHistoryBuilder:o.a.h.s.UserGroupInformation@1674] - PriviledgedActionException as:cdap (auth:KERBEROS) cause:org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) 2016-03-21 23:48:55,670 - WARN [MapReduceRunner-PurchaseHistoryBuilder:o.a.h.s.UserGroupInformation@1674] - PriviledgedActionException as:cdap (auth:KERBEROS) cause:java.io.IOException: java.lang.reflect.UndeclaredThrowableException 2016-03-21 23:48:55,688 - ERROR [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.b.MapReduceRuntimeService@300] - Exception when submitting MapReduce Job: job=PurchaseHistoryBuilder,=namespaceId=default, applicationId=PurchaseHistory, program=PurchaseHistoryBuilder, runid=5424fb96-efbf-11e5-b900-42010af00026 java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) [com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) [co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted 2016-03-21 23:48:55,847 - ERROR [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.ProgramControllerServiceAdapter@81] - Program terminated with exception java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) [co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) [co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted 2016-03-21 23:48:55,879 - ERROR [pcontroller-program:default.PurchaseHistory.mapreduce.PurchaseHistoryBuilder-5424fb96-efbf-11e5-b900-42010af00026:c.c.c.i.a.r.d.AbstractProgramTwillRunnable@311] - Program runner error out. java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted 2016-03-21 23:48:55,912 - ERROR [TwillContainerService:c.c.c.i.a.r.d.AbstractProgramTwillRunnable@323] - Program PurchaseHistoryBuilder execution failed. java.util.concurrent.ExecutionException: java.io.IOException: java.lang.reflect.UndeclaredThrowableException at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:294) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:281) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.distributed.AbstractProgramTwillRunnable.run(AbstractProgramTwillRunnable.java:317) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at org.apache.twill.internal.container.TwillContainerService.doRun(TwillContainerService.java:130) [org.apache.twill.twill-yarn-0.6.0-incubating.jar:0.6.0-incubating] at org.apache.twill.internal.AbstractTwillService.run(AbstractTwillService.java:179) [org.apache.twill.twill-core-0.6.0-incubating.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:52) [com.google.guava.guava-13.0.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) [com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] ... 1 common frames omitted Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted {code}

    Cask Community Issue Tracker | 9 months ago | Chris Gianelloni
    java.util.concurrent.ExecutionException: java.io.IOException: java.lang.reflect.UndeclaredThrowableException
  2. 0

    After enabling HDFS Data at Rest Encryption using Cloudera Manager and a Java KMS, Mapreduce programs appear to fail. The "cdap" user is the KMS admin, as are users in the "sysadmin" group. {code} 2016-03-21 23:48:54,979 - INFO [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.b.MapReduceRuntimeService@289] - Submitting MapReduce Job: job=PurchaseHistoryBuilder,=namespaceId=default, applicationId=PurchaseHistory, program=PurchaseHistoryBuilder, runid=5424fb96-efbf-11e5-b900-42010af00026 2016-03-21 23:48:54,996 - INFO [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.b.i.LocalClientProtocolProvider@42] - Using framework: yarn 2016-03-21 23:48:55,659 - WARN [MapReduceRunner-PurchaseHistoryBuilder:o.a.h.s.UserGroupInformation@1674] - PriviledgedActionException as:cdap (auth:KERBEROS) cause:org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) 2016-03-21 23:48:55,670 - WARN [MapReduceRunner-PurchaseHistoryBuilder:o.a.h.s.UserGroupInformation@1674] - PriviledgedActionException as:cdap (auth:KERBEROS) cause:java.io.IOException: java.lang.reflect.UndeclaredThrowableException 2016-03-21 23:48:55,688 - ERROR [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.b.MapReduceRuntimeService@300] - Exception when submitting MapReduce Job: job=PurchaseHistoryBuilder,=namespaceId=default, applicationId=PurchaseHistory, program=PurchaseHistoryBuilder, runid=5424fb96-efbf-11e5-b900-42010af00026 java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) [com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) [co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted 2016-03-21 23:48:55,847 - ERROR [MapReduceRunner-PurchaseHistoryBuilder:c.c.c.i.a.r.ProgramControllerServiceAdapter@81] - Program terminated with exception java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) [co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) [co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted 2016-03-21 23:48:55,879 - ERROR [pcontroller-program:default.PurchaseHistory.mapreduce.PurchaseHistoryBuilder-5424fb96-efbf-11e5-b900-42010af00026:c.c.c.i.a.r.d.AbstractProgramTwillRunnable@311] - Program runner error out. java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted 2016-03-21 23:48:55,912 - ERROR [TwillContainerService:c.c.c.i.a.r.d.AbstractProgramTwillRunnable@323] - Program PurchaseHistoryBuilder execution failed. java.util.concurrent.ExecutionException: java.io.IOException: java.lang.reflect.UndeclaredThrowableException at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:294) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:281) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.distributed.AbstractProgramTwillRunnable.run(AbstractProgramTwillRunnable.java:317) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at org.apache.twill.internal.container.TwillContainerService.doRun(TwillContainerService.java:130) [org.apache.twill.twill-yarn-0.6.0-incubating.jar:0.6.0-incubating] at org.apache.twill.internal.AbstractTwillService.run(AbstractTwillService.java:179) [org.apache.twill.twill-core-0.6.0-incubating.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:52) [com.google.guava.guava-13.0.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: java.io.IOException: java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:881) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.addDelegationTokens(KeyProviderDelegationTokenExtension.java:86) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.hdfs.DistributedFileSystem.addDelegationTokens(DistributedFileSystem.java:2142) ~[hadoop-hdfs-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:140) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodesInternal(TokenCache.java:100) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) ~[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) [com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387) ~[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na] ... 1 common frames omitted Caused by: java.lang.reflect.UndeclaredThrowableException: null at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1684) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 16 common frames omitted Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:306) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) ~[hadoop-common-2.6.0-cdh5.5.2.jar:na] ... 17 common frames omitted Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_67] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_67] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_67] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67] at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_67] at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261) ~[hadoop-auth-2.6.0-cdh5.5.2.jar:na] ... 28 common frames omitted {code}

    Cask Community Issue Tracker | 9 months ago | Chris Gianelloni
    java.util.concurrent.ExecutionException: java.io.IOException: java.lang.reflect.UndeclaredThrowableException
  3. 0

    Apache NiFi Developer List - Using NiFI with a Secured HBASE setup

    nabble.com | 7 months ago
    org.ietf.jgss.GSSException: No valid credentials provided > (Mechanism level: Failed to find any Kerberos tgt) >         at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Unknown > Source) ~[na:1.8.0_71] >         at > sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Unknown Source) at > sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Unknown Source) at sun.security.jgss.GSSManagerImpl.getMechanismContext(Unknown > Source) ~[na:1.8.0_71] >         at sun.security.jgss.GSSContextImpl.initSecContext(Unknown Source)
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    The explore service is unable to start on a secure 5.x cluster. On cdh 5.2 and 5.3, we see the following exception: {code} 2015-03-21 01:31:50,059 - ERROR [ExploreExecutorService STARTING:c.c.c.c.t.AbstractMasterTwillRunnable$1@144] - Service co.cask.cdap.explore.executor.ExploreExecutorService failed com.google.common.util.concurrent.UncheckedExecutionException: org.apache.hive.service.ServiceException: Unable to login to kerberos with given principal/keytab at com.google.common.util.concurrent.Futures.wrapAndThrowUnchecked(Futures.java:1015) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.Futures.getUnchecked(Futures.java:1001) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.AbstractService.startAndWait(AbstractService.java:220) ~[com.google.guava.guava-13.0.1.jar:na] at com.google.common.util.concurrent.AbstractIdleService.startAndWait(AbstractIdleService.java:106) ~[com.google.guava.guava-13.0.1.jar:na] at co.cask.cdap.explore.executor.ExploreExecutorService.startUp(ExploreExecutorService.java:90) ~[co.cask.cdap.cdap-explore-2.8.0-SNAPSHOT.jar:na] at com.google.common.util.concurrent.AbstractIdleService$1$1.run(AbstractIdleService.java:43) ~[com.google.guava.guava-13.0.1.jar:na] at java.lang.Thread.run(Thread.java:701) ~[na:1.6.0_34] Caused by: org.apache.hive.service.ServiceException: Unable to login to kerberos with given principal/keytab at org.apache.hive.service.cli.CLIService.init(CLIService.java:88) ~[hive-service-0.13.1-cdh5.3.2.jar:0.13.1-cdh5.3.2] at co.cask.cdap.explore.service.hive.BaseHiveExploreService.startUp(BaseHiveExploreService.java:275) ~[co.cask.cdap.cdap-explore-2.8.0-SNAPSHOT.jar:na] ... 2 common frames omitted Caused by: java.io.IOException: HiveServer2 kerberos principal or keytab is not correctly configured at org.apache.hive.service.auth.HiveAuthFactory.loginFromKeytab(HiveAuthFactory.java:183) ~[hive-service-0.13.1-cdh5.3.2.jar:0.13.1-cdh5.3.2] at org.apache.hive.service.cli.CLIService.init(CLIService.java:85) ~[hive-service-0.13.1-cdh5.3.2.jar:0.13.1-cdh5.3.2] ... 3 common frames omitted {code} with cdh 5.0, the service starts up but queries dont go through. For example, a 'show tables' query results in the following exception: {code} 2015-03-21 02:22:21,423 - ERROR [pool-27-thread-1:o.a.t.t.TSaslTransport@296] - SASL negotiation failure javax.security.sasl.SaslException: GSS initiate failed at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:212) ~[na:1.6.0_34] at org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94) ~[org.apache.thrift.libthrift-0.9.0.jar:0.9.0] at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:253) ~[org.apache.thrift.libthrift-0.9.0.jar:0.9.0] at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37) ~[org.apache.thrift.libthrift-0.9.0.jar:0.9.0] at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.6.0_34] at javax.security.auth.Subject.doAs(Subject.java:416) ~[na:1.6.0_34] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) ~[hadoop-common-2.3.0-cdh5.0.0.jar:na] at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:288) ~[hive-metastore-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:169) ~[hive-metastore-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.6.0_34] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) ~[na:1.6.0_34] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:1.6.0_34] at java.lang.reflect.Constructor.newInstance(Constructor.java:534) ~[na:1.6.0_34] at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1161) ~[hive-metastore-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:62) ~[hive-metastore-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:72) ~[hive-metastore-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:2407) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:2418) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.metadata.Hive.getDatabase(Hive.java:1141) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.metadata.Hive.databaseExists(Hive.java:1130) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.exec.DDLTask.showTables(DDLTask.java:2250) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.exec.DDLTask.execute(DDLTask.java:334) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:151) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:65) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1485) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1263) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1091) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.Driver.run(Driver.java:931) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hadoop.hive.ql.Driver.run(Driver.java:926) ~[hive-exec-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:144) ~[hive-service-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hive.service.cli.operation.SQLOperation.access$100(SQLOperation.java:64) ~[hive-service-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at org.apache.hive.service.cli.operation.SQLOperation$1.run(SQLOperation.java:177) ~[hive-service-0.12.0-cdh5.0.0.jar:0.12.0-cdh5.0.0] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.6.0_34] at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) ~[na:1.6.0_34] at java.util.concurrent.FutureTask.run(FutureTask.java:166) ~[na:1.6.0_34] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146) ~[na:1.6.0_34] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.6.0_34] at java.lang.Thread.run(Thread.java:701) ~[na:1.6.0_34] Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.6.0_34] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.6.0_34] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.6.0_34] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:218) ~[na:1.6.0_34] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:213) ~[na:1.6.0_34] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:180) ~[na:1.6.0_34] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:193) ~[na:1.6.0_34] ... 40 common frames omitted {code}

    Cask Community Issue Tracker | 2 years ago | Albert Shau
    javax.security.sasl.SaslException: GSS initiate failed
  6. 0

    No tables get created. I see these exceptions in the explore container: {code} javax.security.sasl.SaslException: GSS initiate failed at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:212) ~[na:1.7.0_75] at org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94) [org.apache.thrift.libthrift-0.9.0.jar:0.9.0] at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:253) ~[org.apache.thrift.libthrift-0.9.0.jar:0.9.0] at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37) [org.apache.thrift.libthrift-0.9.0.jar:0.9.0] at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at java.security.AccessController.doPrivileged(Native Method) [na:1.7.0_75] at javax.security.auth.Subject.doAs(Subject.java:415) [na:1.7.0_75] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554) [hadoop-common-2.3.0-cdh5.1.5.jar:na] at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:296) [hive-metastore-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:169) [hive-metastore-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) [na:1.7.0_75] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) [na:1.7.0_75] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) [na:1.7.0_75] at java.lang.reflect.Constructor.newInstance(Constructor.java:526) [na:1.7.0_75] at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1161) [hive-metastore-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:62) [hive-metastore-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:72) [hive-metastore-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:2394) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:2405) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:953) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:908) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeCreateTable(SemanticAnalyzer.java:9016) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8325) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:317) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:455) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:353) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:996) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:989) [hive-exec-0.12.0-cdh5.1.5.jar:0.12.0-cdh5.1.5] at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:98) [hive-service-0.12.0-cdh5.1.5.jar-1442450923051.jar:0.12.0-cdh5.1.5] at org.apache.hive.service.cli.operation.SQLOperation.run(SQLOperation.java:163) [hive-service-0.12.0-cdh5.1.5.jar-1442450923051.jar:0.12.0-cdh5.1.5] at org.apache.hive.service.cli.session.HiveSessionImpl.runOperationWithLogCapture(HiveSessionImpl.java:524) [hive-service-0.12.0-cdh5.1.5.jar-1442450923051.jar:0.12.0-cdh5.1.5] at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:222) [hive-service-0.12.0-cdh5.1.5.jar-1442450923051.jar:0.12.0-cdh5.1.5] at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:209) [hive-service-0.12.0-cdh5.1.5.jar-1442450923051.jar:0.12.0-cdh5.1.5] at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:182) [hive-service-0.12.0-cdh5.1.5.jar-1442450923051.jar:0.12.0-cdh5.1.5] at co.cask.cdap.explore.service.hive.Hive12CDH5ExploreService.doExecute(Hive12CDH5ExploreService.java:114) [co.cask.cdap.cdap-explore-3.2.0-SNAPSHOT.jar:na] at co.cask.cdap.explore.service.hive.BaseHiveExploreService.execute(BaseHiveExploreService.java:747) [co.cask.cdap.cdap-explore-3.2.0-SNAPSHOT.jar:na] at co.cask.cdap.explore.service.ExploreTableManager.enableDataset(ExploreTableManager.java:234) [co.cask.cdap.cdap-explore-3.2.0-SNAPSHOT.jar:na] at co.cask.cdap.explore.executor.ExploreExecutorHttpHandler.enableDataset(ExploreExecutorHttpHandler.java:162) [co.cask.cdap.cdap-explore-3.2.0-SNAPSHOT.jar:na] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_75] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_75] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_75] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_75] at co.cask.http.HttpMethodInfo.invoke(HttpMethodInfo.java:79) [co.cask.http.netty-http-0.12.0.jar:na] at co.cask.http.HttpDispatcher.messageReceived(HttpDispatcher.java:41) [co.cask.http.netty-http-0.12.0.jar:na] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.messageReceived(SimpleChannelUpstreamHandler.java:124) [io.netty.netty-3.6.6.Final.jar:na] at co.cask.cdap.common.http.AuthenticationChannelHandler.messageReceived(AuthenticationChannelHandler.java:59) [co.cask.cdap.cdap-common-3.2.0-SNAPSHOT.jar:na] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.handler.execution.ChannelUpstreamEventRunnable.doRun(ChannelUpstreamEventRunnable.java:43) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.handler.execution.ChannelEventRunnable.run(ChannelEventRunnable.java:67) [io.netty.netty-3.6.6.Final.jar:na] at org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor$ChildExecutor.run(OrderedMemoryAwareThreadPoolExecutor.java:314) [io.netty.netty-3.6.6.Final.jar:na] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_75] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75] Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.7.0_75] at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) ~[na:1.7.0_75] at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.7.0_75] at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) ~[na:1.7.0_75] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.7.0_75] at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.7.0_75] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:193) ~[na:1.7.0_75] ... 59 common frames omitted {code}

    Cask Community Issue Tracker | 1 year ago | Bhooshan Mogal
    javax.security.sasl.SaslException: GSS initiate failed

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. org.ietf.jgss.GSSException

      No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)

      at sun.security.jgss.krb5.Krb5InitCredential.getInstance()
    2. Java RT
      GSSContextImpl.initSecContext
      1. sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147)[na:1.7.0_67]
      2. sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121)[na:1.7.0_67]
      3. sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187)[na:1.7.0_67]
      4. sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223)[na:1.7.0_67]
      5. sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)[na:1.7.0_67]
      6. sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)[na:1.7.0_67]
      6 frames
    3. Apache Hadoop Auth
      KerberosAuthenticator$1.run
      1. org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:285)[hadoop-auth-2.6.0-cdh5.5.2.jar:na]
      2. org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:261)[hadoop-auth-2.6.0-cdh5.5.2.jar:na]
      2 frames
    4. Java RT
      Subject.doAs
      1. java.security.AccessController.doPrivileged(Native Method)[na:1.7.0_67]
      2. javax.security.auth.Subject.doAs(Subject.java:415)[na:1.7.0_67]
      2 frames
    5. Apache Hadoop Auth
      KerberosAuthenticator.authenticate
      1. org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:261)[hadoop-auth-2.6.0-cdh5.5.2.jar:na]
      2. org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:196)[hadoop-auth-2.6.0-cdh5.5.2.jar:na]
      2 frames
    6. org.apache.hadoop
      DelegationTokenAuthenticator.authenticate
      1. org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:127)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      1 frame
    7. Apache Hadoop Auth
      AuthenticatedURL.openConnection
      1. org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216)[hadoop-auth-2.6.0-cdh5.5.2.jar:na]
      1 frame
    8. org.apache.hadoop
      KMSClientProvider$2.run
      1. org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.doDelegationTokenOperation(DelegationTokenAuthenticator.java:284)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      2. org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.getDelegationToken(DelegationTokenAuthenticator.java:165)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      3. org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.getDelegationToken(DelegationTokenAuthenticatedURL.java:371)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      4. org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:868)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      5. org.apache.hadoop.crypto.key.kms.KMSClientProvider$2.run(KMSClientProvider.java:863)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      5 frames
    9. Java RT
      Subject.doAs
      1. java.security.AccessController.doPrivileged(Native Method)[na:1.7.0_67]
      2. javax.security.auth.Subject.doAs(Subject.java:415)[na:1.7.0_67]
      2 frames
    10. Hadoop
      UserGroupInformation.doAs
      1. org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      1 frame
    11. org.apache.hadoop
      KMSClientProvider.addDelegationTokens
      1. org.apache.hadoop.crypto.key.kms.KMSClientProvider.addDelegationTokens(KMSClientProvider.java:863)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      1 frame
    12. Hadoop
      Job$10.run
      1. org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes(TokenCache.java:80)[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na]
      2. org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:169)[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na]
      3. org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307)[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na]
      4. org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304)[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na]
      4 frames
    13. Java RT
      Subject.doAs
      1. java.security.AccessController.doPrivileged(Native Method)[na:1.7.0_67]
      2. javax.security.auth.Subject.doAs(Subject.java:415)[na:1.7.0_67]
      2 frames
    14. Hadoop
      UserGroupInformation.doAs
      1. org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      1 frame
    15. Hadoop
      Job.submit
      1. org.apache.hadoop.mapreduce.Job.submit(Job.java:1304)[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na]
      1 frame
    16. Java RT
      Subject.doAs
      1. java.security.AccessController.doPrivileged(Native Method)[na:1.7.0_67]
      2. javax.security.auth.Subject.doAs(Subject.java:415)[na:1.7.0_67]
      2 frames
    17. Hadoop
      UserGroupInformation.doAs
      1. org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)[hadoop-common-2.6.0-cdh5.5.2.jar:na]
      1 frame
    18. Hadoop
      Job.submit
      1. org.apache.hadoop.mapreduce.Job.submit(Job.java:1304)[hadoop-mapreduce-client-core-2.6.0-cdh5.5.2.jar:na]
      1 frame
    19. co.cask.cdap
      MapReduceRuntimeService.startUp
      1. co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:291)[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na]
      1 frame
    20. Guava
      AbstractExecutionThreadService$1$1.run
      1. com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47)[com.google.guava.guava-13.0.1.jar:na]
      1 frame
    21. co.cask.cdap
      MapReduceRuntimeService$1$1.run
      1. co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1$1.run(MapReduceRuntimeService.java:387)[co.cask.cdap.cdap-app-fabric-3.3.1.jar:na]
      1 frame
    22. Java RT
      Thread.run
      1. java.lang.Thread.run(Thread.java:745)[na:1.7.0_67]
      1 frame