org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type;

DataStax JIRA | Stephen Qi | 5 months ago
  1. 0

    Table in Cassandra: create table myTable ( ts timestamp, value text ) I'm trying to use CassandraSQLContext.sql("select * from myTable where ts IN (1466708400000,1466643600000)") to query the table, and I got the exception indicating the filter value type of Long is not the same as column type of timestamp. This was not an issue with connector 1.5.1-M2. The exception: ] star_ws - Couldn't perform the aggregation cache - event_agg_h_acme_pull_acme_pull_sduh: org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type; org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type; at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:65) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:57) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:335) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:335) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69) at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:334) at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp$1(QueryPlan.scala:108) at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:118) at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:127) at scala.collection.Iterator$$anon$11.next(Iterator.scala:370) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48) at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:308) at scala.collection.AbstractIterator.to(Iterator.scala:1194) at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:300) at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1194) at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:287) at scala.collection.AbstractIterator.toArray(Iterator.scala:1194) at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:127) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:57) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:50) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:121) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:120) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:120) at scala.collection.immutable.List.foreach(List.scala:381) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:120) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:50) at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44) at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:34) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:133) at org.apache.spark.sql.cassandra.CassandraSQLContext.cassandraSql(CassandraSQLContext.scala:70)

    DataStax JIRA | 5 months ago | Stephen Qi
    org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type;
  2. 0

    Table in Cassandra: {code} create table myTable ( ts timestamp, value text ) {code} I'm trying to use {code} CassandraSQLContext.sql("select * from myTable where ts IN (1466708400000,1466643600000)") {code} to query the table, and I got the exception indicating the filter value type of Long is not the same as column type of timestamp. This was not an issue with connector 1.5.1-M2. The exception: {code} ] star_ws - Couldn't perform the aggregation cache - event_agg_h_acme_pull_acme_pull_sduh: org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type; org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type; at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:65) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:57) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:335) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:335) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69) at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:334) at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp$1(QueryPlan.scala:108) at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:118) at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:127) at scala.collection.Iterator$$anon$11.next(Iterator.scala:370) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48) at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:308) at scala.collection.AbstractIterator.to(Iterator.scala:1194) at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:300) at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1194) at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:287) at scala.collection.AbstractIterator.toArray(Iterator.scala:1194) at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:127) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:57) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:50) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:121) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:120) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:120) at scala.collection.immutable.List.foreach(List.scala:381) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:120) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:50) at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44) at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:34) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:133) at org.apache.spark.sql.cassandra.CassandraSQLContext.cassandraSql(CassandraSQLContext.scala:70) {code}

    DataStax JIRA | 5 months ago | Stephen Qi
    org.apache.spark.sql.AnalysisException: cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type;
  3. 0

    I create a simple dataframe in R and call the summary function on it (standard R, not SparkR). {code} > library(magrittr) > df <- data.frame( date = as.Date("2015-01-01") + 0:99, r = runif(100) ) > df %>% summary date r Min. :2015-01-01 Min. :0.01221 1st Qu.:2015-01-25 1st Qu.:0.30003 Median :2015-02-19 Median :0.46416 Mean :2015-02-19 Mean :0.50350 3rd Qu.:2015-03-16 3rd Qu.:0.73361 Max. :2015-04-10 Max. :0.99618 {code} Notice that the date can be summarised here. In SparkR; this will give an error. {code} > ddf <- createDataFrame(sqlContext, df) > ddf %>% summary Error in invokeJava(isStatic = FALSE, objId$id, methodName, ...) : org.apache.spark.sql.AnalysisException: cannot resolve 'avg(date)' due to data type mismatch: function average requires numeric types, not DateType; at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:61) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:53) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:293) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:293) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51) at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:292) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:290) at org.apache.spark.sql. {code} This is a rather annoying bug since the SparkR documentation currently suggests that dates are now supported in SparkR.

    Apache's JIRA Issue Tracker | 1 year ago | Vincent Warmerdam
    org.apache.spark.sql.AnalysisException: cannot resolve 'avg(date)' due to data type mismatch: function average requires numeric types, not DateType;
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    BIGINT and INT comparison failure in spark sql

    Stack Overflow | 4 months ago | Sai Krishna
    org.apache.spark.sql.AnalysisException: cannot resolve 'my_column[16001]' due to data type mismatch: argument 2 requires bigint type, however, '16001' is of int type.; line 1 pos 43
  6. 0

    GitHub comment 624#173841576

    GitHub | 11 months ago | nsphung
    org.apache.spark.sql.AnalysisException: cannot resolve 'UDF(date)' due to data type mismatch: argument 1 requires string type, however, 'date' is of array<timestamp> type.;

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. org.apache.spark.sql.AnalysisException

      cannot resolve 'ts IN (1466708400000,1466643600000)' due to data type mismatch: Arguments must be same type;

      at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis()
    2. Spark Project Catalyst
      QueryPlan$$anonfun$2.apply
      1. org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
      2. org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:65)
      3. org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:57)
      4. org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:335)
      5. org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:335)
      6. org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69)
      7. org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:334)
      8. org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp$1(QueryPlan.scala:108)
      9. org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:118)
      10. org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:127)
      10 frames
    3. Scala
      AbstractIterator.toArray
      1. scala.collection.Iterator$$anon$11.next(Iterator.scala:370)
      2. scala.collection.Iterator$class.foreach(Iterator.scala:742)
      3. scala.collection.AbstractIterator.foreach(Iterator.scala:1194)
      4. scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)
      5. scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)
      6. scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)
      7. scala.collection.TraversableOnce$class.to(TraversableOnce.scala:308)
      8. scala.collection.AbstractIterator.to(Iterator.scala:1194)
      9. scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:300)
      10. scala.collection.AbstractIterator.toBuffer(Iterator.scala:1194)
      11. scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:287)
      12. scala.collection.AbstractIterator.toArray(Iterator.scala:1194)
      12 frames
    4. Spark Project Catalyst
      TreeNode$$anonfun$foreachUp$1.apply
      1. org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:127)
      2. org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:57)
      3. org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:50)
      4. org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:121)
      5. org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:120)
      6. org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:120)
      6 frames
    5. Scala
      List.foreach
      1. scala.collection.immutable.List.foreach(List.scala:381)
      1 frame
    6. Spark Project Catalyst
      Analyzer.checkAnalysis
      1. org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:120)
      2. org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:50)
      3. org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:44)
      3 frames
    7. Spark Project SQL
      DataFrame.<init>
      1. org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:34)
      2. org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:133)
      2 frames
    8. spark-cassandra-connector
      CassandraSQLContext.cassandraSql
      1. org.apache.spark.sql.cassandra.CassandraSQLContext.cassandraSql(CassandraSQLContext.scala:70)
      1 frame