java.lang.IllegalArgumentException: Comparison method violates its general contract!

Apache's JIRA Issue Tracker | Nathan McCarthy | 1 year ago
  1. 0

    Running an SQL query that has a sub query and multiple left joins fails when there is a CLUSTER BY (which implies a sortBy). This gives the following stack trace; {code} Job aborted due to stage failure: Task 118 in stage 4.0 failed 4 times, most recent failure: Lost task 118.3 in stage 4.0 (TID 18392, node142): java.lang.IllegalArgumentException: Comparison method violates its general contract! at org.apache.spark.util.collection.TimSort$SortState.mergeHi(TimSort.java:900) at org.apache.spark.util.collection.TimSort$SortState.mergeAt(TimSort.java:509) at org.apache.spark.util.collection.TimSort$SortState.mergeCollapse(TimSort.java:435) at org.apache.spark.util.collection.TimSort$SortState.access$200(TimSort.java:307) at org.apache.spark.util.collection.TimSort.sort(TimSort.java:135) at org.apache.spark.util.collection.Sorter.sort(Sorter.scala:37) at org.apache.spark.util.collection.PartitionedPairBuffer.partitionedDestructiveSortedIterator(PartitionedPairBuffer.scala:70) at org.apache.spark.util.collection.ExternalSorter.partitionedIterator(ExternalSorter.scala:690) at org.apache.spark.util.collection.ExternalSorter.iterator(ExternalSorter.scala:708) at org.apache.spark.sql.execution.ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply(basicOperators.scala:222) at org.apache.spark.sql.execution.ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply(basicOperators.scala:218) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:63) at org.apache.spark.scheduler.Task.run(Task.scala:70) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Driver stacktrace: {code} The query looks like; {code} val df = sqlContext.sql("""SELECT CID |, PW_END_DATE |, PROD_NBR_KEY |, SUM(CASE WHEN SUBST_IDX = 1 THEN L13W_SALE END) AS SUB1_L13W_SALE |FROM |(SELECT BASE.CID |, BASE.PW_END_DATE |, BASE.PROD_NBR_KEY |, SUBN.SUBST_IDX |, CASE WHEN IDX.PW_END_DATE BETWEEN DATE_SUB(BASE.PW_END_DATE, 13*7 - 1) AND BASE.PW_END_DATE THEN IDX.TOT_AMT_INCLD_GST END AS L13W_SALE |FROM TESTBASE BASE |LEFT JOIN TABLX SUBN |ON BASE.PROD_NBR_KEY = SUBN.PROD_NBR_KEY AND SUBN.SUBST_IDX <= 3 |LEFT JOIN TABLEF IDX |ON BASE.CRN = IDX.CRN |AND SUBN.CROSS_PROD_NBR = IDX.PROD_NBR_KEY |) SUBSPREM | GROUP BY CRN, PW_END_DATE, PROD_NBR_KEY""".stripMargin) {code}

    Apache's JIRA Issue Tracker | 1 year ago | Nathan McCarthy
    java.lang.IllegalArgumentException: Comparison method violates its general contract!
  2. 0

    Running an SQL query that has a sub query and multiple left joins fails when there is a CLUSTER BY (which implies a sortBy). This gives the following stack trace; {code} Job aborted due to stage failure: Task 118 in stage 4.0 failed 4 times, most recent failure: Lost task 118.3 in stage 4.0 (TID 18392, node142): java.lang.IllegalArgumentException: Comparison method violates its general contract! at org.apache.spark.util.collection.TimSort$SortState.mergeHi(TimSort.java:900) at org.apache.spark.util.collection.TimSort$SortState.mergeAt(TimSort.java:509) at org.apache.spark.util.collection.TimSort$SortState.mergeCollapse(TimSort.java:435) at org.apache.spark.util.collection.TimSort$SortState.access$200(TimSort.java:307) at org.apache.spark.util.collection.TimSort.sort(TimSort.java:135) at org.apache.spark.util.collection.Sorter.sort(Sorter.scala:37) at org.apache.spark.util.collection.PartitionedPairBuffer.partitionedDestructiveSortedIterator(PartitionedPairBuffer.scala:70) at org.apache.spark.util.collection.ExternalSorter.partitionedIterator(ExternalSorter.scala:690) at org.apache.spark.util.collection.ExternalSorter.iterator(ExternalSorter.scala:708) at org.apache.spark.sql.execution.ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply(basicOperators.scala:222) at org.apache.spark.sql.execution.ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply(basicOperators.scala:218) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277) at org.apache.spark.rdd.RDD.iterator(RDD.scala:244) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:63) at org.apache.spark.scheduler.Task.run(Task.scala:70) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Driver stacktrace: {code} The query looks like; {code} val df = sqlContext.sql("""SELECT CID |, PW_END_DATE |, PROD_NBR_KEY |, SUM(CASE WHEN SUBST_IDX = 1 THEN L13W_SALE END) AS SUB1_L13W_SALE |FROM |(SELECT BASE.CID |, BASE.PW_END_DATE |, BASE.PROD_NBR_KEY |, SUBN.SUBST_IDX |, CASE WHEN IDX.PW_END_DATE BETWEEN DATE_SUB(BASE.PW_END_DATE, 13*7 - 1) AND BASE.PW_END_DATE THEN IDX.TOT_AMT_INCLD_GST END AS L13W_SALE |FROM TESTBASE BASE |LEFT JOIN TABLX SUBN |ON BASE.PROD_NBR_KEY = SUBN.PROD_NBR_KEY AND SUBN.SUBST_IDX <= 3 |LEFT JOIN TABLEF IDX |ON BASE.CRN = IDX.CRN |AND SUBN.CROSS_PROD_NBR = IDX.PROD_NBR_KEY |) SUBSPREM | GROUP BY CRN, PW_END_DATE, PROD_NBR_KEY""".stripMargin) {code}

    Apache's JIRA Issue Tracker | 1 year ago | Nathan McCarthy
    java.lang.IllegalArgumentException: Comparison method violates its general contract!
  3. 0

    IllegalArgumentException in ViewerComparator.sort (189)

    Eclipse Bugzilla | 2 years ago | error-reports-inbox
    java.lang.IllegalArgumentException: Comparison method violates its general contract!
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Clipboard failure

    GitHub | 1 year ago | gary-rowe
    java.lang.IllegalArgumentException: Comparison method violates its general contract!
  6. 0

    JGraph/JGraphX : Creating a very complex graph

    Stack Overflow | 3 years ago | user2730789
    java.lang.IllegalArgumentException: Comparison method violates its general contract!

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. java.lang.IllegalArgumentException

      Comparison method violates its general contract!

      at org.apache.spark.util.collection.TimSort$SortState.mergeHi()
    2. Spark
      ExternalSorter.iterator
      1. org.apache.spark.util.collection.TimSort$SortState.mergeHi(TimSort.java:900)
      2. org.apache.spark.util.collection.TimSort$SortState.mergeAt(TimSort.java:509)
      3. org.apache.spark.util.collection.TimSort$SortState.mergeCollapse(TimSort.java:435)
      4. org.apache.spark.util.collection.TimSort$SortState.access$200(TimSort.java:307)
      5. org.apache.spark.util.collection.TimSort.sort(TimSort.java:135)
      6. org.apache.spark.util.collection.Sorter.sort(Sorter.scala:37)
      7. org.apache.spark.util.collection.PartitionedPairBuffer.partitionedDestructiveSortedIterator(PartitionedPairBuffer.scala:70)
      8. org.apache.spark.util.collection.ExternalSorter.partitionedIterator(ExternalSorter.scala:690)
      9. org.apache.spark.util.collection.ExternalSorter.iterator(ExternalSorter.scala:708)
      9 frames
    3. Spark Project SQL
      ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply
      1. org.apache.spark.sql.execution.ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply(basicOperators.scala:222)
      2. org.apache.spark.sql.execution.ExternalSort$$anonfun$doExecute$6$$anonfun$apply$7.apply(basicOperators.scala:218)
      2 frames
    4. Spark
      Executor$TaskRunner.run
      1. org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686)
      2. org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686)
      3. org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
      4. org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
      5. org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
      6. org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
      7. org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
      8. org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
      9. org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
      10. org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
      11. org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
      12. org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
      13. org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
      14. org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
      15. org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:63)
      16. org.apache.spark.scheduler.Task.run(Task.scala:70)
      17. org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
      17 frames
    5. Java RT
      Thread.run
      1. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      2. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      3. java.lang.Thread.run(Thread.java:745)
      3 frames