java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed

JIRA | Alessandro Andrioni | 9 months ago
  1. 0

    Alluxio is giving tons of BlockAlreadyExistsException whenever I do any operations on it using Spark, such as loading and saving files. Some examples: Reading a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").first 16/02/23 13:10:17 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.TaskContextImpl$$anon$1.onTaskCompletion(TaskContextImpl.scala:60) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:79) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:77) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) at org.apache.spark.TaskContextImpl.markTaskCompleted(TaskContextImpl.scala:77) at org.apache.spark.scheduler.Task.run(Task.scala:91) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 21 more {code} Writing a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").saveAsTextFile("alluxio://localhost:19998/sample3.edn") 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more [Stage 9:> (0 + 2) / 2]16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more 16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more {code}

    JIRA | 9 months ago | Alessandro Andrioni
    java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed
  2. 0

    Alluxio is giving tons of BlockAlreadyExistsException whenever I do any operations on it using Spark, such as loading and saving files. Some examples: Reading a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").first 16/02/23 13:10:17 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.TaskContextImpl$$anon$1.onTaskCompletion(TaskContextImpl.scala:60) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:79) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:77) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) at org.apache.spark.TaskContextImpl.markTaskCompleted(TaskContextImpl.scala:77) at org.apache.spark.scheduler.Task.run(Task.scala:91) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 21 more {code} Writing a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").saveAsTextFile("alluxio://localhost:19998/sample3.edn") 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more [Stage 9:> (0 + 2) / 2]16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more 16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more {code}

    JIRA | 9 months ago | Alessandro Andrioni
    java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed
  3. 0

    Re: problem when read data volume that exceed memory size

    Google Groups | 9 months ago | Yupeng Fu
    java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 25,249,710,080 is not available, because it is already committed
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    BlockAlreadyExistsException when writing parquet file to Alluxio from Spark

    Google Groups | 7 months ago | Dragisa Krsmanovic
    alluxio.exception.BlockAlreadyExistsException: Temp blockId 4,898,947,072 is not available, because it is already committed
  6. 0

    How to config Alluxio with using a non-Amazon service provider

    Google Groups | 2 months ago | test520
    java.io.IOException: Failed to cache: alluxio.exception. BlockAlreadyExistsException: Temp blockId 16,777,216 is not available, because it is already committed

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. alluxio.exception.BlockAlreadyExistsException

      Temp blockId 83,886,080 is not available, because it is already committed

      at sun.reflect.NativeConstructorAccessorImpl.newInstance0()
    2. Java RT
      Constructor.newInstance
      1. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
      2. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
      3. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
      4. java.lang.reflect.Constructor.newInstance(Constructor.java:422)
      4 frames
    3. alluxio.exception
      AlluxioException.from
      1. alluxio.exception.AlluxioException.from(AlluxioException.java:72)
      1 frame
    4. alluxio
      AbstractClient.retryRPC
      1. alluxio.AbstractClient.retryRPC(AbstractClient.java:324)
      1 frame
    5. alluxio.client.block
      RemoteBlockOutStream.cancel
      1. alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156)
      2. alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83)
      2 frames
    6. alluxio.client.file
      FileInStream.close
      1. alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261)
      2. alluxio.client.file.FileInStream.close(FileInStream.java:110)
      2 frames
    7. alluxio.hadoop
      HdfsFileInputStream.close
      1. alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117)
      1 frame
    8. Java RT
      FilterInputStream.close
      1. java.io.FilterInputStream.close(FilterInputStream.java:181)
      1 frame
    9. Hadoop
      LineReader.close
      1. org.apache.hadoop.util.LineReader.close(LineReader.java:150)
      1 frame
    10. Hadoop
      LineRecordReader.close
      1. org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284)
      1 frame
    11. Spark
      InterruptibleIterator.hasNext
      1. org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265)
      2. org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66)
      3. org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75)
      4. org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
      4 frames
    12. Scala
      Iterator$$anon$11.hasNext
      1. scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
      2. scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
      2 frames
    13. Spark
      Executor$TaskRunner.run
      1. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197)
      2. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197)
      3. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197)
      4. org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250)
      5. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205)
      6. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185)
      7. org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
      8. org.apache.spark.scheduler.Task.run(Task.scala:89)
      9. org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
      9 frames
    14. Java RT
      Thread.run
      1. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      2. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      3. java.lang.Thread.run(Thread.java:745)
      3 frames