java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block.

JIRA | Alessandro Andrioni | 9 months ago
  1. 0

    Alluxio is giving tons of BlockAlreadyExistsException whenever I do any operations on it using Spark, such as loading and saving files. Some examples: Reading a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").first 16/02/23 13:10:17 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.TaskContextImpl$$anon$1.onTaskCompletion(TaskContextImpl.scala:60) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:79) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:77) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) at org.apache.spark.TaskContextImpl.markTaskCompleted(TaskContextImpl.scala:77) at org.apache.spark.scheduler.Task.run(Task.scala:91) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 21 more {code} Writing a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").saveAsTextFile("alluxio://localhost:19998/sample3.edn") 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more [Stage 9:> (0 + 2) / 2]16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more 16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more {code}

    JIRA | 9 months ago | Alessandro Andrioni
    java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block.
  2. 0

    Alluxio is giving tons of BlockAlreadyExistsException whenever I do any operations on it using Spark, such as loading and saving files. Some examples: Reading a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").first 16/02/23 13:10:17 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.rdd.HadoopRDD$$anon$1$$anonfun$3.apply(HadoopRDD.scala:240) at org.apache.spark.TaskContextImpl$$anon$1.onTaskCompletion(TaskContextImpl.scala:60) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:79) at org.apache.spark.TaskContextImpl$$anonfun$markTaskCompleted$1.apply(TaskContextImpl.scala:77) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) at org.apache.spark.TaskContextImpl.markTaskCompleted(TaskContextImpl.scala:77) at org.apache.spark.scheduler.Task.run(Task.scala:91) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 21 more {code} Writing a file {code:scala} scala> sc.textFile("alluxio://localhost:19998/sample2.edn").saveAsTextFile("alluxio://localhost:19998/sample3.edn") 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more 16/02/23 13:16:22 WARN type: The block with ID 83886080 could not be cached into Alluxio storage. java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:119) at alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128) at alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118) at alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104) at alluxio.client.file.FileInStream.read(FileInStream.java:161) at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 6,344,797,258,071,787,550, address: /172.19.193.117:29999, message: Failed to write block. at alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106) ... 30 more [Stage 9:> (0 + 2) / 2]16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more 16/02/23 13:16:22 WARN HadoopRDD: Exception in RecordReader.close() java.io.IOException: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:85) at alluxio.client.file.FileInStream.closeCacheStream(FileInStream.java:261) at alluxio.client.file.FileInStream.close(FileInStream.java:110) at alluxio.hadoop.HdfsFileInputStream.close(HdfsFileInputStream.java:117) at java.io.FilterInputStream.close(FilterInputStream.java:181) at org.apache.hadoop.util.LineReader.close(LineReader.java:150) at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:284) at org.apache.spark.rdd.HadoopRDD$$anon$1.close(HadoopRDD.scala:265) at org.apache.spark.util.NextIterator.closeIfNeeded(NextIterator.scala:66) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:75) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: alluxio.exception.BlockAlreadyExistsException: Temp blockId 83,886,080 is not available, because it is already committed at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at alluxio.exception.AlluxioException.from(AlluxioException.java:72) at alluxio.AbstractClient.retryRPC(AbstractClient.java:324) at alluxio.client.block.BlockWorkerClient.cancelBlock(BlockWorkerClient.java:156) at alluxio.client.block.RemoteBlockOutStream.cancel(RemoteBlockOutStream.java:83) ... 24 more {code}

    JIRA | 9 months ago | Alessandro Andrioni
    java.io.IOException: java.io.IOException: Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block.
  3. 0

    Tachyon worker is throwing an IOException occassionally

    Google Groups | 8 months ago | Antonio Si
    java.io.IOException: java.io.IOException: error writing blockId: 62724985512263680, userId: 385120561, address: denj51bda04.us.oracle.com/192.168.131.236:29999, message: Failed to write block.
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Tachyon + Spark cache RDD throw Exception while reading cached RDD, it will re-write again...

    Google Groups | 1 year ago | Andy Su
    java.io.IOException: java.io.IOException: error writing blockId: 42949672960, userId: 16, address: ip-10-100-4-47/10.100.4.47:29999, message: Failed to write block.
  6. 0

    Tachyon + Spark cache RDD throw Exception while reading cached RDD, it will re-write again...

    Google Groups | 1 year ago | Andy Su
    java.io.IOException: java.io.IOException: error writing blockId: 246960619520, userId: 9, address: edwutilfc032.17.XXX.XXX/XX.XX.XX.XX:29999, message: Failed to write block.

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. java.io.IOException

      Error writing blockId: 83,886,080, sessionId: 8,168,440,457,697,623,034, address: /172.19.193.117:29999, message: Failed to write block.

      at alluxio.client.netty.NettyRemoteBlockWriter.write()
    2. alluxio.client.netty
      NettyRemoteBlockWriter.write
      1. alluxio.client.netty.NettyRemoteBlockWriter.write(NettyRemoteBlockWriter.java:106)
      1 frame
    3. alluxio.client.block
      BufferedBlockOutStream.write
      1. alluxio.client.block.RemoteBlockOutStream.writeToRemoteBlock(RemoteBlockOutStream.java:128)
      2. alluxio.client.block.RemoteBlockOutStream.flush(RemoteBlockOutStream.java:118)
      3. alluxio.client.block.BufferedBlockOutStream.write(BufferedBlockOutStream.java:104)
      3 frames
    4. alluxio.client.file
      FileInStream.read
      1. alluxio.client.file.FileInStream.read(FileInStream.java:161)
      1 frame
    5. alluxio.hadoop
      HdfsFileInputStream.read
      1. alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:196)
      1 frame
    6. Java RT
      DataInputStream.read
      1. java.io.DataInputStream.read(DataInputStream.java:149)
      1 frame
    7. Hadoop
      UncompressedSplitLineReader.fillBuffer
      1. org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:59)
      1 frame
    8. Hadoop
      LineReader.readLine
      1. org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216)
      2. org.apache.hadoop.util.LineReader.readLine(LineReader.java:174)
      2 frames
    9. Hadoop
      LineRecordReader.next
      1. org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:91)
      2. org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:248)
      3. org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:48)
      3 frames
    10. Spark
      InterruptibleIterator.hasNext
      1. org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:246)
      2. org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:208)
      3. org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73)
      4. org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
      4 frames
    11. Scala
      Iterator$$anon$11.hasNext
      1. scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
      2. scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
      2 frames
    12. Spark
      Executor$TaskRunner.run
      1. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply$mcV$sp(PairRDDFunctions.scala:1197)
      2. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197)
      3. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$6.apply(PairRDDFunctions.scala:1197)
      4. org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250)
      5. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1205)
      6. org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1185)
      7. org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
      8. org.apache.spark.scheduler.Task.run(Task.scala:89)
      9. org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
      9 frames
    13. Java RT
      Thread.run
      1. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      2. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      3. java.lang.Thread.run(Thread.java:745)
      3 frames