java.lang.RuntimeException: Tried to hard link to file that does not exist /mnt/disk5/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-28049-Data.db

Apache's JIRA Issue Tracker | Nikolai Grigoriev | 3 years ago
tip
Do you know that we can give you better hits? Get more relevant results from Samebug’s stack trace search.
  1. 0

    It seems that since recently I have started getting a number of exceptions like "File not found" on all Cassandra nodes. Currently I am getting an exception like this every couple of seconds on each node, for different keyspaces and CFs. I have tried to restart the nodes, tried to scrub them. No luck so far. It seems that scrub cannot complete on any of these nodes, at some point it fails because of the file that it can't find. One one of the nodes currently the "nodetool scrub" command fails instantly and consistently with this exception: {code} # /opt/cassandra/bin/nodetool scrub Exception in thread "main" java.lang.RuntimeException: Tried to hard link to file that does not exist /mnt/disk5/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-28049-Data.db at org.apache.cassandra.io.util.FileUtils.createHardLink(FileUtils.java:75) at org.apache.cassandra.io.sstable.SSTableReader.createLinks(SSTableReader.java:1215) at org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1826) at org.apache.cassandra.db.ColumnFamilyStore.scrub(ColumnFamilyStore.java:1122) at org.apache.cassandra.service.StorageService.scrub(StorageService.java:2159) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:75) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:279) at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112) at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46) at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237) at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138) at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252) at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819) at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801) at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1487) at javax.management.remote.rmi.RMIConnectionImpl.access$300(RMIConnectionImpl.java:97) at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1328) at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1420) at javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:848) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322) at sun.rmi.transport.Transport$1.run(Transport.java:177) at sun.rmi.transport.Transport$1.run(Transport.java:174) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Transport.java:173) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:553) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:808) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:667) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:724) {code} Also I have noticed that the files that are missing are often (or maybe always?) referred to in the log as follows: {quote} WARN 00:06:10,597 At level 3, SSTableReader(path='/mnt/disk5/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-26776-Data.db') [DecoratedKey(-9053060597280257896, 0010f582cddaca974d7198ae30f194ccfd0c00001000000000004b818d00000000000000010000100000000000004000000000000000000300), DecoratedKey(-8855915848970248008, 00103ce153dfeeb547fb881a51adf611f6cf0000100000000000f04f4700000000000000010000100000000000004000000000000000000500)] overlaps SSTableReader(path='/mnt/disk2/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-28022-Data.db') [DecoratedKey(-8964446543595889729, 001043214a8bdcfd46a3b8ea71da2d57bb9a0000100000000001117c0d00000000000000000000100000000000004000000000000000000100), DecoratedKey(-8848132752710859808, 0010d1f6de8039d54218bf5b1e184335df5f000010000000000062e52600000000000000010000100000000000004000000000000000000400)]. This could be caused by a bug in Cassandra 1.1.0 .. 1.1.3 or due to the fact that you have dropped sstables from another node into the data directory. Sending back to L0. If you didn't drop in sstables, and have not yet run scrub, you should do so since you may also have rows out-of-order within an sstable WARN [RMI TCP Connection(2)-10.3.45.158] 2014-02-18 00:06:10,597 LeveledManifest.java (line 171) At level 3, SSTableReader(path='/mnt/disk5/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-26776-Data.db') [DecoratedKey(-9053060597280257896, 0010f582cddaca974d7198ae30f194ccfd0c00001000000000004b818d00000000000000010000100000000000004000000000000000000300), DecoratedKey(-8855915848970248008, 00103ce153dfeeb547fb881a51adf611f6cf0000100000000000f04f4700000000000000010000100000000000004000000000000000000500)] overlaps SSTableReader(path='/mnt/disk2/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-28022-Data.db') [DecoratedKey(-8964446543595889729, 001043214a8bdcfd46a3b8ea71da2d57bb9a0000100000000001117c0d00000000000000000000100000000000004000000000000000000100), DecoratedKey(-8848132752710859808, 0010d1f6de8039d54218bf5b1e184335df5f000010000000000062e52600000000000000010000100000000000004000000000000000000400)]. This could be caused by a bug in Cassandra 1.1.0 .. 1.1.3 or due to the fact that you have dropped sstables from another node into the data directory. Sending back to L0. If you didn't drop in sstables, and have not yet run scrub, you should do so since you may also have rows out-of-order within an sstable {quote} I never had anything but Cassandra 2.0 on these systems. Also I have recreated my test data from scratch with 2.0.4.

    Apache's JIRA Issue Tracker | 3 years ago | Nikolai Grigoriev
    java.lang.RuntimeException: Tried to hard link to file that does not exist /mnt/disk5/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-28049-Data.db
  2. 0

    Porting project JBoss AS 6 to JBoss AS 7

    Stack Overflow | 5 years ago | Rounak
    java.lang.RuntimeException: com.sun.faces.config.ConfigurationException : CONFIGURATION FAILED! Class org.jboss.as.web.deployment.jsf.JandexAnnotationPr ovider is not an instance of com.sun.faces.spi.AnnotationProvider at com.sun.faces.config.ConfigureListener.contextInitialized(ConfigureLi stener.java:292) [jsf-impl.jar:] at org.apache.catalina.core.StandardContext.contextListenerStart(Standar dContext.java:3368) [jbossweb-7.0.1.Final.jar:7.0.2.Final] at org.apache.catalina.core.StandardContext.start(StandardContext.java:3 821) [jbossweb-7.0.1.Final.jar:7.0.2.Final] at org.jboss.as.web.deployment.WebDeploymentService.start(WebDeploymentS ervice.java:70) [jboss-as-web-7.0.2.Final.jar:7.0.2.Final] at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(Se rviceControllerImpl.java:1824) at org.jboss.msc.service.ServiceControllerImpl$StartTask.run(ServiceCont rollerImpl.java:1759) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor. java:1110) [:1.7.0] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor .java:603) [:1.7.0]
  3. 0

    Error initializing GroovyPageView

    Stack Overflow | 3 years ago | mrquad
    org.codehaus.groovy.grails.web.mapping.exceptions.UrlMappingException: Error map ping onto view [/index]: Error initializing GroovyPageView
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Android Asynctask method does not display the data

    Stack Overflow | 3 years ago | user3316775
    java.lang.RuntimeException: An error occured while executing doInBackground() android.os.AsyncTask$3.done(AsyncTask.java:278) java.util.concurrent.FutureTask$Sync.innerSetException(FutureTask.java:273) java.util.concurrent.FutureTask.setException(FutureTask.java:124) java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:307) java.util.concurrent.FutureTask.run(FutureTask.java:137) android.os.AsyncTask$SerialExecutor$1.run(AsyncTask.java:208) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:569)
  6. 0

    Re: MemtablePostFlusher with high number of pending calls?

    apache.org | 2 years ago
    java.lang.RuntimeException: java.lang.RuntimeException: Insufficient disk > space to flush 40009184 bytes at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:34) at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. java.lang.RuntimeException

      Tried to hard link to file that does not exist /mnt/disk5/cassandra/data/mykeyspace_jmeter/test_contacts/mykeyspace_jmeter-test_contacts-jb-28049-Data.db

      at org.apache.cassandra.io.util.FileUtils.createHardLink()
    2. org.apache.cassandra
      StorageService.scrub
      1. org.apache.cassandra.io.util.FileUtils.createHardLink(FileUtils.java:75)
      2. org.apache.cassandra.io.sstable.SSTableReader.createLinks(SSTableReader.java:1215)
      3. org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:1826)
      4. org.apache.cassandra.db.ColumnFamilyStore.scrub(ColumnFamilyStore.java:1122)
      5. org.apache.cassandra.service.StorageService.scrub(StorageService.java:2159)
      5 frames
    3. Java RT
      Thread.run
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      4. java.lang.reflect.Method.invoke(Method.java:606)
      5. sun.reflect.misc.Trampoline.invoke(MethodUtil.java:75)
      6. sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source)
      7. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      8. java.lang.reflect.Method.invoke(Method.java:606)
      9. sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:279)
      10. com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
      11. com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
      12. com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
      13. com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
      14. com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
      15. com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
      16. com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
      17. javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1487)
      18. javax.management.remote.rmi.RMIConnectionImpl.access$300(RMIConnectionImpl.java:97)
      19. javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1328)
      20. javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1420)
      21. javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:848)
      22. sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
      23. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      24. java.lang.reflect.Method.invoke(Method.java:606)
      25. sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322)
      26. sun.rmi.transport.Transport$1.run(Transport.java:177)
      27. sun.rmi.transport.Transport$1.run(Transport.java:174)
      28. java.security.AccessController.doPrivileged(Native Method)
      29. sun.rmi.transport.Transport.serviceCall(Transport.java:173)
      30. sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:553)
      31. sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:808)
      32. sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:667)
      33. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      34. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      35. java.lang.Thread.run(Thread.java:724)
      35 frames