neo4j.conf
dbms.memory.heap.max_size=20g
dbms.memory.heap.initial_size=20g
dbms.memory.pagecache.size=20g
通过spark向neo4j写入数据,前天程序突然卡住,再看了下图数据库中的debug日志,报如下错误,该如何处理
2022-05-07 09:11:41.567+0000 INFO [o.n.c.c.r.RaftReplicator] Replication attempt 1078 to leader MemberId{100be7da}: DistributedOperation{content=TransactionRepresentationReplicatedTransaction{tx=PhysicalTransactionRepresentation[masterId:-1,authorId:-1,timeStarted:1651831156861,latestCommittedTxWhenStarted:10664015,timeCommitted:1651850191237,lockSession:31,additionalHeader:[]commands.length:7002}, globalSession=GlobalSession{sessionId=d90dd626-6850-495b-9b92-e124df66b921, owner=MemberId{100be7da}}, operationId=LocalOperationId{localSessionId=12, sequenceNumber=13210}}
2022-05-07 09:11:42.870+0000 INFO [o.n.c.c.r.RaftReplicator] Replication attempt 1078 to leader MemberId{100be7da}: DistributedOperation{content=TransactionRepresentationReplicatedTransaction{tx=PhysicalTransactionRepresentation[masterId:-1,authorId:-1,timeStarted:1651831159946,latestCommittedTxWhenStarted:10664015,timeCommitted:1651850191238,lockSession:31,additionalHeader:[]commands.length:7002}, globalSession=GlobalSession{sessionId=d90dd626-6850-495b-9b92-e124df66b921, owner=MemberId{100be7da}}, operationId=LocalOperationId{localSessionId=8, sequenceNumber=40832}}
2022-05-07 09:11:46.562+0000 INFO [o.n.c.m.SenderService] Connected: [id: 0x34c7ec0d, L:/10.145.33.139:58186 - R:/10.145.33.139:17000]
2022-05-07 09:11:46.562+0000 INFO [o.n.c.p.h.HandshakeServerInitializer] Installing handshake server on channel [id: 0x4d088919, L:/10.145.33.139:17000 - R:/10.145.33.139:58186]
2022-05-07 09:11:46.562+0000 INFO [o.n.c.p.h.HandshakeClientInitializer] Initiating handshake on channel [id: 0x34c7ec0d, L:/10.145.33.139:58186 - R:/10.145.33.139:17000]
2022-05-07 09:11:46.562+0000 INFO [o.n.c.p.h.HandshakeServerInitializer] Handshake completed on channel [id: 0x4d088919, L:/10.145.33.139:17000 - R:/10.145.33.139:58186]. Installing: raft version:2
2022-05-07 09:11:46.562+0000 INFO [o.n.c.p.h.HandshakeClientInitializer] Handshake completed on channel [id: 0x34c7ec0d, L:/10.145.33.139:58186 - R:/10.145.33.139:17000]. Installing: raft version:2
2022-05-07 09:11:46.563+0000 ERROR [o.n.c.p.h.HandshakeServerInitializer] Exception in outbound for channel: [id: 0x4d088919, L:/10.145.33.139:17000 - R:/10.145.33.139:58186] failed to allocate 16777216 byte(s) of direct memory (used: 21458059271, max: 21474836480)
io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 21458059271, max: 21474836480)
at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:726)
at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:681)
at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:758)
at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:734)
at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:245)
at io.netty.buffer.PoolArena.allocate(PoolArena.java:227)
at io.netty.buffer.PoolArena.allocate(PoolArena.java:147)
at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:342)
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:178)
at io.netty.buffer.AbstractByteBufAllocator.ioBuffer(AbstractByteBufAllocator.java:139)
at io.netty.channel.DefaultMaxMessagesRecvByteBufAllocator$MaxMessageHandle.allocate(DefaultMaxMessagesRecvByteBufAllocator.java:114)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:147)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.lang.Thread.run(Thread.java:748)
at org.neo4j.helpers.NamedThreadFactory$2.run(NamedThreadFactory.java:122)
2022-05-07 09:11:46.563+0000 WARN [o.n.c.m.SenderService] Lost connection to: 10.145.33.139:17000 (/10.145.33.139:17000)
2022-05-07 09:11:46.563+0000 ERROR [o.n.c.p.h.HandshakeClientInitializer] Exception in outbound for channel: [id: 0x34c7ec0d, L:/10.145.33.139:58186 ! R:/10.145.33.139:17000] Connection reset by peer
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.writev0(Native Method)
at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51)
at sun.nio.ch.IOUtil.write(IOUtil.java:148)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504)
at io.netty.channel.socket.nio.NioSocketChannel.doWrite(NioSocketChannel.java:423)
at io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:931)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.flush0(AbstractNioChannel.java:354)
at io.netty.channel.AbstractChannel$AbstractUnsafe.flush(AbstractChannel.java:898)
at io.netty.channel.DefaultChannelPipeline$HeadContext.flush(DefaultChannelPipeline.java:1384)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:749)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:741)
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:727)
at io.netty.handler.stream.ChunkedWriteHandler.doFlush(ChunkedWriteHandler.java:328)
at io.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:133)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:749)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:741)
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:727)
at io.netty.channel.DefaultChannelPipeline.flush(DefaultChannelPipeline.java:979)
at io.netty.channel.AbstractChannel.flush(AbstractChannel.java:241)
at org.neo4j.causalclustering.protocol.handshake.HandshakeClientInitializer.onHandshakeComplete(HandshakeClientInitializer.java:136)
at org.neo4j.causalclustering.protocol.handshake.HandshakeClientInitializer.lambda$initiateHandshake$3(HandshakeClientInitializer.java:115)
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
at org.neo4j.causalclustering.protocol.handshake.HandshakeClient.handle(HandshakeClient.java:179)
at org.neo4j.causalclustering.protocol.handshake.SwitchOverResponse.dispatch(SwitchOverResponse.java:23)
at org.neo4j.causalclustering.protocol.handshake.NettyHandshakeClient.channelRead0(NettyHandshakeClient.java:23)
at org.neo4j.causalclustering.protocol.handshake.NettyHandshakeClient.channelRead0(NettyHandshakeClient.java:11)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.lang.Thread.run(Thread.java:748)
at org.neo4j.helpers.NamedThreadFactory$2.run(NamedThreadFactory.java:122)
2022-05-07 09:11:46.564+0000 ERROR [o.n.c.p.h.HandshakeClientInitializer] Exception in outbound for channel: [id: 0x34c7ec0d, L:/10.145.33.139:58186 ! R:/10.145.33.139:17000]
java.nio.channels.ClosedChannelException
at io.netty.handler.stream.ChunkedWriteHandler.doFlush(ChunkedWriteHandler.java:337)
at io.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:133)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:749)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:741)
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:727)
at io.netty.channel.DefaultChannelPipeline.flush(DefaultChannelPipeline.java:979)
at io.netty.channel.AbstractChannel.flush(AbstractChannel.java:241)
at org.neo4j.causalclustering.protocol.handshake.HandshakeClientInitializer.onHandshakeComplete(HandshakeClientInitializer.java:136)
at org.neo4j.causalclustering.protocol.handshake.HandshakeClientInitializer.lambda$initiateHandshake$3(HandshakeClientInitializer.java:115)
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
at org.neo4j.causalclustering.protocol.handshake.HandshakeClient.handle(HandshakeClient.java:179)
at org.neo4j.causalclustering.protocol.handshake.SwitchOverResponse.dispatch(SwitchOverResponse.java:23)
at org.neo4j.causalclustering.protocol.handshake.NettyHandshakeClient.channelRead0(NettyHandshakeClient.java:23)
at org.neo4j.causalclustering.protocol.handshake.NettyHandshakeClient.channelRead0(NettyHandshakeClient.java:11)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.lang.Thread.run(Thread.java:748)
at org.neo4j.helpers.NamedThreadFactory$2.run(NamedThreadFactory.java:122)