|
记一bug
2021/10/20 21:44:34.713 ERROR [reactor-http-epoll-2] r.n.h.s.HttpServerOperations : [id: 0xa504247f, L:/172.17.0.8:8443 - R:/172.17.0.1:36452] Error finishing response. Closing connection
io.netty.handler.codec.EncoderException: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 503316487, max: 510132224)
at io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:107) ~[netty-codec-4.1.45.Final.jar!/:4.1.45.Final]
Suppressed: reactor.core.publisher.FluxOnAssembly$OnAssemblyException:
Error has been observed at the following site(s):
|_ checkpoint ⇢ org.springframework.cloud.gateway.filter.WeightCalculatorWebFilter [DefaultWebFilterChain]
|_ checkpoint ⇢ HTTP GET "/common/view/viewCount" [ExceptionHandlingWebHandler]
Stack trace:
at io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:107) ~[netty-codec-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.CombinedChannelDuplexHandler.write(CombinedChannelDuplexHandler.java:346) ~[netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:715) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:707) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:790) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:700) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at reactor.netty.http.server.HttpTrafficHandler.write(HttpTrafficHandler.java:320) [reactor-netty-0.9.3.RELEASE.jar!/:0.9.3.RELEASE]
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:715) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:762) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1089) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:384) [netty-transport-native-epoll-4.1.45.Final-linux-x86_64.jar!/:4.1.45.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_111]
Caused by: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 503316487, max: 510132224)
at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:726) ~[netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:681) ~[netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:758) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:734) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:245) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:215) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:147) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:342) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:178) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:115) ~[netty-buffer-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.handler.codec.http.HttpObjectEncoder.encode(HttpObjectEncoder.java:93) ~[netty-codec-http-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:89) ~[netty-codec-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.CombinedChannelDuplexHandler.write(CombinedChannelDuplexHandler.java:346) ~[netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:715) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:707) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:790) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:700) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at reactor.netty.http.server.HttpTrafficHandler.write(HttpTrafficHandler.java:320) [reactor-netty-0.9.3.RELEASE.jar!/:0.9.3.RELEASE]
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:715) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:762) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1089) [netty-transport-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:384) [netty-transport-native-epoll-4.1.45.Final-linux-x86_64.jar!/:4.1.45.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.45.Final.jar!/:4.1.45.Final]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_111]
2021/10/20 21:44:34.813 DEBUG [lettuce-epollEventLoop-4-1] c.s.c.c.DefaultRedisRateLimiter$$EnhancerBySpringCGLIB$$c208318 : response: Response{allowed=true, headers={X-RateLimit-Remaining=997, X-RateLimit-Burst-Capacity=1000, X-RateLimit-Replenish-Rate=100}, tokensRemaining=-1}
原因及解决方法
网关服务使用了Flux,而Flux使用了netty，netty直接使用内存空间，内存有限，如果一直往内存写东西，而不释放的话，必将造成内存溢出。问题代码块：
public class RecorderServerHttpResponseDecorator extends ServerHttpResponseDecorator {
private DataBufferWrapper data = null;
public RecorderServerHttpResponseDecorator(ServerHttpResponse delegate) {
super(delegate);
}
@Override
public Mono writeWith(Publisher body) {
return DataBufferUtilFix.join(Flux.from(body))
.doOnNext(d -> this.data = d)
.flatMap(d -> super.writeWith(copy()));
}
@Override
public Mono writeAndFlushWith(Publisher body) {
return writeWith(Flux.from(body)
.flatMapSequential(p -> p));
}
public Flux copy() {
DataBuffer buffer = this.data.newDataBuffer();
if (buffer == null)
return Flux.empty();
return Flux.just(buffer);
}
}
从代码中可以看到，copy方法中并未对缓存进行释放，修改如下：
public Flux copy() {
//如果data为null 就出错了 正好可以调试
DataBuffer buffer = this.data.newDataBuffer();
if (buffer == null)
return Flux.empty();
try{
return Flux.just(buffer);
} finally { // flux 使用了netty，但是netty分配空间有限，如果不清除缓存，势必造成内存溢出，原代码为：return Flux.just(buffer);
DataBufferUtils.release(buffer);
}
}
|