From e446ac5ed50d1a9fb8f95d3cb604d07edcc605c7 Mon Sep 17 00:00:00 2001 From: genteure Date: Sun, 19 Dec 2021 00:57:32 +0800 Subject: [PATCH] Core: Add disk IO detection --- .../Event/IOStatsEventArgs.cs | 12 ++++ .../Recording/RawDataRecordTask.cs | 11 ++- .../Recording/RecordTaskBase.cs | 72 +++++++++++++------ .../Recording/StandardRecordTask.cs | 13 +++- .../IFlvProcessingContextWriter.cs | 2 +- .../Writer/FlvProcessingContextWriter.cs | 19 ++++- 6 files changed, 104 insertions(+), 25 deletions(-) diff --git a/BililiveRecorder.Core/Event/IOStatsEventArgs.cs b/BililiveRecorder.Core/Event/IOStatsEventArgs.cs index 9600c35..428510a 100644 --- a/BililiveRecorder.Core/Event/IOStatsEventArgs.cs +++ b/BililiveRecorder.Core/Event/IOStatsEventArgs.cs @@ -12,6 +12,18 @@ namespace BililiveRecorder.Core.Event public int NetworkBytesDownloaded { get; set; } + /// + /// mibi-bits per seconds + /// public double NetworkMbps { get; set; } + + public TimeSpan DiskWriteTime { get; set; } + + public int DiskBytesWritten { get; set; } + + /// + /// mibi-bytes per seconds + /// + public double DiskMBps { get; set; } } } diff --git a/BililiveRecorder.Core/Recording/RawDataRecordTask.cs b/BililiveRecorder.Core/Recording/RawDataRecordTask.cs index ee184c6..4f9ad16 100644 --- a/BililiveRecorder.Core/Recording/RawDataRecordTask.cs +++ b/BililiveRecorder.Core/Recording/RawDataRecordTask.cs @@ -60,9 +60,18 @@ namespace BililiveRecorder.Core.Recording if (bytesRead == 0) break; - Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead); + Interlocked.Add(ref this.ioNetworkDownloadedBytes, bytesRead); + this.ioDiskStopwatch.Restart(); await file.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false); + this.ioDiskStopwatch.Stop(); + + lock (this.ioDiskStatsLock) + { + this.ioDiskWriteTime += this.ioDiskStopwatch.Elapsed; + this.ioDiskWrittenBytes += bytesRead; + } + this.ioDiskStopwatch.Reset(); } } catch (OperationCanceledException ex) diff --git a/BililiveRecorder.Core/Recording/RecordTaskBase.cs b/BililiveRecorder.Core/Recording/RecordTaskBase.cs index 5f7fe64..2e3e9a0 100644 --- a/BililiveRecorder.Core/Recording/RecordTaskBase.cs +++ b/BililiveRecorder.Core/Recording/RecordTaskBase.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; @@ -33,9 +34,17 @@ namespace BililiveRecorder.Core.Recording protected bool started = false; protected bool timeoutTriggered = false; - private readonly object fillerStatsLock = new object(); - protected int fillerDownloadedBytes; - private DateTimeOffset fillerStatsLastTrigger; + + private readonly object ioStatsLock = new(); + protected int ioNetworkDownloadedBytes; + + protected Stopwatch ioDiskStopwatch = new(); + protected object ioDiskStatsLock = new(); + protected TimeSpan ioDiskWriteTime; + protected int ioDiskWrittenBytes; + private DateTimeOffset ioDiskWarningTimeout; + + private DateTimeOffset ioStatsLastTrigger; private TimeSpan durationSinceNoDataReceived; protected RecordTaskBase(IRoom room, ILogger logger, IApiClient apiClient) @@ -96,7 +105,7 @@ namespace BililiveRecorder.Core.Recording var stream = await this.GetStreamAsync(fullUrl: fullUrl, timeout: (int)this.room.RoomConfig.TimingStreamConnect).ConfigureAwait(false); - this.fillerStatsLastTrigger = DateTimeOffset.UtcNow; + this.ioStatsLastTrigger = DateTimeOffset.UtcNow; this.durationSinceNoDataReceived = TimeSpan.Zero; this.ct.Register(state => Task.Run(async () => @@ -118,32 +127,55 @@ namespace BililiveRecorder.Core.Recording private void Timer_Elapsed_TriggerIOStats(object sender, ElapsedEventArgs e) { - int bytes; - TimeSpan diff; - DateTimeOffset start, end; + int networkDownloadBytes, diskWriteBytes; + TimeSpan durationDiff, diskWriteTime; + DateTimeOffset startTime, endTime; - lock (this.fillerStatsLock) + + lock (this.ioStatsLock) // 锁 timer elapsed 事件本身防止并行运行 { - bytes = Interlocked.Exchange(ref this.fillerDownloadedBytes, 0); - end = DateTimeOffset.UtcNow; - start = this.fillerStatsLastTrigger; - this.fillerStatsLastTrigger = end; - diff = end - start; + // networks + networkDownloadBytes = Interlocked.Exchange(ref this.ioNetworkDownloadedBytes, 0); // 锁网络统计 + endTime = DateTimeOffset.UtcNow; + startTime = this.ioStatsLastTrigger; + this.ioStatsLastTrigger = endTime; + durationDiff = endTime - startTime; - this.durationSinceNoDataReceived = bytes > 0 ? TimeSpan.Zero : this.durationSinceNoDataReceived + diff; + this.durationSinceNoDataReceived = networkDownloadBytes > 0 ? TimeSpan.Zero : this.durationSinceNoDataReceived + durationDiff; + + // disks + lock (this.ioDiskStatsLock) // 锁硬盘统计 + { + diskWriteTime = this.ioDiskWriteTime; + diskWriteBytes = this.ioDiskWrittenBytes; + this.ioDiskWriteTime = TimeSpan.Zero; + this.ioDiskWrittenBytes = 0; + } } - var mbps = bytes * (8d / 1024d / 1024d) / diff.TotalSeconds; + var netMbps = networkDownloadBytes * (8d / 1024d / 1024d) / durationDiff.TotalSeconds; + var diskMBps = diskWriteBytes / (1024d * 1024d) / diskWriteTime.TotalSeconds; this.OnIOStats(new IOStatsEventArgs { - NetworkBytesDownloaded = bytes, - Duration = diff, - StartTime = start, - EndTime = end, - NetworkMbps = mbps + NetworkBytesDownloaded = networkDownloadBytes, + Duration = durationDiff, + StartTime = startTime, + EndTime = endTime, + NetworkMbps = netMbps, + DiskBytesWritten = diskWriteBytes, + DiskWriteTime = diskWriteTime, + DiskMBps = diskMBps, }); + var now = DateTimeOffset.Now; + if (diskWriteBytes > 0 && this.ioDiskWarningTimeout < now && (diskWriteTime.TotalSeconds > 1d || diskMBps < 2d)) + { + // 硬盘 IO 可能不能满足录播 + this.ioDiskWarningTimeout = now + TimeSpan.FromMinutes(2); // 最多每 2 分钟提醒一次 + this.logger.Warning("检测到硬盘写入速度较慢可能影响录播，请检查是否有其他软件或游戏正在使用硬盘"); + } + if ((!this.timeoutTriggered) && (this.durationSinceNoDataReceived.TotalMilliseconds > this.room.RoomConfig.TimingWatchdogTimeout)) { this.timeoutTriggered = true; diff --git a/BililiveRecorder.Core/Recording/StandardRecordTask.cs b/BililiveRecorder.Core/Recording/StandardRecordTask.cs index e6bae24..65d6517 100644 --- a/BililiveRecorder.Core/Recording/StandardRecordTask.cs +++ b/BililiveRecorder.Core/Recording/StandardRecordTask.cs @@ -125,7 +125,7 @@ namespace BililiveRecorder.Core.Recording if (bytesRead == 0) break; writer.Advance(bytesRead); - Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead); + Interlocked.Add(ref this.ioNetworkDownloadedBytes, bytesRead); } catch (Exception ex) { @@ -167,7 +167,16 @@ namespace BililiveRecorder.Core.Recording if (this.context.Comments.Count > 0) this.logger.Debug("修复逻辑输出 {@Comments}", this.context.Comments); - await this.writer.WriteAsync(this.context).ConfigureAwait(false); + this.ioDiskStopwatch.Restart(); + var bytesWritten = await this.writer.WriteAsync(this.context).ConfigureAwait(false); + this.ioDiskStopwatch.Stop(); + + lock (this.ioDiskStatsLock) + { + this.ioDiskWriteTime += this.ioDiskStopwatch.Elapsed; + this.ioDiskWrittenBytes += bytesWritten; + } + this.ioDiskStopwatch.Reset(); if (this.context.Actions.Any(x => x is PipelineDisconnectAction)) { diff --git a/BililiveRecorder.Flv/IFlvProcessingContextWriter.cs b/BililiveRecorder.Flv/IFlvProcessingContextWriter.cs index 6675c0f..0f8b207 100644 --- a/BililiveRecorder.Flv/IFlvProcessingContextWriter.cs +++ b/BililiveRecorder.Flv/IFlvProcessingContextWriter.cs @@ -12,6 +12,6 @@ namespace BililiveRecorder.Flv event EventHandler FileClosed; - Task WriteAsync(FlvProcessingContext context); + Task WriteAsync(FlvProcessingContext context); } } diff --git a/BililiveRecorder.Flv/Writer/FlvProcessingContextWriter.cs b/BililiveRecorder.Flv/Writer/FlvProcessingContextWriter.cs index 4d55044..9e2eda0 100644 --- a/BililiveRecorder.Flv/Writer/FlvProcessingContextWriter.cs +++ b/BililiveRecorder.Flv/Writer/FlvProcessingContextWriter.cs @@ -25,6 +25,8 @@ namespace BililiveRecorder.Flv.Writer private KeyframesScriptDataValue? keyframesScriptDataValue = null; private double lastDuration; + private int bytesWrittenByCurrentWriteCall { get; set; } + public event EventHandler? FileClosed; public Action? BeforeScriptTagWrite { get; set; } @@ -37,7 +39,7 @@ namespace BililiveRecorder.Flv.Writer this.disableKeyframes = disableKeyframes; } - public async Task WriteAsync(FlvProcessingContext context) + public async Task WriteAsync(FlvProcessingContext context) { if (this.state == WriterState.Invalid) throw new InvalidOperationException("FlvProcessingContextWriter is in a invalid state."); @@ -70,11 +72,16 @@ namespace BililiveRecorder.Flv.Writer this.semaphoreSlim.Release(); } + var bytesWritten = this.bytesWrittenByCurrentWriteCall; + this.bytesWrittenByCurrentWriteCall = 0; + // Dispose tags foreach (var action in context.Actions) if (action is PipelineDataAction dataAction) foreach (var tag in dataAction.Tags) tag.BinaryData?.Dispose(); + + return bytesWritten; } #region Flv Writer Implementation @@ -214,6 +221,8 @@ namespace BililiveRecorder.Flv.Writer await this.tagWriter.WriteTag(this.nextAudioHeaderTag).ConfigureAwait(false); } + this.bytesWrittenByCurrentWriteCall += (int)this.tagWriter.FileSize; + this.state = WriterState.Writing; } @@ -236,9 +245,13 @@ namespace BililiveRecorder.Flv.Writer var duration = tags[tags.Count - 1].Timestamp / 1000d; this.lastDuration = duration; + var beforeFileSize = this.tagWriter.FileSize; + foreach (var tag in tags) await this.tagWriter.WriteTag(tag).ConfigureAwait(false); + this.bytesWrittenByCurrentWriteCall += (int)(this.tagWriter.FileSize - beforeFileSize); + await this.RewriteScriptTagImpl(duration, firstTag.IsKeyframeData(), firstTag.Timestamp, pos).ConfigureAwait(false); } @@ -255,7 +268,11 @@ namespace BililiveRecorder.Flv.Writer throw new InvalidOperationException($"Can't write data tag with current state ({this.state})"); } + var beforeFileSize = this.tagWriter.FileSize; + await this.tagWriter.WriteTag(endAction.Tag).ConfigureAwait(false); + + this.bytesWrittenByCurrentWriteCall += (int)(this.tagWriter.FileSize - beforeFileSize); } #endregion