Core: Add disk IO detection

BililiveRecorder.Core/Event/IOStatsEventArgs.cs

@@ -12,6 +12,18 @@ namespace BililiveRecorder.Core.Event
 
         public int NetworkBytesDownloaded { get; set; }
 
+        /// <summary>
+        /// mibi-bits per seconds
+        /// </summary>
         public double NetworkMbps { get; set; }
+
+        public TimeSpan DiskWriteTime { get; set; }
+
+        public int DiskBytesWritten { get; set; }
+
+        /// <summary>
+        /// mibi-bytes per seconds
+        /// </summary>
+        public double DiskMBps { get; set; }
     }
 }

BililiveRecorder.Core/Recording/RawDataRecordTask.cs

@@ -60,9 +60,18 @@ namespace BililiveRecorder.Core.Recording
                     if (bytesRead == 0)
                         break;
 
-                    Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead);
+                    Interlocked.Add(ref this.ioNetworkDownloadedBytes, bytesRead);
 
+                    this.ioDiskStopwatch.Restart();
                     await file.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false);
+                    this.ioDiskStopwatch.Stop();
+
+                    lock (this.ioDiskStatsLock)
+                    {
+                        this.ioDiskWriteTime += this.ioDiskStopwatch.Elapsed;
+                        this.ioDiskWrittenBytes += bytesRead;
+                    }
+                    this.ioDiskStopwatch.Reset();
                 }
             }
             catch (OperationCanceledException ex)

BililiveRecorder.Core/Recording/RecordTaskBase.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -33,9 +34,17 @@ namespace BililiveRecorder.Core.Recording
         protected bool started = false;
         protected bool timeoutTriggered = false;
 
-        private readonly object fillerStatsLock = new object();
-        protected int fillerDownloadedBytes;
-        private DateTimeOffset fillerStatsLastTrigger;
+
+        private readonly object ioStatsLock = new();
+        protected int ioNetworkDownloadedBytes;
+
+        protected Stopwatch ioDiskStopwatch = new();
+        protected object ioDiskStatsLock = new();
+        protected TimeSpan ioDiskWriteTime;
+        protected int ioDiskWrittenBytes;
+        private DateTimeOffset ioDiskWarningTimeout;
+
+        private DateTimeOffset ioStatsLastTrigger;
         private TimeSpan durationSinceNoDataReceived;
 
         protected RecordTaskBase(IRoom room, ILogger logger, IApiClient apiClient)
@@ -96,7 +105,7 @@ namespace BililiveRecorder.Core.Recording
 
             var stream = await this.GetStreamAsync(fullUrl: fullUrl, timeout: (int)this.room.RoomConfig.TimingStreamConnect).ConfigureAwait(false);
 
-            this.fillerStatsLastTrigger = DateTimeOffset.UtcNow;
+            this.ioStatsLastTrigger = DateTimeOffset.UtcNow;
             this.durationSinceNoDataReceived = TimeSpan.Zero;
 
             this.ct.Register(state => Task.Run(async () =>
@@ -118,32 +127,55 @@ namespace BililiveRecorder.Core.Recording
 
         private void Timer_Elapsed_TriggerIOStats(object sender, ElapsedEventArgs e)
         {
-            int bytes;
-            TimeSpan diff;
-            DateTimeOffset start, end;
+            int networkDownloadBytes, diskWriteBytes;
+            TimeSpan durationDiff, diskWriteTime;
+            DateTimeOffset startTime, endTime;
 
-            lock (this.fillerStatsLock)
+
+            lock (this.ioStatsLock) // 锁 timer elapsed 事件本身防止并行运行
             {
-                bytes = Interlocked.Exchange(ref this.fillerDownloadedBytes, 0);
-                end = DateTimeOffset.UtcNow;
-                start = this.fillerStatsLastTrigger;
-                this.fillerStatsLastTrigger = end;
-                diff = end - start;
+                // networks
+                networkDownloadBytes = Interlocked.Exchange(ref this.ioNetworkDownloadedBytes, 0); // 锁网络统计
+                endTime = DateTimeOffset.UtcNow;
+                startTime = this.ioStatsLastTrigger;
+                this.ioStatsLastTrigger = endTime;
+                durationDiff = endTime - startTime;
 
-                this.durationSinceNoDataReceived = bytes > 0 ? TimeSpan.Zero : this.durationSinceNoDataReceived + diff;
+                this.durationSinceNoDataReceived = networkDownloadBytes > 0 ? TimeSpan.Zero : this.durationSinceNoDataReceived + durationDiff;
+
+                // disks
+                lock (this.ioDiskStatsLock) // 锁硬盘统计
+                {
+                    diskWriteTime = this.ioDiskWriteTime;
+                    diskWriteBytes = this.ioDiskWrittenBytes;
+                    this.ioDiskWriteTime = TimeSpan.Zero;
+                    this.ioDiskWrittenBytes = 0;
+                }
             }
 
-            var mbps = bytes * (8d / 1024d / 1024d) / diff.TotalSeconds;
+            var netMbps = networkDownloadBytes * (8d / 1024d / 1024d) / durationDiff.TotalSeconds;
+            var diskMBps = diskWriteBytes / (1024d * 1024d) / diskWriteTime.TotalSeconds;
 
             this.OnIOStats(new IOStatsEventArgs
             {
-                NetworkBytesDownloaded = bytes,
-                Duration = diff,
-                StartTime = start,
-                EndTime = end,
-                NetworkMbps = mbps
+                NetworkBytesDownloaded = networkDownloadBytes,
+                Duration = durationDiff,
+                StartTime = startTime,
+                EndTime = endTime,
+                NetworkMbps = netMbps,
+                DiskBytesWritten = diskWriteBytes,
+                DiskWriteTime = diskWriteTime,
+                DiskMBps = diskMBps,
             });
 
+            var now = DateTimeOffset.Now;
+            if (diskWriteBytes > 0 && this.ioDiskWarningTimeout < now && (diskWriteTime.TotalSeconds > 1d || diskMBps < 2d))
+            {
+                // 硬盘 IO 可能不能满足录播
+                this.ioDiskWarningTimeout = now + TimeSpan.FromMinutes(2); // 最多每 2 分钟提醒一次
+                this.logger.Warning("检测到硬盘写入速度较慢可能影响录播，请检查是否有其他软件或游戏正在使用硬盘");
+            }
+
             if ((!this.timeoutTriggered) && (this.durationSinceNoDataReceived.TotalMilliseconds > this.room.RoomConfig.TimingWatchdogTimeout))
             {
                 this.timeoutTriggered = true;

BililiveRecorder.Core/Recording/StandardRecordTask.cs

@@ -125,7 +125,7 @@ namespace BililiveRecorder.Core.Recording
                         if (bytesRead == 0)
                             break;
                         writer.Advance(bytesRead);
-                        Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead);
+                        Interlocked.Add(ref this.ioNetworkDownloadedBytes, bytesRead);
                     }
                     catch (Exception ex)
                     {
@@ -167,7 +167,16 @@ namespace BililiveRecorder.Core.Recording
                     if (this.context.Comments.Count > 0)
                         this.logger.Debug("修复逻辑输出 {@Comments}", this.context.Comments);
 
-                    await this.writer.WriteAsync(this.context).ConfigureAwait(false);
+                    this.ioDiskStopwatch.Restart();
+                    var bytesWritten = await this.writer.WriteAsync(this.context).ConfigureAwait(false);
+                    this.ioDiskStopwatch.Stop();
+
+                    lock (this.ioDiskStatsLock)
+                    {
+                        this.ioDiskWriteTime += this.ioDiskStopwatch.Elapsed;
+                        this.ioDiskWrittenBytes += bytesWritten;
+                    }
+                    this.ioDiskStopwatch.Reset();
 
                     if (this.context.Actions.Any(x => x is PipelineDisconnectAction))
                     {

BililiveRecorder.Flv/IFlvProcessingContextWriter.cs

@@ -12,6 +12,6 @@ namespace BililiveRecorder.Flv
 
         event EventHandler<FileClosedEventArgs> FileClosed;
 
-        Task WriteAsync(FlvProcessingContext context);
+        Task<int> WriteAsync(FlvProcessingContext context);
     }
 }

BililiveRecorder.Flv/Writer/FlvProcessingContextWriter.cs

@@ -25,6 +25,8 @@ namespace BililiveRecorder.Flv.Writer
         private KeyframesScriptDataValue? keyframesScriptDataValue = null;
         private double lastDuration;
 
+        private int bytesWrittenByCurrentWriteCall { get; set; }
+
         public event EventHandler<FileClosedEventArgs>? FileClosed;
 
         public Action<ScriptTagBody>? BeforeScriptTagWrite { get; set; }
@@ -37,7 +39,7 @@ namespace BililiveRecorder.Flv.Writer
             this.disableKeyframes = disableKeyframes;
         }
 
-        public async Task WriteAsync(FlvProcessingContext context)
+        public async Task<int> WriteAsync(FlvProcessingContext context)
         {
             if (this.state == WriterState.Invalid)
                 throw new InvalidOperationException("FlvProcessingContextWriter is in a invalid state.");
@@ -70,11 +72,16 @@ namespace BililiveRecorder.Flv.Writer
                 this.semaphoreSlim.Release();
             }
 
+            var bytesWritten = this.bytesWrittenByCurrentWriteCall;
+            this.bytesWrittenByCurrentWriteCall = 0;
+
             // Dispose tags
             foreach (var action in context.Actions)
                 if (action is PipelineDataAction dataAction)
                     foreach (var tag in dataAction.Tags)
                         tag.BinaryData?.Dispose();
+
+            return bytesWritten;
         }
 
         #region Flv Writer Implementation
@@ -214,6 +221,8 @@ namespace BililiveRecorder.Flv.Writer
                     await this.tagWriter.WriteTag(this.nextAudioHeaderTag).ConfigureAwait(false);
             }
 
+            this.bytesWrittenByCurrentWriteCall += (int)this.tagWriter.FileSize;
+
             this.state = WriterState.Writing;
         }
 
@@ -236,9 +245,13 @@ namespace BililiveRecorder.Flv.Writer
             var duration = tags[tags.Count - 1].Timestamp / 1000d;
             this.lastDuration = duration;
 
+            var beforeFileSize = this.tagWriter.FileSize;
+
             foreach (var tag in tags)
                 await this.tagWriter.WriteTag(tag).ConfigureAwait(false);
 
+            this.bytesWrittenByCurrentWriteCall += (int)(this.tagWriter.FileSize - beforeFileSize);
+
             await this.RewriteScriptTagImpl(duration, firstTag.IsKeyframeData(), firstTag.Timestamp, pos).ConfigureAwait(false);
         }
 
@@ -255,7 +268,11 @@ namespace BililiveRecorder.Flv.Writer
                     throw new InvalidOperationException($"Can't write data tag with current state ({this.state})");
             }
 
+            var beforeFileSize = this.tagWriter.FileSize;
+
             await this.tagWriter.WriteTag(endAction.Tag).ConfigureAwait(false);
+
+            this.bytesWrittenByCurrentWriteCall += (int)(this.tagWriter.FileSize - beforeFileSize);
         }
         #endregion