Core: Add raw data recording mode

This commit is contained in:
Genteure 2021-04-20 22:28:52 +08:00
parent a0a3fd9044
commit 560e8f22e7
11 changed files with 758 additions and 533 deletions

View File

@ -28,6 +28,14 @@ namespace BililiveRecorder.Core.Config.V2
[JsonProperty(nameof(AutoRecord)), EditorBrowsable(EditorBrowsableState.Never)]
public Optional<bool> OptionalAutoRecord { get => this.GetPropertyValueOptional<bool>(nameof(this.AutoRecord)); set => this.SetPropertyValueOptional(value, nameof(this.AutoRecord)); }
/// <summary>
/// 录制模式
/// </summary>
public RecordMode RecordMode { get => this.GetPropertyValue<RecordMode>(); set => this.SetPropertyValue(value); }
public bool HasRecordMode { get => this.GetPropertyHasValue(nameof(this.RecordMode)); set => this.SetPropertyHasValue<RecordMode>(value, nameof(this.RecordMode)); }
[JsonProperty(nameof(RecordMode)), EditorBrowsable(EditorBrowsableState.Never)]
public Optional<RecordMode> OptionalRecordMode { get => this.GetPropertyValueOptional<RecordMode>(nameof(this.RecordMode)); set => this.SetPropertyValueOptional(value, nameof(this.RecordMode)); }
/// <summary>
/// 录制文件自动切割模式
/// </summary>
@ -245,6 +253,14 @@ namespace BililiveRecorder.Core.Config.V2
[JsonProperty(nameof(WpfShowTitleAndArea)), EditorBrowsable(EditorBrowsableState.Never)]
public Optional<bool> OptionalWpfShowTitleAndArea { get => this.GetPropertyValueOptional<bool>(nameof(this.WpfShowTitleAndArea)); set => this.SetPropertyValueOptional(value, nameof(this.WpfShowTitleAndArea)); }
/// <summary>
/// 录制模式
/// </summary>
public RecordMode RecordMode { get => this.GetPropertyValue<RecordMode>(); set => this.SetPropertyValue(value); }
public bool HasRecordMode { get => this.GetPropertyHasValue(nameof(this.RecordMode)); set => this.SetPropertyHasValue<RecordMode>(value, nameof(this.RecordMode)); }
[JsonProperty(nameof(RecordMode)), EditorBrowsable(EditorBrowsableState.Never)]
public Optional<RecordMode> OptionalRecordMode { get => this.GetPropertyValueOptional<RecordMode>(nameof(this.RecordMode)); set => this.SetPropertyValueOptional(value, nameof(this.RecordMode)); }
/// <summary>
/// 录制文件自动切割模式
/// </summary>
@ -332,6 +348,8 @@ namespace BililiveRecorder.Core.Config.V2
public bool WpfShowTitleAndArea => true;
public RecordMode RecordMode => RecordMode.Standard;
public CuttingMode CuttingMode => CuttingMode.Disabled;
public uint CuttingNumber => 100;

View File

@ -0,0 +1,8 @@
namespace BililiveRecorder.Core.Config.V2
{
public enum RecordMode : int
{
Standard = 0,
RawData = 1,
}
}

View File

@ -77,6 +77,11 @@ module.exports = {
"desc": "是否启用自动录制",
"default": "default",
"without_global": true
}, {
"name": "RecordMode",
"type": "RecordMode",
"desc": "录制模式",
"default": "RecordMode.Standard"
}, {
"name": "CuttingMode",
"type": "CuttingMode",

View File

@ -0,0 +1,115 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using BililiveRecorder.Core.Api;
using BililiveRecorder.Core.Event;
using Serilog;
namespace BililiveRecorder.Core.Recording
{
public class RawDataRecordTask : RecordTaskBase
{
private RecordFileOpeningEventArgs? fileOpeningEventArgs;
public RawDataRecordTask(IRoom room,
ILogger logger,
IApiClient apiClient)
: base(room: room,
logger: logger?.ForContext<RawDataRecordTask>().ForContext(LoggingContext.RoomId, room.RoomConfig.RoomId)!,
apiClient: apiClient)
{
}
public override void SplitOutput() { }
public override void RequestStop() => this.cts.Cancel();
protected override void StartRecordingLoop(Stream stream)
{
var paths = this.CreateFileName();
try
{ Directory.CreateDirectory(Path.GetDirectoryName(paths.fullPath)); }
catch (Exception) { }
this.fileOpeningEventArgs = new RecordFileOpeningEventArgs(this.room)
{
SessionId = this.SessionId,
FullPath = paths.fullPath,
RelativePath = paths.relativePath,
FileOpenTime = DateTimeOffset.Now,
};
this.OnRecordFileOpening(this.fileOpeningEventArgs);
var file = new FileStream(paths.fullPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete);
_ = Task.Run(async () => await this.WriteStreamToFileAsync(stream, file).ConfigureAwait(false));
}
private async Task WriteStreamToFileAsync(Stream stream, FileStream file)
{
try
{
var buffer = new byte[1024 * 8];
this.timer.Start();
while (!this.ct.IsCancellationRequested)
{
var bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length, this.ct).ConfigureAwait(false);
if (bytesRead == 0)
break;
Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead);
await file.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false);
}
}
catch (OperationCanceledException ex)
{
this.logger.Debug(ex, "录制被取消");
}
catch (IOException ex)
{
this.logger.Warning(ex, "录制时发生IO错误");
}
catch (Exception ex)
{
this.logger.Warning(ex, "录制时发生了错误");
}
finally
{
this.logger.Debug("录制退出");
this.timer.Stop();
this.cts.Cancel();
try
{
var openingEventArgs = this.fileOpeningEventArgs;
if (openingEventArgs is not null)
this.OnRecordFileClosed(new RecordFileClosedEventArgs(this.room)
{
SessionId = this.SessionId,
FullPath = openingEventArgs.FullPath,
RelativePath = openingEventArgs.RelativePath,
FileOpenTime = openingEventArgs.FileOpenTime,
FileCloseTime = DateTimeOffset.Now,
Duration = 0,
FileSize = file.Length,
});
}
catch (Exception ex)
{
this.logger.Warning(ex, "Error calling OnRecordFileClosed");
}
stream.Dispose();
file.Dispose();
this.OnRecordSessionEnded(EventArgs.Empty);
}
}
}
}

View File

@ -1,529 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Pipelines;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Timers;
using BililiveRecorder.Core.Api;
using BililiveRecorder.Core.Event;
using BililiveRecorder.Core.ProcessingRules;
using BililiveRecorder.Flv;
using BililiveRecorder.Flv.Amf;
using BililiveRecorder.Flv.Pipeline;
using Serilog;
using Timer = System.Timers.Timer;
namespace BililiveRecorder.Core.Recording
{
public class RecordTask : IRecordTask
{
private const string HttpHeaderAccept = "*/*";
private const string HttpHeaderOrigin = "https://live.bilibili.com";
private const string HttpHeaderReferer = "https://live.bilibili.com/";
private const string HttpHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36";
private const int timer_inverval = 2;
private readonly Timer timer = new Timer(1000 * timer_inverval);
private readonly Random random = new Random();
private readonly CancellationTokenSource cts = new CancellationTokenSource();
private readonly CancellationToken ct;
private readonly IRoom room;
private readonly ILogger logger;
private readonly IApiClient apiClient;
private readonly IFlvTagReaderFactory flvTagReaderFactory;
private readonly ITagGroupReaderFactory tagGroupReaderFactory;
private readonly IFlvProcessingContextWriterFactory writerFactory;
private readonly ProcessingDelegate pipeline;
private readonly IFlvWriterTargetProvider targetProvider;
private readonly StatsRule statsRule;
private readonly SplitRule splitFileRule;
private readonly FlvProcessingContext context = new FlvProcessingContext();
private readonly IDictionary<object, object?> session = new Dictionary<object, object?>();
private bool started = false;
private Task? filler;
private ITagGroupReader? reader;
private IFlvProcessingContextWriter? writer;
private readonly object fillerStatsLock = new object();
private int fillerDownloadedBytes;
private DateTimeOffset fillerStatsLastTrigger;
private TimeSpan durationSinceNoDataReceived;
public RecordTask(IRoom room,
ILogger logger,
IProcessingPipelineBuilder builder,
IApiClient apiClient,
IFlvTagReaderFactory flvTagReaderFactory,
ITagGroupReaderFactory tagGroupReaderFactory,
IFlvProcessingContextWriterFactory writerFactory)
{
this.room = room ?? throw new ArgumentNullException(nameof(room));
this.logger = logger?.ForContext<RecordTask>().ForContext(LoggingContext.RoomId, this.room.RoomConfig.RoomId) ?? throw new ArgumentNullException(nameof(logger));
this.apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
this.flvTagReaderFactory = flvTagReaderFactory ?? throw new ArgumentNullException(nameof(flvTagReaderFactory));
this.tagGroupReaderFactory = tagGroupReaderFactory ?? throw new ArgumentNullException(nameof(tagGroupReaderFactory));
this.writerFactory = writerFactory ?? throw new ArgumentNullException(nameof(writerFactory));
if (builder is null)
throw new ArgumentNullException(nameof(builder));
this.ct = this.cts.Token;
this.statsRule = new StatsRule();
this.splitFileRule = new SplitRule();
this.statsRule.StatsUpdated += this.StatsRule_StatsUpdated;
this.pipeline = builder
.Add(this.statsRule)
.Add(this.splitFileRule)
.AddDefault()
.AddRemoveFillerData()
.Build();
this.targetProvider = new WriterTargetProvider(this.room, this.logger.ForContext(LoggingContext.RoomId, this.room.RoomConfig.RoomId), paths =>
{
this.logger.ForContext(LoggingContext.RoomId, this.room.RoomConfig.RoomId).Debug("输出路径 {Path}", paths.fullPath);
var e = new RecordFileOpeningEventArgs(this.room)
{
SessionId = this.SessionId,
FullPath = paths.fullPath,
RelativePath = paths.relativePath,
FileOpenTime = DateTimeOffset.Now,
};
RecordFileOpening?.Invoke(this, e);
return e;
});
this.timer.Elapsed += this.Timer_Elapsed_TriggerStats;
}
public Guid SessionId { get; } = Guid.NewGuid();
public event EventHandler<NetworkingStatsEventArgs>? NetworkingStats;
public event EventHandler<RecordingStatsEventArgs>? RecordingStats;
public event EventHandler<RecordFileOpeningEventArgs>? RecordFileOpening;
public event EventHandler<RecordFileClosedEventArgs>? RecordFileClosed;
public event EventHandler? RecordSessionEnded;
public void SplitOutput() => this.splitFileRule.SetSplitBeforeFlag();
public void RequestStop() => this.cts.Cancel();
public async Task StartAsync()
{
if (this.started)
throw new InvalidOperationException("Only one StartAsync call allowed per instance.");
this.started = true;
var fullUrl = await this.FetchStreamUrlAsync().ConfigureAwait(false);
this.logger.Information("连接直播服务器 {Host}", new Uri(fullUrl).Host);
this.logger.Debug("直播流地址 {Url}", fullUrl);
var stream = await this.GetStreamAsync(fullUrl).ConfigureAwait(false);
var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false));
this.reader = this.tagGroupReaderFactory.CreateTagGroupReader(this.flvTagReaderFactory.CreateFlvTagReader(pipe.Reader));
this.writer = this.writerFactory.CreateWriter(this.targetProvider);
this.writer.BeforeScriptTagWrite = this.Writer_BeforeScriptTagWrite;
this.writer.FileClosed += (sender, e) =>
{
var openingEventArgs = (RecordFileOpeningEventArgs)e.State!;
RecordFileClosed?.Invoke(this, new RecordFileClosedEventArgs(this.room)
{
SessionId = this.SessionId,
FullPath = openingEventArgs.FullPath,
RelativePath = openingEventArgs.RelativePath,
FileOpenTime = openingEventArgs.FileOpenTime,
FileCloseTime = DateTimeOffset.Now,
Duration = e.Duration,
FileSize = e.FileSize,
});
};
this.fillerStatsLastTrigger = DateTimeOffset.UtcNow;
this.durationSinceNoDataReceived = TimeSpan.Zero;
this.filler = Task.Run(async () => await this.FillPipeAsync(stream, pipe.Writer).ConfigureAwait(false));
_ = Task.Run(this.RecordingLoopAsync);
}
private async Task FillPipeAsync(Stream stream, PipeWriter writer)
{
const int minimumBufferSize = 1024;
this.timer.Start();
Exception? exception = null;
try
{
while (!this.ct.IsCancellationRequested)
{
var memory = writer.GetMemory(minimumBufferSize);
try
{
var bytesRead = await stream.ReadAsync(memory, this.ct).ConfigureAwait(false);
if (bytesRead == 0)
break;
writer.Advance(bytesRead);
Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead);
}
catch (Exception ex)
{
exception = ex;
break;
}
var result = await writer.FlushAsync(this.ct).ConfigureAwait(false);
if (result.IsCompleted)
break;
}
}
finally
{
this.timer.Stop();
stream.Dispose();
await writer.CompleteAsync(exception).ConfigureAwait(false);
}
}
private void Timer_Elapsed_TriggerStats(object sender, ElapsedEventArgs e)
{
int bytes;
TimeSpan diff;
DateTimeOffset start, end;
lock (this.fillerStatsLock)
{
bytes = Interlocked.Exchange(ref this.fillerDownloadedBytes, 0);
end = DateTimeOffset.UtcNow;
start = this.fillerStatsLastTrigger;
this.fillerStatsLastTrigger = end;
diff = end - start;
this.durationSinceNoDataReceived = bytes > 0 ? TimeSpan.Zero : this.durationSinceNoDataReceived + diff;
}
var mbps = bytes * 8d / 1024d / 1024d / diff.TotalSeconds;
NetworkingStats?.Invoke(this, new NetworkingStatsEventArgs
{
BytesDownloaded = bytes,
Duration = diff,
StartTime = start,
EndTime = end,
Mbps = mbps
});
if (this.durationSinceNoDataReceived.TotalMilliseconds > this.room.RoomConfig.TimingWatchdogTimeout)
{
this.logger.Warning("直播服务器未断开连接但停止发送直播数据,将会主动断开连接");
this.RequestStop();
}
}
private void Writer_BeforeScriptTagWrite(ScriptTagBody scriptTagBody)
{
if (scriptTagBody.Values.Count == 2 && scriptTagBody.Values[1] is ScriptDataEcmaArray value)
{
var now = DateTimeOffset.Now;
value["Title"] = (ScriptDataString)this.room.Title;
value["Artist"] = (ScriptDataString)$"{this.room.Name} ({this.room.RoomConfig.RoomId})";
value["Comment"] = (ScriptDataString)
($"B站直播间 {this.room.RoomConfig.RoomId} 的直播录像\n" +
$"主播名: {this.room.Name}\n" +
$"直播标题: {this.room.Title}\n" +
$"直播分区: {this.room.AreaNameParent}·{this.room.AreaNameChild}\n" +
$"录制时间: {now:O}\n" +
$"\n" +
$"使用 B站录播姬 录制 https://rec.danmuji.org\n" +
$"录播姬版本: {GitVersionInformation.FullSemVer}");
value["BililiveRecorder"] = new ScriptDataEcmaArray
{
["RecordedBy"] = (ScriptDataString)"BililiveRecorder B站录播姬",
["RecorderVersion"] = (ScriptDataString)GitVersionInformation.FullSemVer,
["StartTime"] = (ScriptDataDate)now,
["RoomId"] = (ScriptDataString)this.room.RoomConfig.RoomId.ToString(),
["ShortId"] = (ScriptDataString)this.room.ShortId.ToString(),
["Name"] = (ScriptDataString)this.room.Name,
["StreamTitle"] = (ScriptDataString)this.room.Title,
["AreaNameParent"] = (ScriptDataString)this.room.AreaNameParent,
["AreaNameChild"] = (ScriptDataString)this.room.AreaNameChild,
};
}
}
private async Task RecordingLoopAsync()
{
try
{
if (this.reader is null) return;
if (this.writer is null) return;
while (!this.ct.IsCancellationRequested)
{
var group = await this.reader.ReadGroupAsync(this.ct).ConfigureAwait(false);
if (group is null)
break;
this.context.Reset(group, this.session);
this.pipeline(this.context);
if (this.context.Comments.Count > 0)
this.logger.Debug("修复逻辑输出 {@Comments}", this.context.Comments);
await this.writer.WriteAsync(this.context).ConfigureAwait(false);
if (this.context.Actions.Any(x => x is PipelineDisconnectAction))
{
this.logger.Information("根据修复逻辑的要求结束录制");
break;
}
}
}
catch (OperationCanceledException ex)
{
this.logger.Debug(ex, "录制被取消");
}
catch (IOException ex)
{
this.logger.Warning(ex, "录制时发生IO错误");
}
catch (Exception ex)
{
this.logger.Warning(ex, "录制时发生未知错误");
}
finally
{
this.logger.Debug("录制退出");
this.reader?.Dispose();
this.reader = null;
this.writer?.Dispose();
this.writer = null;
this.cts.Cancel();
RecordSessionEnded?.Invoke(this, EventArgs.Empty);
}
}
private async Task<Stream> GetStreamAsync(string fullUrl)
{
var client = CreateHttpClient();
while (true)
{
var resp = await client.GetAsync(fullUrl,
HttpCompletionOption.ResponseHeadersRead,
new CancellationTokenSource((int)this.room.RoomConfig.TimingStreamConnect).Token)
.ConfigureAwait(false);
switch (resp.StatusCode)
{
case System.Net.HttpStatusCode.OK:
{
this.logger.Debug("开始接收直播流");
var stream = await resp.Content.ReadAsStreamAsync().ConfigureAwait(false);
return stream;
}
case System.Net.HttpStatusCode.Moved:
case System.Net.HttpStatusCode.Redirect:
{
fullUrl = resp.Headers.Location.OriginalString;
this.logger.Debug("跳转到 {Url}", fullUrl);
resp.Dispose();
break;
}
default:
throw new Exception(string.Format("尝试下载直播流时服务器返回了 ({0}){1}", resp.StatusCode, resp.ReasonPhrase));
}
}
}
private async Task<string> FetchStreamUrlAsync()
{
var apiResp = await this.apiClient.GetStreamUrlAsync(this.room.RoomConfig.RoomId).ConfigureAwait(false);
var url_data = apiResp?.Data?.PlayurlInfo?.Playurl?.Streams;
if (url_data is null)
throw new Exception("playurl is null");
var url_http_stream_flv_avc =
url_data.FirstOrDefault(x => x.ProtocolName == "http_stream")
?.Formats?.FirstOrDefault(x => x.FormatName == "flv")
?.Codecs?.FirstOrDefault(x => x.CodecName == "avc");
if (url_http_stream_flv_avc is null)
throw new Exception("no supported stream url");
if (url_http_stream_flv_avc.CurrentQn != 10000)
this.logger.Warning("当前录制的画质是 {CurrentQn}", url_http_stream_flv_avc.CurrentQn);
var url_infos = url_http_stream_flv_avc.UrlInfos;
if (url_infos is null || url_infos.Length == 0)
throw new Exception("no url_info");
var url_info = url_infos[this.random.Next(url_infos.Length)];
var fullUrl = url_info.Host + url_http_stream_flv_avc.BaseUrl + url_info.Extra;
return fullUrl;
}
private static HttpClient CreateHttpClient()
{
var httpClient = new HttpClient(new HttpClientHandler
{
AllowAutoRedirect = false
});
var headers = httpClient.DefaultRequestHeaders;
headers.Add("Accept", HttpHeaderAccept);
headers.Add("Origin", HttpHeaderOrigin);
headers.Add("Referer", HttpHeaderReferer);
headers.Add("User-Agent", HttpHeaderUserAgent);
return httpClient;
}
private void StatsRule_StatsUpdated(object sender, RecordingStatsEventArgs e)
{
switch (this.room.RoomConfig.CuttingMode)
{
case Config.V2.CuttingMode.ByTime:
if (e.FileMaxTimestamp > this.room.RoomConfig.CuttingNumber * (60u * 1000u))
this.splitFileRule.SetSplitBeforeFlag();
break;
case Config.V2.CuttingMode.BySize:
if ((e.CurrnetFileSize + (e.OutputVideoByteCount * 1.1) + e.OutputAudioByteCount) / (1024d * 1024d) > this.room.RoomConfig.CuttingNumber)
this.splitFileRule.SetSplitBeforeFlag();
break;
}
RecordingStats?.Invoke(this, e);
}
internal class WriterTargetProvider : IFlvWriterTargetProvider
{
private static readonly Random random = new Random();
private readonly IRoom room;
private readonly ILogger logger;
private readonly Func<(string fullPath, string relativePath), object> OnNewFile;
private string last_path = string.Empty;
public WriterTargetProvider(IRoom room, ILogger logger, Func<(string fullPath, string relativePath), object> onNewFile)
{
this.room = room ?? throw new ArgumentNullException(nameof(room));
this.logger = logger?.ForContext<WriterTargetProvider>() ?? throw new ArgumentNullException(nameof(logger));
this.OnNewFile = onNewFile ?? throw new ArgumentNullException(nameof(onNewFile));
}
public (Stream stream, object state) CreateOutputStream()
{
var paths = this.FormatFilename(this.room.RoomConfig.RecordFilenameFormat!);
try
{ Directory.CreateDirectory(Path.GetDirectoryName(paths.fullPath)); }
catch (Exception) { }
this.last_path = paths.fullPath;
var state = this.OnNewFile(paths);
var stream = new FileStream(paths.fullPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete);
return (stream, state);
}
public Stream CreateAlternativeHeaderStream()
{
var path = string.IsNullOrWhiteSpace(this.last_path)
? Path.ChangeExtension(this.FormatFilename(this.room.RoomConfig.RecordFilenameFormat!).fullPath, "headers.txt")
: Path.ChangeExtension(this.last_path, "headers.txt");
try
{ Directory.CreateDirectory(Path.GetDirectoryName(path)); }
catch (Exception) { }
var stream = new FileStream(path, FileMode.Append, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return stream;
}
private (string fullPath, string relativePath) FormatFilename(string formatString)
{
var now = DateTime.Now;
var date = now.ToString("yyyyMMdd");
var time = now.ToString("HHmmss");
var randomStr = random.Next(100, 999).ToString();
var relativePath = formatString
.Replace(@"{date}", date)
.Replace(@"{time}", time)
.Replace(@"{random}", randomStr)
.Replace(@"{roomid}", this.room.RoomConfig.RoomId.ToString())
.Replace(@"{title}", RemoveInvalidFileName(this.room.Title))
.Replace(@"{name}", RemoveInvalidFileName(this.room.Name))
.Replace(@"{parea}", RemoveInvalidFileName(this.room.AreaNameParent))
.Replace(@"{area}", RemoveInvalidFileName(this.room.AreaNameChild))
;
if (!relativePath.EndsWith(".flv", StringComparison.OrdinalIgnoreCase))
relativePath += ".flv";
relativePath = RemoveInvalidFileName(relativePath, ignore_slash: true);
var workDirectory = this.room.RoomConfig.WorkDirectory;
var fullPath = Path.Combine(workDirectory, relativePath);
fullPath = Path.GetFullPath(fullPath);
if (!CheckIsWithinPath(workDirectory!, Path.GetDirectoryName(fullPath)))
{
this.logger.Warning("录制文件位置超出允许范围,请检查设置。将写入到默认路径。");
relativePath = Path.Combine(this.room.RoomConfig.RoomId.ToString(), $"{this.room.RoomConfig.RoomId}-{date}-{time}-{randomStr}.flv");
fullPath = Path.Combine(workDirectory, relativePath);
}
if (File.Exists(fullPath))
{
this.logger.Warning("录制文件名冲突,请检查设置。将写入到默认路径。");
relativePath = Path.Combine(this.room.RoomConfig.RoomId.ToString(), $"{this.room.RoomConfig.RoomId}-{date}-{time}-{randomStr}.flv");
fullPath = Path.Combine(workDirectory, relativePath);
}
return (fullPath, relativePath);
}
internal static string RemoveInvalidFileName(string input, bool ignore_slash = false)
{
foreach (var c in Path.GetInvalidFileNameChars())
if (!ignore_slash || c != '\\' && c != '/')
input = input.Replace(c, '_');
return input;
}
internal static bool CheckIsWithinPath(string parent, string child)
{
if (parent is null || child is null)
return false;
parent = parent.Replace('/', '\\');
if (!parent.EndsWith("\\"))
parent += "\\";
parent = Path.GetFullPath(parent);
child = child.Replace('/', '\\');
if (!child.EndsWith("\\"))
child += "\\";
child = Path.GetFullPath(child);
return child.StartsWith(parent, StringComparison.Ordinal);
}
}
}
}

View File

@ -0,0 +1,283 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Timers;
using BililiveRecorder.Core.Api;
using BililiveRecorder.Core.Event;
using Serilog;
using Timer = System.Timers.Timer;
namespace BililiveRecorder.Core.Recording
{
public abstract class RecordTaskBase : IRecordTask
{
private const string HttpHeaderAccept = "*/*";
private const string HttpHeaderOrigin = "https://live.bilibili.com";
private const string HttpHeaderReferer = "https://live.bilibili.com/";
private const string HttpHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36";
private const int timer_inverval = 2;
protected readonly Timer timer = new Timer(1000 * timer_inverval);
protected readonly Random random = new Random();
protected readonly CancellationTokenSource cts = new CancellationTokenSource();
protected readonly CancellationToken ct;
protected readonly IRoom room;
protected readonly ILogger logger;
protected readonly IApiClient apiClient;
protected bool started = false;
private readonly object fillerStatsLock = new object();
protected int fillerDownloadedBytes;
private DateTimeOffset fillerStatsLastTrigger;
private TimeSpan durationSinceNoDataReceived;
protected RecordTaskBase(IRoom room, ILogger logger, IApiClient apiClient)
{
this.room = room ?? throw new ArgumentNullException(nameof(room));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
this.ct = this.cts.Token;
this.timer.Elapsed += this.Timer_Elapsed_TriggerStats;
}
public Guid SessionId { get; } = Guid.NewGuid();
#region Events
public event EventHandler<NetworkingStatsEventArgs>? NetworkingStats;
public event EventHandler<RecordingStatsEventArgs>? RecordingStats;
public event EventHandler<RecordFileOpeningEventArgs>? RecordFileOpening;
public event EventHandler<RecordFileClosedEventArgs>? RecordFileClosed;
public event EventHandler? RecordSessionEnded;
protected void OnNetworkingStats(NetworkingStatsEventArgs e) => NetworkingStats?.Invoke(this, e);
protected void OnRecordingStats(RecordingStatsEventArgs e) => RecordingStats?.Invoke(this, e);
protected void OnRecordFileOpening(RecordFileOpeningEventArgs e) => RecordFileOpening?.Invoke(this, e);
protected void OnRecordFileClosed(RecordFileClosedEventArgs e) => RecordFileClosed?.Invoke(this, e);
protected void OnRecordSessionEnded(EventArgs e) => RecordSessionEnded?.Invoke(this, e);
#endregion
public virtual void RequestStop() => this.cts.Cancel();
public virtual void SplitOutput() { }
public async virtual Task StartAsync()
{
if (this.started)
throw new InvalidOperationException("Only one StartAsync call allowed per instance.");
this.started = true;
var fullUrl = await this.FetchStreamUrlAsync(this.room.RoomConfig.RoomId).ConfigureAwait(false);
this.logger.Information("连接直播服务器 {Host}", new Uri(fullUrl).Host);
this.logger.Debug("直播流地址 {Url}", fullUrl);
var stream = await this.GetStreamAsync(fullUrl: fullUrl, timeout: (int)this.room.RoomConfig.TimingStreamConnect).ConfigureAwait(false);
this.fillerStatsLastTrigger = DateTimeOffset.UtcNow;
this.durationSinceNoDataReceived = TimeSpan.Zero;
this.StartRecordingLoop(stream);
}
protected abstract void StartRecordingLoop(Stream stream);
private void Timer_Elapsed_TriggerStats(object sender, ElapsedEventArgs e)
{
int bytes;
TimeSpan diff;
DateTimeOffset start, end;
lock (this.fillerStatsLock)
{
bytes = Interlocked.Exchange(ref this.fillerDownloadedBytes, 0);
end = DateTimeOffset.UtcNow;
start = this.fillerStatsLastTrigger;
this.fillerStatsLastTrigger = end;
diff = end - start;
this.durationSinceNoDataReceived = bytes > 0 ? TimeSpan.Zero : this.durationSinceNoDataReceived + diff;
}
var mbps = bytes * (8d / 1024d / 1024d) / diff.TotalSeconds;
this.OnNetworkingStats(new NetworkingStatsEventArgs
{
BytesDownloaded = bytes,
Duration = diff,
StartTime = start,
EndTime = end,
Mbps = mbps
});
if (this.durationSinceNoDataReceived.TotalMilliseconds > this.room.RoomConfig.TimingWatchdogTimeout)
{
this.logger.Warning("直播服务器未断开连接但停止发送直播数据,将会主动断开连接");
this.RequestStop();
}
}
#region File Name
protected (string fullPath, string relativePath) CreateFileName()
{
var formatString = room.RoomConfig.RecordFilenameFormat!;
var now = DateTime.Now;
var date = now.ToString("yyyyMMdd");
var time = now.ToString("HHmmss");
var randomStr = random.Next(100, 999).ToString();
var relativePath = formatString
.Replace(@"{date}", date)
.Replace(@"{time}", time)
.Replace(@"{random}", randomStr)
.Replace(@"{roomid}", this.room.RoomConfig.RoomId.ToString())
.Replace(@"{title}", RemoveInvalidFileName(this.room.Title))
.Replace(@"{name}", RemoveInvalidFileName(this.room.Name))
.Replace(@"{parea}", RemoveInvalidFileName(this.room.AreaNameParent))
.Replace(@"{area}", RemoveInvalidFileName(this.room.AreaNameChild))
;
if (!relativePath.EndsWith(".flv", StringComparison.OrdinalIgnoreCase))
relativePath += ".flv";
relativePath = RemoveInvalidFileName(relativePath, ignore_slash: true);
var workDirectory = this.room.RoomConfig.WorkDirectory;
var fullPath = Path.Combine(workDirectory, relativePath);
fullPath = Path.GetFullPath(fullPath);
if (!CheckIsWithinPath(workDirectory!, Path.GetDirectoryName(fullPath)))
{
this.logger.Warning("录制文件位置超出允许范围,请检查设置。将写入到默认路径。");
relativePath = Path.Combine(this.room.RoomConfig.RoomId.ToString(), $"{this.room.RoomConfig.RoomId}-{date}-{time}-{randomStr}.flv");
fullPath = Path.Combine(workDirectory, relativePath);
}
if (File.Exists(fullPath))
{
this.logger.Warning("录制文件名冲突,请检查设置。将写入到默认路径。");
relativePath = Path.Combine(this.room.RoomConfig.RoomId.ToString(), $"{this.room.RoomConfig.RoomId}-{date}-{time}-{randomStr}.flv");
fullPath = Path.Combine(workDirectory, relativePath);
}
return (fullPath, relativePath);
}
internal static string RemoveInvalidFileName(string input, bool ignore_slash = false)
{
foreach (var c in Path.GetInvalidFileNameChars())
if (!ignore_slash || c != '\\' && c != '/')
input = input.Replace(c, '_');
return input;
}
internal static bool CheckIsWithinPath(string parent, string child)
{
if (parent is null || child is null)
return false;
parent = parent.Replace('/', '\\');
if (!parent.EndsWith("\\"))
parent += "\\";
parent = Path.GetFullPath(parent);
child = child.Replace('/', '\\');
if (!child.EndsWith("\\"))
child += "\\";
child = Path.GetFullPath(child);
return child.StartsWith(parent, StringComparison.Ordinal);
}
#endregion
#region Api Requests
private static HttpClient CreateHttpClient()
{
var httpClient = new HttpClient(new HttpClientHandler
{
AllowAutoRedirect = false
});
var headers = httpClient.DefaultRequestHeaders;
headers.Add("Accept", HttpHeaderAccept);
headers.Add("Origin", HttpHeaderOrigin);
headers.Add("Referer", HttpHeaderReferer);
headers.Add("User-Agent", HttpHeaderUserAgent);
return httpClient;
}
protected async Task<string> FetchStreamUrlAsync(int roomid)
{
var apiResp = await this.apiClient.GetStreamUrlAsync(roomid: roomid).ConfigureAwait(false);
var url_data = apiResp?.Data?.PlayurlInfo?.Playurl?.Streams;
if (url_data is null)
throw new Exception("playurl is null");
var url_http_stream_flv_avc =
url_data.FirstOrDefault(x => x.ProtocolName == "http_stream")
?.Formats?.FirstOrDefault(x => x.FormatName == "flv")
?.Codecs?.FirstOrDefault(x => x.CodecName == "avc");
if (url_http_stream_flv_avc is null)
throw new Exception("no supported stream url");
if (url_http_stream_flv_avc.CurrentQn != 10000)
this.logger.Warning("当前录制的画质是 {CurrentQn}", url_http_stream_flv_avc.CurrentQn);
var url_infos = url_http_stream_flv_avc.UrlInfos;
if (url_infos is null || url_infos.Length == 0)
throw new Exception("no url_info");
var url_info = url_infos[this.random.Next(url_infos.Length)];
var fullUrl = url_info.Host + url_http_stream_flv_avc.BaseUrl + url_info.Extra;
return fullUrl;
}
protected async Task<Stream> GetStreamAsync(string fullUrl, int timeout)
{
var client = CreateHttpClient();
while (true)
{
var resp = await client.GetAsync(fullUrl,
HttpCompletionOption.ResponseHeadersRead,
new CancellationTokenSource(timeout).Token)
.ConfigureAwait(false);
switch (resp.StatusCode)
{
case System.Net.HttpStatusCode.OK:
{
this.logger.Debug("开始接收直播流");
var stream = await resp.Content.ReadAsStreamAsync().ConfigureAwait(false);
return stream;
}
case System.Net.HttpStatusCode.Moved:
case System.Net.HttpStatusCode.Redirect:
{
fullUrl = resp.Headers.Location.OriginalString;
this.logger.Debug("跳转到 {Url}", fullUrl);
resp.Dispose();
break;
}
default:
throw new Exception(string.Format("尝试下载直播流时服务器返回了 ({0}){1}", resp.StatusCode, resp.ReasonPhrase));
}
}
}
#endregion
}
}

View File

@ -1,4 +1,5 @@
using System;
using BililiveRecorder.Core.Config.V2;
using Microsoft.Extensions.DependencyInjection;
namespace BililiveRecorder.Core.Recording
@ -13,6 +14,10 @@ namespace BililiveRecorder.Core.Recording
}
public IRecordTask CreateRecordTask(IRoom room) =>
ActivatorUtilities.CreateInstance<RecordTask>(this.serviceProvider, room);
room.RoomConfig.RecordMode switch
{
RecordMode.RawData => ActivatorUtilities.CreateInstance<RawDataRecordTask>(this.serviceProvider, room),
_ => ActivatorUtilities.CreateInstance<StandardRecordTask>(this.serviceProvider, room)
};
}
}

View File

@ -0,0 +1,294 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Pipelines;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BililiveRecorder.Core.Api;
using BililiveRecorder.Core.Event;
using BililiveRecorder.Core.ProcessingRules;
using BililiveRecorder.Flv;
using BililiveRecorder.Flv.Amf;
using BililiveRecorder.Flv.Pipeline;
using Serilog;
namespace BililiveRecorder.Core.Recording
{
public class StandardRecordTask : RecordTaskBase
{
private readonly IFlvTagReaderFactory flvTagReaderFactory;
private readonly ITagGroupReaderFactory tagGroupReaderFactory;
private readonly IFlvProcessingContextWriterFactory writerFactory;
private readonly ProcessingDelegate pipeline;
private readonly IFlvWriterTargetProvider targetProvider;
private readonly StatsRule statsRule;
private readonly SplitRule splitFileRule;
private readonly FlvProcessingContext context = new FlvProcessingContext();
private readonly IDictionary<object, object?> session = new Dictionary<object, object?>();
private ITagGroupReader? reader;
private IFlvProcessingContextWriter? writer;
public StandardRecordTask(IRoom room,
ILogger logger,
IProcessingPipelineBuilder builder,
IApiClient apiClient,
IFlvTagReaderFactory flvTagReaderFactory,
ITagGroupReaderFactory tagGroupReaderFactory,
IFlvProcessingContextWriterFactory writerFactory)
: base(room: room,
logger: logger?.ForContext<StandardRecordTask>().ForContext(LoggingContext.RoomId, room.RoomConfig.RoomId)!,
apiClient: apiClient)
{
this.flvTagReaderFactory = flvTagReaderFactory ?? throw new ArgumentNullException(nameof(flvTagReaderFactory));
this.tagGroupReaderFactory = tagGroupReaderFactory ?? throw new ArgumentNullException(nameof(tagGroupReaderFactory));
this.writerFactory = writerFactory ?? throw new ArgumentNullException(nameof(writerFactory));
if (builder is null)
throw new ArgumentNullException(nameof(builder));
this.statsRule = new StatsRule();
this.splitFileRule = new SplitRule();
this.statsRule.StatsUpdated += this.StatsRule_StatsUpdated;
this.pipeline = builder
.Add(this.statsRule)
.Add(this.splitFileRule)
.AddDefault()
.AddRemoveFillerData()
.Build();
this.targetProvider = new WriterTargetProvider(this, paths =>
{
this.logger.ForContext(LoggingContext.RoomId, this.room.RoomConfig.RoomId).Debug("输出路径 {Path}", paths.fullPath);
var e = new RecordFileOpeningEventArgs(this.room)
{
SessionId = this.SessionId,
FullPath = paths.fullPath,
RelativePath = paths.relativePath,
FileOpenTime = DateTimeOffset.Now,
};
this.OnRecordFileOpening(e);
return e;
});
}
public override void SplitOutput() => this.splitFileRule.SetSplitBeforeFlag();
protected override void StartRecordingLoop(Stream stream)
{
var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false));
this.reader = this.tagGroupReaderFactory.CreateTagGroupReader(this.flvTagReaderFactory.CreateFlvTagReader(pipe.Reader));
this.writer = this.writerFactory.CreateWriter(this.targetProvider);
this.writer.BeforeScriptTagWrite = this.Writer_BeforeScriptTagWrite;
this.writer.FileClosed += (sender, e) =>
{
var openingEventArgs = (RecordFileOpeningEventArgs)e.State!;
this.OnRecordFileClosed(new RecordFileClosedEventArgs(this.room)
{
SessionId = this.SessionId,
FullPath = openingEventArgs.FullPath,
RelativePath = openingEventArgs.RelativePath,
FileOpenTime = openingEventArgs.FileOpenTime,
FileCloseTime = DateTimeOffset.Now,
Duration = e.Duration,
FileSize = e.FileSize,
});
};
_ = Task.Run(async () => await this.FillPipeAsync(stream, pipe.Writer).ConfigureAwait(false));
_ = Task.Run(this.RecordingLoopAsync);
}
private async Task FillPipeAsync(Stream stream, PipeWriter writer)
{
const int minimumBufferSize = 1024;
this.timer.Start();
Exception? exception = null;
try
{
while (!this.ct.IsCancellationRequested)
{
var memory = writer.GetMemory(minimumBufferSize);
try
{
var bytesRead = await stream.ReadAsync(memory, this.ct).ConfigureAwait(false);
if (bytesRead == 0)
break;
writer.Advance(bytesRead);
Interlocked.Add(ref this.fillerDownloadedBytes, bytesRead);
}
catch (Exception ex)
{
exception = ex;
break;
}
var result = await writer.FlushAsync(this.ct).ConfigureAwait(false);
if (result.IsCompleted)
break;
}
}
finally
{
this.timer.Stop();
stream.Dispose();
await writer.CompleteAsync(exception).ConfigureAwait(false);
}
}
private async Task RecordingLoopAsync()
{
try
{
if (this.reader is null) return;
if (this.writer is null) return;
while (!this.ct.IsCancellationRequested)
{
var group = await this.reader.ReadGroupAsync(this.ct).ConfigureAwait(false);
if (group is null)
break;
this.context.Reset(group, this.session);
this.pipeline(this.context);
if (this.context.Comments.Count > 0)
this.logger.Debug("修复逻辑输出 {@Comments}", this.context.Comments);
await this.writer.WriteAsync(this.context).ConfigureAwait(false);
if (this.context.Actions.Any(x => x is PipelineDisconnectAction))
{
this.logger.Information("根据修复逻辑的要求结束录制");
break;
}
}
}
catch (OperationCanceledException ex)
{
this.logger.Debug(ex, "录制被取消");
}
catch (IOException ex)
{
this.logger.Warning(ex, "录制时发生IO错误");
}
catch (Exception ex)
{
this.logger.Warning(ex, "录制时发生了错误");
}
finally
{
this.logger.Debug("录制退出");
this.reader?.Dispose();
this.reader = null;
this.writer?.Dispose();
this.writer = null;
this.cts.Cancel();
this.OnRecordSessionEnded(EventArgs.Empty);
}
}
private void Writer_BeforeScriptTagWrite(ScriptTagBody scriptTagBody)
{
if (scriptTagBody.Values.Count == 2 && scriptTagBody.Values[1] is ScriptDataEcmaArray value)
{
var now = DateTimeOffset.Now;
value["Title"] = (ScriptDataString)this.room.Title;
value["Artist"] = (ScriptDataString)$"{this.room.Name} ({this.room.RoomConfig.RoomId})";
value["Comment"] = (ScriptDataString)
($"B站直播间 {this.room.RoomConfig.RoomId} 的直播录像\n" +
$"主播名: {this.room.Name}\n" +
$"直播标题: {this.room.Title}\n" +
$"直播分区: {this.room.AreaNameParent}·{this.room.AreaNameChild}\n" +
$"录制时间: {now:O}\n" +
$"\n" +
$"使用 B站录播姬 录制 https://rec.danmuji.org\n" +
$"录播姬版本: {GitVersionInformation.FullSemVer}");
value["BililiveRecorder"] = new ScriptDataEcmaArray
{
["RecordedBy"] = (ScriptDataString)"BililiveRecorder B站录播姬",
["RecorderVersion"] = (ScriptDataString)GitVersionInformation.FullSemVer,
["StartTime"] = (ScriptDataDate)now,
["RoomId"] = (ScriptDataString)this.room.RoomConfig.RoomId.ToString(),
["ShortId"] = (ScriptDataString)this.room.ShortId.ToString(),
["Name"] = (ScriptDataString)this.room.Name,
["StreamTitle"] = (ScriptDataString)this.room.Title,
["AreaNameParent"] = (ScriptDataString)this.room.AreaNameParent,
["AreaNameChild"] = (ScriptDataString)this.room.AreaNameChild,
};
}
}
private void StatsRule_StatsUpdated(object sender, RecordingStatsEventArgs e)
{
switch (this.room.RoomConfig.CuttingMode)
{
case Config.V2.CuttingMode.ByTime:
if (e.FileMaxTimestamp > this.room.RoomConfig.CuttingNumber * (60u * 1000u))
this.splitFileRule.SetSplitBeforeFlag();
break;
case Config.V2.CuttingMode.BySize:
if ((e.CurrnetFileSize + (e.OutputVideoByteCount * 1.1) + e.OutputAudioByteCount) / (1024d * 1024d) > this.room.RoomConfig.CuttingNumber)
this.splitFileRule.SetSplitBeforeFlag();
break;
}
this.OnRecordingStats(e);
}
internal class WriterTargetProvider : IFlvWriterTargetProvider
{
private readonly StandardRecordTask task;
private readonly Func<(string fullPath, string relativePath), object> OnNewFile;
private string last_path = string.Empty;
public WriterTargetProvider(StandardRecordTask task, Func<(string fullPath, string relativePath), object> onNewFile)
{
this.task = task ?? throw new ArgumentNullException(nameof(task));
this.OnNewFile = onNewFile ?? throw new ArgumentNullException(nameof(onNewFile));
}
public (Stream stream, object state) CreateOutputStream()
{
var paths = this.task.CreateFileName();
try
{ Directory.CreateDirectory(Path.GetDirectoryName(paths.fullPath)); }
catch (Exception) { }
this.last_path = paths.fullPath;
var state = this.OnNewFile(paths);
var stream = new FileStream(paths.fullPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete);
return (stream, state);
}
public Stream CreateAlternativeHeaderStream()
{
var path = string.IsNullOrWhiteSpace(this.last_path)
? Path.ChangeExtension(this.task.CreateFileName().fullPath, "headers.txt")
: Path.ChangeExtension(this.last_path, "headers.txt");
try
{ Directory.CreateDirectory(Path.GetDirectoryName(path)); }
catch (Exception) { }
var stream = new FileStream(path, FileMode.Append, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return stream;
}
}
}
}

View File

@ -46,6 +46,20 @@
</local:SettingWithDefault>
</StackPanel>
</GroupBox>
<GroupBox Header="录制模式">
<StackPanel>
<local:SettingWithDefault IsSettingNotUsingDefault="{Binding HasRecordMode}">
<StackPanel>
<RadioButton GroupName="RecordMode" Content="标准模式"
IsChecked="{Binding Path=RecordMode, Converter={StaticResource EnumToBooleanConverter},
ConverterParameter={x:Static confiv2:RecordMode.Standard}}" />
<RadioButton GroupName="RecordMode" Content="原始数据模式"
IsChecked="{Binding Path=RecordMode, Converter={StaticResource EnumToBooleanConverter},
ConverterParameter={x:Static confiv2:RecordMode.RawData}}" />
</StackPanel>
</local:SettingWithDefault>
</StackPanel>
</GroupBox>
<GroupBox Header="{l:Loc Settings_Splitting_Title}">
<StackPanel>
<local:SettingWithDefault IsSettingNotUsingDefault="{Binding HasCuttingMode}">

View File

@ -41,6 +41,18 @@
</StackPanel>
</StackPanel>
</GroupBox>
<GroupBox Header="录制模式">
<StackPanel>
<StackPanel>
<RadioButton GroupName="RecordMode" Content="标准模式"
IsChecked="{Binding Path=RecordMode, Converter={StaticResource EnumToBooleanConverter},
ConverterParameter={x:Static confiv2:RecordMode.Standard}}" />
<RadioButton GroupName="RecordMode" Content="原始数据模式"
IsChecked="{Binding Path=RecordMode, Converter={StaticResource EnumToBooleanConverter},
ConverterParameter={x:Static confiv2:RecordMode.RawData}}" />
</StackPanel>
</StackPanel>
</GroupBox>
<GroupBox Header="{l:Loc Settings_Splitting_Title}">
<StackPanel>
<RadioButton GroupName="Splitting" Name="CutDisabledRadioButton" Content="{l:Loc Settings_Splitting_RadioButton_Disabled}"

View File

@ -32,7 +32,7 @@ namespace BililiveRecorder.Core.UnitTests.Recording
public void Test(string parent, string child, bool result)
{
// TODO fix path tests
Assert.Equal(result, Core.Recording.RecordTask.WriterTargetProvider.CheckIsWithinPath(parent, Path.GetDirectoryName(child)!));
Assert.Equal(result, Core.Recording.RecordTaskBase.CheckIsWithinPath(parent, Path.GetDirectoryName(child)!));
}
}
}