From ccdba838f8866df15b19d2f3a7b6e38409db158e Mon Sep 17 00:00:00 2001 From: genteure Date: Sat, 20 Nov 2021 13:19:23 +0800 Subject: [PATCH] Toolbox: Add data hashes to exported xml.gz --- .../RecyclableMemoryStreamProvider.cs | 2 +- .../BililiveRecorder.Flv.csproj | 1 + .../DefaultMemoryStreamProvider.cs | 2 +- BililiveRecorder.Flv/H264Nalu.cs | 6 +++ BililiveRecorder.Flv/IMemoryStreamProvider.cs | 2 +- BililiveRecorder.Flv/Tag.cs | 41 +++++++++++++++++-- .../RecyclableMemoryStreamProvider.cs | 2 +- .../Tool/Export/ExportHandler.cs | 12 ++++-- .../FlvTests/TagTests.cs | 1 + .../TestRecyclableMemoryStreamProvider.cs | 2 +- 10 files changed, 59 insertions(+), 12 deletions(-) diff --git a/BililiveRecorder.Core/RecyclableMemoryStreamProvider.cs b/BililiveRecorder.Core/RecyclableMemoryStreamProvider.cs index e0820a6..d5f788b 100644 --- a/BililiveRecorder.Core/RecyclableMemoryStreamProvider.cs +++ b/BililiveRecorder.Core/RecyclableMemoryStreamProvider.cs @@ -28,6 +28,6 @@ namespace BililiveRecorder.Core //}; } - public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag); + public MemoryStream CreateMemoryStream(string tag) => this.manager.GetStream(tag); } } diff --git a/BililiveRecorder.Flv/BililiveRecorder.Flv.csproj b/BililiveRecorder.Flv/BililiveRecorder.Flv.csproj index 8a620aa..c2deeae 100644 --- a/BililiveRecorder.Flv/BililiveRecorder.Flv.csproj +++ b/BililiveRecorder.Flv/BililiveRecorder.Flv.csproj @@ -8,6 +8,7 @@ + diff --git a/BililiveRecorder.Flv/DefaultMemoryStreamProvider.cs b/BililiveRecorder.Flv/DefaultMemoryStreamProvider.cs index ef185ae..18bd77b 100644 --- a/BililiveRecorder.Flv/DefaultMemoryStreamProvider.cs +++ b/BililiveRecorder.Flv/DefaultMemoryStreamProvider.cs @@ -4,6 +4,6 @@ namespace BililiveRecorder.Flv { public class DefaultMemoryStreamProvider : IMemoryStreamProvider { - public Stream CreateMemoryStream(string tag) => new MemoryStream(); + public MemoryStream CreateMemoryStream(string tag) => new MemoryStream(); } } diff --git a/BililiveRecorder.Flv/H264Nalu.cs b/BililiveRecorder.Flv/H264Nalu.cs index 2fdf4ae..01683c8 100644 --- a/BililiveRecorder.Flv/H264Nalu.cs +++ b/BililiveRecorder.Flv/H264Nalu.cs @@ -84,6 +84,12 @@ namespace BililiveRecorder.Flv /// [XmlAttribute] public H264NaluType Type { get; set; } + + /// + /// nal_unit data hash + /// + [XmlAttribute] + public string? NaluHash { get; set; } } /// diff --git a/BililiveRecorder.Flv/IMemoryStreamProvider.cs b/BililiveRecorder.Flv/IMemoryStreamProvider.cs index 0c6329b..f065bc6 100644 --- a/BililiveRecorder.Flv/IMemoryStreamProvider.cs +++ b/BililiveRecorder.Flv/IMemoryStreamProvider.cs @@ -4,6 +4,6 @@ namespace BililiveRecorder.Flv { public interface IMemoryStreamProvider { - Stream CreateMemoryStream(string tag); + MemoryStream CreateMemoryStream(string tag); } } diff --git a/BililiveRecorder.Flv/Tag.cs b/BililiveRecorder.Flv/Tag.cs index da986cf..400c574 100644 --- a/BililiveRecorder.Flv/Tag.cs +++ b/BililiveRecorder.Flv/Tag.cs @@ -3,8 +3,10 @@ using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; +using System.Runtime.CompilerServices; using System.Xml.Serialization; using BililiveRecorder.Flv.Amf; +using FastHashes; namespace BililiveRecorder.Flv { @@ -26,8 +28,11 @@ namespace BililiveRecorder.Flv [XmlAttribute] public int Timestamp { get; set; } + [XmlAttribute] + public string? DataHash { get; set; } + [XmlIgnore] - public Stream? BinaryData { get; set; } + public MemoryStream? BinaryData { get; set; } [XmlElement] public ScriptTagBody? ScriptData { get; set; } @@ -43,7 +48,7 @@ namespace BililiveRecorder.Flv : BinaryConvertUtilities.StreamToHexString(this.BinaryData); set { - Stream? new_stream = null; + MemoryStream? new_stream = null; if (value != null) new_stream = BinaryConvertUtilities.HexStringToMemoryStream(value); @@ -52,10 +57,13 @@ namespace BililiveRecorder.Flv } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ShouldSerializeBinaryDataForSerializationUseOnly() => 0 != (this.Flag & TagFlag.Header); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ShouldSerializeScriptData() => this.Type == TagType.Script; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ShouldSerializeNalus() => this.Type == TagType.Video && (0 == (this.Flag & TagFlag.Header)); public override string ToString() => this.DebuggerDisplay; @@ -64,7 +72,7 @@ namespace BililiveRecorder.Flv public Tag Clone() => this.Clone(null); public Tag Clone(IMemoryStreamProvider? provider = null) { - Stream? binaryData = null; + MemoryStream? binaryData = null; if (this.BinaryData != null) { binaryData = provider?.CreateMemoryStream(nameof(Tag) + ":" + nameof(Clone)) ?? new MemoryStream(); @@ -88,12 +96,37 @@ namespace BililiveRecorder.Flv Size = this.Size, Index = this.Index, Timestamp = this.Timestamp, + DataHash = this.DataHash, BinaryData = binaryData, ScriptData = scriptData, Nalus = this.Nalus is null ? null : new List(this.Nalus), }; } + private static readonly FarmHash64 farmHash64 = new FarmHash64(); + + public string? UpdateDataHash() + { + if (this.BinaryData is null) + { + this.DataHash = null; + } + else + { + var buffer = this.BinaryData.GetBuffer(); + this.DataHash = BinaryConvertUtilities.ByteArrayToHexString(farmHash64.ComputeHash(buffer, (int)this.BinaryData.Length)); + + if (this.Nalus?.Count > 0) + { + foreach (var nalu in this.Nalus) + { + nalu.NaluHash = BinaryConvertUtilities.ByteArrayToHexString(farmHash64.ComputeHash(buffer, nalu.StartPosition, (int)nalu.FullSize)); + } + } + } + return this.DataHash; + } + private string DebuggerDisplay => string.Format("{0}, {1}{2}{3}, TS={4}, Size={5}", this.Type switch { @@ -158,7 +191,7 @@ namespace BililiveRecorder.Flv return new string(result); } - internal static Stream HexStringToMemoryStream(string hex) + internal static MemoryStream HexStringToMemoryStream(string hex) { var stream = new MemoryStream(hex.Length / 2); for (var i = 0; i < hex.Length; i += 2) diff --git a/BililiveRecorder.ToolBox/RecyclableMemoryStreamProvider.cs b/BililiveRecorder.ToolBox/RecyclableMemoryStreamProvider.cs index 07ac800..65911d4 100644 --- a/BililiveRecorder.ToolBox/RecyclableMemoryStreamProvider.cs +++ b/BililiveRecorder.ToolBox/RecyclableMemoryStreamProvider.cs @@ -28,6 +28,6 @@ namespace BililiveRecorder.ToolBox //}; } - public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag); + public MemoryStream CreateMemoryStream(string tag) => this.manager.GetStream(tag); } } diff --git a/BililiveRecorder.ToolBox/Tool/Export/ExportHandler.cs b/BililiveRecorder.ToolBox/Tool/Export/ExportHandler.cs index 425f522..aed043e 100644 --- a/BililiveRecorder.ToolBox/Tool/Export/ExportHandler.cs +++ b/BililiveRecorder.ToolBox/Tool/Export/ExportHandler.cs @@ -15,7 +15,7 @@ namespace BililiveRecorder.ToolBox.Tool.Export public class ExportHandler : ICommandHandler { private static readonly ILogger logger = Log.ForContext(); - + public string Name => "Export"; public async Task> Handle(ExportRequest request, CancellationToken cancellationToken, ProgressCallback? progress) @@ -66,11 +66,17 @@ namespace BililiveRecorder.ToolBox.Tool.Export var count = 0; var tags = new List(); var memoryStreamProvider = new RecyclableMemoryStreamProvider(); - using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: true, logger: logger); + using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: false, logger: logger); while (!cancellationToken.IsCancellationRequested) { var tag = await reader.ReadTagAsync(cancellationToken).ConfigureAwait(false); - if (tag is null) break; + if (tag is null) + break; + + tag.UpdateDataHash(); + if (!tag.ShouldSerializeBinaryDataForSerializationUseOnly()) + tag.BinaryData?.Dispose(); + tags.Add(tag); if (count++ % 300 == 0 && progress is not null) diff --git a/test/BililiveRecorder.Flv.Tests/FlvTests/TagTests.cs b/test/BililiveRecorder.Flv.Tests/FlvTests/TagTests.cs index 18a7856..25b6636 100644 --- a/test/BililiveRecorder.Flv.Tests/FlvTests/TagTests.cs +++ b/test/BililiveRecorder.Flv.Tests/FlvTests/TagTests.cs @@ -31,6 +31,7 @@ namespace BililiveRecorder.Flv.Tests.FlvTests Assert.Equal(source.Index, result.Index); Assert.Equal(source.Size, result.Size); Assert.Equal(source.Timestamp, result.Timestamp); + Assert.Equal(source.DataHash, result.DataHash); Assert.NotSame(source.BinaryData, result.BinaryData); Assert.Equal(source.BinaryDataForSerializationUseOnly, result.BinaryDataForSerializationUseOnly); diff --git a/test/BililiveRecorder.Flv.Tests/TestRecyclableMemoryStreamProvider.cs b/test/BililiveRecorder.Flv.Tests/TestRecyclableMemoryStreamProvider.cs index bd46d73..c5979fe 100644 --- a/test/BililiveRecorder.Flv.Tests/TestRecyclableMemoryStreamProvider.cs +++ b/test/BililiveRecorder.Flv.Tests/TestRecyclableMemoryStreamProvider.cs @@ -12,6 +12,6 @@ namespace BililiveRecorder.Flv.Tests MaximumFreeLargePoolBytes = 64 * 1024 * 32, }; - public Stream CreateMemoryStream(string tag) => manager.GetStream(tag); + public MemoryStream CreateMemoryStream(string tag) => manager.GetStream(tag); } }