Austin/BililiveRecorder
1
0
Fork 0
mirror of https://github.com/BililiveRecorder/BililiveRecorder.git synced 2024-11-15 19:22:19 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Toolbox: Add data hashes to exported xml.gz

Browse Source
This commit is contained in:
genteure 2021-11-20 13:19:23 +08:00
parent 8ad4962cff
commit ccdba838f8
10 changed files with 59 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BililiveRecorder.Core/RecyclableMemoryStreamProvider.cs
View File

@ -28,6 +28,6 @@ namespace BililiveRecorder.Core
//};
}
public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
public MemoryStream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
}
}

1
BililiveRecorder.Flv/BililiveRecorder.Flv.csproj
View File

@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FastHashes" Version="2.4.0" />
<PackageReference Include="JsonSubTypes" Version="1.8.0" />
<PackageReference Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="5.0.0" />

2
BililiveRecorder.Flv/DefaultMemoryStreamProvider.cs
View File

@ -4,6 +4,6 @@ namespace BililiveRecorder.Flv
{
public class DefaultMemoryStreamProvider : IMemoryStreamProvider
{
public Stream CreateMemoryStream(string tag) => new MemoryStream();
public MemoryStream CreateMemoryStream(string tag) => new MemoryStream();
}
}

6
BililiveRecorder.Flv/H264Nalu.cs
View File

@ -84,6 +84,12 @@ namespace BililiveRecorder.Flv
/// </summary>
[XmlAttribute]
public H264NaluType Type { get; set; }
/// <summary>
/// nal_unit data hash
/// </summary>
[XmlAttribute]
public string? NaluHash { get; set; }
}
/// <summary>

2
BililiveRecorder.Flv/IMemoryStreamProvider.cs
View File

@ -4,6 +4,6 @@ namespace BililiveRecorder.Flv
{
public interface IMemoryStreamProvider
{
Stream CreateMemoryStream(string tag);
MemoryStream CreateMemoryStream(string tag);
}
}

41
BililiveRecorder.Flv/Tag.cs
View File

@ -3,8 +3,10 @@ using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using System.Xml.Serialization;
using BililiveRecorder.Flv.Amf;
using FastHashes;
namespace BililiveRecorder.Flv
{
@ -26,8 +28,11 @@ namespace BililiveRecorder.Flv
[XmlAttribute]
public int Timestamp { get; set; }
[XmlAttribute]
public string? DataHash { get; set; }
[XmlIgnore]
public Stream? BinaryData { get; set; }
public MemoryStream? BinaryData { get; set; }
[XmlElement]
public ScriptTagBody? ScriptData { get; set; }
@ -43,7 +48,7 @@ namespace BililiveRecorder.Flv
: BinaryConvertUtilities.StreamToHexString(this.BinaryData);
set
{
Stream? new_stream = null;
MemoryStream? new_stream = null;
if (value != null)
new_stream = BinaryConvertUtilities.HexStringToMemoryStream(value);
@ -52,10 +57,13 @@ namespace BililiveRecorder.Flv
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ShouldSerializeBinaryDataForSerializationUseOnly() => 0 != (this.Flag & TagFlag.Header);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ShouldSerializeScriptData() => this.Type == TagType.Script;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ShouldSerializeNalus() => this.Type == TagType.Video && (0 == (this.Flag & TagFlag.Header));
public override string ToString() => this.DebuggerDisplay;
@ -64,7 +72,7 @@ namespace BililiveRecorder.Flv
public Tag Clone() => this.Clone(null);
public Tag Clone(IMemoryStreamProvider? provider = null)
{
Stream? binaryData = null;
MemoryStream? binaryData = null;
if (this.BinaryData != null)
{
binaryData = provider?.CreateMemoryStream(nameof(Tag) + ":" + nameof(Clone)) ?? new MemoryStream();
@ -88,12 +96,37 @@ namespace BililiveRecorder.Flv
Size = this.Size,
Index = this.Index,
Timestamp = this.Timestamp,
DataHash = this.DataHash,
BinaryData = binaryData,
ScriptData = scriptData,
Nalus = this.Nalus is null ? null : new List<H264Nalu>(this.Nalus),
};
}
private static readonly FarmHash64 farmHash64 = new FarmHash64();
public string? UpdateDataHash()
{
if (this.BinaryData is null)
{
this.DataHash = null;
}
else
{
var buffer = this.BinaryData.GetBuffer();
this.DataHash = BinaryConvertUtilities.ByteArrayToHexString(farmHash64.ComputeHash(buffer, (int)this.BinaryData.Length));
if (this.Nalus?.Count > 0)
{
foreach (var nalu in this.Nalus)
{
nalu.NaluHash = BinaryConvertUtilities.ByteArrayToHexString(farmHash64.ComputeHash(buffer, nalu.StartPosition, (int)nalu.FullSize));
}
}
}
return this.DataHash;
}
private string DebuggerDisplay => string.Format("{0}, {1}{2}{3}, TS={4}, Size={5}",
this.Type switch
{
@ -158,7 +191,7 @@ namespace BililiveRecorder.Flv
return new string(result);
}
internal static Stream HexStringToMemoryStream(string hex)
internal static MemoryStream HexStringToMemoryStream(string hex)
{
var stream = new MemoryStream(hex.Length / 2);
for (var i = 0; i < hex.Length; i += 2)

2
BililiveRecorder.ToolBox/RecyclableMemoryStreamProvider.cs
View File

@ -28,6 +28,6 @@ namespace BililiveRecorder.ToolBox
//};
}
public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
public MemoryStream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
}
}

10
BililiveRecorder.ToolBox/Tool/Export/ExportHandler.cs
View File

@ -66,11 +66,17 @@ namespace BililiveRecorder.ToolBox.Tool.Export
var count = 0;
var tags = new List<Tag>();
var memoryStreamProvider = new RecyclableMemoryStreamProvider();
using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: true, logger: logger);
using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: false, logger: logger);
while (!cancellationToken.IsCancellationRequested)
{
var tag = await reader.ReadTagAsync(cancellationToken).ConfigureAwait(false);
if (tag is null) break;
if (tag is null)
break;
tag.UpdateDataHash();
if (!tag.ShouldSerializeBinaryDataForSerializationUseOnly())
tag.BinaryData?.Dispose();
tags.Add(tag);
if (count++ % 300 == 0 && progress is not null)

1
test/BililiveRecorder.Flv.Tests/FlvTests/TagTests.cs
View File

@ -31,6 +31,7 @@ namespace BililiveRecorder.Flv.Tests.FlvTests
Assert.Equal(source.Index, result.Index);
Assert.Equal(source.Size, result.Size);
Assert.Equal(source.Timestamp, result.Timestamp);
Assert.Equal(source.DataHash, result.DataHash);
Assert.NotSame(source.BinaryData, result.BinaryData);
Assert.Equal(source.BinaryDataForSerializationUseOnly, result.BinaryDataForSerializationUseOnly);

2
test/BililiveRecorder.Flv.Tests/TestRecyclableMemoryStreamProvider.cs
View File

@ -12,6 +12,6 @@ namespace BililiveRecorder.Flv.Tests
MaximumFreeLargePoolBytes = 64 * 1024 * 32,
};
public Stream CreateMemoryStream(string tag) => manager.GetStream(tag);
public MemoryStream CreateMemoryStream(string tag) => manager.GetStream(tag);
}
}