FLV: UpdateTimestampOffsetRule is now functional

This commit is contained in:
Genteure 2021-04-22 22:40:40 +08:00
parent 5f4c9633bd
commit afd8f7a2d3
13 changed files with 259 additions and 148 deletions

View File

@ -4,16 +4,20 @@ using Microsoft.IO;
namespace BililiveRecorder.Core
{
public class RecyclableMemoryStreamProvider : IMemoryStreamProvider
internal class RecyclableMemoryStreamProvider : IMemoryStreamProvider
{
private readonly RecyclableMemoryStreamManager manager = new RecyclableMemoryStreamManager(32 * 1024, 64 * 1024, 64 * 1024 * 32)
{
MaximumFreeSmallPoolBytes = 64 * 1024 * 1024,
MaximumFreeLargePoolBytes = 64 * 1024 * 32,
};
private readonly RecyclableMemoryStreamManager manager;
public RecyclableMemoryStreamProvider()
{
const int K = 1024;
const int M = K * K;
this.manager = new RecyclableMemoryStreamManager(32 * K, 64 * K, 64 * K * 32)
{
MaximumFreeSmallPoolBytes = 32 * M,
MaximumFreeLargePoolBytes = 64 * K * 32,
};
//manager.StreamFinalized += () =>
//{
// Debug.WriteLine("TestRecyclableMemoryStreamProvider: Stream Finalized");

View File

@ -1,4 +1,3 @@
using System;
using System.Collections.Generic;
using System.Linq;
using BililiveRecorder.Flv.Pipeline;
@ -11,11 +10,14 @@ namespace BililiveRecorder.Flv.Grouping.Rules
public bool AppendWith(Tag tag, LinkedList<Tag> tags, out LinkedList<Tag>? leftover)
{
var flag = tag.IsNonKeyframeData()
|| (tag.IsKeyframeData() && tags.All(x => x.IsNonKeyframeData()))
|| (tag.Type == TagType.Audio && tag.Flag == TagFlag.Header && tags.All(x => x.Type != TagType.Audio || x.Flag == TagFlag.Header));
var shouldAppend =
// Tag 是非关键帧数据
tag.IsNonKeyframeData()
// 或是音频头,并且之前未出现过音频数据
|| (tag.Type == TagType.Audio && tag.Flag == TagFlag.Header && tags.All(x => x.Type != TagType.Audio || x.Flag == TagFlag.Header));
// || (tag.IsKeyframeData() && tags.All(x => x.IsNonKeyframeData()))
if (flag)
if (shouldAppend)
{
tags.AddLast(tag);
leftover = null;
@ -23,40 +25,9 @@ namespace BililiveRecorder.Flv.Grouping.Rules
}
else
{
var ts = tag.Timestamp;
var lastAudio = tags.LastOrDefault(x => x.Type == TagType.Audio);
bool predicate(Tag x) => x.Type == TagType.Audio && x.Timestamp >= ts;
if (tag.IsKeyframeData() && lastAudio is not null && Math.Abs(tag.Timestamp - lastAudio.Timestamp) <= 50 && tags.Any(predicate))
{
{
leftover = new LinkedList<Tag>();
foreach (var item in tags.Where(predicate))
leftover.AddLast(item);
leftover.AddLast(tag);
}
// tags.RemoveAll(predicate);
{
var node = tags.First;
while (node != null)
{
var next = node.Next;
if (predicate(node.Value))
tags.Remove(node);
node = next;
}
}
return false;
}
else
{
leftover = new LinkedList<Tag>();
leftover.AddLast(tag);
return false;
}
leftover = new LinkedList<Tag>();
leftover.AddLast(tag);
return false;
}
}

View File

@ -6,9 +6,8 @@ namespace BililiveRecorder.Flv.Pipeline.Rules
{
public class UpdateTimestampOffsetRule : ISimpleProcessingRule
{
private const int MAX_ALLOWED_DIFF = 1000 * 10; // 10 seconds
private static readonly ProcessingComment comment1 = new ProcessingComment(CommentType.Unrepairable, "GOP 内音频或视频时间戳不连续");
private static readonly ProcessingComment comment2 = new ProcessingComment(CommentType.Unrepairable, "出现了无法计算偏移量的音视频偏移");
public void Run(FlvProcessingContext context, Action next)
{
@ -29,39 +28,121 @@ namespace BililiveRecorder.Flv.Pipeline.Rules
yield break;
}
// 这个问题可能不能稳定修复,如果是在录直播最好还是断开重连,获取正常的直播流
// TODO 确认修复效果
yield return PipelineDisconnectAction.Instance;
if (!(this.CheckIfNormal(data.Tags.Where(x => x.Type == TagType.Audio)) && this.CheckIfNormal(data.Tags.Where(x => x.Type == TagType.Video))))
{
// 音频或视频自身就有问题,没救了
yield return PipelineDisconnectAction.Instance;
context.AddComment(comment1);
yield break;
}
else
{
var audio = data.Tags.First(x => x.Type == TagType.Audio);
var video = data.Tags.First(x => x.Type == TagType.Video);
var oc = new OffsetCalculator();
var diff = audio.Timestamp - video.Timestamp;
foreach (var tag in data.Tags)
oc.AddTag(tag);
if (diff > 50)
if (oc.Calculate(out var videoOffset))
{
context.AddComment(new ProcessingComment(CommentType.TimestampOffset, $"音视频时间戳偏移, A: {audio.Timestamp}, V: {video.Timestamp}, D: {diff}"));
foreach (var tag in data.Tags.Where(x => x.Type == TagType.Audio))
if (videoOffset != 0)
{
tag.Timestamp -= diff;
context.AddComment(new ProcessingComment(CommentType.TimestampOffset, $"音视频时间戳偏移, D: {videoOffset}"));
foreach (var tag in data.Tags)
if (tag.Type == TagType.Video)
tag.Timestamp += videoOffset;
}
yield return data;
yield break;
}
else
{
context.AddComment(comment2);
yield return PipelineDisconnectAction.Instance;
yield break;
}
// 因为上面已经检查了音频或视频单独不存在时间戳跳变问题,所以可以进行排序
data.Tags = data.Tags.OrderBy(x => x.Timestamp).ToList();
yield return data;
}
}
else
yield return action;
}
/// <summary>
/// 音视频偏差量计算
/// </summary>
private class OffsetCalculator
{
/*
*
*
* 使
* */
private Tag? lastAudio = null;
private readonly Stack<Tag> tags = new Stack<Tag>();
private int maxOffset = int.MaxValue;
private int minOffset = int.MinValue;
public void AddTag(Tag tag)
{
if (tag.Type == TagType.Audio)
{
this.ReduceOffsetRange(this.lastAudio, tag);
this.lastAudio = tag;
}
else if (tag.Type == TagType.Video)
{
this.tags.Push(tag);
}
else
throw new ArgumentException("unexpected tag type");
}
public bool Calculate(out int offset)
{
this.ReduceOffsetRange(this.lastAudio, null);
this.lastAudio = null;
if (this.minOffset == this.maxOffset)
{
offset = this.minOffset;
return true;
}
else if (this.minOffset <= this.maxOffset)
{
offset = (this.minOffset + this.maxOffset) / 2;
return true;
}
else
{
offset = 0;
return false;
}
}
private void ReduceOffsetRange(Tag? leftAudio, Tag? rightAudio)
{
while (this.tags.Count > 0)
{
var video = this.tags.Pop();
if (leftAudio is not null)
{
var min = leftAudio.Timestamp - video.Timestamp;
if (this.minOffset < min)
this.minOffset = min;
}
if (rightAudio is not null)
{
var max = rightAudio.Timestamp - video.Timestamp;
if (this.maxOffset > max)
this.maxOffset = max;
}
}
}
}
}
}

View File

@ -6,6 +6,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.1" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="1.4.0" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20574.7" />
</ItemGroup>

View File

@ -50,7 +50,7 @@ namespace BililiveRecorder.ToolBox.Commands
FileStream? flvFileStream = null;
try
{
var memoryStreamProvider = new DefaultMemoryStreamProvider();
var memoryStreamProvider = new RecyclableMemoryStreamProvider();
var comments = new List<ProcessingComment>();
var context = new FlvProcessingContext();
var session = new Dictionary<object, object?>();

View File

@ -65,7 +65,8 @@ namespace BililiveRecorder.ToolBox.Commands
{
var count = 0;
var tags = new List<Tag>();
using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), new DefaultMemoryStreamProvider(), skipData: true, logger: logger);
var memoryStreamProvider = new RecyclableMemoryStreamProvider();
using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: true, logger: logger);
while (true)
{
var tag = await reader.ReadTagAsync(default).ConfigureAwait(false);

View File

@ -53,7 +53,7 @@ namespace BililiveRecorder.ToolBox.Commands
FileStream? flvFileStream = null;
try
{
var memoryStreamProvider = new DefaultMemoryStreamProvider();
var memoryStreamProvider = new RecyclableMemoryStreamProvider();
var comments = new List<ProcessingComment>();
var context = new FlvProcessingContext();
var session = new Dictionary<object, object?>();

View File

@ -0,0 +1,33 @@
using System.IO;
using BililiveRecorder.Flv;
using Microsoft.IO;
namespace BililiveRecorder.ToolBox
{
internal class RecyclableMemoryStreamProvider : IMemoryStreamProvider
{
private readonly RecyclableMemoryStreamManager manager;
public RecyclableMemoryStreamProvider()
{
const int K = 1024;
const int M = K * K;
this.manager = new RecyclableMemoryStreamManager(32 * K, 64 * K, 64 * K * 32)
{
MaximumFreeSmallPoolBytes = 32 * M,
MaximumFreeLargePoolBytes = 64 * K * 32,
};
//manager.StreamFinalized += () =>
//{
// Debug.WriteLine("TestRecyclableMemoryStreamProvider: Stream Finalized");
//};
//manager.StreamDisposed += () =>
//{
// // Debug.WriteLine("TestRecyclableMemoryStreamProvider: Stream Disposed");
//};
}
public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
}
}

View File

@ -16,90 +16,92 @@
mc:Ignorable="d" DataContext="{x:Null}"
d:DesignHeight="600" d:DesignWidth="900"
Title="ToolboxAutoFixPage">
<Grid Margin="20">
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
</Grid.RowDefinitions>
<DockPanel Margin="0,0,0,10">
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Content="修复失败?">
<ui:FlyoutService.Flyout>
<ui:Flyout Placement="LeftEdgeAlignedTop">
<StackPanel>
<TextBlock Text="如果修复出错、或在修复后依旧有问题"/>
<TextBlock Text="请发邮件到 rec@danmuji.org 反馈"/>
<TextBlock Text="在邮件内附上此处导出的分析数据"/>
<TextBlock Text="并尽量详细描述视频的问题、所使用的录播姬版本"/>
<TextBlock Text="请分析未经修复的原始文件,而不是录播姬修复后的文件"/>
<TextBlock><Hyperlink NavigateUri=""><Run Text="更多信息请点击这里"/></Hyperlink></TextBlock>
<Button Margin="0,15,0,10" HorizontalAlignment="Center" Click="Export_Button_Click">
<StackPanel Orientation="Horizontal">
<ui:PathIcon Margin="0,0,5,0" Style="{StaticResource PathIconDataExport}"/>
<TextBlock Text="导出原始分析数据"/>
</StackPanel>
</Button>
</StackPanel>
</ui:Flyout>
</ui:FlyoutService.Flyout>
</Button>
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Margin="5,0" Click="Fix_Button_Click">
<StackPanel Orientation="Horizontal">
<ui:PathIcon Height="14" Margin="0,0,5,0" Style="{StaticResource PathIconDataAutoFix}"/>
<TextBlock Text="修复"/>
</StackPanel>
</Button>
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Click="Analyze_Button_Click">
<StackPanel Orientation="Horizontal">
<ui:PathIcon Height="14" Margin="0,0,5,0" Style="{StaticResource PathIconDataMagnifyScan}"/>
<TextBlock Text="分析"/>
</StackPanel>
</Button>
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Margin="0,0,5,0" Content="选择..." Click="SelectFile_Button_Click"/>
<TextBox ui:ControlHelper.PlaceholderText="FLV 文件" ui:TextBoxHelper.IsDeleteButtonVisible="False" x:Name="FileNameTextBox"/>
</DockPanel>
<Border Grid.Row="1" BorderThickness="1" CornerRadius="5" x:Name="analyzeResultDisplayArea" DataContext="{x:Null}"
<Border Background="Transparent" AllowDrop="True" Drop="FileNameTextBox_Drop">
<Grid Margin="20" >
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
</Grid.RowDefinitions>
<DockPanel Margin="0,0,0,10">
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Content="修复失败?">
<ui:FlyoutService.Flyout>
<ui:Flyout Placement="LeftEdgeAlignedTop">
<StackPanel>
<TextBlock Text="如果修复出错、或在修复后依旧有问题"/>
<TextBlock Text="请发邮件到 rec@danmuji.org 反馈"/>
<TextBlock Text="在邮件内附上此处导出的分析数据"/>
<TextBlock Text="并尽量详细描述视频的问题、所使用的录播姬版本"/>
<TextBlock Text="请分析未经修复的原始文件,而不是录播姬修复后的文件"/>
<TextBlock><Hyperlink NavigateUri=""><Run Text="更多信息请点击这里"/></Hyperlink></TextBlock>
<Button Margin="0,15,0,10" HorizontalAlignment="Center" Click="Export_Button_Click">
<StackPanel Orientation="Horizontal">
<ui:PathIcon Margin="0,0,5,0" Style="{StaticResource PathIconDataExport}"/>
<TextBlock Text="导出原始分析数据"/>
</StackPanel>
</Button>
</StackPanel>
</ui:Flyout>
</ui:FlyoutService.Flyout>
</Button>
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Margin="5,0" Click="Fix_Button_Click">
<StackPanel Orientation="Horizontal">
<ui:PathIcon Height="14" Margin="0,0,5,0" Style="{StaticResource PathIconDataAutoFix}"/>
<TextBlock Text="修复"/>
</StackPanel>
</Button>
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Click="Analyze_Button_Click">
<StackPanel Orientation="Horizontal">
<ui:PathIcon Height="14" Margin="0,0,5,0" Style="{StaticResource PathIconDataMagnifyScan}"/>
<TextBlock Text="分析"/>
</StackPanel>
</Button>
<Button VerticalAlignment="Bottom" DockPanel.Dock="Right" Margin="0,0,5,0" Content="选择..." Click="SelectFile_Button_Click"/>
<TextBox ui:ControlHelper.PlaceholderText="FLV 文件" ui:TextBoxHelper.IsDeleteButtonVisible="False" x:Name="FileNameTextBox" />
</DockPanel>
<Border Grid.Row="1" BorderThickness="1" CornerRadius="5" x:Name="analyzeResultDisplayArea" DataContext="{x:Null}"
Background="{DynamicResource SystemControlBackgroundAltHighBrush}"
BorderBrush="{DynamicResource SystemControlBackgroundAccentBrush}">
<Border.Resources>
<DataTemplate x:Key="NullAnalyzeResult">
<StackPanel Margin="16">
<TextBlock Text="点击分析按钮开始分析" FontSize="26" VerticalAlignment="Center" HorizontalAlignment="Center"/>
<TextBlock Text="注:不分析也可以进行修复操作" VerticalAlignment="Center" HorizontalAlignment="Center"/>
</StackPanel>
</DataTemplate>
<DataTemplate x:Key="NormalAnalyzeResult" DataType="{x:Type tool:AnalyzeResponse}">
<Grid Margin="5">
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
</Grid.RowDefinitions>
<TextBox Grid.Row="0" IsReadOnly="True" Text="{Binding InputPath}" ui:ControlHelper.Header="文件:"/>
<TextBlock Grid.Row="1" HorizontalAlignment="Center" FontSize="24" Text="无需修复" Foreground="Green"
<Border.Resources>
<DataTemplate x:Key="NullAnalyzeResult">
<StackPanel Margin="16">
<TextBlock Text="点击分析按钮开始分析" FontSize="26" VerticalAlignment="Center" HorizontalAlignment="Center"/>
<TextBlock Text="注:不分析也可以进行修复操作" VerticalAlignment="Center" HorizontalAlignment="Center"/>
</StackPanel>
</DataTemplate>
<DataTemplate x:Key="NormalAnalyzeResult" DataType="{x:Type tool:AnalyzeResponse}">
<Grid Margin="5">
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
</Grid.RowDefinitions>
<TextBox Grid.Row="0" IsReadOnly="True" Text="{Binding InputPath}" ui:ControlHelper.Header="文件:"/>
<TextBlock Grid.Row="1" HorizontalAlignment="Center" FontSize="24" Text="无需修复" Foreground="Green"
Visibility="{Binding NeedFix,Converter={StaticResource InvertBooleanToVisibilityCollapsedConverter}}"/>
<TextBlock Grid.Row="1" HorizontalAlignment="Center" FontSize="24" Text="需要修复" Foreground="Red"
<TextBlock Grid.Row="1" HorizontalAlignment="Center" FontSize="24" Text="需要修复" Foreground="Red"
Visibility="{Binding NeedFix,Converter={StaticResource BooleanToVisibilityCollapsedConverter}}"/>
<StackPanel Grid.Row="2" HorizontalAlignment="Center"
<StackPanel Grid.Row="2" HorizontalAlignment="Center"
Visibility="{Binding Unrepairable,Converter={StaticResource BooleanToVisibilityCollapsedConverter}}">
<TextBlock HorizontalAlignment="Center" FontSize="20" Text="文件内存在录播姬无法自动修复的问题" Foreground="Red"/>
<TextBlock HorizontalAlignment="Center" Text="请点击“修复失败?”按钮并反馈本问题"/>
</StackPanel>
<StackPanel Grid.Row="3" HorizontalAlignment="Center" Margin="10">
<TextBlock Text="{Binding OutputFileCount,StringFormat=修复将会输出 {0} 个文件}" Margin="0,0,0,5"/>
<TextBlock Text="{Binding IssueTypeTimestampOffset,StringFormat=时间戳错位问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeTimestampJump,StringFormat=时间戳跳变问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeDecodingHeader,StringFormat=分辨率、解码问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeRepeatingData,StringFormat=重复片段 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeOther,StringFormat=其他问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeUnrepairable,StringFormat=无法修复的问题 {0} 处}"/>
</StackPanel>
</Grid>
</DataTemplate>
<c:NullValueTemplateSelector x:Key="SelectorTemplate" Normal="{StaticResource NormalAnalyzeResult}" Null="{StaticResource NullAnalyzeResult}"/>
</Border.Resources>
<ContentControl Content="{Binding}" ContentTemplateSelector="{StaticResource SelectorTemplate}"/>
</Border>
</Grid>
<TextBlock HorizontalAlignment="Center" FontSize="20" Text="文件内存在录播姬无法自动修复的问题" Foreground="Red"/>
<TextBlock HorizontalAlignment="Center" Text="请点击 修复失败? 按钮并反馈本问题"/>
</StackPanel>
<StackPanel Grid.Row="3" HorizontalAlignment="Center" Margin="10">
<TextBlock Text="{Binding OutputFileCount,StringFormat=修复将会输出 {0} 个文件}" Margin="0,0,0,5"/>
<TextBlock Text="{Binding IssueTypeTimestampOffset,StringFormat=时间戳错位问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeTimestampJump,StringFormat=时间戳跳变问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeDecodingHeader,StringFormat=分辨率、解码问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeRepeatingData,StringFormat=重复片段 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeOther,StringFormat=其他问题 {0} 处}"/>
<TextBlock Text="{Binding IssueTypeUnrepairable,StringFormat=无法修复的问题 {0} 处}"/>
</StackPanel>
</Grid>
</DataTemplate>
<c:NullValueTemplateSelector x:Key="SelectorTemplate" Normal="{StaticResource NormalAnalyzeResult}" Null="{StaticResource NullAnalyzeResult}"/>
</Border.Resources>
<ContentControl Content="{Binding}" ContentTemplateSelector="{StaticResource SelectorTemplate}"/>
</Border>
</Grid>
</Border>
</ui:Page>

View File

@ -275,5 +275,19 @@ namespace BililiveRecorder.WPF.Pages
catch (Exception) { }
}
}
private void FileNameTextBox_Drop(object sender, DragEventArgs e)
{
try
{
if (e.Data.GetDataPresent(DataFormats.FileDrop))
{
var files = (string[])e.Data.GetData(DataFormats.FileDrop);
this.FileNameTextBox.Text = files[0];
}
}
catch (Exception)
{ }
}
}
}

View File

@ -104,7 +104,7 @@ namespace BililiveRecorder.Flv.RuleTests.Integrated
Assert.Equal(0, tags[2].Timestamp);
Assert.Equal(TagFlag.Header, tags[2].Flag);
Assert.Equal(0, tags[3].Timestamp);
Assert.InRange(tags[3].Timestamp, 0, 50);
}
protected void AssertTagsAlmostEqual(List<Tag> expected, List<Tag> actual)

View File

@ -1,4 +1,7 @@
{
"AllowedComments": {
"TimestampOffset": 105
},
"Files": [
{
"VideoHeaderData": "17000000000164001F030100176764001FACB402802DD0800000030080000018478C195001000468EF3CB0",

View File

@ -2,6 +2,7 @@
"AlternativeHeaderCount": 0,
"AllowedComments": {
"DecodingHeader": 1,
"TimestampOffset": 1,
"TimestampJump": 1
},
"Files": [