深度图像中的重影效果消除

如何解决深度图像中的重影效果消除

我正在执行以下深度过滤器，以消除深度立体图像中的闪烁。

该过滤器可完美地处理低数据增量，如闪烁，可达到10-20。但是有很明显的重影效果。

输入是来自立体摄像机的两个深度图像。

这是执行闪烁删除的脚本

using UnityEngine;

using Unity.Jobs;

using Unity.Collections;

using Unity.Burst;

using sl;



using UnityEngine.UI;

using System;

public class DepthFilter : MonoBehaviour

{

const int HISTORY_CAPACITY = 3;



readonly static int _dataDeltaID = Shader.PropertyToID("_dataDelta");

readonly static int[] _historyIDs = new int[] {

Shader.PropertyToID("_HistoryA"),Shader.PropertyToID("_HistoryB"),Shader.PropertyToID("_HistoryC"),};



public Material filterMaterialAsset;



Material _filterMaterialInstance;



RenderTexture[] _historyPages;

public ZEDToOpenCVRetriever imageRetriever;

private const int width = 1280;

private const int height = 720;

int _oldestPageLeft = 0;

int _oldestPageRight = 0;

public GameObject frame_left;

public GameObject frame_right;

private RenderTexture src_left;

private RenderTexture src_right;

private RenderTexture _outputTexture;

private RenderTexture dest_left;

private RenderTexture dest_right;

private Texture2D final_depth;

public uint dataDelta = 50;

private void Start()

{

if (!imageRetriever) imageRetriever = ZEDToOpenCVRetriever.GetInstance();

imageRetriever.OnImageUpdated_GPU += ImageUpdated;



_outputTexture = new RenderTexture(width,height,0);



dest_left = new RenderTexture(width,0);



dest_right = new RenderTexture(width,0);



frame_left.GetComponent<MeshRenderer>().material.SetTexture("_DepthXYZTex",dest_left);

frame_right.GetComponent<MeshRenderer>().material.SetTexture("_DepthXYZTex",dest_right);



src_left = new RenderTexture(width,0);

src_right = new RenderTexture(width,0);



}

private void ImageUpdated(ref Texture2D zedGPULeftTexture,ref Texture2D zedGPURightTexture)

{

RenderTexture.active = src_left;

// Copy your texture ref to the render texture

Graphics.Blit(zedGPULeftTexture,src_left);

Filter(ref src_left,dest_left,dataDelta);



RenderTexture.active = src_right;

// Copy your texture ref to the render texture

Graphics.Blit(zedGPURightTexture,src_right);

Filter(ref src_right,dest_right,dataDelta);





}

public void Filter(ref RenderTexture source,RenderTexture destination,uint dataDelta)

{



// If we haven't initialized our history buffer,do so.

// Match the size/format of whatever render target

// we were passed as a source.

if (_historyPages == null)

{

_filterMaterialInstance = Instantiate(filterMaterialAsset);

_historyPages = new RenderTexture[HISTORY_CAPACITY];

for (int i = 0; i < HISTORY_CAPACITY; i++)

{

_historyPages[i] = new RenderTexture(source);

_filterMaterialInstance.SetTexture(_historyIDs[i],_historyPages[i]);



}

}




_filterMaterialInstance.SetFloat(_dataDeltaID,dataDelta);



// Apply the shader filter to the source and current history,// and write the result into destination.

Graphics.Blit(source,destination,_filterMaterialInstance);



// Our source becomes the newest sample in our history.

// (Or,if it's supposed to be your destination,use that instead).

// The oldest history page takes its place as a new writable source texture.

var newPage = source;

source = _historyPages[_oldestPageLeft];

_historyPages[_oldestPageLeft] = newPage;

_filterMaterialInstance.SetTexture(_historyIDs[_oldestPageLeft],newPage);



_oldestPageLeft = (_oldestPageLeft + 1) % HISTORY_CAPACITY;

}

}

着色器：

Shader "Unlit/DepthFilter"

{

Properties

{

_MainTex("Texture",2D) = "white" {}

_HistoryA("History A",2D) = "white" {}

_HistoryB("History B",2D) = "white" {}

_HistoryC("History C",2D) = "white" {}




_dataDelta("Data Delta",range(0,256)) = 256

}

SubShader

{

Tags { "RenderType" = "Opaque" }

LOD 100



Pass

{

CGPROGRAM

#pragma vertex vert

#pragma fragment frag



#include "UnityCG.cginc"



struct appdata

{

float4 vertex : POSITION;

float2 uv : TEXCOORD0;

};



struct v2f

{

float2 uv : TEXCOORD0;

float4 vertex : SV_POSITION;

};



sampler2D _MainTex;

sampler2D _HistoryA;

sampler2D _HistoryB;

sampler2D _HistoryC;




float _dataDelta;



v2f vert(appdata v)

{

v2f o;

o.vertex = UnityObjectToClipPos(v.vertex);

o.uv.x = v.uv.x;

o.uv.y = v.uv.y;

return o;

}



fixed4 frag(v2f i) : SV_Target

{

fixed source = tex2D(_MainTex,i.uv).r;



// Accumulate the history into a total.

float average = tex2D(_HistoryA,i.uv).r;

average += tex2D(_HistoryB,i.uv).r;

average += tex2D(_HistoryC,i.uv).r;

average += tex2D(_HistoryD,i.uv).r;



// Divide by our sample count to get an average.

average /= 3.0f;



// Write into the output texture if and only if

// abs(source - average) < _dataDelta.

clip(_dataDelta / 256.0f - abs(source - average));

return source;

}

ENDCG

}

}

}

我认为问题在于剪辑像素。我试图添加更多历史记录页面，但这没有帮助。

这是一个视频https://youtu.be/fXGE_YvCD4M​

我问一个专家，他提出了以下建议，但我不知道如何实现

您可以改为估计每个先前的历史记录框架中的视线，然后将其穿过深度缓冲区以查看其相交的位置。然后使用该射线深度累计到您的平均值。那将是一种重新投射的形式。

当相机静止不动时，所有历史视线都将与当前视线匹配，因此其作用类似于您的当前滤镜。当您的相机运动时，它可以补偿运动。

与我们每帧更新历史记录缓冲区的方式相同，您还需要更新代表反视图投影矩阵的着色器变量的集合。在您的顶点着色器中，您可以通过这些矩阵来变换屏幕位置，以获取光线的方向，以在每个像素上进行插值。

然后，在片段着色器中，编写一个raymarcher，将三个历史射线行进到各自的深度缓冲区中