The proper way to transmit a texture between compositor effects?

Help Shaders
,
1

Godot Version

4.7.1.stable

Question

Hi! I’m making a CompositorEffect that requires a second camera and viewport to transmit a texture to my main compositor effect. I have it all set up already, it’s just one texture is consistently failing to display properly, and sometimes even throws an error to the godot console.

I have setup the auxiliary “donor” effect as such:

@tool
class_name AuxEffect extends CompositorEffect

var rd: RenderingDevice
var shader: RID
var main_pipeline: RID
var auxtex: Array[RID]
var depthex: Array[RID]
var scene_buffers: RenderSceneBuffersRD

var started: bool = false

func _notification(what: int) -> void:
	if what == NOTIFICATION_PREDELETE and shader.is_valid() and shader:
		RenderingServer.free_rid(shader)

func _init() -> void:
	needs_motion_vectors = false
	effect_callback_type = CompositorEffect.EFFECT_CALLBACK_TYPE_POST_TRANSPARENT

func _render_callback(_effect_callback_type: int, render_data: RenderData) -> void:
	scene_buffers = render_data.get_render_scene_buffers()
	if not scene_buffers: return
	
	auxtex.resize(scene_buffers.get_view_count())
	depthex.resize(scene_buffers.get_view_count())
	
	if not started:
		started = true # This delays the shader by one render callback
		return
	
	for view in scene_buffers.get_view_count():
		var aux: RID = scene_buffers.get_color_layer(view)
		var depth: RID = scene_buffers.get_depth_layer(view)
		
		auxtex[view] = aux
		depthex[view] = depth

And a helper autoload class that only exists because you can’t access the scene tree from within a compositor effect. It has an array of viewports because the editor can have up to 4 viewports simultaneously, and this approach prevents the effect from bugging out when adding those extra viewports:

@tool
extends Node

var mask_tex: Array[Array]
var depth_tex: Array[Array]
var aux_effects: Array[AuxEffect]
var aux_viewports: Array[SubViewport]
var aux_scene_buffers: Array[RenderSceneBuffersRD]
var tree: SceneTree
var currently_edited_scene: Node
var aux_viewport_scene = preload("res://assets/scenes/aux_camera.tscn")
var hash_array: Array[StringName] = ["One", "Two", "Three", "Four"]

func _ready():
	if Engine.is_editor_hint():
		start_values.call_deferred()
		for i in range(4):
			mask_tex.append([RID()])
			depth_tex.append([RID()])
			aux_effects.append(null)


func start_values() -> void:
	if Engine.is_editor_hint():
		for aux_viewport in aux_viewports:
			if is_instance_valid(aux_viewport):
				if aux_viewport.get_parent() == currently_edited_scene:
					currently_edited_scene.remove_child(aux_viewport)
				aux_viewport.queue_free()
		
		aux_viewports.clear()
		aux_effects.clear()
		mask_tex.clear()
		depth_tex.clear()
		aux_scene_buffers.clear()
		
		if currently_edited_scene is WorldEnvironment:
			for index in range(4):
				if not currently_edited_scene.find_child(str(hash_array[index].hash()), false, false):
					aux_viewports.append(aux_viewport_scene.instantiate())
					aux_viewports[index].name = str(hash_array[index].hash())
					aux_viewports[index].size = EditorInterface.get_editor_viewport_3d(index).get_size()
					currently_edited_scene.add_child(aux_viewports[index])
                    
					aux_effects.append(aux_viewports[index].get_camera_3d().compositor.compositor_effects[0])
					mask_tex.append(aux_effects[index].auxtex)
					depth_tex.append(aux_effects[index].depthex)
					aux_scene_buffers.append(aux_effects[index].scene_buffers)


func _process(_delta: float) -> void:
	if Engine.is_editor_hint():
		var previously_edited_scene: Node = currently_edited_scene
		currently_edited_scene = EditorInterface.get_edited_scene_root()
		if currently_edited_scene != previously_edited_scene and previously_edited_scene:
			for child_index in range(4):
				var child = previously_edited_scene.find_child(str(hash_array[child_index].hash()), false, false)
				if child in aux_viewports:
					previously_edited_scene.remove_child(child)
					child.queue_free()
					aux_viewports.remove_at(aux_viewports.find(child))
			
			tree = currently_edited_scene.get_tree()
			if aux_viewports.size() < 4:
				start_values()
				return
				
			for index in range(aux_viewports.size()):
				var current_viewport: SubViewport = EditorInterface.get_editor_viewport_3d(index)
				if current_viewport != aux_viewports[index]:
					start_values()
					return
		elif not previously_edited_scene and currently_edited_scene:
			start_values()
		
		if currently_edited_scene is WorldEnvironment:
			for index in range(4):
				var editor_viewport: SubViewport = EditorInterface.get_editor_viewport_3d(index)
				aux_viewports[index].size = editor_viewport.size
				
				var aux_camera: Camera3D = aux_viewports[index].get_node("AuxCamera")
				var editor_camera: Camera3D = editor_viewport.get_camera_3d()
				aux_camera.transform = editor_camera.transform
				aux_camera.fov = editor_camera.fov
				
				mask_tex[index] = aux_effects[index].auxtex
				depth_tex[index] = aux_effects[index].depthex
				aux_scene_buffers[index] = aux_effects[index].scene_buffers

And the main effect. Only binding 4 matters to my problem, as it’s the one with errors:

@tool
class_name TrueDatamosh extends CompositorEffect

var rd: RenderingDevice
var shader: RID
var main_pipeline: RID

var nearest_sampler: RID
var fmt: RDTextureFormat
var size: Vector2i = Vector2i(1920, 1080)

var prevtex: Array[RID] = []

func _notification(what: int) -> void:
	if what == NOTIFICATION_PREDELETE and shader:
		if shader.is_valid():
			RenderingServer.free_rid(shader)


func initialize_cs() -> void:
	rd = RenderingServer.get_rendering_device()
	if not rd:
		#print("Failed to get RD.")
		return
	#print("RD got successfully.")
	
	var glsl_file: RDShaderFile = load("res://assets/shaders/true_datamosh.glsl")
	shader = rd.shader_create_from_spirv(glsl_file.get_spirv())
	if not shader:
		#print("Failed to get shader.")
		return
	#print("Shader got successfully.")
	
	main_pipeline = rd.compute_pipeline_create(shader)
	if not main_pipeline:
		#print("Failed to get pipeline.")
		return
	#print("Pipeline got successfully.")


func _init() -> void:
	needs_motion_vectors = true
	effect_callback_type = CompositorEffect.EFFECT_CALLBACK_TYPE_POST_TRANSPARENT
	RenderingServer.call_on_render_thread(initialize_cs)
	prevtex.clear()
	
	var sampler_state: RDSamplerState = RDSamplerState.new()
	sampler_state.min_filter = RenderingDevice.SAMPLER_FILTER_NEAREST
	sampler_state.mag_filter = RenderingDevice.SAMPLER_FILTER_NEAREST
	nearest_sampler = rd.sampler_create(sampler_state)
	
	fmt = RDTextureFormat.new()
	fmt.width = size.x
	fmt.height = size.y
	fmt.format = RenderingDevice.DATA_FORMAT_R16G16B16A16_SFLOAT
	fmt.usage_bits = RenderingDevice.TEXTURE_USAGE_CAN_UPDATE_BIT \
					| RenderingDevice.TEXTURE_USAGE_STORAGE_BIT \
					| RenderingDevice.TEXTURE_USAGE_CAN_COPY_FROM_BIT \
					| RenderingDevice.TEXTURE_USAGE_CPU_READ_BIT \
					| RenderingDevice.TEXTURE_USAGE_SAMPLING_BIT
	
	prevtex.append(rd.texture_create(fmt, RDTextureView.new(), []))

func _render_callback(_effect_callback_type: int, render_data: RenderData) -> void:
	if ShaderGlobals.aux_viewports.size() > 0:
		if not rd or not shader or not main_pipeline:
			initialize_cs()
			return
			
		var masktex: RID
		var maskdepth: RID
		
		var scene_buffers: RenderSceneBuffersRD = render_data.get_render_scene_buffers()
		if not scene_buffers: return
		
		size = scene_buffers.get_internal_size()
		if size.x == 0 or size.y == 0: return
		
		fmt.width = size.x
		fmt.height = size.y
		
		@warning_ignore_start("integer_division")
		var x_groups: int = (size.x - 1) / 16 + 1
		var y_groups: int = (size.y - 1) / 16 + 1
		@warning_ignore_restore("integer_division")
		
		var push_constants: PackedFloat32Array
		push_constants.append(size.x)
		push_constants.append(size.y)
		
		for view in scene_buffers.get_view_count():
			var screentex: RID = scene_buffers.get_color_layer(view)
			var motiontex: RID = scene_buffers.get_velocity_layer(view)
			var depthext: RID = scene_buffers.get_depth_layer(view)
			
			var render_target: RID = render_data.get_render_scene_buffers().get_render_target()
			for index in range(4):
				var viewport: SubViewport = EditorInterface.get_editor_viewport_3d(index)
				var viewport_RID: RID = viewport.get_viewport_rid()
				var viewport_render_target_RID: RID = RenderingServer.viewport_get_render_target(viewport_RID)
				if render_target == viewport_render_target_RID:
					masktex = ShaderGlobals.mask_tex[index][view]
					#masktex = ShaderGlobals.aux_scene_buffers[index].get_color_layer(view)
					maskdepth = ShaderGlobals.depth_tex[index][view]
					#maskdepth = ShaderGlobals.aux_scene_buffers[index].get_color_layer(view)
					break
			
			var uniform_mask: RDUniform = RDUniform.new()
			uniform_mask.uniform_type = RenderingDevice.UNIFORM_TYPE_IMAGE
			uniform_mask.binding = 4
			uniform_mask.add_id(masktex)
            
			var image_uniform_set: RID
			image_uniform_set = UniformSetCacheRD.get_cache(shader, 0, [uniform_screen, uniform_vector, uniform_depth, uniform_prev, uniform_mask, uniform_maskdepth])
			
			var compute_list: int = rd.compute_list_begin()
			
			rd.compute_list_bind_compute_pipeline(compute_list, main_pipeline)
			rd.compute_list_bind_uniform_set(compute_list, image_uniform_set, 0)
			rd.compute_list_set_push_constant(compute_list, push_constants.to_byte_array(), push_constants.size() * 4)
			rd.compute_list_dispatch(compute_list, x_groups, y_groups, 1)
			rd.compute_list_end()

My main problem now is that I get a black screen, if I use imageStore(screen_tex, giid, mask_tex), and half the time the editor will throw errors to the console, always with binding 4, which I can’t figure out what is really wrong with it. I tried to get the scene buffers directly, or just the textures, and I still can’t figure out how to properly do it.

Can someone here please help me? There is no documentation in any of this CompositorEffect thingy…

2

I think the binding 4 errors come from holding the raw get_color_layer RID, that texture belongs to the aux viewport’s internal buffers and gets freed and recreated when they rebuild, like on a resize, so the stored RID can go stale by the time the main effect binds it.

What if the aux effect copied the color layer into a texture it creates and owns with rd.texture_create, and shared that RID instead? An owned texture survives the buffer rebuilds, so binding 4 always gets something valid. Roughly, create it once with the same format as the color buffer, then each callback:

rd.texture_copy(scene_buffers.get_color_layer(view), auxtex[view], Vector3.ZERO, Vector3.ZERO, Vector3(size.x, size.y, 1), 0, 0, 0, 0)
3

That errors in Source texture requires the 'RenderingDevice.TEXTURE_USAGE_CAN_COPY_FROM_BIT' to be set to be retrieved., which is an issue I remember solving, but I can’t recall how I solved it…

4

You actually can:

Engine.get_main_loop()
5

From what I’ve found the built in color and depth textures don’t have that bit set, so texture_copy can’t read from them directly. There’s a Godot issue where someone hit the same wall trying to copy the depth layer out: [Compositor] unable to access depth texture in shader · Issue #99493 · godotengine/godot · GitHub

Maybe try to copy in a tiny compute pass. The color layer does have the storage bit, that’s how compositor effects bind it in the first place, so a minimal shader that binds the color layer as one image and your owned texture as another and does imageStore(dest, giid, imageLoad(src, giid)) gets the same result without needing any copy usage bits. I’m pretty sure the aux effect would dispatch that pass in its callback and the main effect keeps reading the owned texture.

6

And then what? Just Engine.get_main_loop().get_tree().root?

7

Well depends what you need from it. Engine::get_main_loop() returns the same thing Node::get_tree() would. You have the access to the scene tree object. SceneTree is the main loop. If you look at the class reference you’ll see that SceneTree inherits MainLoop

8

Oh, that simple. I’ll definitely check that tomorrow, then, it’s getting late now.

9

That’s weird, because if I test the depth texture with my method, it just works. On top of that, this shader was working before I tried to add the functionality of supporting more than one viewport and scene switching (it was arguably unusable in an actual project without those features, so I’m working on them), I just don’t know what happened since then and now that it just stopped working.

On top of that, it to this day has an issue in the editor that some fragments result in NaN unless you remove the effect and readd it to the scene, only in the editor, but never ingame.

Doing this whole project while documentation blind is getting out of hand, but I know it’s going to be worth it in the end.

10

Since depth comes through and color doesn’t, what if you printed the RIDs themselves right before the uniform set gets built:

print(rd.texture_is_valid(masktex), " ", rd.texture_is_valid(maskdepth))

If masktex comes back false on the frames that error, then it’s the handoff from the aux effect rather than anything in the shader. If it’s valid, then I’d look at the format and usage bits on it next.

The other thing I’m wondering about is the update mode on the aux SubViewport. If it’s still on the default I think it might not be redrawing every frame in the editor, and that could leave auxtex holding a color layer that never gets refreshed while depth happens to land fine.

11

I already checked the process mode to always, and I’ve tried printing the RIDs, and they were all normal. Tomorrow, I’ll try again and get more time with the project and report back here.

I haven’t tried usage bits yet, I should’ve thought of messing with that sooner, but oh well, will try that tomorrow.

12

Sounds good. Maybe try this tomorrow too on both the aux texture and the main viewport’s own color layer:

var f := rd.texture_get_format(masktex)
print(f.usage_bits, " ", f.format, " ", f.width, " ", f.height)
13

Printing that format results in

523 96 890 686
523 96 891 686
(2) 523 96 39 39

This is running once per view, per editor viewport, so it’s outputting more than one value.

Do I really need a tiny shader to copy the texture? It was working before, I don’t understand enough about the compositor, and there isn’t enough documentation for me to work this out on my own lol

14

Seems that masktex and maskdepth are only assigned inside the render target match, so if that loop doesn’t find a hit they stay empty RIDs and still get handed to get_cache. Maybe try bailing out right after that loop:

if not masktex.is_valid() or not maskdepth.is_valid():
	continue

I’m wondering if that’s the intermittent part, since the callback fires for every viewport that uses that compositor, not just the four editor 3D ones.

On the copy question, there’s rd.texture_copy if you only want to move pixels, so not necessarily a shader. The catch is the source needs TEXTURE_USAGE_CAN_COPY_FROM_BIT and the destination needs TEXTURE_USAGE_CAN_COPY_TO_BIT, and I’m not certain the 3D color buffer is created with those, so it might just error and tell you.

What are the four numbers in that print? If two of them are the mask and two are the main viewport’s internal size, and they aren’t lining up, that could explain the black screen, since your dispatch is sized off get_internal_size() and the giid would be running past the mask texture.

15

image
image1795×698 335 KB

The two values above seem to be from the other viewport I have open in the editor, it seems. There’s 4 viewports at all times in the editor, so that’s why it keeps outputting 4 values.

Where do I put the continue in the code? I’m putting more debug code in the source effect, but now that I think about it, I should do it on the target effect.

523 96 891 342

It only outputs once because of the code to get the texture using a break after finding a match. It runs only once per viewport, too, which is weird that it’s getting those same values every time.

16

Try with the continue in the outer view loop, right after your render target loop closes and before you build uniform_mask:

for view in scene_buffers.get_view_count():
	masktex = RID()
	maskdepth = RID()

	# your existing loop over the four editor viewports here

	if not masktex.is_valid() or not maskdepth.is_valid():
		continue

	var uniform_mask: RDUniform = RDUniform.new()

Clearing them at the top is worth adding too, since masktex and maskdepth are declared once at the top of the callback. On a multi view pass a view that finds no match would carry the previous view’s RIDs through and the guard wouldn’t catch it.

On the print, the last two numbers are following your editor viewport size, they line up with the 891 x 686 in your screenshot overlay, and 891 x 342 reads like a horizontal split. The first two sit at 523 96 no matter what. Which pair is the mask? If it’s the 523 96 one, then that texture isn’t following the resize, and I’m wondering if that’s your black screen, since the dispatch is sized off get_internal_size() and the giid would run past it.

17

I did that and it still outputs a black screen.

func _render_callback(_effect_callback_type: int, render_data: RenderData) -> void:
#...
	for view in scene_buffers.get_view_count():
		var screentex: RID = scene_buffers.get_color_layer(view)
		var motiontex: RID = scene_buffers.get_velocity_layer(view)
		var depthext: RID = scene_buffers.get_depth_layer(view)
		
		var render_target: RID = render_data.get_render_scene_buffers().get_render_target()
		for index in range(4):
			var viewport: SubViewport = EditorInterface.get_editor_viewport_3d(index)
			var viewport_RID: RID = viewport.get_viewport_rid()
			var viewport_render_target_RID: RID = RenderingServer.viewport_get_render_target(viewport_RID)
			
			if render_target == viewport_render_target_RID:
				
				masktex = ShaderGlobals.mask_tex[index][view]
				maskdepth = ShaderGlobals.depth_tex[index][view]
				
				break
		if not masktex.is_valid() or not maskdepth.is_valid():
			continue

The shader is dispatched after that, and doing imageStore(screen_tex, uv, mask_tex); on glsl is what’s doing the black screen, which probably means the color layer is getting wiped at some point.

I think it’s not persisting after being drawn because the texture is being cleared after the draw is done, even with the clear_mode in the subviewport set to never…

18

Also, note that doing imageStore(screen_tex, uv, vec4(vec3(depth_tex), 1.0)); outputs just fine… I’m really puzzled here…

image
image1795×698 50.3 KB

19

Post a MRP.

20

Hmm, the 523 is sampling plus color attachment plus storage plus input attachment, and 96 is R16G16B16A16_SFLOAT. So there’s no copy from bit on it, which settles the texture_copy dead end from earlier, but the storage bit is there, so binding it as an image is legitimate and the RID itself looks healthy.

Seems like it might just be the shader, which is the one piece that hasn’t been in the thread yet. Could you post the binding declarations from true_datamosh.glsl, binding 4 especially? An image2D you read from needs a format qualifier and it has to match the texture, so if binding 4 is declared as anything other than layout(rgba16f) you’d get this exact pair of symptoms, validation noise on that binding and zeros coming back.

Fits the depth working too. A depth format can’t carry the storage bit, so maskdepth has to be going in as a sampler, and samplers don’t have the format matching requirement that images do.