OS.execute_with_pipe() buffer limitation?

Godot Version

4.3

Question

I start an external executable with OS.execute_with_pipe() then watch it with a new thread to get its STDIO, works ok with threads that have like 100 lines, but when it takes more time, it stops responding. In other programming languages, we can increase the buffer size for that kind of thing, searched for it but couldn’t find anywhere. Thanks.

Just couple of questions. Does it work with just OS.execute().
Also did you check the stderr.

Here is a class I use for it:

class_name ExecPipeClass

var pipe: FileAccess
var stderr: FileAccess
var pid: int
var thread: Thread
var info
signal pipe_in_progress

func exec_using_pipe(bin: String, args: PackedStringArray):
#func init_exec_with_pipe(bin: String, args: PackedStringArray):
	var info = OS.execute_with_pipe(bin, args)
	pipe = info["stdio"]
	stderr=info["stderr"]
	pid=info["pid"]
	thread = Thread.new()
	
func start():
	thread.start(_thread_func)
	
func _thread_func():
	# read stdin and report to log.
	var line:=""
	var pipe_err
	var std_err
	var count=0
	if !pipe.is_open():
		pipe_in_progress.emit.call_deferred("Error opening .", pipe.get_error())
	
	while pipe.is_open():
		pipe_err=pipe.get_error() 
		if pipe_err == OK:
			line=pipe.get_line()
			count+=1
			pipe_in_progress.emit.call_deferred(line)
			pass
		else:
			line=stderr.get_line()
			if line!="":
				pipe_in_progress.emit.call_deferred(line)
			else:
				break
	pipe_in_progress.emit.call_deferred(null)

func clean_thread():
	if thread.is_alive():
		thread.wait_to_finish()
	pipe.close()
	OS.kill(pid)

thank you for replying. i didn’t use OS.execute() because i thought it was blocking. but i guess if i put that in a separate function and run it with a thread it would work. not sure if i could get the stdio from it though.

my code is very similar to your suggestion, i check pipe line and stderror with a signal just like while pipe.isopen(). the weird thing is if the stdio is relatively small, it works just fine. if the executed program’s IO gets longer it doesn’t. i had a similar problem in the past with nodejs and solved it with increasing the buffer size, but couldn’t find anything similar in godot. perhaps it’s well hidden :slight_smile:

What is the external application you are calling? Could it be you need to flush the some buffer once in a while?

it is a bat file i created that runs a cli program in windows, it’s for data recovery. the program needs quotes for its arguments and i’ve found out that i can’t pass them in OS.execute_with_pipe(), so as a not-so-great solution i wrote the code to a bat file and started it instead.

it’d be great to know how i can flush the buffer.

You may want to read this bug report and upvote it.

1 Like

here is a workaround:

extends Control

signal execsignal

var timer: Timer
var o = []
var pipe
var pid = -1
var stds

# Called when the node enters the scene tree for the first time.
func _ready() -> void:
	execsignal.connect(exec_signal)
	mytimer()
	executeCommand()

func exec_signal(s) -> void:
	print(s)
	
	
func checkActive() -> bool:
	var oo = []
	# var cmd = "tasklist /FI \"PID eq " + str(pid) + "\""
	# print(cmd)
	OS.execute("cmd.exe", ["/c", "tasklist"], oo)
	var x = (oo[0]).strip_edges().strip_escapes()
	#find the pid in the output
	if x.find(str(pid)) == -1:
		return false
	else:
		return true
func readFile() -> void:
	#check if pipe is still running with pid
	if pid != -1:
		if checkActive() == false:
			print("Process is dead")
			pipe = null
			pid = -1
			timer.stop()
	else:
		print("No process")
	
	
	if pipe:
		o = []
		OS.execute("powershell.exe", ["-command", "type", "-Tail 1", "test.log"], o,false,false)
		var x = (o[0]).strip_edges().strip_escapes()
		execsignal.emit.call_deferred(x)
	else:
		print("No pipe")
	
	
func executeCommand() -> void:
	var tthread = Thread.new()
	tthread.start(mycommander)
	tthread.wait_to_finish()
	

func mycommander() -> void:
	pipe = OS.execute_with_pipe("cmd.exe", ["/c", "ping", "-n", "20", "localhost", ">", "test.log"])
	pid = pipe.pid
	stds = pipe.stdio
	
	
		
func mytimer() -> void:
	print("Timer started")
	timer = Timer.new()
	timer.set_wait_time(1)
	timer.set_one_shot(false)
	timer.connect("timeout", readFile)
	add_child(timer)
	timer.start()

func _exit_tree() -> void:
	if pipe:
		OS.kill(pipe)
		print("Killed")

for anyone interested, the solution was to use OS.create_process(), it doesn’t get stuck like OS.execute_with_pipe() though you need to manually watch its output with another function (timer in my case).

thanks for all input and your time.