From e1daab789d203c99ebf9790957ccce7b3429f332 Mon Sep 17 00:00:00 2001 From: Gert Hulselmans Date: Tue, 23 Dec 2025 11:57:01 +0100 Subject: [PATCH] Add `pipe` function to read/write to pipes. Add `pipe` function to read and write to pipes with the help of `popen`. This allows the following: $ cat pipe_command_tar.codon print("Read tar file list") with pipe("tar tzvf codon-linux-x86_64.tar.gz", "r") as fh: i = 0 for line in fh: if i == 15: break print(line, end="") i += 1 print("Gunzip tar file and compress with zstd") with pipe("gzip -cd codon-linux-x86_64.tar.gz", "r") as in_fh: with pipe("zstd -c -9 - > codon-linux-x86_64.tar.zst", "w") as out_fh: chunk = in_fh.read(4096*4) while chunk: out_fh.write(chunk) chunk = in_fh.read(4096*4) $ codon run pipe_command_tar.codon Read tar file list drwxr-xr-x root/root 0 2025-11-28 01:00 codon-deploy-linux-x86_64/ drwxr-xr-x root/root 0 2025-11-28 01:49 codon-deploy-linux-x86_64/python/ -rw-r--r-- root/root 2692 2025-11-28 00:32 codon-deploy-linux-x86_64/python/setup.py -rw-r--r-- root/root 32 2025-11-28 00:32 codon-deploy-linux-x86_64/python/.gitignore -rw-r--r-- root/root 184 2025-11-28 00:32 codon-deploy-linux-x86_64/python/README.md -rw-r--r-- root/root 69 2025-11-28 00:32 codon-deploy-linux-x86_64/python/pyproject.toml drwxr-xr-x root/root 0 2025-11-28 01:00 codon-deploy-linux-x86_64/python/codon/ -rw-r--r-- root/root 134 2025-11-28 00:37 codon-deploy-linux-x86_64/python/codon/version.py -rw-r--r-- root/root 10317 2025-11-28 00:32 codon-deploy-linux-x86_64/python/codon/decorator.py -rw-r--r-- root/root 2541 2025-11-28 00:32 codon-deploy-linux-x86_64/python/codon/jit.pyx -rw-r--r-- root/root 636 2025-11-28 00:32 codon-deploy-linux-x86_64/python/codon/jit.pxd -rw-r--r-- root/root 279 2025-11-28 00:32 codon-deploy-linux-x86_64/python/codon/__init__.py -rw-r--r-- root/root 20 2025-11-28 00:32 codon-deploy-linux-x86_64/python/MANIFEST.in drwxr-xr-x root/root 0 2025-11-28 01:00 codon-deploy-linux-x86_64/python/codon_jit.egg-info/ -rw-r--r-- root/root 6 2025-11-28 01:00 codon-deploy-linux-x86_64/python/codon_jit.egg-info/top_level.txt Gunzip tar file and compress with zstd There are a few downsides to using popen: - popen spawns a shell to execute the "command" and failures of finding the actual command are not 100% reliable to detect. The shell return exit code 127 if if can't find the command and exit code 126 if it is not an executable, but if for some reason the actual command would use the same exit codes, there is no way to communicate that. - popen only captures stdout and not stderr So in the long run it is probably better to have a proper implementation that can also capture stderr (and avoids deadlocks when reading both streams) and also captures the real exit code instead of always spawning a shell. --- stdlib/internal/__init__.codon | 2 +- stdlib/internal/c_stubs.codon | 10 +++++++++ stdlib/internal/file.codon | 40 +++++++++++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/stdlib/internal/__init__.codon b/stdlib/internal/__init__.codon index cd3789a66..18c7d495f 100644 --- a/stdlib/internal/__init__.codon +++ b/stdlib/internal/__init__.codon @@ -40,7 +40,7 @@ from internal.str import * from internal.sort import sorted from openmp import Ident as __OMPIdent, for_par, for_par as par -from internal.file import File, gzFile, open, gzopen +from internal.file import File, gzFile, open, gzopen, pipe from internal.gpu import _gpu_loop_outline_template from pickle import pickle, unpickle from internal.dlopen import dlsym as _dlsym diff --git a/stdlib/internal/c_stubs.codon b/stdlib/internal/c_stubs.codon index 44de5f9f3..e29de8269 100644 --- a/stdlib/internal/c_stubs.codon +++ b/stdlib/internal/c_stubs.codon @@ -629,6 +629,16 @@ def fflush(a: cobj) -> None: def getline(a: Ptr[cobj], b: Ptr[int], c: cobj) -> int: pass +@nocapture +@C +def popen(a: cobj, b: cobj) -> cobj: + pass + +@nocapture +@C +def pclose(a: cobj) -> int: + pass + # from C import exit(int) diff --git a/stdlib/internal/file.codon b/stdlib/internal/file.codon index 215605aea..28fded740 100644 --- a/stdlib/internal/file.codon +++ b/stdlib/internal/file.codon @@ -6,21 +6,37 @@ class File: sz: int buf: Ptr[byte] fp: cobj + is_pipe: bool def __init__(self, fp: cobj): self.fp = fp + self.is_pipe = False self._reset() def __init__(self, path: str, mode: str): self.fp = _C.fopen(path.c_str(), mode.c_str()) if not self.fp: raise IOError(f"file {path} could not be opened") + self.is_pipe = False self._reset() def __init__(self, fd: int, mode: str): self.fp = _C.fdopen(fd, mode.c_str()) if not self.fp: raise IOError(f"file descriptor {fd} could not be opened") + self.is_pipe = False + self._reset() + + def __init__(self, cmd: str, mode: str, is_pipe: bool): + if not is_pipe: + self.__init__(cmd, mode) + return None + # Add "FD_CLOEXEC" flag to mode. + mode = mode + "e" if "e" not in mode else mode + self.fp = _C.popen(cmd.c_str(), mode.c_str()) + if not self.fp: + raise IOError(f"shell for executing {cmd} could not be opened") + self.is_pipe = True self._reset() def _errcheck(self, msg: str): @@ -66,12 +82,16 @@ class File: def tell(self) -> int: self._ensure_open() + if self.is_pipe: + raise IOError("\"tell\" cannot be used on a pipe") ret = _C.ftell(self.fp) self._errcheck("error in tell") return ret def seek(self, offset: int, whence: int): self._ensure_open() + if self.is_pipe: + raise IOError("\"seek\" cannot be used on a pipe") _C.fseek(self.fp, offset, i32(whence)) self._errcheck("error in seek") @@ -81,11 +101,26 @@ class File: def close(self): if self.fp: - _C.fclose(self.fp) + if self.is_pipe: + exit_code = int((_C.pclose(self.fp) & 0xff00) >> 8) + else: + _C.fclose(self.fp) self.fp = cobj() if self.buf: _C.free(self.buf) self._reset() + if self.is_pipe: + if exit_code == 127: + raise IOError("command not found") + elif exit_code == 126: + raise IOError("command not executable") + elif exit_code == -1: + raise IOError("failed to close pipe") + elif exit_code == 141: + # Ignore broken pipe exit code. + pass + elif exit_code != 0: + raise IOError(f"command failed with exit code: {exit_code}") def _ensure_open(self): if not self.fp: @@ -401,6 +436,9 @@ class bzFile: def open(path, mode: str = "r") -> File: return File(path, mode) +def pipe(path, mode: str = "r") -> File: + return File(path, mode, True) + def gzopen(path: str, mode: str = "r") -> gzFile: return gzFile(path, mode)