无缓冲Python子流程管道

我一直在尝试围绕子流程实现包装器，如下所示：

def ans_cmd_stream_color(inputcmd):
"""Driver function for local ansible commands.

Stream stdout to stdout and log file with color.
Runs <inputcmd> via subprocess.
Returns return code, stdout, stderr as dict.
"""
fullcmd = inputcmd
create_debug('Enabling colorful ansible output.', LOGGER)
create_info('Running command: ' + fullcmd, LOGGER, True)
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' + fullcmd,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
                     shell=True)
stdout_l = []
stderr_l = []
rcode = 0
# Regex black magic
ansi_escape = re.compile(r'\x1b[^m]*m')
# Get the unbuffered IO action going.
try:
    # Non blocking
    reads = [p.stdout.fileno(), p.stderr.fileno()]
    ret = select.select(reads, [], [])
    # Print line by line
    while True:
        for fd in ret[0]:
            if fd == p.stdout.fileno():
                line = p.stdout.readline()
                sys.stdout.write(line.encode('utf-8'))
                stdout_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
            if fd == p.stderr.fileno():
                line = p.stdout.readline()
                sys.stderr.write(line.encode('utf-8'))
                stderr_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
        # Break when the process is done.
        if p.poll() is not None:
            rcode = p.returncode
            break
except BaseException as e:
    raise e
outstr = ''.join(stdout_l)
errstr = ''.join(stderr_l)
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n')
expstr = errstr.strip('ERROR: ')
if len(expstr) >= 1:
    create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER,
                True)
    if rcode == 0:
        rcode = 1
else:
    create_info(outstr + '\n', LOGGER)
    if rcode == 0:
        create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER,
                    True)
    expstr = False
ret_dict = {inputcmd: {}}
ret_dict[inputcmd]['rcode'] = rcode
ret_dict[inputcmd]['stdout'] = outstr
ret_dict[inputcmd]['stderr'] = expstr
return copy.deepcopy(ret_dict)

这并不理想。有什么想法或者有人遇到过这个问题吗

更新：使用ansible，您需要禁用子流程的缓冲以遵守缓冲设置：

def ans_cmd_stream_color(inputcmd):
"""Driver function for local ansible commands.

Stream stdout to stdout and log file with color.
Runs <inputcmd> via subprocess.
Returns return code, stdout, stderr as dict.
"""
fullcmd = inputcmd
create_debug('Enabling colorful ansible output.', LOGGER)
create_info('Running command: ' + fullcmd, LOGGER, True)
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' +
                     'export PYTHONUNBUFFERED=1; ' + fullcmd,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
                     shell=True)
stdout_l = []
stderr_l = []
rcode = 0
# Regex black magic
ansi_escape = re.compile(r'\x1b[^m]*m')
# Get the unbuffered IO action going.
try:
    # Non blocking
    reads = [p.stdout.fileno(), p.stderr.fileno()]
    ret = select.select(reads, [], [])
    # Print line by line
    while True:
        for fd in ret[0]:
            if fd == p.stdout.fileno():
                line = p.stdout.readline()
                sys.stdout.write(line.encode('utf-8'))
                stdout_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
            if fd == p.stderr.fileno():
                line = p.stdout.readline()
                sys.stderr.write(line.encode('utf-8'))
                stderr_l.append(ansi_escape.sub('',
                                                line.encode('utf-8'))
                                )
        # Break when the process is done.
        if p.poll() is not None:
            rcode = p.returncode
            break
except BaseException as e:
    raise e
outstr = ''.join(stdout_l)
errstr = ''.join(stderr_l)
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n')
expstr = errstr.strip('ERROR: ')
if len(expstr) >= 1:
    create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER,
                True)
    if rcode == 0:
        rcode = 1
else:
    create_info(outstr + '\n', LOGGER)
    if rcode == 0:
        create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER,
                    True)
    expstr = False
ret_dict = {inputcmd: {}}
ret_dict[inputcmd]['rcode'] = rcode
ret_dict[inputcmd]['stdout'] = outstr
ret_dict[inputcmd]['stderr'] = expstr
return copy.deepcopy(ret_dict)

---

您可能应该直接从子流程管道中读取。类似于以下内容的内容将从标准输出读取到信息记录器，并将标准错误读取到错误记录器

import logging, subprocess
logging.basicConfig(level=logging.INFO)
proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
cont = True
while cont:
    cont = False
    line = proc.stdout.readline()
    if not line == b"":
        out = line.decode("utf-8").rstrip()
        logging.info(out)
        cont = True

    line = proc.stderr.readline()
    if not line == b"":
        out = line.decode("utf-8").rstrip()
        logging.error(out)
        cont = True

    if not cont and proc.poll() is not None:
        break

---

为了解决缓冲问题，根据，从属Python脚本必须显式刷新缓冲区，或者必须将环境变量PYTHONUNBUFFERED设置为非空字符串。

不幸的是，直接从管道读取也会缓冲，无法解决获取逐行输出的问题。可以通过不跳过循环的其余部分来处理输出的交错，就像我最初所做的那样。在原始版本中，缓冲区为0，应该可以实现这一点。您确定外部进程没有缓冲吗？是的，100%-os.environ[PYTHONUNBUFFERED]=1解决了这个问题。我最初使用proc.stderr.readline，但后来切换到direct io.open方法来调试输入缓冲区。我意识到subprocess.PIPE文件对象本身存在缓冲。从subprocess中的注释来看：bufsize（如果给定）与内置开放函数的相应参数具有相同的含义：0表示未缓冲，1表示行缓冲，任何其他正值表示使用大约该大小的缓冲。一个负的bufsize意味着使用系统默认值。似乎出于这样或那样的原因这是不被尊重的。这就是你让我难堪的地方。在过去，我通过使用buffer=0来解决类似的问题，或者，如果不起作用，请确保未缓冲从属命令。1-不要将解决方案放入问题中，而是将其作为答案发布，以允许对解决方案进行评论和投票。2-在代码阻塞时选择调用。select循环可能更简单。如果可以合并stdout/stderr，那么。3-您是否在询问如何在任意子流程中取消缓冲内部缓冲区？这在一般情况下是不可能的。虽然-u适用于pythonurelated：要将envvar传递给子级，请使用Popen…，env=dictos.environ，ANSIBLE\u FORCE\u COLOR='true'，pythonunbuffer='1'

import logging, subprocess
logging.basicConfig(level=logging.INFO)
proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
cont = True
while cont:
    cont = False
    line = proc.stdout.readline()
    if not line == b"":
        out = line.decode("utf-8").rstrip()
        logging.info(out)
        cont = True

    line = proc.stderr.readline()
    if not line == b"":
        out = line.decode("utf-8").rstrip()
        logging.error(out)
        cont = True

    if not cont and proc.poll() is not None:
        break