diff --git a/gclient.py b/gclient.py index 7e6b878fc..d7f02091c 100644 --- a/gclient.py +++ b/gclient.py @@ -1197,8 +1197,8 @@ def Main(argv): options.entries_filename = options.config_filename + '_entries' if options.jobs < 1: parser.error('--jobs must be 1 or higher') - # Useful for --jobs. - options.stdout = sys.stdout + # Always autoflush so buildbot doesn't kill us during lengthy operations. + options.stdout = gclient_utils.StdoutAutoFlush(sys.stdout) # These hacks need to die. if not hasattr(options, 'revisions'): diff --git a/gclient_utils.py b/gclient_utils.py index d21ade51b..9d14bd255 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -285,6 +285,30 @@ def CheckCallAndFilterAndHeader(args, always=False, **kwargs): return CheckCallAndFilter(args, **kwargs) +class StdoutAutoFlush(object): + """Automatically flush after N seconds.""" + def __init__(self, stdout, delay=10): + self.lock = threading.Lock() + self.stdout = stdout + self.delay = delay + self.last_flushed_at = time.time() + self.stdout.flush() + + def write(self, out): + """Thread-safe.""" + self.stdout.write(out) + should_flush = False + with self.lock: + if (time.time() - self.last_flushed_at) > self.delay: + should_flush = True + self.last_flushed_at = time.time() + if should_flush: + self.stdout.flush() + + def flush(self): + self.stdout.flush() + + def CheckCallAndFilter(args, stdout=None, filter_fn=None, print_stdout=None, call_filter_on_first_line=False, **kwargs): @@ -308,7 +332,6 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None, **kwargs) # Do a flush of stdout before we begin reading from the subprocess's stdout - last_flushed_at = time.time() stdout.flush() # Also, we need to forward stdout to prevent weird re-ordering of output. @@ -329,12 +352,6 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None, else: filter_fn(in_line) in_line = '' - # Flush at least 10 seconds between line writes. We wait at least 10 - # seconds to avoid overloading the reader that called us with output, - # which can slow busy readers down. - if (time.time() - last_flushed_at) > 10: - last_flushed_at = time.time() - stdout.flush() in_byte = kid.stdout.read(1) # Flush the rest of buffered output. This is only an issue with # stdout/stderr not ending with a \n. diff --git a/tests/gclient_utils_test.py b/tests/gclient_utils_test.py index 4b230cb21..1af302153 100755 --- a/tests/gclient_utils_test.py +++ b/tests/gclient_utils_test.py @@ -28,7 +28,7 @@ class GclientUtilsUnittest(GclientUtilBase): 'GetGClientRootAndEntries', 'GetNamedNodeText', 'GetNodeNamedAttributeText', 'PathDifference', 'ParseXML', 'Popen', 'PrintableObject', 'RemoveDirectory', 'SplitUrlRevision', - 'SyntaxErrorToError', 'WorkItem', + 'StdoutAutoFlush', 'SyntaxErrorToError', 'WorkItem', 'errno', 'logging', 'os', 're', 'stat', 'subprocess', 'sys', 'threading', 'time', 'xml', ]