[owners] Limit number of concurrent Gerrit connections

R=gavinmak@google.com

Fixed: 319469729
Change-Id: I955fa17c80cee25bc2163a8bca0234cdbb659f62
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5186760
Auto-Submit: Josip Sokcevic <sokcevic@chromium.org>
Commit-Queue: Gavin Mak <gavinmak@google.com>
Reviewed-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>

gerrit_util.py

@@ -49,6 +49,9 @@ MIN_BACKOFF = 1.8
 # This is parameterized primarily to enable GerritTestCase.
 GERRIT_PROTOCOL = 'https'
 
+# Controls how many concurrent Gerrit connections there can be.
+MAX_CONCURRENT_CONNECTION = 20
+
 
 def time_sleep(seconds):
     # Use this so that it can be mocked in tests without interfering with python

git_cl.py

@@ -81,7 +81,6 @@ GIT_HASH_RE = re.compile(r'\b([a-f0-9]{6})[a-f0-9]{34}\b', flags=re.I)
 GITCOOKIES_REDACT_RE = re.compile(r'1/.*')
 
 MAX_ATTEMPTS = 3
-MAX_CONCURRENT_CONNECTION = 20
 
 # The maximum number of traces we will keep. Multiplied by 3 since we store
 # 3 files per trace.
@@ -3928,7 +3927,7 @@ def get_cl_statuses(changes, fine_grained, max_processes=None):
 
     If max_processes is specified, it is used as the maximum number of processes
     to spawn to fetch CL status from the server. Otherwise 1 process per branch
-    is spawned, up to max of MAX_CONCURRENT_CONNECTION.
+    is spawned, up to max of gerrit_util.MAX_CONCURRENT_CONNECTION.
 
     See GetStatus() for a list of possible statuses.
     """
@@ -3957,7 +3956,7 @@ def get_cl_statuses(changes, fine_grained, max_processes=None):
                               cl.GetIssue())
             raise
 
-    threads_count = min(MAX_CONCURRENT_CONNECTION, len(changes))
+    threads_count = min(gerrit_util.MAX_CONCURRENT_CONNECTION, len(changes))
     if max_processes:
         threads_count = max(1, min(threads_count, max_processes))
     logging.debug('querying %d CLs using %d threads', len(changes),

owners_client.py

@@ -48,7 +48,10 @@ class OwnersClient(object):
 
         Returns a dictionary {path: [owners]}.
         """
-        with git_common.ScopedPool(kind='threads') as pool:
+        if not paths:
+            return dict()
+        nproc = min(gerrit_util.MAX_CONCURRENT_CONNECTION, len(paths))
+        with git_common.ScopedPool(nproc, kind='threads') as pool:
             return dict(
                 pool.imap_unordered(lambda p: (p, self.ListOwners(p)), paths))