[scm] Implement caching for git config

Bug: 1501984
Change-Id: I751a1f08eb9ae7141b8b4e1517731ed553328c03
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5252497
Commit-Queue: Aravind Vasudevan <aravindvasudev@google.com>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>

scm.py

@@ -3,12 +3,14 @@
 # found in the LICENSE file.
 """SCM-specific utility classes."""
 
+from collections import defaultdict
 import glob
 import io
 import os
 import platform
 import re
 import sys
+from typing import Mapping, List
 
 import gclient_utils
 import subprocess2
@@ -98,6 +100,47 @@ class GIT(object):
     current_version = None
     rev_parse_cache = {}
 
+    # Maps cwd -> {config key, [config values]}
+    # This cache speeds up all `git config ...` operations by only running a
+    # single subcommand, which can greatly accelerate things like
+    # git-map-branches.
+    _CONFIG_CACHE: Mapping[str, Mapping[str, List[str]]] = {}
+
+    @staticmethod
+    def _load_config(cwd: str) -> Mapping[str, List[str]]:
+        """Loads git config for the given cwd.
+
+        The calls to this method are cached in-memory for performance. The
+        config is only reloaded on cache misses.
+
+        Args:
+            cwd: path to fetch `git config` for.
+
+        Returns:
+            A dict mapping git config keys to a list of its values.
+        """
+        if cwd not in GIT._CONFIG_CACHE:
+            try:
+                rawConfig = GIT.Capture(['config', '--list'],
+                                        cwd=cwd,
+                                        strip_out=False)
+            except subprocess2.CalledProcessError:
+                return {}
+
+            cfg = defaultdict(list)
+            for line in rawConfig.splitlines():
+                key, value = map(str.strip, line.split('=', 1))
+                cfg[key].append(value)
+
+            GIT._CONFIG_CACHE[cwd] = cfg
+
+        return GIT._CONFIG_CACHE[cwd]
+
+    @staticmethod
+    def _clear_config(cwd: str) -> None:
+        GIT._CONFIG_CACHE.pop(cwd, None)
+
+
     @staticmethod
     def ApplyEnvVars(kwargs):
         env = kwargs.pop('env', None) or os.environ.copy()
@@ -167,11 +210,29 @@ class GIT(object):
 
     @staticmethod
     def GetConfig(cwd, key, default=None):
-        try:
+        values = GIT._load_config(cwd).get(key, None)
-            return GIT.Capture(['config', key], cwd=cwd)
+        if not values:
-        except subprocess2.CalledProcessError:
             return default
 
+        return values[-1]
+
+    @staticmethod
+    def GetConfigBool(cwd, key):
+        return GIT.GetConfig(cwd, key) == 'true'
+
+    @staticmethod
+    def GetConfigList(cwd, key):
+        return GIT._load_config(cwd).get(key, [])
+
+    @staticmethod
+    def YieldConfigRegexp(cwd, pattern):
+        """Yields (key, value) pairs for any config keys matching `pattern`."""
+        p = re.compile(pattern)
+        for name, values in GIT._load_config(cwd).items():
+            if p.match(name):
+                for value in values:
+                    yield name, value
+
     @staticmethod
     def GetBranchConfig(cwd, branch, key, default=None):
         assert branch, 'A branch must be given'
@@ -179,11 +240,42 @@ class GIT(object):
         return GIT.GetConfig(cwd, key, default)
 
     @staticmethod
-    def SetConfig(cwd, key, value=None):
+    def SetConfig(cwd,
-        if value is None:
+                  key,
-            args = ['config', '--unset', key]
+                  value=None,
+                  *,
+                  value_pattern=None,
+                  modify_all=False,
+                  scope='local'):
+        """Sets or unsets one or more config values.
+
+        Args:
+            cwd: path to fetch `git config` for.
+            key: The specific config key to affect.
+            value: The value to set. If this is None, `key` will be unset.
+            value_pattern: For use with `all=True`, allows further filtering of
+                the set or unset operation based on the currently configured
+                value. Ignored for `all=False`.
+            modify_all: If True, this will change a set operation to
+                `--replace-all`, and will change an unset operation to
+                `--unset-all`.
+            scope: By default this is the local scope, but could be `system`,
+                `global`, or `worktree`, depending on which config scope you
+                want to affect.
+        """
+        GIT._clear_config(cwd)
+
+        args = ['config', f'--{scope}']
+        if value:
+            if modify_all:
+                args.append('--replace-all')
+
+            args.extend([key, value])
         else:
-            args = ['config', key, value]
+            args.extend(['--unset' + ('-all' if modify_all else ''), key])
+
+        if modify_all and value_pattern:
+            args.append(value_pattern)
         GIT.Capture(args, cwd=cwd)
 
     @staticmethod

tests/scm_unittest.py

@@ -29,10 +29,11 @@ class GitWrapperTestCase(unittest.TestCase):
 
     @mock.patch('scm.GIT.Capture')
     def testGetEmail(self, mockCapture):
-        mockCapture.return_value = 'mini@me.com'
+        mockCapture.return_value = 'user.email = mini@me.com'
         self.assertEqual(scm.GIT.GetEmail(self.root_dir), 'mini@me.com')
-        mockCapture.assert_called_with(['config', 'user.email'],
+        mockCapture.assert_called_with(['config', '--list'],
-                                       cwd=self.root_dir)
+                                       cwd=self.root_dir,
+                                       strip_out=False)
 
     def testRefToRemoteRef(self):
         remote = 'origin'
@@ -211,6 +212,50 @@ class RealGitTest(fake_repos.FakeReposTestBase):
         self.assertEqual('default-value',
                          scm.GIT.GetConfig(self.cwd, key, 'default-value'))
 
+    def testGetSetConfigBool(self):
+        key = 'scm.test-key'
+        self.assertFalse(scm.GIT.GetConfigBool(self.cwd, key))
+
+        scm.GIT.SetConfig(self.cwd, key, 'true')
+        self.assertTrue(scm.GIT.GetConfigBool(self.cwd, key))
+
+        scm.GIT.SetConfig(self.cwd, key)
+        self.assertFalse(scm.GIT.GetConfigBool(self.cwd, key))
+
+    def testGetSetConfigList(self):
+        key = 'scm.test-key'
+        self.assertListEqual([], scm.GIT.GetConfigList(self.cwd, key))
+
+        scm.GIT.SetConfig(self.cwd, key, 'foo')
+        scm.GIT.Capture(['config', '--add', key, 'bar'], cwd=self.cwd)
+        self.assertListEqual(['foo', 'bar'],
+                             scm.GIT.GetConfigList(self.cwd, key))
+
+        scm.GIT.SetConfig(self.cwd, key, modify_all=True, value_pattern='^f')
+        self.assertListEqual(['bar'], scm.GIT.GetConfigList(self.cwd, key))
+
+        scm.GIT.SetConfig(self.cwd, key)
+        self.assertListEqual([], scm.GIT.GetConfigList(self.cwd, key))
+
+    def testYieldConfigRegexp(self):
+        key1 = 'scm.aaa'
+        key2 = 'scm.aaab'
+
+        config = scm.GIT.YieldConfigRegexp(self.cwd, key1)
+        with self.assertRaises(StopIteration):
+            next(config)
+
+        scm.GIT.SetConfig(self.cwd, key1, 'foo')
+        scm.GIT.SetConfig(self.cwd, key2, 'bar')
+        scm.GIT.Capture(['config', '--add', key2, 'baz'], cwd=self.cwd)
+
+        config = scm.GIT.YieldConfigRegexp(self.cwd, '^scm\\.aaa')
+        self.assertEqual((key1, 'foo'), next(config))
+        self.assertEqual((key2, 'bar'), next(config))
+        self.assertEqual((key2, 'baz'), next(config))
+        with self.assertRaises(StopIteration):
+            next(config)
+
     def testGetSetBranchConfig(self):
         branch = scm.GIT.GetBranch(self.cwd)
         key = 'scm.test-key'