From 28be1ff85450d22df9cf709b9fa501e057306235 Mon Sep 17 00:00:00 2001 From: Alex Drexler Date: Wed, 25 Mar 2020 23:11:50 +0000 Subject: [PATCH] Use console url for recursive gsutil operations. The storage.* links do not work unless an individual object is specified. This isn't the case for recursive upload/copy operations. When uploading multiple objects recursively, we should use the console link instead. Bug: None Test: Updated recipe expectations. Recipe-Nontrivial-Roll: build Change-Id: I511f04f367f6c20c538b3c2093a40807323d7b75 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2118421 Reviewed-by: Robbie Iannucci Commit-Queue: Alex Drexler --- recipes/README.recipes.md | 16 ++++---- recipes/recipe_modules/gsutil/api.py | 22 +++++++---- .../gsutil/examples/full.expected/basic.json | 38 +++++++++++++++++++ .../recipe_modules/gsutil/examples/full.py | 14 +++++++ 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index dc68b077a5..74eba7b2a9 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -656,23 +656,23 @@ Arguments: * name (str) - Name of the step to use. Defaults to the first non-flag token in the cmd. -— **def [cat](/recipes/recipe_modules/gsutil/api.py#108)(self, url, args=None, \*\*kwargs):** +— **def [cat](/recipes/recipe_modules/gsutil/api.py#109)(self, url, args=None, \*\*kwargs):** -— **def [copy](/recipes/recipe_modules/gsutil/api.py#122)(self, source_bucket, source, dest_bucket, dest, args=None, link_name='gsutil.copy', metadata=None, unauthenticated_url=False, \*\*kwargs):** +— **def [copy](/recipes/recipe_modules/gsutil/api.py#123)(self, source_bucket, source, dest_bucket, dest, args=None, link_name='gsutil.copy', metadata=None, unauthenticated_url=False, \*\*kwargs):** -— **def [download](/recipes/recipe_modules/gsutil/api.py#94)(self, bucket, source, dest, args=None, \*\*kwargs):** +— **def [download](/recipes/recipe_modules/gsutil/api.py#95)(self, bucket, source, dest, args=None, \*\*kwargs):** -— **def [download\_url](/recipes/recipe_modules/gsutil/api.py#101)(self, url, dest, args=None, \*\*kwargs):** +— **def [download\_url](/recipes/recipe_modules/gsutil/api.py#102)(self, url, dest, args=None, \*\*kwargs):**   **@property**
— **def [gsutil\_py\_path](/recipes/recipe_modules/gsutil/api.py#10)(self):** -— **def [list](/recipes/recipe_modules/gsutil/api.py#139)(self, url, args=None, \*\*kwargs):** +— **def [list](/recipes/recipe_modules/gsutil/api.py#142)(self, url, args=None, \*\*kwargs):** -— **def [remove\_url](/recipes/recipe_modules/gsutil/api.py#153)(self, url, args=None, \*\*kwargs):** +— **def [remove\_url](/recipes/recipe_modules/gsutil/api.py#156)(self, url, args=None, \*\*kwargs):** -— **def [signurl](/recipes/recipe_modules/gsutil/api.py#146)(self, private_key_file, bucket, dest, args=None, \*\*kwargs):** +— **def [signurl](/recipes/recipe_modules/gsutil/api.py#149)(self, private_key_file, bucket, dest, args=None, \*\*kwargs):** -— **def [stat](/recipes/recipe_modules/gsutil/api.py#115)(self, url, args=None, \*\*kwargs):** +— **def [stat](/recipes/recipe_modules/gsutil/api.py#116)(self, url, args=None, \*\*kwargs):** — **def [upload](/recipes/recipe_modules/gsutil/api.py#78)(self, source, bucket, dest, args=None, link_name='gsutil.upload', metadata=None, unauthenticated_url=False, \*\*kwargs):** ### *recipe_modules* / [infra\_paths](/recipes/recipe_modules/infra_paths) diff --git a/recipes/recipe_modules/gsutil/api.py b/recipes/recipe_modules/gsutil/api.py index c222290a8b..0c4c47199e 100644 --- a/recipes/recipe_modules/gsutil/api.py +++ b/recipes/recipe_modules/gsutil/api.py @@ -87,8 +87,9 @@ class GSUtilApi(recipe_api.RecipeApi): result = self(cmd, name, **kwargs) if link_name: + is_dir = '-r' in args or '--recursive' in args result.presentation.links[link_name] = self._http_url( - bucket, dest, unauthenticated_url=unauthenticated_url) + bucket, dest, is_directory=is_dir, is_anonymous=unauthenticated_url) return result def download(self, bucket, source, dest, args=None, **kwargs): @@ -132,8 +133,10 @@ class GSUtilApi(recipe_api.RecipeApi): result = self(cmd, name, **kwargs) if link_name: + is_dir = '-r' in args or '--recursive' in args result.presentation.links[link_name] = self._http_url( - dest_bucket, dest, unauthenticated_url=unauthenticated_url) + dest_bucket, dest, is_directory=is_dir, + is_anonymous=unauthenticated_url) return result def list(self, url, args=None, **kwargs): @@ -179,12 +182,17 @@ class GSUtilApi(recipe_api.RecipeApi): raise AssertionError("%s cannot be normalized" % url) @classmethod - def _http_url(cls, bucket, dest, unauthenticated_url=False): - if unauthenticated_url: - base = 'https://storage.googleapis.com/%s/%s' + def _http_url(cls, bucket, dest, is_directory=False, is_anonymous=False): + if is_directory: + # Use GCP console. + url_template = 'https://console.cloud.google.com/storage/browser/%s/%s' + elif is_anonymous: + # Use unauthenticated object viewer. + url_template = 'https://storage.googleapis.com/%s/%s' else: - base = 'https://storage.cloud.google.com/%s/%s' - return base % (bucket, dest) + # Use authenticated object viewer. + url_template = 'https://storage.cloud.google.com/%s/%s' + return url_template % (bucket, dest) @staticmethod def _get_metadata_field(name, provider_prefix=None): diff --git a/recipes/recipe_modules/gsutil/examples/full.expected/basic.json b/recipes/recipe_modules/gsutil/examples/full.expected/basic.json index 0511da7456..4dd1e107c8 100644 --- a/recipes/recipe_modules/gsutil/examples/full.expected/basic.json +++ b/recipes/recipe_modules/gsutil/examples/full.expected/basic.json @@ -52,6 +52,44 @@ "@@@STEP_LINK@gsutil.upload@https://storage.googleapis.com/example/some/random/path/to/boom@@@" ] }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[TMP_BASE]", + "gs://example/some/random/path" + ], + "infra_step": true, + "name": "gsutil upload -r", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/example/some/random/path@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "--recursive", + "[TMP_BASE]", + "gs://example/some/other/random/path" + ], + "infra_step": true, + "name": "gsutil upload --recursive", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/example/some/other/random/path@@@" + ] + }, { "cmd": [ "python", diff --git a/recipes/recipe_modules/gsutil/examples/full.py b/recipes/recipe_modules/gsutil/examples/full.py index 2e2279d2df..558b36e208 100644 --- a/recipes/recipe_modules/gsutil/examples/full.py +++ b/recipes/recipe_modules/gsutil/examples/full.py @@ -36,6 +36,20 @@ def RunSteps(api): multithreaded=True, use_retry_wrapper=False) + # Upload directory contents. + api.gsutil.upload( + api.path['tmp_base'], + bucket, + 'some/random/path', + args=['-r'], + name='upload -r') + api.gsutil.upload( + api.path['tmp_base'], + bucket, + 'some/other/random/path', + args=['--recursive'], + name='upload --recursive') + api.gsutil(['cp', 'gs://%s/some/random/path/**' % bucket, 'gs://%s/staging' % bucket])